diff --git a/src/cache/store.zig b/src/cache/store.zig index b32d3c6..66b6fa7 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -67,6 +67,17 @@ pub const Ttl = struct { /// month. pub const tickers_funds: i64 = 30 * s_per_day; pub const tickers_companies: i64 = 30 * s_per_day; + + /// How long a genuine Tiingo 404 suppresses further Tiingo + /// attempts for a symbol (see `CandleMeta.tiingo_retry_after_s`). + /// + /// Long enough that a symbol Tiingo genuinely does not carry + /// costs about one wasted 404 per month, short enough that + /// coverage changes get picked up without operator action. + /// Call sites layer `jitter_pct` on top so a batch of symbols + /// demoted in the same window does not all re-probe on the same + /// day. + pub const tiingo_backoff: i64 = 30 * s_per_day; }; /// Cache TTL specification with optional per-key expiration jitter. @@ -894,7 +905,10 @@ pub const Store = struct { /// candle metadata, stamping its `#!expires=` with `expires_at_s` /// (the caller computes the market-aware freshness boundary; see /// `market.nextCandleExpiry`). - pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void { + /// + /// `attrs` supplies the provider-state fields; `last_close` and + /// `last_date` are derived from the newest candle written. + pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, attrs: CandleMetaAttrs, expires_at_s: i64) void { if (serializeCandles(self.allocator, candles, .{})) |srf_data| { defer self.allocator.free(srf_data); self.writeRaw(symbol, .candles_daily, srf_data) catch |err| { @@ -906,7 +920,13 @@ pub const Store = struct { if (candles.len > 0) { const last = candles[candles.len - 1]; - self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s); + self.updateCandleMeta(symbol, .{ + .last_close = last.close, + .last_date = last.date, + .provider = attrs.provider, + .fail_count = attrs.fail_count, + .tiingo_retry_after_s = attrs.tiingo_retry_after_s, + }, expires_at_s); } } @@ -914,7 +934,14 @@ pub const Store = struct { /// Falls back to a full rewrite if append fails (e.g. file doesn't exist). /// Also updates candle metadata, stamping its `#!expires=` with /// `expires_at_s` (caller-computed market-aware boundary). - pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void { + /// + /// `meta` supplies every metadata field except `last_close` and + /// `last_date`, which are derived from the newest appended candle. + /// Callers are expected to pass the symbol's *existing* meta so + /// that fields describing the series as a whole survive an append + /// unchanged - appending bars does not re-derive anything about + /// the rows already on disk. + pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, meta: CandleMeta, expires_at_s: i64) void { if (new_candles.len == 0) return; if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| { @@ -943,23 +970,25 @@ pub const Store = struct { } const last = new_candles[new_candles.len - 1]; - self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s); + var next = meta; + next.last_close = last.close; + next.last_date = last.date; + self.updateCandleMeta(symbol, next, expires_at_s); } - /// Write (or refresh) candle metadata with a specific provider source. + /// Write (or refresh) candle metadata. /// /// `expires_at_s` is the absolute Unix-seconds freshness boundary for /// the `#!expires=` directive, computed by the caller via /// `market.nextCandleExpiry` (close-anchored) or a short retry. The /// cache layer no longer reads the wall clock for this - the boundary /// is market-domain knowledge owned by the caller. - pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void { - const meta = CandleMeta{ - .last_close = last_close, - .last_date = last_date, - .provider = provider, - .fail_count = fail_count, - }; + /// + /// Takes the whole `CandleMeta` rather than a field-per-parameter + /// list so that adding a metadata field does not churn every call + /// site. Callers that mean "same metadata, one field changed" + /// should copy the value they read and mutate the one field. + pub fn updateCandleMeta(self: *Store, symbol: []const u8, meta: CandleMeta, expires_at_s: i64) void { if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires_at_s })) |meta_data| { defer self.allocator.free(meta_data); self.writeRaw(symbol, .candles_meta, meta_data) catch |err| { @@ -1348,13 +1377,20 @@ pub const Store = struct { pub const CandleMeta = struct { last_close: f64, last_date: Date, - /// Which provider sourced the candle data. **No default - /// value on purpose** - SRF auto-elides fields whose value - /// equals their default, which would hide the provider line - /// when it equaled the implicit default. We want every cache - /// file to record its provider explicitly so cache inspection - /// can always answer "where did this come from?". Construction - /// sites must pass the provider explicitly. + /// Which provider sourced the candle data. Pure provenance - + /// this field answers "where did these bars come from?" and + /// nothing else. It deliberately does **not** drive provider + /// routing; see `tiingo_retry_after_s` for that. Conflating + /// the two is what produced the one-way drift to Yahoo that + /// this field's value used to cause. + /// + /// **No default value on purpose** - SRF auto-elides fields + /// whose value equals their default, which would hide the + /// provider line when it equaled the implicit default. We + /// want every cache file to record its provider explicitly + /// so cache inspection can always answer "where did this + /// come from?". Construction sites must pass the provider + /// explicitly. /// /// Cache compatibility: pre-2026-05 caches that elided the /// provider field will fail to deserialize after this change @@ -1369,6 +1405,44 @@ pub const Store = struct { /// Incremented on ServerError; reset to 0 on success. When >= 3, the /// symbol is degraded to a fallback provider until Tiingo recovers. fail_count: u8 = 0, + /// Unix-seconds instant before which Tiingo should not be + /// consulted for this symbol. `0` (the default) means "no + /// backoff - always try Tiingo first". + /// + /// Set **only** when Tiingo returns a genuine 404, which is a + /// fact about coverage rather than about the request. A 400 / + /// 402 / malformed body falls back to Yahoo for that one call + /// but leaves this field alone, so the next call retries + /// Tiingo. Cleared back to `0` the moment Tiingo serves the + /// symbol again. + /// + /// Why this exists: routing used to key off `provider == + /// .yahoo`, which a single non-transient Tiingo failure would + /// latch permanently - Yahoo was then tried first, succeeded, + /// and rewrote `provider = .yahoo`, so Tiingo was never + /// consulted again. An explicit, expiring signal makes the + /// demotion recoverable and records *why* it happened. + /// + /// Defaulted so legacy caches (which lack the field) parse + /// cleanly and simply behave as "no backoff". + tiingo_retry_after_s: i64 = 0, + }; + + /// The subset of `CandleMeta` that describes provider state rather + /// than the candle data itself. + /// + /// `cacheCandles` takes this instead of a whole `CandleMeta` + /// because a full fetch may be a cold start with no prior metadata + /// to copy - there is no honest `last_close` / `last_date` for the + /// caller to supply, and those get derived from the candles being + /// written anyway. `appendCandles` by contrast always has the + /// existing meta in hand (it read `last_date` to decide what to + /// fetch), so it takes the full struct and preserves everything it + /// does not derive. + pub const CandleMetaAttrs = struct { + provider: CandleProvider, + fail_count: u8 = 0, + tiingo_retry_after_s: i64 = 0, }; pub const CandleProvider = enum { @@ -3585,6 +3659,107 @@ test "deserializeCandleMeta fails on old cache that elided provider field" { try std.testing.expectError(error.InvalidData, result); } +test "CandleMeta.tiingo_retry_after_s defaults to 0 and is elided when unset" { + // The field is defaulted precisely so caches written before it + // existed keep parsing. SRF elides default-valued fields, so an + // unset backoff costs no bytes and reads back as "no backoff". + const allocator = std.testing.allocator; + const meta = Store.CandleMeta{ + .last_close = 100.0, + .last_date = Date.fromYmd(2026, 8, 14), + .provider = .tiingo, + }; + try std.testing.expectEqual(@as(i64, 0), meta.tiingo_retry_after_s); + + const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 }); + defer allocator.free(data); + try std.testing.expect(std.mem.indexOf(u8, data, "tiingo_retry_after_s") == null); +} + +test "CandleMeta.tiingo_retry_after_s round-trips when armed" { + const allocator = std.testing.allocator; + const meta = Store.CandleMeta{ + .last_close = 100.0, + .last_date = Date.fromYmd(2026, 8, 14), + .provider = .yahoo, + .tiingo_retry_after_s = 1_790_000_000, + }; + const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 }); + defer allocator.free(data); + try std.testing.expect(std.mem.indexOf(u8, data, "tiingo_retry_after_s:num:1790000000") != null); + + const parsed = try Store.deserializeCandleMeta(allocator, data); + try std.testing.expectEqual(@as(i64, 1_790_000_000), parsed.tiingo_retry_after_s); + try std.testing.expectEqual(Store.CandleProvider.yahoo, parsed.provider); +} + +test "legacy candles_meta without tiingo_retry_after_s still parses" { + // The whole migration story for this field: unlike `provider` + // (which is deliberately default-less and therefore wipes old + // caches), a defaulted field must NOT break existing caches. If + // this ever regresses, every cached symbol takes a full re-fetch + // on first use - and for symbols Tiingo does not carry, the + // cold-start path would write a negative-cache marker over a + // perfectly good candle history. + const allocator = std.testing.allocator; + const legacy = + \\#!srfv1 + \\#!expires=1787000100 + \\#!created=1786748010 + \\last_close:num:776.34,last_date::2026-08-14,provider::yahoo + \\ + ; + const parsed = try Store.deserializeCandleMeta(allocator, legacy); + try std.testing.expectEqual(@as(i64, 0), parsed.tiingo_retry_after_s); + try std.testing.expectEqual(Store.CandleProvider.yahoo, parsed.provider); + try std.testing.expect(parsed.last_date.eql(Date.fromYmd(2026, 8, 14))); +} + +test "appendCandles preserves caller-supplied provider state" { + // appendCandles derives only last_close/last_date; every other + // metadata field must pass through untouched. Commit-2's + // `adj_basis` depends on this exact property, so pin it now. + const io = std.testing.io; + const allocator = std.testing.allocator; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var store = Store.init(io, allocator, dir_path); + const seed = [_]Candle{.{ + .date = Date.fromYmd(2026, 8, 13), + .open = 1, + .high = 1, + .low = 1, + .close = 1, + .adj_close = 1, + .volume = 1, + }}; + store.cacheCandles("SMPL", seed[0..], .{ .provider = .yahoo, .tiingo_retry_after_s = 1_790_000_000 }, 9_999_999_999); + + const before = store.readCandleMeta("SMPL") orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(@as(i64, 1_790_000_000), before.meta.tiingo_retry_after_s); + + const more = [_]Candle{.{ + .date = Date.fromYmd(2026, 8, 14), + .open = 2, + .high = 2, + .low = 2, + .close = 2, + .adj_close = 2, + .volume = 2, + }}; + store.appendCandles("SMPL", more[0..], before.meta, 9_999_999_999); + + const after = store.readCandleMeta("SMPL") orelse return error.TestUnexpectedResult; + try std.testing.expectEqual(@as(i64, 1_790_000_000), after.meta.tiingo_retry_after_s); + try std.testing.expectEqual(Store.CandleProvider.yahoo, after.meta.provider); + // ...while the derived fields did advance. + try std.testing.expect(after.meta.last_date.eql(Date.fromYmd(2026, 8, 14))); + try std.testing.expectApproxEqAbs(@as(f64, 2), after.meta.last_close, 0.001); +} + // ── writeRaw / appendRaw atomicity ─────────────────────────── // // A concurrent reader hitting a cache file mid-write must never see a diff --git a/src/commands/diagnose.zig b/src/commands/diagnose.zig index 30e95ae..5bef635 100644 --- a/src/commands/diagnose.zig +++ b/src/commands/diagnose.zig @@ -239,6 +239,14 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void { try out.print(", {s}", .{if (obs.local_fresh) "TTL still in the future" else "TTL lapsed"}); try out.print(", {s}", .{@tagName(m.meta.provider)}); if (m.meta.fail_count > 0) try out.print(", {d} consecutive failures", .{m.meta.fail_count}); + // A live Tiingo backoff means this symbol is being served by + // Yahoo on purpose (Tiingo 404'd it), not by accident. Surface + // it so a provider that looks "wrong" can be explained. + if (m.meta.tiingo_retry_after_s > ctx.now_s) { + try out.print(", Tiingo backoff until {f}", .{Date.fromEpoch(m.meta.tiingo_retry_after_s)}); + } else if (m.meta.tiingo_retry_after_s != 0) { + try out.print(", Tiingo backoff lapsed (retries next refresh)", .{}); + } try out.print("\n", .{}); } else { try out.print("local nothing cached\n", .{}); diff --git a/src/service.zig b/src/service.zig index 2666c39..2d401fc 100644 --- a/src/service.zig +++ b/src/service.zig @@ -133,6 +133,17 @@ pub fn isPermanentProviderFailure(err: anyerror) bool { return err == error.NotFound; } +/// Spread applied to `Ttl.tiingo_backoff` when a symbol is demoted off +/// Tiingo by a 404. +/// +/// Policy lives here rather than in `TtlSpec` (see its doc comment). +/// 7% of 30 days is roughly +/-2 days, so a batch of symbols demoted in +/// the same window re-probes across five distinct days instead of all +/// on one. Sized against the cron cadence: with the refresh running +/// twice a day, five days of spread keeps the re-probe burst to a +/// handful of extra 404s per run. +const tiingo_backoff_jitter_pct: u8 = 7; + /// Result of a CUSIP-to-ticker lookup (provider-agnostic). pub const CusipResult = OpenFigi.FigiResult; @@ -652,7 +663,7 @@ pub const DataService = struct { // (market-aware next post-close / NAV-availability time). const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds(); const kind = market.classify(symbol); - s.cacheCandles(symbol, triple.candles, .tiingo, 0, expiryAfterFetch(now_s, kind, triple.candles)); + s.cacheCandles(symbol, triple.candles, .{ .provider = .tiingo }, expiryAfterFetch(now_s, kind, triple.candles)); } // Dividends and splits use the supplement write path: Tiingo's // view merges into existing (typically Polygon-sourced) records @@ -678,6 +689,73 @@ pub const DataService = struct { // ── Public data methods ────────────────────────────────────── + /// What a candle fetch learned about Tiingo's coverage of a symbol. + /// + /// This is deliberately three-valued. The old code collapsed + /// "Tiingo does not carry this symbol" and "Tiingo failed this + /// request" into a single permanent demotion to Yahoo, which is + /// how 22 of 32 cached symbols drifted off Tiingo. Only + /// `.not_found` is a statement about the symbol; everything else + /// is a statement about one HTTP call and must not be remembered. + pub const TiingoCoverage = enum { + /// Tiingo served the request - it definitely covers this symbol. + covered, + /// Tiingo returned a genuine 404 - it does not carry this symbol. + not_found, + /// Tiingo was not consulted, or failed for a reason that says + /// nothing about coverage (active backoff, no API key, 400, + /// 402, malformed body). Nothing new learned; leave any + /// existing backoff exactly as it was. + unknown, + }; + + /// Fold a fetch's outcome into a symbol's candle metadata. + /// + /// Called on any *successful* candle fetch, so `fail_count` resets + /// to 0. The interesting part is `tiingo_retry_after_s`: + /// + /// - `.covered` -> clear the backoff. Tiingo just served this + /// symbol, so any prior 404 is stale news. This + /// is what walks a previously-demoted symbol + /// back onto Tiingo. + /// - `.not_found` -> arm the backoff at `Ttl.tiingo_backoff` with + /// per-symbol jitter, so a batch demoted in the + /// same window does not all re-probe on one day. + /// - `.unknown` -> leave it untouched. Either Tiingo was never + /// asked, or it failed for a reason that says + /// nothing about coverage. + fn applyTiingoCoverage( + meta: cache.Store.CandleMeta, + symbol: []const u8, + now_s: i64, + provider: cache.Store.CandleProvider, + coverage: TiingoCoverage, + ) cache.Store.CandleMeta { + var next = meta; + next.provider = provider; + next.fail_count = 0; + + switch (coverage) { + .covered => { + if (meta.tiingo_retry_after_s != 0) { + log.info("{s}: provider converted {t} -> tiingo, clearing Tiingo backoff", .{ symbol, meta.provider }); + } + next.tiingo_retry_after_s = 0; + }, + .not_found => { + next.tiingo_retry_after_s = cache.computeExpires( + now_s, + .{ .seconds = cache.Ttl.tiingo_backoff, .jitter_pct = tiingo_backoff_jitter_pct }, + symbol, + ); + log.info("{s}: Tiingo NotFound, backing off until {f}", .{ symbol, Date.fromEpoch(next.tiingo_retry_after_s) }); + }, + .unknown => {}, + } + + return next; + } + /// Fetch candles from providers with error classification. /// /// Error handling: @@ -685,32 +763,42 @@ pub const DataService = struct { /// - NotFound/ParseError/InvalidResponse from Tiingo -> try Yahoo (symbol-level issue) /// - Unauthorized -> TransientError (config problem, stop refresh) /// - /// The `preferred` param controls incremental fetch consistency: use the same - /// provider that sourced the existing cache data. + /// `prefer_yahoo` skips the Tiingo attempt entirely. Callers derive + /// it from `CandleMeta.tiingo_retry_after_s` - i.e. "Tiingo told us + /// 404 recently, don't waste the call". It is NOT derived from + /// which provider sourced the cache; see the `provider` field's + /// doc comment for why that distinction matters. fn fetchCandlesFromProviders( self: *DataService, symbol: []const u8, from: Date, to: Date, - preferred: cache.Store.CandleProvider, - ) (DataError || error{NotFound})!struct { candles: []Candle, provider: cache.Store.CandleProvider } { - // If preferred is Yahoo (degraded symbol), try Yahoo first - if (preferred == .yahoo) { + prefer_yahoo: bool, + ) (DataError || error{NotFound})!struct { + candles: []Candle, + provider: cache.Store.CandleProvider, + tiingo_coverage: TiingoCoverage, + } { + // Under an active Tiingo backoff, go straight to Yahoo. + if (prefer_yahoo) { if (self.getProvider(Yahoo)) |yh| { if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| { - log.debug("{s}: candles from Yahoo (preferred)", .{symbol}); - return .{ .candles = candles, .provider = .yahoo }; + log.debug("{s}: candles from Yahoo (Tiingo backoff active)", .{symbol}); + return .{ .candles = candles, .provider = .yahoo, .tiingo_coverage = .unknown }; } else |err| { - log.warn("{s}: Yahoo (preferred) failed: {s}", .{ symbol, @errorName(err) }); + log.warn("{s}: Yahoo (Tiingo backoff active) failed: {s}", .{ symbol, @errorName(err) }); } } else |_| {} } - // Primary: Tiingo + // Primary: Tiingo. `coverage` accumulates what this attempt + // taught us, so the Yahoo fallback below can report it back + // to the caller without re-deriving it from the error. + var coverage: TiingoCoverage = .unknown; if (self.getProvider(Tiingo)) |tg| { if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| { log.debug("{s}: candles from Tiingo", .{symbol}); - return .{ .candles = candles, .provider = .tiingo }; + return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered }; } else |err| { log.warn("{s}: Tiingo failed: {s}", .{ symbol, @errorName(err) }); @@ -725,7 +813,7 @@ pub const DataService = struct { self.rateLimitBackoff(); if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| { log.debug("{s}: candles from Tiingo (after rate limit backoff)", .{symbol}); - return .{ .candles = candles, .provider = .tiingo }; + return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered }; } else |retry_err| { log.warn("{s}: Tiingo retry after backoff failed: {s}", .{ symbol, @errorName(retry_err) }); if (retry_err == error.RateLimited) { @@ -733,7 +821,7 @@ pub const DataService = struct { self.rateLimitBackoff(); if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| { log.debug("{s}: candles from Tiingo (after second backoff)", .{symbol}); - return .{ .candles = candles, .provider = .tiingo }; + return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered }; } else |_| {} } // Exhausted rate limit retries - treat as transient @@ -746,19 +834,31 @@ pub const DataService = struct { return DataError.TransientError; } - // NotFound, ParseError, InvalidResponse - symbol-level issue, try Yahoo - log.info("{s}: Tiingo does not have this symbol, trying Yahoo", .{symbol}); + // NotFound, ParseError, InvalidResponse - fall back to + // Yahoo for this call. Only a genuine 404 is a + // statement about Tiingo's coverage; a 400 / 402 / + // malformed body says something about this request and + // must not earn a remembered demotion. Mirrors the rule + // `isPermanentProviderFailure` already applies to the + // negative cache. + if (isPermanentProviderFailure(err)) { + coverage = .not_found; + log.info("{s}: Tiingo does not carry this symbol, trying Yahoo", .{symbol}); + } else { + log.info("{s}: Tiingo request failed ({s}) - not a coverage verdict, trying Yahoo for this call only", .{ symbol, @errorName(err) }); + } } } else |_| { log.warn("{s}: Tiingo provider not available (no API key?)", .{symbol}); } - // Fallback: Yahoo (symbol not on Tiingo) - if (preferred != .yahoo) { + // Fallback: Yahoo. Skipped when we already tried Yahoo first + // (active backoff) and it failed - no point asking twice. + if (!prefer_yahoo) { if (self.getProvider(Yahoo)) |yh| { if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| { log.info("{s}: candles from Yahoo (Tiingo fallback)", .{symbol}); - return .{ .candles = candles, .provider = .yahoo }; + return .{ .candles = candles, .provider = .yahoo, .tiingo_coverage = coverage }; } else |err| { log.warn("{s}: Yahoo fallback also failed: {s}", .{ symbol, @errorName(err) }); } @@ -959,18 +1059,20 @@ pub const DataService = struct { // boundary while the lag check reported it lagging (the // Friday-17:00 deadlock that exited 75 every retry). if (!market.shouldRefresh(now_s, kind, m.last_date)) { - s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count, expires); + s.updateCandleMeta(symbol, m, expires); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } else { // Incremental fetch from day after last cached candle self.assertNetworkAllowed("getCandles incremental fetchCandlesFromProviders"); - const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, m.provider) catch |err| { + const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, now_s < m.tiingo_retry_after_s) catch |err| { if (err == DataError.TransientError) { // Increment fail_count for this symbol const new_fail_count = m.fail_count +| 1; // saturating add log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count }); - s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count, now_s + market.short_retry_s); + var degraded = m; + degraded.fail_count = new_fail_count; + s.updateCandleMeta(symbol, degraded, now_s + market.short_retry_s); // If degraded (fail_count >= 3), return stale data rather than failing if (new_fail_count >= 3) { @@ -987,6 +1089,10 @@ pub const DataService = struct { }; const new_candles = result.candles; + // Fold what this fetch learned about Tiingo coverage + // into the metadata we're about to write. + const next_meta = applyTiingoCoverage(m, symbol, now_s, result.provider, result.tiingo_coverage); + if (new_candles.len == 0) { // No new candles. Either a genuine non-trading-day // gap (weekend/holiday), the provider hasn't posted @@ -999,7 +1105,7 @@ pub const DataService = struct { // an un-modeled closure stops thrashing and waits // for the next real session. self.allocator.free(new_candles); - s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0, market.staleCandleExpiry(now_s, kind, m.last_date)); + s.updateCandleMeta(symbol, next_meta, market.staleCandleExpiry(now_s, kind, m.last_date)); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } else { @@ -1007,7 +1113,7 @@ pub const DataService = struct { // TTL via `expiryAfterFetch`, NOT the precomputed // next-boundary `expires`: getting a bar back does not // mean getting the RIGHT bar back. - s.appendCandles(symbol, new_candles, result.provider, 0, expiryAfterFetch(now_s, kind, new_candles)); + s.appendCandles(symbol, new_candles, next_meta, expiryAfterFetch(now_s, kind, new_candles)); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| { self.allocator.free(new_candles); return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; @@ -1050,8 +1156,9 @@ pub const DataService = struct { // Transient: increment fail_count on existing meta so // we know to back off if this keeps happening. if (meta_result) |mr| { - const new_fail_count = mr.meta.fail_count +| 1; - s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count, now_s + market.short_retry_s); + var degraded = mr.meta; + degraded.fail_count = mr.meta.fail_count +| 1; + s.updateCandleMeta(symbol, degraded, now_s + market.short_retry_s); } return DataError.TransientError; } @@ -3791,7 +3898,7 @@ test "getCandles offline mode returns cached data without network" { .{ .date = Date.fromYmd(2026, 5, 19), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 }, .{ .date = Date.fromYmd(2026, 5, 20), .open = 104, .high = 106, .low = 103, .close = 105, .adj_close = 105, .volume = 1100 }, }; - store.cacheCandles("TEST", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); + store.cacheCandles("TEST", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); // Set the test guard: any network call would panic. We expect // the offline-mode path NOT to touch the network. @@ -3946,7 +4053,7 @@ test "loadAllPrices offline mode skips network and returns cached" { var fresh_candles = [_]Candle{ .{ .date = Date.fromYmd(2026, 5, 20), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 }, }; - store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); + store.cacheCandles("FRESH", fresh_candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); // Symbol with no cache at all. // (no setup needed - just passes a symbol that doesn't exist) @@ -3972,6 +4079,118 @@ test "loadAllPrices offline mode skips network and returns cached" { try std.testing.expectEqual(@as(usize, 1), result.failed_count); } +// ── Tiingo coverage bookkeeping ────────────────────────────── +// +// Regression suite for the one-way drift to Yahoo. The old code +// routed on `CandleMeta.provider == .yahoo`, so any non-transient +// Tiingo failure latched permanently: Yahoo was tried first, +// succeeded, rewrote `provider = .yahoo`, and Tiingo was never asked +// again. 22 of 32 cached symbols had drifted off Tiingo that way. + +test "applyTiingoCoverage: .covered clears an armed Tiingo backoff" { + const armed = cache.Store.CandleMeta{ + .last_close = 100, + .last_date = Date.fromYmd(2026, 8, 14), + .provider = .yahoo, + .fail_count = 2, + .tiingo_retry_after_s = 1_790_000_000, + }; + const next = DataService.applyTiingoCoverage(armed, "SMPL", 1_787_000_000, .tiingo, .covered); + + // Tiingo just served the symbol, so the prior 404 is stale news. + try std.testing.expectEqual(@as(i64, 0), next.tiingo_retry_after_s); + try std.testing.expectEqual(cache.Store.CandleProvider.tiingo, next.provider); + // A successful fetch also resets the transient-failure counter. + try std.testing.expectEqual(@as(u8, 0), next.fail_count); +} + +test "applyTiingoCoverage: .not_found arms a jittered backoff ~30 days out" { + const now_s: i64 = 1_787_000_000; + const meta = cache.Store.CandleMeta{ + .last_close = 100, + .last_date = Date.fromYmd(2026, 8, 14), + .provider = .tiingo, + }; + const next = DataService.applyTiingoCoverage(meta, "SMPL", now_s, .yahoo, .not_found); + + // Yahoo answered, so provenance records Yahoo... + try std.testing.expectEqual(cache.Store.CandleProvider.yahoo, next.provider); + // ...and the backoff lands 30 days out, +/- the jitter window. + const base = now_s + cache.Ttl.tiingo_backoff; + const max_offset = @divFloor(cache.Ttl.tiingo_backoff * @as(i64, tiingo_backoff_jitter_pct), 100); + try std.testing.expect(next.tiingo_retry_after_s >= base - max_offset); + try std.testing.expect(next.tiingo_retry_after_s <= base + max_offset); + // The spread must be meaningful but bounded: roughly +/-2 days. + try std.testing.expect(max_offset >= 2 * std.time.s_per_day); + try std.testing.expect(max_offset <= 3 * std.time.s_per_day); +} + +test "applyTiingoCoverage: .unknown leaves the backoff untouched" { + // A 400 / 402 / malformed body says nothing about coverage. It + // must neither arm a backoff (that's the drift bug) nor clear an + // existing one (that would defeat the 404 we already recorded). + const now_s: i64 = 1_787_000_000; + + const unarmed = cache.Store.CandleMeta{ + .last_close = 100, + .last_date = Date.fromYmd(2026, 8, 14), + .provider = .tiingo, + }; + const a = DataService.applyTiingoCoverage(unarmed, "SMPL", now_s, .yahoo, .unknown); + try std.testing.expectEqual(@as(i64, 0), a.tiingo_retry_after_s); + + var armed = unarmed; + armed.tiingo_retry_after_s = 1_790_000_000; + const b = DataService.applyTiingoCoverage(armed, "SMPL", now_s, .yahoo, .unknown); + try std.testing.expectEqual(@as(i64, 1_790_000_000), b.tiingo_retry_after_s); +} + +test "applyTiingoCoverage: backoff jitter is deterministic per symbol" { + // Deterministic-by-symbol (Wyhash via computeExpires), NOT random: + // repeated writes for the same symbol must not drift the deadline, + // and distinct symbols must land on different days so a batch + // demoted in one window does not all re-probe together. + const now_s: i64 = 1_787_000_000; + const meta = cache.Store.CandleMeta{ + .last_close = 100, + .last_date = Date.fromYmd(2026, 8, 14), + .provider = .tiingo, + }; + + const a1 = DataService.applyTiingoCoverage(meta, "SMPLA", now_s, .yahoo, .not_found); + const a2 = DataService.applyTiingoCoverage(meta, "SMPLA", now_s, .yahoo, .not_found); + try std.testing.expectEqual(a1.tiingo_retry_after_s, a2.tiingo_retry_after_s); + + // Spread check across a handful of symbols: they must not all + // collapse onto the same instant. + const syms = [_][]const u8{ "SMPLA", "SMPLB", "SMPLC", "SMPLD", "SMPLE", "SMPLF" }; + var seen: [syms.len]i64 = undefined; + for (syms, 0..) |sym, i| { + seen[i] = DataService.applyTiingoCoverage(meta, sym, now_s, .yahoo, .not_found).tiingo_retry_after_s; + } + var distinct: usize = 0; + for (seen, 0..) |v, i| { + var is_new = true; + for (seen[0..i]) |prev| { + if (prev == v) is_new = false; + } + if (is_new) distinct += 1; + } + try std.testing.expect(distinct > 1); +} + +test "isPermanentProviderFailure gates which Tiingo errors are remembered" { + // The rule that Commit 1 reuses for the Yahoo-demotion decision: + // only a genuine 404 is a statement about the symbol. Everything + // else describes one HTTP call. + try std.testing.expect(isPermanentProviderFailure(error.NotFound)); + try std.testing.expect(!isPermanentProviderFailure(error.InvalidResponse)); + try std.testing.expect(!isPermanentProviderFailure(error.PaymentRequired)); + try std.testing.expect(!isPermanentProviderFailure(error.ParseError)); + try std.testing.expect(!isPermanentProviderFailure(error.RateLimited)); + try std.testing.expect(!isPermanentProviderFailure(error.Unauthorized)); +} + test "loadAllPrices force_refresh tops up without wiping the candle cache" { // Regression: force_refresh must mean "ignore TTL + incremental // top-up", NOT "delete the cache and re-download from scratch". @@ -3998,7 +4217,7 @@ test "loadAllPrices force_refresh tops up without wiping the candle cache" { var candles = [_]Candle{ .{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 }, }; - store.cacheCandles("HELD", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); + store.cacheCandles("HELD", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); // Any provider/network attempt now panics. If force_refresh wiped // the cache (old behavior), getCandles would fall through to a full @@ -5114,7 +5333,7 @@ test "serverBarRegression: blocks a regression and reports both dates" { var s = cache.Store.init(io, allocator, dir_path); // Local copy holds Monday's bar. - s.updateCandleMeta("AAPL", 100.0, Date.fromYmd(2026, 8, 10), .tiingo, 0, 9999999999); + s.updateCandleMeta("AAPL", .{ .last_close = 100.0, .last_date = Date.fromYmd(2026, 8, 10), .provider = .tiingo }, 9999999999); const older = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-07,provider::tiingo\n"; const same = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-10,provider::tiingo\n";