stop convergence to Yahoo for equity coverage
This commit is contained in:
parent
44e8c65df4
commit
6c75ebf1ec
GPG key ID:
A7B62D657EF764F8
3 changed files with 451 additions and 49 deletions
213
src/cache/store.zig
vendored
213
src/cache/store.zig
vendored
|
|
@ -67,6 +67,17 @@ pub const Ttl = struct {
|
|||
/// month.
|
||||
pub const tickers_funds: i64 = 30 * s_per_day;
|
||||
pub const tickers_companies: i64 = 30 * s_per_day;
|
||||
|
||||
/// How long a genuine Tiingo 404 suppresses further Tiingo
|
||||
/// attempts for a symbol (see `CandleMeta.tiingo_retry_after_s`).
|
||||
///
|
||||
/// Long enough that a symbol Tiingo genuinely does not carry
|
||||
/// costs about one wasted 404 per month, short enough that
|
||||
/// coverage changes get picked up without operator action.
|
||||
/// Call sites layer `jitter_pct` on top so a batch of symbols
|
||||
/// demoted in the same window does not all re-probe on the same
|
||||
/// day.
|
||||
pub const tiingo_backoff: i64 = 30 * s_per_day;
|
||||
};
|
||||
|
||||
/// Cache TTL specification with optional per-key expiration jitter.
|
||||
|
|
@ -894,7 +905,10 @@ pub const Store = struct {
|
|||
/// candle metadata, stamping its `#!expires=` with `expires_at_s`
|
||||
/// (the caller computes the market-aware freshness boundary; see
|
||||
/// `market.nextCandleExpiry`).
|
||||
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||
///
|
||||
/// `attrs` supplies the provider-state fields; `last_close` and
|
||||
/// `last_date` are derived from the newest candle written.
|
||||
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, attrs: CandleMetaAttrs, expires_at_s: i64) void {
|
||||
if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
|
||||
defer self.allocator.free(srf_data);
|
||||
self.writeRaw(symbol, .candles_daily, srf_data) catch |err| {
|
||||
|
|
@ -906,7 +920,13 @@ pub const Store = struct {
|
|||
|
||||
if (candles.len > 0) {
|
||||
const last = candles[candles.len - 1];
|
||||
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
|
||||
self.updateCandleMeta(symbol, .{
|
||||
.last_close = last.close,
|
||||
.last_date = last.date,
|
||||
.provider = attrs.provider,
|
||||
.fail_count = attrs.fail_count,
|
||||
.tiingo_retry_after_s = attrs.tiingo_retry_after_s,
|
||||
}, expires_at_s);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -914,7 +934,14 @@ pub const Store = struct {
|
|||
/// Falls back to a full rewrite if append fails (e.g. file doesn't exist).
|
||||
/// Also updates candle metadata, stamping its `#!expires=` with
|
||||
/// `expires_at_s` (caller-computed market-aware boundary).
|
||||
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||
///
|
||||
/// `meta` supplies every metadata field except `last_close` and
|
||||
/// `last_date`, which are derived from the newest appended candle.
|
||||
/// Callers are expected to pass the symbol's *existing* meta so
|
||||
/// that fields describing the series as a whole survive an append
|
||||
/// unchanged - appending bars does not re-derive anything about
|
||||
/// the rows already on disk.
|
||||
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, meta: CandleMeta, expires_at_s: i64) void {
|
||||
if (new_candles.len == 0) return;
|
||||
|
||||
if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| {
|
||||
|
|
@ -943,23 +970,25 @@ pub const Store = struct {
|
|||
}
|
||||
|
||||
const last = new_candles[new_candles.len - 1];
|
||||
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
|
||||
var next = meta;
|
||||
next.last_close = last.close;
|
||||
next.last_date = last.date;
|
||||
self.updateCandleMeta(symbol, next, expires_at_s);
|
||||
}
|
||||
|
||||
/// Write (or refresh) candle metadata with a specific provider source.
|
||||
/// Write (or refresh) candle metadata.
|
||||
///
|
||||
/// `expires_at_s` is the absolute Unix-seconds freshness boundary for
|
||||
/// the `#!expires=` directive, computed by the caller via
|
||||
/// `market.nextCandleExpiry` (close-anchored) or a short retry. The
|
||||
/// cache layer no longer reads the wall clock for this - the boundary
|
||||
/// is market-domain knowledge owned by the caller.
|
||||
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||
const meta = CandleMeta{
|
||||
.last_close = last_close,
|
||||
.last_date = last_date,
|
||||
.provider = provider,
|
||||
.fail_count = fail_count,
|
||||
};
|
||||
///
|
||||
/// Takes the whole `CandleMeta` rather than a field-per-parameter
|
||||
/// list so that adding a metadata field does not churn every call
|
||||
/// site. Callers that mean "same metadata, one field changed"
|
||||
/// should copy the value they read and mutate the one field.
|
||||
pub fn updateCandleMeta(self: *Store, symbol: []const u8, meta: CandleMeta, expires_at_s: i64) void {
|
||||
if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires_at_s })) |meta_data| {
|
||||
defer self.allocator.free(meta_data);
|
||||
self.writeRaw(symbol, .candles_meta, meta_data) catch |err| {
|
||||
|
|
@ -1348,13 +1377,20 @@ pub const Store = struct {
|
|||
pub const CandleMeta = struct {
|
||||
last_close: f64,
|
||||
last_date: Date,
|
||||
/// Which provider sourced the candle data. **No default
|
||||
/// value on purpose** - SRF auto-elides fields whose value
|
||||
/// equals their default, which would hide the provider line
|
||||
/// when it equaled the implicit default. We want every cache
|
||||
/// file to record its provider explicitly so cache inspection
|
||||
/// can always answer "where did this come from?". Construction
|
||||
/// sites must pass the provider explicitly.
|
||||
/// Which provider sourced the candle data. Pure provenance -
|
||||
/// this field answers "where did these bars come from?" and
|
||||
/// nothing else. It deliberately does **not** drive provider
|
||||
/// routing; see `tiingo_retry_after_s` for that. Conflating
|
||||
/// the two is what produced the one-way drift to Yahoo that
|
||||
/// this field's value used to cause.
|
||||
///
|
||||
/// **No default value on purpose** - SRF auto-elides fields
|
||||
/// whose value equals their default, which would hide the
|
||||
/// provider line when it equaled the implicit default. We
|
||||
/// want every cache file to record its provider explicitly
|
||||
/// so cache inspection can always answer "where did this
|
||||
/// come from?". Construction sites must pass the provider
|
||||
/// explicitly.
|
||||
///
|
||||
/// Cache compatibility: pre-2026-05 caches that elided the
|
||||
/// provider field will fail to deserialize after this change
|
||||
|
|
@ -1369,6 +1405,44 @@ pub const Store = struct {
|
|||
/// Incremented on ServerError; reset to 0 on success. When >= 3, the
|
||||
/// symbol is degraded to a fallback provider until Tiingo recovers.
|
||||
fail_count: u8 = 0,
|
||||
/// Unix-seconds instant before which Tiingo should not be
|
||||
/// consulted for this symbol. `0` (the default) means "no
|
||||
/// backoff - always try Tiingo first".
|
||||
///
|
||||
/// Set **only** when Tiingo returns a genuine 404, which is a
|
||||
/// fact about coverage rather than about the request. A 400 /
|
||||
/// 402 / malformed body falls back to Yahoo for that one call
|
||||
/// but leaves this field alone, so the next call retries
|
||||
/// Tiingo. Cleared back to `0` the moment Tiingo serves the
|
||||
/// symbol again.
|
||||
///
|
||||
/// Why this exists: routing used to key off `provider ==
|
||||
/// .yahoo`, which a single non-transient Tiingo failure would
|
||||
/// latch permanently - Yahoo was then tried first, succeeded,
|
||||
/// and rewrote `provider = .yahoo`, so Tiingo was never
|
||||
/// consulted again. An explicit, expiring signal makes the
|
||||
/// demotion recoverable and records *why* it happened.
|
||||
///
|
||||
/// Defaulted so legacy caches (which lack the field) parse
|
||||
/// cleanly and simply behave as "no backoff".
|
||||
tiingo_retry_after_s: i64 = 0,
|
||||
};
|
||||
|
||||
/// The subset of `CandleMeta` that describes provider state rather
|
||||
/// than the candle data itself.
|
||||
///
|
||||
/// `cacheCandles` takes this instead of a whole `CandleMeta`
|
||||
/// because a full fetch may be a cold start with no prior metadata
|
||||
/// to copy - there is no honest `last_close` / `last_date` for the
|
||||
/// caller to supply, and those get derived from the candles being
|
||||
/// written anyway. `appendCandles` by contrast always has the
|
||||
/// existing meta in hand (it read `last_date` to decide what to
|
||||
/// fetch), so it takes the full struct and preserves everything it
|
||||
/// does not derive.
|
||||
pub const CandleMetaAttrs = struct {
|
||||
provider: CandleProvider,
|
||||
fail_count: u8 = 0,
|
||||
tiingo_retry_after_s: i64 = 0,
|
||||
};
|
||||
|
||||
pub const CandleProvider = enum {
|
||||
|
|
@ -3585,6 +3659,107 @@ test "deserializeCandleMeta fails on old cache that elided provider field" {
|
|||
try std.testing.expectError(error.InvalidData, result);
|
||||
}
|
||||
|
||||
test "CandleMeta.tiingo_retry_after_s defaults to 0 and is elided when unset" {
|
||||
// The field is defaulted precisely so caches written before it
|
||||
// existed keep parsing. SRF elides default-valued fields, so an
|
||||
// unset backoff costs no bytes and reads back as "no backoff".
|
||||
const allocator = std.testing.allocator;
|
||||
const meta = Store.CandleMeta{
|
||||
.last_close = 100.0,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
try std.testing.expectEqual(@as(i64, 0), meta.tiingo_retry_after_s);
|
||||
|
||||
const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 });
|
||||
defer allocator.free(data);
|
||||
try std.testing.expect(std.mem.indexOf(u8, data, "tiingo_retry_after_s") == null);
|
||||
}
|
||||
|
||||
test "CandleMeta.tiingo_retry_after_s round-trips when armed" {
|
||||
const allocator = std.testing.allocator;
|
||||
const meta = Store.CandleMeta{
|
||||
.last_close = 100.0,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .yahoo,
|
||||
.tiingo_retry_after_s = 1_790_000_000,
|
||||
};
|
||||
const data = try Store.serializeCandleMeta(std.testing.io, allocator, meta, .{ .expires = 1234567890 });
|
||||
defer allocator.free(data);
|
||||
try std.testing.expect(std.mem.indexOf(u8, data, "tiingo_retry_after_s:num:1790000000") != null);
|
||||
|
||||
const parsed = try Store.deserializeCandleMeta(allocator, data);
|
||||
try std.testing.expectEqual(@as(i64, 1_790_000_000), parsed.tiingo_retry_after_s);
|
||||
try std.testing.expectEqual(Store.CandleProvider.yahoo, parsed.provider);
|
||||
}
|
||||
|
||||
test "legacy candles_meta without tiingo_retry_after_s still parses" {
|
||||
// The whole migration story for this field: unlike `provider`
|
||||
// (which is deliberately default-less and therefore wipes old
|
||||
// caches), a defaulted field must NOT break existing caches. If
|
||||
// this ever regresses, every cached symbol takes a full re-fetch
|
||||
// on first use - and for symbols Tiingo does not carry, the
|
||||
// cold-start path would write a negative-cache marker over a
|
||||
// perfectly good candle history.
|
||||
const allocator = std.testing.allocator;
|
||||
const legacy =
|
||||
\\#!srfv1
|
||||
\\#!expires=1787000100
|
||||
\\#!created=1786748010
|
||||
\\last_close:num:776.34,last_date::2026-08-14,provider::yahoo
|
||||
\\
|
||||
;
|
||||
const parsed = try Store.deserializeCandleMeta(allocator, legacy);
|
||||
try std.testing.expectEqual(@as(i64, 0), parsed.tiingo_retry_after_s);
|
||||
try std.testing.expectEqual(Store.CandleProvider.yahoo, parsed.provider);
|
||||
try std.testing.expect(parsed.last_date.eql(Date.fromYmd(2026, 8, 14)));
|
||||
}
|
||||
|
||||
test "appendCandles preserves caller-supplied provider state" {
|
||||
// appendCandles derives only last_close/last_date; every other
|
||||
// metadata field must pass through untouched. Commit-2's
|
||||
// `adj_basis` depends on this exact property, so pin it now.
|
||||
const io = std.testing.io;
|
||||
const allocator = std.testing.allocator;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
var store = Store.init(io, allocator, dir_path);
|
||||
const seed = [_]Candle{.{
|
||||
.date = Date.fromYmd(2026, 8, 13),
|
||||
.open = 1,
|
||||
.high = 1,
|
||||
.low = 1,
|
||||
.close = 1,
|
||||
.adj_close = 1,
|
||||
.volume = 1,
|
||||
}};
|
||||
store.cacheCandles("SMPL", seed[0..], .{ .provider = .yahoo, .tiingo_retry_after_s = 1_790_000_000 }, 9_999_999_999);
|
||||
|
||||
const before = store.readCandleMeta("SMPL") orelse return error.TestUnexpectedResult;
|
||||
try std.testing.expectEqual(@as(i64, 1_790_000_000), before.meta.tiingo_retry_after_s);
|
||||
|
||||
const more = [_]Candle{.{
|
||||
.date = Date.fromYmd(2026, 8, 14),
|
||||
.open = 2,
|
||||
.high = 2,
|
||||
.low = 2,
|
||||
.close = 2,
|
||||
.adj_close = 2,
|
||||
.volume = 2,
|
||||
}};
|
||||
store.appendCandles("SMPL", more[0..], before.meta, 9_999_999_999);
|
||||
|
||||
const after = store.readCandleMeta("SMPL") orelse return error.TestUnexpectedResult;
|
||||
try std.testing.expectEqual(@as(i64, 1_790_000_000), after.meta.tiingo_retry_after_s);
|
||||
try std.testing.expectEqual(Store.CandleProvider.yahoo, after.meta.provider);
|
||||
// ...while the derived fields did advance.
|
||||
try std.testing.expect(after.meta.last_date.eql(Date.fromYmd(2026, 8, 14)));
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 2), after.meta.last_close, 0.001);
|
||||
}
|
||||
|
||||
// ── writeRaw / appendRaw atomicity ───────────────────────────
|
||||
//
|
||||
// A concurrent reader hitting a cache file mid-write must never see a
|
||||
|
|
|
|||
|
|
@ -239,6 +239,14 @@ pub fn run(ctx: *framework.RunCtx, parsed: ParsedArgs) !void {
|
|||
try out.print(", {s}", .{if (obs.local_fresh) "TTL still in the future" else "TTL lapsed"});
|
||||
try out.print(", {s}", .{@tagName(m.meta.provider)});
|
||||
if (m.meta.fail_count > 0) try out.print(", {d} consecutive failures", .{m.meta.fail_count});
|
||||
// A live Tiingo backoff means this symbol is being served by
|
||||
// Yahoo on purpose (Tiingo 404'd it), not by accident. Surface
|
||||
// it so a provider that looks "wrong" can be explained.
|
||||
if (m.meta.tiingo_retry_after_s > ctx.now_s) {
|
||||
try out.print(", Tiingo backoff until {f}", .{Date.fromEpoch(m.meta.tiingo_retry_after_s)});
|
||||
} else if (m.meta.tiingo_retry_after_s != 0) {
|
||||
try out.print(", Tiingo backoff lapsed (retries next refresh)", .{});
|
||||
}
|
||||
try out.print("\n", .{});
|
||||
} else {
|
||||
try out.print("local nothing cached\n", .{});
|
||||
|
|
|
|||
279
src/service.zig
279
src/service.zig
|
|
@ -133,6 +133,17 @@ pub fn isPermanentProviderFailure(err: anyerror) bool {
|
|||
return err == error.NotFound;
|
||||
}
|
||||
|
||||
/// Spread applied to `Ttl.tiingo_backoff` when a symbol is demoted off
|
||||
/// Tiingo by a 404.
|
||||
///
|
||||
/// Policy lives here rather than in `TtlSpec` (see its doc comment).
|
||||
/// 7% of 30 days is roughly +/-2 days, so a batch of symbols demoted in
|
||||
/// the same window re-probes across five distinct days instead of all
|
||||
/// on one. Sized against the cron cadence: with the refresh running
|
||||
/// twice a day, five days of spread keeps the re-probe burst to a
|
||||
/// handful of extra 404s per run.
|
||||
const tiingo_backoff_jitter_pct: u8 = 7;
|
||||
|
||||
/// Result of a CUSIP-to-ticker lookup (provider-agnostic).
|
||||
pub const CusipResult = OpenFigi.FigiResult;
|
||||
|
||||
|
|
@ -652,7 +663,7 @@ pub const DataService = struct {
|
|||
// (market-aware next post-close / NAV-availability time).
|
||||
const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds();
|
||||
const kind = market.classify(symbol);
|
||||
s.cacheCandles(symbol, triple.candles, .tiingo, 0, expiryAfterFetch(now_s, kind, triple.candles));
|
||||
s.cacheCandles(symbol, triple.candles, .{ .provider = .tiingo }, expiryAfterFetch(now_s, kind, triple.candles));
|
||||
}
|
||||
// Dividends and splits use the supplement write path: Tiingo's
|
||||
// view merges into existing (typically Polygon-sourced) records
|
||||
|
|
@ -678,6 +689,73 @@ pub const DataService = struct {
|
|||
|
||||
// ── Public data methods ──────────────────────────────────────
|
||||
|
||||
/// What a candle fetch learned about Tiingo's coverage of a symbol.
|
||||
///
|
||||
/// This is deliberately three-valued. The old code collapsed
|
||||
/// "Tiingo does not carry this symbol" and "Tiingo failed this
|
||||
/// request" into a single permanent demotion to Yahoo, which is
|
||||
/// how 22 of 32 cached symbols drifted off Tiingo. Only
|
||||
/// `.not_found` is a statement about the symbol; everything else
|
||||
/// is a statement about one HTTP call and must not be remembered.
|
||||
pub const TiingoCoverage = enum {
|
||||
/// Tiingo served the request - it definitely covers this symbol.
|
||||
covered,
|
||||
/// Tiingo returned a genuine 404 - it does not carry this symbol.
|
||||
not_found,
|
||||
/// Tiingo was not consulted, or failed for a reason that says
|
||||
/// nothing about coverage (active backoff, no API key, 400,
|
||||
/// 402, malformed body). Nothing new learned; leave any
|
||||
/// existing backoff exactly as it was.
|
||||
unknown,
|
||||
};
|
||||
|
||||
/// Fold a fetch's outcome into a symbol's candle metadata.
|
||||
///
|
||||
/// Called on any *successful* candle fetch, so `fail_count` resets
|
||||
/// to 0. The interesting part is `tiingo_retry_after_s`:
|
||||
///
|
||||
/// - `.covered` -> clear the backoff. Tiingo just served this
|
||||
/// symbol, so any prior 404 is stale news. This
|
||||
/// is what walks a previously-demoted symbol
|
||||
/// back onto Tiingo.
|
||||
/// - `.not_found` -> arm the backoff at `Ttl.tiingo_backoff` with
|
||||
/// per-symbol jitter, so a batch demoted in the
|
||||
/// same window does not all re-probe on one day.
|
||||
/// - `.unknown` -> leave it untouched. Either Tiingo was never
|
||||
/// asked, or it failed for a reason that says
|
||||
/// nothing about coverage.
|
||||
fn applyTiingoCoverage(
|
||||
meta: cache.Store.CandleMeta,
|
||||
symbol: []const u8,
|
||||
now_s: i64,
|
||||
provider: cache.Store.CandleProvider,
|
||||
coverage: TiingoCoverage,
|
||||
) cache.Store.CandleMeta {
|
||||
var next = meta;
|
||||
next.provider = provider;
|
||||
next.fail_count = 0;
|
||||
|
||||
switch (coverage) {
|
||||
.covered => {
|
||||
if (meta.tiingo_retry_after_s != 0) {
|
||||
log.info("{s}: provider converted {t} -> tiingo, clearing Tiingo backoff", .{ symbol, meta.provider });
|
||||
}
|
||||
next.tiingo_retry_after_s = 0;
|
||||
},
|
||||
.not_found => {
|
||||
next.tiingo_retry_after_s = cache.computeExpires(
|
||||
now_s,
|
||||
.{ .seconds = cache.Ttl.tiingo_backoff, .jitter_pct = tiingo_backoff_jitter_pct },
|
||||
symbol,
|
||||
);
|
||||
log.info("{s}: Tiingo NotFound, backing off until {f}", .{ symbol, Date.fromEpoch(next.tiingo_retry_after_s) });
|
||||
},
|
||||
.unknown => {},
|
||||
}
|
||||
|
||||
return next;
|
||||
}
|
||||
|
||||
/// Fetch candles from providers with error classification.
|
||||
///
|
||||
/// Error handling:
|
||||
|
|
@ -685,32 +763,42 @@ pub const DataService = struct {
|
|||
/// - NotFound/ParseError/InvalidResponse from Tiingo -> try Yahoo (symbol-level issue)
|
||||
/// - Unauthorized -> TransientError (config problem, stop refresh)
|
||||
///
|
||||
/// The `preferred` param controls incremental fetch consistency: use the same
|
||||
/// provider that sourced the existing cache data.
|
||||
/// `prefer_yahoo` skips the Tiingo attempt entirely. Callers derive
|
||||
/// it from `CandleMeta.tiingo_retry_after_s` - i.e. "Tiingo told us
|
||||
/// 404 recently, don't waste the call". It is NOT derived from
|
||||
/// which provider sourced the cache; see the `provider` field's
|
||||
/// doc comment for why that distinction matters.
|
||||
fn fetchCandlesFromProviders(
|
||||
self: *DataService,
|
||||
symbol: []const u8,
|
||||
from: Date,
|
||||
to: Date,
|
||||
preferred: cache.Store.CandleProvider,
|
||||
) (DataError || error{NotFound})!struct { candles: []Candle, provider: cache.Store.CandleProvider } {
|
||||
// If preferred is Yahoo (degraded symbol), try Yahoo first
|
||||
if (preferred == .yahoo) {
|
||||
prefer_yahoo: bool,
|
||||
) (DataError || error{NotFound})!struct {
|
||||
candles: []Candle,
|
||||
provider: cache.Store.CandleProvider,
|
||||
tiingo_coverage: TiingoCoverage,
|
||||
} {
|
||||
// Under an active Tiingo backoff, go straight to Yahoo.
|
||||
if (prefer_yahoo) {
|
||||
if (self.getProvider(Yahoo)) |yh| {
|
||||
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.debug("{s}: candles from Yahoo (preferred)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .yahoo };
|
||||
log.debug("{s}: candles from Yahoo (Tiingo backoff active)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .yahoo, .tiingo_coverage = .unknown };
|
||||
} else |err| {
|
||||
log.warn("{s}: Yahoo (preferred) failed: {s}", .{ symbol, @errorName(err) });
|
||||
log.warn("{s}: Yahoo (Tiingo backoff active) failed: {s}", .{ symbol, @errorName(err) });
|
||||
}
|
||||
} else |_| {}
|
||||
}
|
||||
|
||||
// Primary: Tiingo
|
||||
// Primary: Tiingo. `coverage` accumulates what this attempt
|
||||
// taught us, so the Yahoo fallback below can report it back
|
||||
// to the caller without re-deriving it from the error.
|
||||
var coverage: TiingoCoverage = .unknown;
|
||||
if (self.getProvider(Tiingo)) |tg| {
|
||||
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.debug("{s}: candles from Tiingo", .{symbol});
|
||||
return .{ .candles = candles, .provider = .tiingo };
|
||||
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
|
||||
} else |err| {
|
||||
log.warn("{s}: Tiingo failed: {s}", .{ symbol, @errorName(err) });
|
||||
|
||||
|
|
@ -725,7 +813,7 @@ pub const DataService = struct {
|
|||
self.rateLimitBackoff();
|
||||
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.debug("{s}: candles from Tiingo (after rate limit backoff)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .tiingo };
|
||||
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
|
||||
} else |retry_err| {
|
||||
log.warn("{s}: Tiingo retry after backoff failed: {s}", .{ symbol, @errorName(retry_err) });
|
||||
if (retry_err == error.RateLimited) {
|
||||
|
|
@ -733,7 +821,7 @@ pub const DataService = struct {
|
|||
self.rateLimitBackoff();
|
||||
if (tg.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.debug("{s}: candles from Tiingo (after second backoff)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .tiingo };
|
||||
return .{ .candles = candles, .provider = .tiingo, .tiingo_coverage = .covered };
|
||||
} else |_| {}
|
||||
}
|
||||
// Exhausted rate limit retries - treat as transient
|
||||
|
|
@ -746,19 +834,31 @@ pub const DataService = struct {
|
|||
return DataError.TransientError;
|
||||
}
|
||||
|
||||
// NotFound, ParseError, InvalidResponse - symbol-level issue, try Yahoo
|
||||
log.info("{s}: Tiingo does not have this symbol, trying Yahoo", .{symbol});
|
||||
// NotFound, ParseError, InvalidResponse - fall back to
|
||||
// Yahoo for this call. Only a genuine 404 is a
|
||||
// statement about Tiingo's coverage; a 400 / 402 /
|
||||
// malformed body says something about this request and
|
||||
// must not earn a remembered demotion. Mirrors the rule
|
||||
// `isPermanentProviderFailure` already applies to the
|
||||
// negative cache.
|
||||
if (isPermanentProviderFailure(err)) {
|
||||
coverage = .not_found;
|
||||
log.info("{s}: Tiingo does not carry this symbol, trying Yahoo", .{symbol});
|
||||
} else {
|
||||
log.info("{s}: Tiingo request failed ({s}) - not a coverage verdict, trying Yahoo for this call only", .{ symbol, @errorName(err) });
|
||||
}
|
||||
}
|
||||
} else |_| {
|
||||
log.warn("{s}: Tiingo provider not available (no API key?)", .{symbol});
|
||||
}
|
||||
|
||||
// Fallback: Yahoo (symbol not on Tiingo)
|
||||
if (preferred != .yahoo) {
|
||||
// Fallback: Yahoo. Skipped when we already tried Yahoo first
|
||||
// (active backoff) and it failed - no point asking twice.
|
||||
if (!prefer_yahoo) {
|
||||
if (self.getProvider(Yahoo)) |yh| {
|
||||
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
|
||||
log.info("{s}: candles from Yahoo (Tiingo fallback)", .{symbol});
|
||||
return .{ .candles = candles, .provider = .yahoo };
|
||||
return .{ .candles = candles, .provider = .yahoo, .tiingo_coverage = coverage };
|
||||
} else |err| {
|
||||
log.warn("{s}: Yahoo fallback also failed: {s}", .{ symbol, @errorName(err) });
|
||||
}
|
||||
|
|
@ -959,18 +1059,20 @@ pub const DataService = struct {
|
|||
// boundary while the lag check reported it lagging (the
|
||||
// Friday-17:00 deadlock that exited 75 every retry).
|
||||
if (!market.shouldRefresh(now_s, kind, m.last_date)) {
|
||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count, expires);
|
||||
s.updateCandleMeta(symbol, m, expires);
|
||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
||||
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
} else {
|
||||
// Incremental fetch from day after last cached candle
|
||||
self.assertNetworkAllowed("getCandles incremental fetchCandlesFromProviders");
|
||||
const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, m.provider) catch |err| {
|
||||
const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, now_s < m.tiingo_retry_after_s) catch |err| {
|
||||
if (err == DataError.TransientError) {
|
||||
// Increment fail_count for this symbol
|
||||
const new_fail_count = m.fail_count +| 1; // saturating add
|
||||
log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count });
|
||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count, now_s + market.short_retry_s);
|
||||
var degraded = m;
|
||||
degraded.fail_count = new_fail_count;
|
||||
s.updateCandleMeta(symbol, degraded, now_s + market.short_retry_s);
|
||||
|
||||
// If degraded (fail_count >= 3), return stale data rather than failing
|
||||
if (new_fail_count >= 3) {
|
||||
|
|
@ -987,6 +1089,10 @@ pub const DataService = struct {
|
|||
};
|
||||
const new_candles = result.candles;
|
||||
|
||||
// Fold what this fetch learned about Tiingo coverage
|
||||
// into the metadata we're about to write.
|
||||
const next_meta = applyTiingoCoverage(m, symbol, now_s, result.provider, result.tiingo_coverage);
|
||||
|
||||
if (new_candles.len == 0) {
|
||||
// No new candles. Either a genuine non-trading-day
|
||||
// gap (weekend/holiday), the provider hasn't posted
|
||||
|
|
@ -999,7 +1105,7 @@ pub const DataService = struct {
|
|||
// an un-modeled closure stops thrashing and waits
|
||||
// for the next real session.
|
||||
self.allocator.free(new_candles);
|
||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0, market.staleCandleExpiry(now_s, kind, m.last_date));
|
||||
s.updateCandleMeta(symbol, next_meta, market.staleCandleExpiry(now_s, kind, m.last_date));
|
||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
||||
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
} else {
|
||||
|
|
@ -1007,7 +1113,7 @@ pub const DataService = struct {
|
|||
// TTL via `expiryAfterFetch`, NOT the precomputed
|
||||
// next-boundary `expires`: getting a bar back does not
|
||||
// mean getting the RIGHT bar back.
|
||||
s.appendCandles(symbol, new_candles, result.provider, 0, expiryAfterFetch(now_s, kind, new_candles));
|
||||
s.appendCandles(symbol, new_candles, next_meta, expiryAfterFetch(now_s, kind, new_candles));
|
||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r| {
|
||||
self.allocator.free(new_candles);
|
||||
return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
|
|
@ -1050,8 +1156,9 @@ pub const DataService = struct {
|
|||
// Transient: increment fail_count on existing meta so
|
||||
// we know to back off if this keeps happening.
|
||||
if (meta_result) |mr| {
|
||||
const new_fail_count = mr.meta.fail_count +| 1;
|
||||
s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count, now_s + market.short_retry_s);
|
||||
var degraded = mr.meta;
|
||||
degraded.fail_count = mr.meta.fail_count +| 1;
|
||||
s.updateCandleMeta(symbol, degraded, now_s + market.short_retry_s);
|
||||
}
|
||||
return DataError.TransientError;
|
||||
}
|
||||
|
|
@ -3791,7 +3898,7 @@ test "getCandles offline mode returns cached data without network" {
|
|||
.{ .date = Date.fromYmd(2026, 5, 19), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||
.{ .date = Date.fromYmd(2026, 5, 20), .open = 104, .high = 106, .low = 103, .close = 105, .adj_close = 105, .volume = 1100 },
|
||||
};
|
||||
store.cacheCandles("TEST", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
store.cacheCandles("TEST", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
|
||||
// Set the test guard: any network call would panic. We expect
|
||||
// the offline-mode path NOT to touch the network.
|
||||
|
|
@ -3946,7 +4053,7 @@ test "loadAllPrices offline mode skips network and returns cached" {
|
|||
var fresh_candles = [_]Candle{
|
||||
.{ .date = Date.fromYmd(2026, 5, 20), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||
};
|
||||
store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
store.cacheCandles("FRESH", fresh_candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
|
||||
// Symbol with no cache at all.
|
||||
// (no setup needed - just passes a symbol that doesn't exist)
|
||||
|
|
@ -3972,6 +4079,118 @@ test "loadAllPrices offline mode skips network and returns cached" {
|
|||
try std.testing.expectEqual(@as(usize, 1), result.failed_count);
|
||||
}
|
||||
|
||||
// ── Tiingo coverage bookkeeping ──────────────────────────────
|
||||
//
|
||||
// Regression suite for the one-way drift to Yahoo. The old code
|
||||
// routed on `CandleMeta.provider == .yahoo`, so any non-transient
|
||||
// Tiingo failure latched permanently: Yahoo was tried first,
|
||||
// succeeded, rewrote `provider = .yahoo`, and Tiingo was never asked
|
||||
// again. 22 of 32 cached symbols had drifted off Tiingo that way.
|
||||
|
||||
test "applyTiingoCoverage: .covered clears an armed Tiingo backoff" {
|
||||
const armed = cache.Store.CandleMeta{
|
||||
.last_close = 100,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .yahoo,
|
||||
.fail_count = 2,
|
||||
.tiingo_retry_after_s = 1_790_000_000,
|
||||
};
|
||||
const next = DataService.applyTiingoCoverage(armed, "SMPL", 1_787_000_000, .tiingo, .covered);
|
||||
|
||||
// Tiingo just served the symbol, so the prior 404 is stale news.
|
||||
try std.testing.expectEqual(@as(i64, 0), next.tiingo_retry_after_s);
|
||||
try std.testing.expectEqual(cache.Store.CandleProvider.tiingo, next.provider);
|
||||
// A successful fetch also resets the transient-failure counter.
|
||||
try std.testing.expectEqual(@as(u8, 0), next.fail_count);
|
||||
}
|
||||
|
||||
test "applyTiingoCoverage: .not_found arms a jittered backoff ~30 days out" {
|
||||
const now_s: i64 = 1_787_000_000;
|
||||
const meta = cache.Store.CandleMeta{
|
||||
.last_close = 100,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
const next = DataService.applyTiingoCoverage(meta, "SMPL", now_s, .yahoo, .not_found);
|
||||
|
||||
// Yahoo answered, so provenance records Yahoo...
|
||||
try std.testing.expectEqual(cache.Store.CandleProvider.yahoo, next.provider);
|
||||
// ...and the backoff lands 30 days out, +/- the jitter window.
|
||||
const base = now_s + cache.Ttl.tiingo_backoff;
|
||||
const max_offset = @divFloor(cache.Ttl.tiingo_backoff * @as(i64, tiingo_backoff_jitter_pct), 100);
|
||||
try std.testing.expect(next.tiingo_retry_after_s >= base - max_offset);
|
||||
try std.testing.expect(next.tiingo_retry_after_s <= base + max_offset);
|
||||
// The spread must be meaningful but bounded: roughly +/-2 days.
|
||||
try std.testing.expect(max_offset >= 2 * std.time.s_per_day);
|
||||
try std.testing.expect(max_offset <= 3 * std.time.s_per_day);
|
||||
}
|
||||
|
||||
test "applyTiingoCoverage: .unknown leaves the backoff untouched" {
|
||||
// A 400 / 402 / malformed body says nothing about coverage. It
|
||||
// must neither arm a backoff (that's the drift bug) nor clear an
|
||||
// existing one (that would defeat the 404 we already recorded).
|
||||
const now_s: i64 = 1_787_000_000;
|
||||
|
||||
const unarmed = cache.Store.CandleMeta{
|
||||
.last_close = 100,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
const a = DataService.applyTiingoCoverage(unarmed, "SMPL", now_s, .yahoo, .unknown);
|
||||
try std.testing.expectEqual(@as(i64, 0), a.tiingo_retry_after_s);
|
||||
|
||||
var armed = unarmed;
|
||||
armed.tiingo_retry_after_s = 1_790_000_000;
|
||||
const b = DataService.applyTiingoCoverage(armed, "SMPL", now_s, .yahoo, .unknown);
|
||||
try std.testing.expectEqual(@as(i64, 1_790_000_000), b.tiingo_retry_after_s);
|
||||
}
|
||||
|
||||
test "applyTiingoCoverage: backoff jitter is deterministic per symbol" {
|
||||
// Deterministic-by-symbol (Wyhash via computeExpires), NOT random:
|
||||
// repeated writes for the same symbol must not drift the deadline,
|
||||
// and distinct symbols must land on different days so a batch
|
||||
// demoted in one window does not all re-probe together.
|
||||
const now_s: i64 = 1_787_000_000;
|
||||
const meta = cache.Store.CandleMeta{
|
||||
.last_close = 100,
|
||||
.last_date = Date.fromYmd(2026, 8, 14),
|
||||
.provider = .tiingo,
|
||||
};
|
||||
|
||||
const a1 = DataService.applyTiingoCoverage(meta, "SMPLA", now_s, .yahoo, .not_found);
|
||||
const a2 = DataService.applyTiingoCoverage(meta, "SMPLA", now_s, .yahoo, .not_found);
|
||||
try std.testing.expectEqual(a1.tiingo_retry_after_s, a2.tiingo_retry_after_s);
|
||||
|
||||
// Spread check across a handful of symbols: they must not all
|
||||
// collapse onto the same instant.
|
||||
const syms = [_][]const u8{ "SMPLA", "SMPLB", "SMPLC", "SMPLD", "SMPLE", "SMPLF" };
|
||||
var seen: [syms.len]i64 = undefined;
|
||||
for (syms, 0..) |sym, i| {
|
||||
seen[i] = DataService.applyTiingoCoverage(meta, sym, now_s, .yahoo, .not_found).tiingo_retry_after_s;
|
||||
}
|
||||
var distinct: usize = 0;
|
||||
for (seen, 0..) |v, i| {
|
||||
var is_new = true;
|
||||
for (seen[0..i]) |prev| {
|
||||
if (prev == v) is_new = false;
|
||||
}
|
||||
if (is_new) distinct += 1;
|
||||
}
|
||||
try std.testing.expect(distinct > 1);
|
||||
}
|
||||
|
||||
test "isPermanentProviderFailure gates which Tiingo errors are remembered" {
|
||||
// The rule that Commit 1 reuses for the Yahoo-demotion decision:
|
||||
// only a genuine 404 is a statement about the symbol. Everything
|
||||
// else describes one HTTP call.
|
||||
try std.testing.expect(isPermanentProviderFailure(error.NotFound));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.InvalidResponse));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.PaymentRequired));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.ParseError));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.RateLimited));
|
||||
try std.testing.expect(!isPermanentProviderFailure(error.Unauthorized));
|
||||
}
|
||||
|
||||
test "loadAllPrices force_refresh tops up without wiping the candle cache" {
|
||||
// Regression: force_refresh must mean "ignore TTL + incremental
|
||||
// top-up", NOT "delete the cache and re-download from scratch".
|
||||
|
|
@ -3998,7 +4217,7 @@ test "loadAllPrices force_refresh tops up without wiping the candle cache" {
|
|||
var candles = [_]Candle{
|
||||
.{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||
};
|
||||
store.cacheCandles("HELD", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
store.cacheCandles("HELD", candles[0..], .{ .provider = .tiingo }, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||
|
||||
// Any provider/network attempt now panics. If force_refresh wiped
|
||||
// the cache (old behavior), getCandles would fall through to a full
|
||||
|
|
@ -5114,7 +5333,7 @@ test "serverBarRegression: blocks a regression and reports both dates" {
|
|||
|
||||
var s = cache.Store.init(io, allocator, dir_path);
|
||||
// Local copy holds Monday's bar.
|
||||
s.updateCandleMeta("AAPL", 100.0, Date.fromYmd(2026, 8, 10), .tiingo, 0, 9999999999);
|
||||
s.updateCandleMeta("AAPL", .{ .last_close = 100.0, .last_date = Date.fromYmd(2026, 8, 10), .provider = .tiingo }, 9999999999);
|
||||
|
||||
const older = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-07,provider::tiingo\n";
|
||||
const same = "#!srfv1\nlast_close:num:99.00,last_date::2026-08-10,provider::tiingo\n";
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue