diff --git a/TODO.md b/TODO.md index 11103ad..a8b5679 100644 --- a/TODO.md +++ b/TODO.md @@ -518,30 +518,13 @@ Implementation notes: - `Tiingo.fetchQuotes` returns an array whose order is NOT guaranteed to match the request order, so key results by the returned `ticker` field, not by position. -- Depends on honoring Tiingo's 50 req/hour limit (see below): routing - bursty live traffic through Tiingo without that enforcement risks - starving the candle/EOD path. +- Tiingo-sourced live quotes would share Tiingo's 50/hour token bucket + (`RateLimiter.perHour`, wired into the provider). A batched quote + call is 1 request, but heavy `r` use plus candle refreshes draw from + the same hourly budget, so watch for contention. - Tiingo websocket streaming would be the natural follow-on for true push-based real-time, replacing poll-on-`r` entirely. -## Rate limiter doesn't honor Tiingo's 50 req/hour limit - priority MEDIUM - -Tiingo's free tier caps at both 1,000 req/day AND 50 req/hour, but the -`net/RateLimiter.zig` token bucket (and the `tiingo.zig` header note, -"no per-minute restriction") only accounts for the daily ceiling. The -hourly limit is unenforced today. - -This bites the candle path hardest: candle fetches are per-symbol -(`/tiingo/daily/{sym}/prices`), so a cold-cache load of a ~40-name -portfolio fires ~40 requests in one burst - most of the hourly budget -in a single app open. Nothing throttles that; we'd just start taking -429s mid-load. - -Fix: extend the limiter to enforce a rolling 50/hour window for Tiingo -(in addition to the daily count), backing off or queueing when the -window is full. Directly relevant to the live-quote-provider knob -above: any Tiingo-sourced live quotes would share this same bucket. - ## Audit: em-dash sentinel usage across all tables — priority LOW The codebase uses `—` (em-dash) as the canonical "no data" sentinel diff --git a/src/net/RateLimiter.zig b/src/net/RateLimiter.zig index 849a69d..1559fa8 100644 --- a/src/net/RateLimiter.zig +++ b/src/net/RateLimiter.zig @@ -50,6 +50,19 @@ pub fn perDay(io: std.Io, n: u32) RateLimiter { return init(io, n, std.time.ns_per_day); } +/// Convenience: N requests per hour. Starts with a full bucket (like +/// `perDay`, unlike `perMinute`) so a burst up to N runs unthrottled — +/// e.g. a nightly refresh fetching one candle file per held symbol — +/// while sustained usage beyond N/hour blocks via `acquire`. Use for +/// providers whose binding limit is hourly (Tiingo free tier: 50/hour). +/// +/// In-memory and per-process: it caps a single run's burst at N, which +/// is the common case (one cron invocation). It does not coordinate +/// across separate process launches within the same hour. +pub fn perHour(io: std.Io, n: u32) RateLimiter { + return init(io, n, std.time.ns_per_hour); +} + /// Try to acquire a token. Returns true if granted, false if rate-limited. /// Caller should sleep and retry if false. pub fn tryAcquire(self: *RateLimiter) bool { @@ -122,6 +135,17 @@ test "rate limiter perDay keeps full burst" { try std.testing.expect(!rl.tryAcquire()); } +test "rate limiter perHour allows a full-N burst then throttles" { + var rl = RateLimiter.perHour(std.testing.io, 50); + // Full bucket: a 50-request burst (e.g. a nightly candle refresh) + // goes through without pacing. + for (0..50) |_| { + try std.testing.expect(rl.tryAcquire()); + } + // The 51st within the hour is throttled. + try std.testing.expect(!rl.tryAcquire()); +} + test "rate limiter exhaustion" { var rl = RateLimiter.init(std.testing.io, 2, std.time.ns_per_s); try std.testing.expect(rl.tryAcquire()); diff --git a/src/providers/tiingo.zig b/src/providers/tiingo.zig index b6f51bb..68d8a2d 100644 --- a/src/providers/tiingo.zig +++ b/src/providers/tiingo.zig @@ -1,6 +1,9 @@ //! Tiingo provider -- official REST API for end-of-day prices and corporate actions. //! -//! Free tier: 1,000 requests/day, no per-minute restriction. +//! Free tier: 50 requests/hour and 1,000 requests/day. We enforce the +//! hourly cap with a 50/hour token bucket (`RateLimiter.perHour`, +//! which starts full so a nightly refresh burst runs unthrottled); +//! the daily ceiling is far from binding for zfin's bursty usage. //! Covers stocks, ETFs, and mutual funds with same-day NAV updates //! (mutual fund NAVs available after midnight ET). //! @@ -46,6 +49,7 @@ const std = @import("std"); const http = @import("../net/http.zig"); +const RateLimiter = @import("../net/RateLimiter.zig"); const Date = @import("../Date.zig"); const Candle = @import("../models/candle.zig").Candle; const Dividend = @import("../models/dividend.zig").Dividend; @@ -70,12 +74,17 @@ pub const Tiingo = struct { client: http.Client, allocator: std.mem.Allocator, api_key: []const u8, + /// Free-tier hourly cap (50/hour). Starts full so a nightly + /// refresh burst (one candle file per held symbol) isn't paced; + /// sustained usage beyond 50/hour blocks in `acquire`. + rate_limiter: RateLimiter, pub fn init(io: std.Io, allocator: std.mem.Allocator, api_key: []const u8) Tiingo { return .{ .client = http.Client.init(io, allocator), .allocator = allocator, .api_key = api_key, + .rate_limiter = RateLimiter.perHour(io, 50), }; } @@ -112,6 +121,10 @@ pub const Tiingo = struct { }); defer allocator.free(url); + // Honor Tiingo's 50/hour free-tier cap. Blocks only once a + // single run has spent its 50-token burst within the hour. + self.rate_limiter.acquire(); + var response = try self.client.get(url); defer response.deinit(); diff --git a/src/service.zig b/src/service.zig index a008b2b..6b83d9d 100644 --- a/src/service.zig +++ b/src/service.zig @@ -1884,13 +1884,13 @@ pub const DataService = struct { .options => if (self.cboe) |*cboe| cboe.rate_limiter.estimateWaitNs() else return null, // EDGAR-served: ETF metrics, entity facts, ticker maps. .etf_metrics, .entity_facts, .tickers_funds, .tickers_companies => if (self.edgar) |*e| e.rate_limiter.estimateWaitNs() else return null, - // No proactive token-bucket limiter for these. Tiingo - // (candles) has a 1000/day quota enforced reactively - // via 429-then-backoff in `getCandles`; Wikidata - // (classification) has no published quota; the `meta` - // type isn't fetched. Nothing useful to wait for at the - // call site, so report 0. - .candles_daily, .candles_meta, .classification, .meta => 0, + // Tiingo-served candles: 50/hour token bucket. When Tiingo + // isn't instantiated (no key), candles fall back to keyless + // Yahoo with no proactive limiter, so report 0 rather than + // null. `candles_meta` shares Tiingo's budget; `meta` isn't + // fetched; Wikidata (classification) has no published quota. + .candles_daily, .candles_meta => if (self.tg) |*tg| tg.rate_limiter.estimateWaitNs() else 0, + .classification, .meta => 0, }; return if (ns == 0) 0 else @max(1, ns / std.time.ns_per_s); }