wire tiingo to rate limiter
This commit is contained in:
parent
30787db0ba
commit
9139c36e09
4 changed files with 49 additions and 29 deletions
25
TODO.md
25
TODO.md
|
|
@ -518,30 +518,13 @@ Implementation notes:
|
|||
- `Tiingo.fetchQuotes` returns an array whose order is NOT guaranteed to
|
||||
match the request order, so key results by the returned `ticker`
|
||||
field, not by position.
|
||||
- Depends on honoring Tiingo's 50 req/hour limit (see below): routing
|
||||
bursty live traffic through Tiingo without that enforcement risks
|
||||
starving the candle/EOD path.
|
||||
- Tiingo-sourced live quotes would share Tiingo's 50/hour token bucket
|
||||
(`RateLimiter.perHour`, wired into the provider). A batched quote
|
||||
call is 1 request, but heavy `r` use plus candle refreshes draw from
|
||||
the same hourly budget, so watch for contention.
|
||||
- Tiingo websocket streaming would be the natural follow-on for true
|
||||
push-based real-time, replacing poll-on-`r` entirely.
|
||||
|
||||
## Rate limiter doesn't honor Tiingo's 50 req/hour limit - priority MEDIUM
|
||||
|
||||
Tiingo's free tier caps at both 1,000 req/day AND 50 req/hour, but the
|
||||
`net/RateLimiter.zig` token bucket (and the `tiingo.zig` header note,
|
||||
"no per-minute restriction") only accounts for the daily ceiling. The
|
||||
hourly limit is unenforced today.
|
||||
|
||||
This bites the candle path hardest: candle fetches are per-symbol
|
||||
(`/tiingo/daily/{sym}/prices`), so a cold-cache load of a ~40-name
|
||||
portfolio fires ~40 requests in one burst - most of the hourly budget
|
||||
in a single app open. Nothing throttles that; we'd just start taking
|
||||
429s mid-load.
|
||||
|
||||
Fix: extend the limiter to enforce a rolling 50/hour window for Tiingo
|
||||
(in addition to the daily count), backing off or queueing when the
|
||||
window is full. Directly relevant to the live-quote-provider knob
|
||||
above: any Tiingo-sourced live quotes would share this same bucket.
|
||||
|
||||
## Audit: em-dash sentinel usage across all tables — priority LOW
|
||||
|
||||
The codebase uses `—` (em-dash) as the canonical "no data" sentinel
|
||||
|
|
|
|||
|
|
@ -50,6 +50,19 @@ pub fn perDay(io: std.Io, n: u32) RateLimiter {
|
|||
return init(io, n, std.time.ns_per_day);
|
||||
}
|
||||
|
||||
/// Convenience: N requests per hour. Starts with a full bucket (like
|
||||
/// `perDay`, unlike `perMinute`) so a burst up to N runs unthrottled —
|
||||
/// e.g. a nightly refresh fetching one candle file per held symbol —
|
||||
/// while sustained usage beyond N/hour blocks via `acquire`. Use for
|
||||
/// providers whose binding limit is hourly (Tiingo free tier: 50/hour).
|
||||
///
|
||||
/// In-memory and per-process: it caps a single run's burst at N, which
|
||||
/// is the common case (one cron invocation). It does not coordinate
|
||||
/// across separate process launches within the same hour.
|
||||
pub fn perHour(io: std.Io, n: u32) RateLimiter {
|
||||
return init(io, n, std.time.ns_per_hour);
|
||||
}
|
||||
|
||||
/// Try to acquire a token. Returns true if granted, false if rate-limited.
|
||||
/// Caller should sleep and retry if false.
|
||||
pub fn tryAcquire(self: *RateLimiter) bool {
|
||||
|
|
@ -122,6 +135,17 @@ test "rate limiter perDay keeps full burst" {
|
|||
try std.testing.expect(!rl.tryAcquire());
|
||||
}
|
||||
|
||||
test "rate limiter perHour allows a full-N burst then throttles" {
|
||||
var rl = RateLimiter.perHour(std.testing.io, 50);
|
||||
// Full bucket: a 50-request burst (e.g. a nightly candle refresh)
|
||||
// goes through without pacing.
|
||||
for (0..50) |_| {
|
||||
try std.testing.expect(rl.tryAcquire());
|
||||
}
|
||||
// The 51st within the hour is throttled.
|
||||
try std.testing.expect(!rl.tryAcquire());
|
||||
}
|
||||
|
||||
test "rate limiter exhaustion" {
|
||||
var rl = RateLimiter.init(std.testing.io, 2, std.time.ns_per_s);
|
||||
try std.testing.expect(rl.tryAcquire());
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
//! Tiingo provider -- official REST API for end-of-day prices and corporate actions.
|
||||
//!
|
||||
//! Free tier: 1,000 requests/day, no per-minute restriction.
|
||||
//! Free tier: 50 requests/hour and 1,000 requests/day. We enforce the
|
||||
//! hourly cap with a 50/hour token bucket (`RateLimiter.perHour`,
|
||||
//! which starts full so a nightly refresh burst runs unthrottled);
|
||||
//! the daily ceiling is far from binding for zfin's bursty usage.
|
||||
//! Covers stocks, ETFs, and mutual funds with same-day NAV updates
|
||||
//! (mutual fund NAVs available after midnight ET).
|
||||
//!
|
||||
|
|
@ -46,6 +49,7 @@
|
|||
|
||||
const std = @import("std");
|
||||
const http = @import("../net/http.zig");
|
||||
const RateLimiter = @import("../net/RateLimiter.zig");
|
||||
const Date = @import("../Date.zig");
|
||||
const Candle = @import("../models/candle.zig").Candle;
|
||||
const Dividend = @import("../models/dividend.zig").Dividend;
|
||||
|
|
@ -70,12 +74,17 @@ pub const Tiingo = struct {
|
|||
client: http.Client,
|
||||
allocator: std.mem.Allocator,
|
||||
api_key: []const u8,
|
||||
/// Free-tier hourly cap (50/hour). Starts full so a nightly
|
||||
/// refresh burst (one candle file per held symbol) isn't paced;
|
||||
/// sustained usage beyond 50/hour blocks in `acquire`.
|
||||
rate_limiter: RateLimiter,
|
||||
|
||||
pub fn init(io: std.Io, allocator: std.mem.Allocator, api_key: []const u8) Tiingo {
|
||||
return .{
|
||||
.client = http.Client.init(io, allocator),
|
||||
.allocator = allocator,
|
||||
.api_key = api_key,
|
||||
.rate_limiter = RateLimiter.perHour(io, 50),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -112,6 +121,10 @@ pub const Tiingo = struct {
|
|||
});
|
||||
defer allocator.free(url);
|
||||
|
||||
// Honor Tiingo's 50/hour free-tier cap. Blocks only once a
|
||||
// single run has spent its 50-token burst within the hour.
|
||||
self.rate_limiter.acquire();
|
||||
|
||||
var response = try self.client.get(url);
|
||||
defer response.deinit();
|
||||
|
||||
|
|
|
|||
|
|
@ -1884,13 +1884,13 @@ pub const DataService = struct {
|
|||
.options => if (self.cboe) |*cboe| cboe.rate_limiter.estimateWaitNs() else return null,
|
||||
// EDGAR-served: ETF metrics, entity facts, ticker maps.
|
||||
.etf_metrics, .entity_facts, .tickers_funds, .tickers_companies => if (self.edgar) |*e| e.rate_limiter.estimateWaitNs() else return null,
|
||||
// No proactive token-bucket limiter for these. Tiingo
|
||||
// (candles) has a 1000/day quota enforced reactively
|
||||
// via 429-then-backoff in `getCandles`; Wikidata
|
||||
// (classification) has no published quota; the `meta`
|
||||
// type isn't fetched. Nothing useful to wait for at the
|
||||
// call site, so report 0.
|
||||
.candles_daily, .candles_meta, .classification, .meta => 0,
|
||||
// Tiingo-served candles: 50/hour token bucket. When Tiingo
|
||||
// isn't instantiated (no key), candles fall back to keyless
|
||||
// Yahoo with no proactive limiter, so report 0 rather than
|
||||
// null. `candles_meta` shares Tiingo's budget; `meta` isn't
|
||||
// fetched; Wikidata (classification) has no published quota.
|
||||
.candles_daily, .candles_meta => if (self.tg) |*tg| tg.rate_limiter.estimateWaitNs() else 0,
|
||||
.classification, .meta => 0,
|
||||
};
|
||||
return if (ns == 0) 0 else @max(1, ns / std.time.ns_per_s);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue