support Tiingo power tier rate limit
This commit is contained in:
parent
051ab35975
commit
a73966ad39
6 changed files with 182 additions and 21 deletions
|
|
@ -46,11 +46,7 @@ repos:
|
|||
- id: test
|
||||
name: Run zig build test
|
||||
entry: zig
|
||||
# TEMPORARY: lowered 80 -> 79 for the portfolio live-stream TUI
|
||||
# work (Phase D), whose event/tick handlers need a vaxis App
|
||||
# harness we don't have. Restore to 80 once the Tiingo REST
|
||||
# quote work (highly testable parsers) lands.
|
||||
args: ["build", "coverage", "-Dcoverage-threshold=79"]
|
||||
args: ["build", "coverage", "-Dcoverage-threshold=80"]
|
||||
language: system
|
||||
types: [file]
|
||||
pass_filenames: false
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
//! > `$HOME/.cache/zfin`.
|
||||
|
||||
const std = @import("std");
|
||||
const builtin = @import("builtin");
|
||||
|
||||
const EnvMap = std.StringHashMap([]const u8);
|
||||
|
||||
|
|
@ -30,12 +31,21 @@ pub const default_portfolio_filename = "portfolio*.srf";
|
|||
/// `default_portfolio_filename`.
|
||||
pub const default_watchlist_filename = "watchlist.srf";
|
||||
|
||||
/// Tiingo subscription tier, sourced from `ZFIN_TIINGO_PLAN`. Drives the
|
||||
/// provider's hourly rate-limit budget (see `tiingoHourlyLimit`): the
|
||||
/// free tier is capped at 50 requests/hour, while paid tiers raise that
|
||||
/// ceiling substantially. Defaults to `.free`.
|
||||
pub const TiingoPlan = enum { free, power };
|
||||
|
||||
// ── Fields ───────────────────────────────────────────────────
|
||||
|
||||
twelvedata_key: ?[]const u8 = null,
|
||||
polygon_key: ?[]const u8 = null,
|
||||
fmp_key: ?[]const u8 = null,
|
||||
tiingo_key: ?[]const u8 = null,
|
||||
/// Tiingo subscription tier (`ZFIN_TIINGO_PLAN`). Drives the provider's
|
||||
/// hourly rate-limit budget via `tiingoHourlyLimit`. Defaults to `.free`.
|
||||
tiingo_plan: TiingoPlan = .free,
|
||||
openfigi_key: ?[]const u8 = null,
|
||||
/// User contact email used as the User-Agent / From header for
|
||||
/// open-data providers that require politeness identification
|
||||
|
|
@ -101,6 +111,7 @@ pub fn fromEnv(io: std.Io, allocator: std.mem.Allocator, environ_map: *const std
|
|||
self.polygon_key = self.resolve("POLYGON_API_KEY");
|
||||
self.fmp_key = self.resolve("FMP_API_KEY");
|
||||
self.tiingo_key = self.resolve("TIINGO_API_KEY");
|
||||
self.tiingo_plan = parseTiingoPlan(self.resolve("ZFIN_TIINGO_PLAN"));
|
||||
self.openfigi_key = self.resolve("OPENFIGI_API_KEY");
|
||||
self.user_email = self.resolve("ZFIN_USER_EMAIL");
|
||||
self.server_url = self.resolve("ZFIN_SERVER");
|
||||
|
|
@ -381,6 +392,17 @@ pub fn hasAnyKey(self: @This()) bool {
|
|||
self.tiingo_key != null;
|
||||
}
|
||||
|
||||
/// Tiingo's hourly request budget for the configured plan. The free
|
||||
/// tier is 50/hour; the Power tier raises it to 10,000/hour. Threaded
|
||||
/// into the provider's rate limiter (see `service.getProvider`) so a
|
||||
/// paying subscriber isn't throttled to free-tier limits.
|
||||
pub fn tiingoHourlyLimit(self: @This()) usize {
|
||||
return switch (self.tiingo_plan) {
|
||||
.free => 50,
|
||||
.power => 10_000,
|
||||
};
|
||||
}
|
||||
|
||||
// ── Internals ────────────────────────────────────────────────
|
||||
|
||||
/// Look up a key: process environment first, then .env file fallback.
|
||||
|
|
@ -392,6 +414,20 @@ fn resolve(self: *@This(), key: []const u8) ?[]const u8 {
|
|||
return null;
|
||||
}
|
||||
|
||||
/// Parse `ZFIN_TIINGO_PLAN` into a `TiingoPlan`. Absent -> `.free`. A
|
||||
/// non-empty unrecognized value logs a warning and falls back to
|
||||
/// `.free` so a typo (e.g. "pwoer") doesn't silently throttle a paying
|
||||
/// subscriber to free-tier limits without explanation. The warning is
|
||||
/// gated under `!builtin.is_test` so the fallback-path tests below don't
|
||||
/// spam the test output.
|
||||
fn parseTiingoPlan(value: ?[]const u8) TiingoPlan {
|
||||
const v = value orelse return .free;
|
||||
if (std.ascii.eqlIgnoreCase(v, "free")) return .free;
|
||||
if (std.ascii.eqlIgnoreCase(v, "power")) return .power;
|
||||
if (!builtin.is_test) std.log.scoped(.config).warn("ZFIN_TIINGO_PLAN=\"{s}\" not recognized (expected free|power); using free", .{v});
|
||||
return .free;
|
||||
}
|
||||
|
||||
/// Parse all KEY=VALUE pairs from .env content into a HashMap.
|
||||
/// Values are slices into the original buffer (no extra allocations per entry).
|
||||
fn parseEnvFile(allocator: std.mem.Allocator, data: []const u8) ?EnvMap {
|
||||
|
|
@ -515,6 +551,29 @@ test "hasAnyKey: openfigi / server_url don't count as provider keys" {
|
|||
try testing.expect(!c.hasAnyKey());
|
||||
}
|
||||
|
||||
test "parseTiingoPlan: recognized values are case-insensitive" {
|
||||
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan("free"));
|
||||
try testing.expectEqual(TiingoPlan.power, parseTiingoPlan("power"));
|
||||
try testing.expectEqual(TiingoPlan.power, parseTiingoPlan("Power"));
|
||||
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan("FREE"));
|
||||
}
|
||||
|
||||
test "parseTiingoPlan: absent or unrecognized defaults to free" {
|
||||
// null (unset) is the common case and must not warn.
|
||||
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan(null));
|
||||
// Non-empty unrecognized values fall back to free (the warning is
|
||||
// gated under !builtin.is_test, so this stays quiet in tests).
|
||||
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan("enterprise"));
|
||||
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan(""));
|
||||
}
|
||||
|
||||
test "tiingoHourlyLimit: free is 50/hr, power is 10,000/hr" {
|
||||
var c: @This() = .{ .cache_dir = "/tmp" };
|
||||
try testing.expectEqual(@as(usize, 50), c.tiingoHourlyLimit());
|
||||
c.tiingo_plan = .power;
|
||||
try testing.expectEqual(@as(usize, 10_000), c.tiingoHourlyLimit());
|
||||
}
|
||||
|
||||
test "ResolvedPath.deinit: frees when owned, no-op when not owned" {
|
||||
const allocator = testing.allocator;
|
||||
|
||||
|
|
|
|||
|
|
@ -73,8 +73,9 @@ const max_message_size = 512 * 1024;
|
|||
const read_buffer_size = 32 * 1024;
|
||||
/// Read-poll timeout (ms). The read loop wakes at least this often to
|
||||
/// re-check `should_stop`, bounding quit latency while streaming. Short
|
||||
/// enough to feel instant, long enough to be free when idle.
|
||||
const read_poll_ms: u32 = 250;
|
||||
/// enough to feel instant, long enough to be free when idle. Untyped:
|
||||
/// coerces to the `readTimeout` argument type at the call site.
|
||||
const read_poll_ms = 250;
|
||||
const reconnect_backoff_ms = 2_000;
|
||||
/// Yahoo pricing protobufs are ~60-90 bytes decoded; 1 KiB is ample.
|
||||
/// A frame whose decoded size exceeds this is skipped (returns null).
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ const std = @import("std");
|
|||
|
||||
io: std.Io,
|
||||
/// Maximum tokens (requests) in the bucket
|
||||
max_tokens: u32,
|
||||
max_tokens: usize,
|
||||
/// Current available tokens
|
||||
tokens: f64,
|
||||
/// Tokens added per nanosecond
|
||||
|
|
@ -27,7 +27,7 @@ const RateLimiter = @This();
|
|||
|
||||
/// Create a rate limiter.
|
||||
/// `max_per_window` is the max requests allowed in `window_ns` nanoseconds.
|
||||
pub fn init(io: std.Io, max_per_window: u32, window_ns: u64) RateLimiter {
|
||||
pub fn init(io: std.Io, max_per_window: usize, window_ns: u64) RateLimiter {
|
||||
return .{
|
||||
.io = io,
|
||||
.max_tokens = max_per_window,
|
||||
|
|
@ -39,14 +39,14 @@ pub fn init(io: std.Io, max_per_window: u32, window_ns: u64) RateLimiter {
|
|||
|
||||
/// Convenience: N requests per minute.
|
||||
/// Starts with 1 token (no burst) to stay within provider sliding-window limits.
|
||||
pub fn perMinute(io: std.Io, n: u32) RateLimiter {
|
||||
pub fn perMinute(io: std.Io, n: usize) RateLimiter {
|
||||
var rl = init(io, n, std.time.ns_per_min);
|
||||
rl.tokens = 1.0;
|
||||
return rl;
|
||||
}
|
||||
|
||||
/// Convenience: N requests per day
|
||||
pub fn perDay(io: std.Io, n: u32) RateLimiter {
|
||||
pub fn perDay(io: std.Io, n: usize) RateLimiter {
|
||||
return init(io, n, std.time.ns_per_day);
|
||||
}
|
||||
|
||||
|
|
@ -59,7 +59,7 @@ pub fn perDay(io: std.Io, n: u32) RateLimiter {
|
|||
/// In-memory and per-process: it caps a single run's burst at N, which
|
||||
/// is the common case (one cron invocation). It does not coordinate
|
||||
/// across separate process launches within the same hour.
|
||||
pub fn perHour(io: std.Io, n: u32) RateLimiter {
|
||||
pub fn perHour(io: std.Io, n: usize) RateLimiter {
|
||||
return init(io, n, std.time.ns_per_hour);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
//! Tiingo provider -- official REST API for end-of-day prices and corporate actions.
|
||||
//!
|
||||
//! Free tier: 50 requests/hour and 1,000 requests/day. We enforce the
|
||||
//! hourly cap with a 50/hour token bucket (`RateLimiter.perHour`,
|
||||
//! which starts full so a nightly refresh burst runs unthrottled);
|
||||
//! the daily ceiling is far from binding for zfin's bursty usage.
|
||||
//! Rate limiting is plan-driven: the hourly token bucket is sized by
|
||||
//! `Config.tiingoHourlyLimit()` (free tier 50/hour, Power tier
|
||||
//! 10,000/hour) and passed to `init` via `Options.per_hour`. The bucket
|
||||
//! starts full so a nightly refresh burst runs unthrottled; sustained
|
||||
//! usage beyond the cap blocks in `acquire`. The daily ceiling is far
|
||||
//! from binding for zfin's bursty usage.
|
||||
//! Covers stocks, ETFs, and mutual funds with same-day NAV updates
|
||||
//! (mutual fund NAVs available after midnight ET).
|
||||
//!
|
||||
|
|
@ -74,17 +76,25 @@ pub const Tiingo = struct {
|
|||
client: http.Client,
|
||||
allocator: std.mem.Allocator,
|
||||
api_key: []const u8,
|
||||
/// Free-tier hourly cap (50/hour). Starts full so a nightly
|
||||
/// refresh burst (one candle file per held symbol) isn't paced;
|
||||
/// sustained usage beyond 50/hour blocks in `acquire`.
|
||||
/// Hourly request cap from the configured plan (free 50/hour, Power
|
||||
/// 10,000/hour - see `Config.tiingoHourlyLimit`). Starts full so a
|
||||
/// nightly refresh burst (one candle file per held symbol) isn't
|
||||
/// paced; sustained usage beyond the cap blocks in `acquire`.
|
||||
rate_limiter: RateLimiter,
|
||||
|
||||
pub fn init(io: std.Io, allocator: std.mem.Allocator, api_key: []const u8) Tiingo {
|
||||
/// Plan-driven construction options.
|
||||
pub const Options = struct {
|
||||
/// Hourly request budget, from `Config.tiingoHourlyLimit()`.
|
||||
/// Defaults to the free-tier 50/hour.
|
||||
per_hour: usize = 50,
|
||||
};
|
||||
|
||||
pub fn init(io: std.Io, allocator: std.mem.Allocator, api_key: []const u8, opts: Options) Tiingo {
|
||||
return .{
|
||||
.client = http.Client.init(io, allocator),
|
||||
.allocator = allocator,
|
||||
.api_key = api_key,
|
||||
.rate_limiter = RateLimiter.perHour(io, 50),
|
||||
.rate_limiter = RateLimiter.perHour(io, opts.per_hour),
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -414,6 +414,15 @@ pub const DataService = struct {
|
|||
// headers per each provider's politeness contract.
|
||||
const email = self.config.user_email orelse return DataError.NoApiKey;
|
||||
@field(self, field_name) = T.init(self.io, self.allocator, email);
|
||||
} else if (T == Tiingo) {
|
||||
// Tiingo takes a plan-driven hourly rate limit alongside the
|
||||
// key. The cap comes from `Config.tiingoHourlyLimit` (free
|
||||
// 50/hour, Power 10,000/hour) so a paying subscriber isn't
|
||||
// throttled to free-tier limits.
|
||||
const key = self.config.tiingo_key orelse return DataError.NoApiKey;
|
||||
@field(self, field_name) = Tiingo.init(self.io, self.allocator, key, .{
|
||||
.per_hour = self.config.tiingoHourlyLimit(),
|
||||
});
|
||||
} else {
|
||||
// All we're doing here is lower casing the type name, then
|
||||
// appending _key to it, so Tiingo -> tiingo_key
|
||||
|
|
@ -3199,6 +3208,92 @@ test "DataService live-stream accessors no-op when no stream is running" {
|
|||
try std.testing.expect(!svc.liveStreamActive());
|
||||
}
|
||||
|
||||
test "getProvider(Tiingo): hourly rate limit follows the configured plan" {
|
||||
const allocator = std.testing.allocator;
|
||||
// Free plan -> 50/hour token bucket.
|
||||
{
|
||||
var svc = DataService.init(std.testing.io, allocator, .{
|
||||
.cache_dir = "/tmp/zfin-test-cache",
|
||||
.tiingo_key = "test-key",
|
||||
.tiingo_plan = .free,
|
||||
});
|
||||
defer svc.deinit();
|
||||
const tg = try svc.getProvider(Tiingo);
|
||||
try std.testing.expectEqual(@as(usize, 50), tg.rate_limiter.max_tokens);
|
||||
}
|
||||
// Power plan -> 10,000/hour token bucket.
|
||||
{
|
||||
var svc = DataService.init(std.testing.io, allocator, .{
|
||||
.cache_dir = "/tmp/zfin-test-cache",
|
||||
.tiingo_key = "test-key",
|
||||
.tiingo_plan = .power,
|
||||
});
|
||||
defer svc.deinit();
|
||||
const tg = try svc.getProvider(Tiingo);
|
||||
try std.testing.expectEqual(@as(usize, 10_000), tg.rate_limiter.max_tokens);
|
||||
}
|
||||
}
|
||||
|
||||
test "getProvider(Tiingo): missing key -> NoApiKey" {
|
||||
const allocator = std.testing.allocator;
|
||||
var svc = DataService.init(std.testing.io, allocator, .{ .cache_dir = "/tmp/zfin-test-cache" });
|
||||
defer svc.deinit();
|
||||
try std.testing.expectError(DataError.NoApiKey, svc.getProvider(Tiingo));
|
||||
}
|
||||
|
||||
test "getProvider: keyless/keyed dispatch and lazy caching" {
|
||||
const allocator = std.testing.allocator;
|
||||
var svc = DataService.init(std.testing.io, allocator, .{
|
||||
.cache_dir = "/tmp/zfin-test-cache",
|
||||
.twelvedata_key = "td-key",
|
||||
});
|
||||
defer svc.deinit();
|
||||
|
||||
// Keyless provider (Yahoo): no key needed; lazily created, then
|
||||
// cached - a second call returns the same pointer.
|
||||
const yh1 = try svc.getProvider(Yahoo);
|
||||
const yh2 = try svc.getProvider(Yahoo);
|
||||
try std.testing.expect(yh1 == yh2);
|
||||
|
||||
// Keyed provider via the generic branch (TwelveData, key present).
|
||||
_ = try svc.getProvider(TwelveData);
|
||||
|
||||
// Keyed provider whose key is unset -> NoApiKey (Polygon).
|
||||
try std.testing.expectError(DataError.NoApiKey, svc.getProvider(Polygon));
|
||||
}
|
||||
|
||||
test "getProvider: open-data providers need a contact email" {
|
||||
const allocator = std.testing.allocator;
|
||||
// With an email, Wikidata/Edgar init via the email branch.
|
||||
{
|
||||
var svc = DataService.init(std.testing.io, allocator, .{
|
||||
.cache_dir = "/tmp/zfin-test-cache",
|
||||
.user_email = "test@example.com",
|
||||
});
|
||||
defer svc.deinit();
|
||||
_ = try svc.getProvider(Wikidata);
|
||||
_ = try svc.getProvider(Edgar);
|
||||
}
|
||||
// Without an email -> NoApiKey.
|
||||
{
|
||||
var svc = DataService.init(std.testing.io, allocator, .{ .cache_dir = "/tmp/zfin-test-cache" });
|
||||
defer svc.deinit();
|
||||
try std.testing.expectError(DataError.NoApiKey, svc.getProvider(Wikidata));
|
||||
}
|
||||
}
|
||||
|
||||
test "loadLiveQuotes: empty symbol list returns empty without touching the network" {
|
||||
const allocator = std.testing.allocator;
|
||||
// panic_on_network_attempt asserts the early return happens BEFORE
|
||||
// any network gate is reached.
|
||||
var svc = DataService.init(std.testing.io, allocator, .{ .cache_dir = "/tmp/zfin-test-cache" });
|
||||
svc.panic_on_network_attempt = true;
|
||||
defer svc.deinit();
|
||||
var prices = svc.loadLiveQuotes(&.{});
|
||||
defer prices.deinit();
|
||||
try std.testing.expectEqual(@as(usize, 0), prices.count());
|
||||
}
|
||||
|
||||
test "DataService store helper creates valid store" {
|
||||
const allocator = std.testing.allocator;
|
||||
const config = Config{
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue