support Tiingo power tier rate limit

This commit is contained in:
Emil Lerch 2026-06-29 20:53:42 -07:00
parent 051ab35975
commit a73966ad39
Signed by: lobo
GPG key ID: A7B62D657EF764F8
6 changed files with 182 additions and 21 deletions

View file

@ -46,11 +46,7 @@ repos:
- id: test
name: Run zig build test
entry: zig
# TEMPORARY: lowered 80 -> 79 for the portfolio live-stream TUI
# work (Phase D), whose event/tick handlers need a vaxis App
# harness we don't have. Restore to 80 once the Tiingo REST
# quote work (highly testable parsers) lands.
args: ["build", "coverage", "-Dcoverage-threshold=79"]
args: ["build", "coverage", "-Dcoverage-threshold=80"]
language: system
types: [file]
pass_filenames: false

View file

@ -14,6 +14,7 @@
//! > `$HOME/.cache/zfin`.
const std = @import("std");
const builtin = @import("builtin");
const EnvMap = std.StringHashMap([]const u8);
@ -30,12 +31,21 @@ pub const default_portfolio_filename = "portfolio*.srf";
/// `default_portfolio_filename`.
pub const default_watchlist_filename = "watchlist.srf";
/// Tiingo subscription tier, sourced from `ZFIN_TIINGO_PLAN`. Drives the
/// provider's hourly rate-limit budget (see `tiingoHourlyLimit`): the
/// free tier is capped at 50 requests/hour, while paid tiers raise that
/// ceiling substantially. Defaults to `.free`.
pub const TiingoPlan = enum { free, power };
// Fields
twelvedata_key: ?[]const u8 = null,
polygon_key: ?[]const u8 = null,
fmp_key: ?[]const u8 = null,
tiingo_key: ?[]const u8 = null,
/// Tiingo subscription tier (`ZFIN_TIINGO_PLAN`). Drives the provider's
/// hourly rate-limit budget via `tiingoHourlyLimit`. Defaults to `.free`.
tiingo_plan: TiingoPlan = .free,
openfigi_key: ?[]const u8 = null,
/// User contact email used as the User-Agent / From header for
/// open-data providers that require politeness identification
@ -101,6 +111,7 @@ pub fn fromEnv(io: std.Io, allocator: std.mem.Allocator, environ_map: *const std
self.polygon_key = self.resolve("POLYGON_API_KEY");
self.fmp_key = self.resolve("FMP_API_KEY");
self.tiingo_key = self.resolve("TIINGO_API_KEY");
self.tiingo_plan = parseTiingoPlan(self.resolve("ZFIN_TIINGO_PLAN"));
self.openfigi_key = self.resolve("OPENFIGI_API_KEY");
self.user_email = self.resolve("ZFIN_USER_EMAIL");
self.server_url = self.resolve("ZFIN_SERVER");
@ -381,6 +392,17 @@ pub fn hasAnyKey(self: @This()) bool {
self.tiingo_key != null;
}
/// Tiingo's hourly request budget for the configured plan. The free
/// tier is 50/hour; the Power tier raises it to 10,000/hour. Threaded
/// into the provider's rate limiter (see `service.getProvider`) so a
/// paying subscriber isn't throttled to free-tier limits.
pub fn tiingoHourlyLimit(self: @This()) usize {
return switch (self.tiingo_plan) {
.free => 50,
.power => 10_000,
};
}
// Internals
/// Look up a key: process environment first, then .env file fallback.
@ -392,6 +414,20 @@ fn resolve(self: *@This(), key: []const u8) ?[]const u8 {
return null;
}
/// Parse `ZFIN_TIINGO_PLAN` into a `TiingoPlan`. Absent -> `.free`. A
/// non-empty unrecognized value logs a warning and falls back to
/// `.free` so a typo (e.g. "pwoer") doesn't silently throttle a paying
/// subscriber to free-tier limits without explanation. The warning is
/// gated under `!builtin.is_test` so the fallback-path tests below don't
/// spam the test output.
fn parseTiingoPlan(value: ?[]const u8) TiingoPlan {
const v = value orelse return .free;
if (std.ascii.eqlIgnoreCase(v, "free")) return .free;
if (std.ascii.eqlIgnoreCase(v, "power")) return .power;
if (!builtin.is_test) std.log.scoped(.config).warn("ZFIN_TIINGO_PLAN=\"{s}\" not recognized (expected free|power); using free", .{v});
return .free;
}
/// Parse all KEY=VALUE pairs from .env content into a HashMap.
/// Values are slices into the original buffer (no extra allocations per entry).
fn parseEnvFile(allocator: std.mem.Allocator, data: []const u8) ?EnvMap {
@ -515,6 +551,29 @@ test "hasAnyKey: openfigi / server_url don't count as provider keys" {
try testing.expect(!c.hasAnyKey());
}
test "parseTiingoPlan: recognized values are case-insensitive" {
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan("free"));
try testing.expectEqual(TiingoPlan.power, parseTiingoPlan("power"));
try testing.expectEqual(TiingoPlan.power, parseTiingoPlan("Power"));
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan("FREE"));
}
test "parseTiingoPlan: absent or unrecognized defaults to free" {
// null (unset) is the common case and must not warn.
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan(null));
// Non-empty unrecognized values fall back to free (the warning is
// gated under !builtin.is_test, so this stays quiet in tests).
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan("enterprise"));
try testing.expectEqual(TiingoPlan.free, parseTiingoPlan(""));
}
test "tiingoHourlyLimit: free is 50/hr, power is 10,000/hr" {
var c: @This() = .{ .cache_dir = "/tmp" };
try testing.expectEqual(@as(usize, 50), c.tiingoHourlyLimit());
c.tiingo_plan = .power;
try testing.expectEqual(@as(usize, 10_000), c.tiingoHourlyLimit());
}
test "ResolvedPath.deinit: frees when owned, no-op when not owned" {
const allocator = testing.allocator;

View file

@ -73,8 +73,9 @@ const max_message_size = 512 * 1024;
const read_buffer_size = 32 * 1024;
/// Read-poll timeout (ms). The read loop wakes at least this often to
/// re-check `should_stop`, bounding quit latency while streaming. Short
/// enough to feel instant, long enough to be free when idle.
const read_poll_ms: u32 = 250;
/// enough to feel instant, long enough to be free when idle. Untyped:
/// coerces to the `readTimeout` argument type at the call site.
const read_poll_ms = 250;
const reconnect_backoff_ms = 2_000;
/// Yahoo pricing protobufs are ~60-90 bytes decoded; 1 KiB is ample.
/// A frame whose decoded size exceeds this is skipped (returns null).

View file

@ -15,7 +15,7 @@ const std = @import("std");
io: std.Io,
/// Maximum tokens (requests) in the bucket
max_tokens: u32,
max_tokens: usize,
/// Current available tokens
tokens: f64,
/// Tokens added per nanosecond
@ -27,7 +27,7 @@ const RateLimiter = @This();
/// Create a rate limiter.
/// `max_per_window` is the max requests allowed in `window_ns` nanoseconds.
pub fn init(io: std.Io, max_per_window: u32, window_ns: u64) RateLimiter {
pub fn init(io: std.Io, max_per_window: usize, window_ns: u64) RateLimiter {
return .{
.io = io,
.max_tokens = max_per_window,
@ -39,14 +39,14 @@ pub fn init(io: std.Io, max_per_window: u32, window_ns: u64) RateLimiter {
/// Convenience: N requests per minute.
/// Starts with 1 token (no burst) to stay within provider sliding-window limits.
pub fn perMinute(io: std.Io, n: u32) RateLimiter {
pub fn perMinute(io: std.Io, n: usize) RateLimiter {
var rl = init(io, n, std.time.ns_per_min);
rl.tokens = 1.0;
return rl;
}
/// Convenience: N requests per day
pub fn perDay(io: std.Io, n: u32) RateLimiter {
pub fn perDay(io: std.Io, n: usize) RateLimiter {
return init(io, n, std.time.ns_per_day);
}
@ -59,7 +59,7 @@ pub fn perDay(io: std.Io, n: u32) RateLimiter {
/// In-memory and per-process: it caps a single run's burst at N, which
/// is the common case (one cron invocation). It does not coordinate
/// across separate process launches within the same hour.
pub fn perHour(io: std.Io, n: u32) RateLimiter {
pub fn perHour(io: std.Io, n: usize) RateLimiter {
return init(io, n, std.time.ns_per_hour);
}

View file

@ -1,9 +1,11 @@
//! Tiingo provider -- official REST API for end-of-day prices and corporate actions.
//!
//! Free tier: 50 requests/hour and 1,000 requests/day. We enforce the
//! hourly cap with a 50/hour token bucket (`RateLimiter.perHour`,
//! which starts full so a nightly refresh burst runs unthrottled);
//! the daily ceiling is far from binding for zfin's bursty usage.
//! Rate limiting is plan-driven: the hourly token bucket is sized by
//! `Config.tiingoHourlyLimit()` (free tier 50/hour, Power tier
//! 10,000/hour) and passed to `init` via `Options.per_hour`. The bucket
//! starts full so a nightly refresh burst runs unthrottled; sustained
//! usage beyond the cap blocks in `acquire`. The daily ceiling is far
//! from binding for zfin's bursty usage.
//! Covers stocks, ETFs, and mutual funds with same-day NAV updates
//! (mutual fund NAVs available after midnight ET).
//!
@ -74,17 +76,25 @@ pub const Tiingo = struct {
client: http.Client,
allocator: std.mem.Allocator,
api_key: []const u8,
/// Free-tier hourly cap (50/hour). Starts full so a nightly
/// refresh burst (one candle file per held symbol) isn't paced;
/// sustained usage beyond 50/hour blocks in `acquire`.
/// Hourly request cap from the configured plan (free 50/hour, Power
/// 10,000/hour - see `Config.tiingoHourlyLimit`). Starts full so a
/// nightly refresh burst (one candle file per held symbol) isn't
/// paced; sustained usage beyond the cap blocks in `acquire`.
rate_limiter: RateLimiter,
pub fn init(io: std.Io, allocator: std.mem.Allocator, api_key: []const u8) Tiingo {
/// Plan-driven construction options.
pub const Options = struct {
/// Hourly request budget, from `Config.tiingoHourlyLimit()`.
/// Defaults to the free-tier 50/hour.
per_hour: usize = 50,
};
pub fn init(io: std.Io, allocator: std.mem.Allocator, api_key: []const u8, opts: Options) Tiingo {
return .{
.client = http.Client.init(io, allocator),
.allocator = allocator,
.api_key = api_key,
.rate_limiter = RateLimiter.perHour(io, 50),
.rate_limiter = RateLimiter.perHour(io, opts.per_hour),
};
}

View file

@ -414,6 +414,15 @@ pub const DataService = struct {
// headers per each provider's politeness contract.
const email = self.config.user_email orelse return DataError.NoApiKey;
@field(self, field_name) = T.init(self.io, self.allocator, email);
} else if (T == Tiingo) {
// Tiingo takes a plan-driven hourly rate limit alongside the
// key. The cap comes from `Config.tiingoHourlyLimit` (free
// 50/hour, Power 10,000/hour) so a paying subscriber isn't
// throttled to free-tier limits.
const key = self.config.tiingo_key orelse return DataError.NoApiKey;
@field(self, field_name) = Tiingo.init(self.io, self.allocator, key, .{
.per_hour = self.config.tiingoHourlyLimit(),
});
} else {
// All we're doing here is lower casing the type name, then
// appending _key to it, so Tiingo -> tiingo_key
@ -3199,6 +3208,92 @@ test "DataService live-stream accessors no-op when no stream is running" {
try std.testing.expect(!svc.liveStreamActive());
}
test "getProvider(Tiingo): hourly rate limit follows the configured plan" {
const allocator = std.testing.allocator;
// Free plan -> 50/hour token bucket.
{
var svc = DataService.init(std.testing.io, allocator, .{
.cache_dir = "/tmp/zfin-test-cache",
.tiingo_key = "test-key",
.tiingo_plan = .free,
});
defer svc.deinit();
const tg = try svc.getProvider(Tiingo);
try std.testing.expectEqual(@as(usize, 50), tg.rate_limiter.max_tokens);
}
// Power plan -> 10,000/hour token bucket.
{
var svc = DataService.init(std.testing.io, allocator, .{
.cache_dir = "/tmp/zfin-test-cache",
.tiingo_key = "test-key",
.tiingo_plan = .power,
});
defer svc.deinit();
const tg = try svc.getProvider(Tiingo);
try std.testing.expectEqual(@as(usize, 10_000), tg.rate_limiter.max_tokens);
}
}
test "getProvider(Tiingo): missing key -> NoApiKey" {
const allocator = std.testing.allocator;
var svc = DataService.init(std.testing.io, allocator, .{ .cache_dir = "/tmp/zfin-test-cache" });
defer svc.deinit();
try std.testing.expectError(DataError.NoApiKey, svc.getProvider(Tiingo));
}
test "getProvider: keyless/keyed dispatch and lazy caching" {
const allocator = std.testing.allocator;
var svc = DataService.init(std.testing.io, allocator, .{
.cache_dir = "/tmp/zfin-test-cache",
.twelvedata_key = "td-key",
});
defer svc.deinit();
// Keyless provider (Yahoo): no key needed; lazily created, then
// cached - a second call returns the same pointer.
const yh1 = try svc.getProvider(Yahoo);
const yh2 = try svc.getProvider(Yahoo);
try std.testing.expect(yh1 == yh2);
// Keyed provider via the generic branch (TwelveData, key present).
_ = try svc.getProvider(TwelveData);
// Keyed provider whose key is unset -> NoApiKey (Polygon).
try std.testing.expectError(DataError.NoApiKey, svc.getProvider(Polygon));
}
test "getProvider: open-data providers need a contact email" {
const allocator = std.testing.allocator;
// With an email, Wikidata/Edgar init via the email branch.
{
var svc = DataService.init(std.testing.io, allocator, .{
.cache_dir = "/tmp/zfin-test-cache",
.user_email = "test@example.com",
});
defer svc.deinit();
_ = try svc.getProvider(Wikidata);
_ = try svc.getProvider(Edgar);
}
// Without an email -> NoApiKey.
{
var svc = DataService.init(std.testing.io, allocator, .{ .cache_dir = "/tmp/zfin-test-cache" });
defer svc.deinit();
try std.testing.expectError(DataError.NoApiKey, svc.getProvider(Wikidata));
}
}
test "loadLiveQuotes: empty symbol list returns empty without touching the network" {
const allocator = std.testing.allocator;
// panic_on_network_attempt asserts the early return happens BEFORE
// any network gate is reached.
var svc = DataService.init(std.testing.io, allocator, .{ .cache_dir = "/tmp/zfin-test-cache" });
svc.panic_on_network_attempt = true;
defer svc.deinit();
var prices = svc.loadLiveQuotes(&.{});
defer prices.deinit();
try std.testing.expectEqual(@as(usize, 0), prices.count());
}
test "DataService store helper creates valid store" {
const allocator = std.testing.allocator;
const config = Config{