Compare commits
6 commits
641a88b0b7
...
4d65cc45f4
| Author | SHA1 | Date | |
|---|---|---|---|
| 4d65cc45f4 | |||
| 60435b645f | |||
| b9416dab7f | |||
| 95d57d43ec | |||
| 4639edd813 | |||
| f54faf4732 |
4 changed files with 343 additions and 82 deletions
284
src/cache/store.zig
vendored
284
src/cache/store.zig
vendored
|
|
@ -214,19 +214,46 @@ pub const DataType = enum {
|
|||
};
|
||||
}
|
||||
|
||||
pub fn ttl(self: DataType) i64 {
|
||||
/// TTL specification for this data type, including any per-key
|
||||
/// jitter policy. The cache layer owns the policy because it
|
||||
/// owns both the constants (`Ttl.X`) and the mechanism that
|
||||
/// applies them (`computeExpires`); call sites just delegate.
|
||||
///
|
||||
/// Jitter assignments:
|
||||
///
|
||||
/// - 11% on dividends/splits (14d base, ~3d total spread).
|
||||
/// Tuned so a daily cron sees roughly 1/3 of a portfolio's
|
||||
/// symbols expire each day instead of all in lockstep.
|
||||
///
|
||||
/// - 8% on the longer-TTL types (classification 90d,
|
||||
/// etf_metrics 90d, entity_facts 30d, ticker maps 30d).
|
||||
/// Same thundering-herd defense; smaller percentage
|
||||
/// because the absolute spread on a 30d/90d base is
|
||||
/// already large in days.
|
||||
///
|
||||
/// - 0% on the rest. options/earnings/etf_profile either
|
||||
/// have natural cadence spread or are short-TTL enough
|
||||
/// that jitter would exceed meaningful drift.
|
||||
pub fn ttl(self: DataType) TtlSpec {
|
||||
return switch (self) {
|
||||
.dividends => Ttl.dividends,
|
||||
.splits => Ttl.splits,
|
||||
.options => Ttl.options,
|
||||
.earnings => Ttl.earnings,
|
||||
.etf_profile => Ttl.etf_profile,
|
||||
.classification => Ttl.classification,
|
||||
.etf_metrics => Ttl.etf_metrics,
|
||||
.entity_facts => Ttl.entity_facts,
|
||||
.tickers_funds => Ttl.tickers_funds,
|
||||
.tickers_companies => Ttl.tickers_companies,
|
||||
.candles_daily, .candles_meta, .meta => 0,
|
||||
.dividends => .{ .seconds = Ttl.dividends, .jitter_pct = 11 },
|
||||
.splits => .{ .seconds = Ttl.splits, .jitter_pct = 11 },
|
||||
.options => .{ .seconds = Ttl.options },
|
||||
.earnings => .{ .seconds = Ttl.earnings },
|
||||
.etf_profile => .{ .seconds = Ttl.etf_profile },
|
||||
.classification => .{ .seconds = Ttl.classification, .jitter_pct = 8 },
|
||||
.etf_metrics => .{ .seconds = Ttl.etf_metrics, .jitter_pct = 8 },
|
||||
.entity_facts => .{ .seconds = Ttl.entity_facts, .jitter_pct = 8 },
|
||||
.tickers_funds => .{ .seconds = Ttl.tickers_funds, .jitter_pct = 8 },
|
||||
.tickers_companies => .{ .seconds = Ttl.tickers_companies, .jitter_pct = 8 },
|
||||
// Sentinel: these types have their own writers
|
||||
// (`cacheCandles` for the candle pair, `writeNegative`
|
||||
// for `meta`) that don't go through the generic
|
||||
// `write()` / `writeWithSource()` path. Calling
|
||||
// `.ttl()` on one of them is a misuse — replace this
|
||||
// `unreachable` with `@compileError` once the call
|
||||
// graph is locked down enough to enforce at comptime.
|
||||
.candles_daily, .candles_meta, .meta => unreachable,
|
||||
};
|
||||
}
|
||||
};
|
||||
|
|
@ -521,11 +548,15 @@ pub const Store = struct {
|
|||
}
|
||||
}
|
||||
|
||||
if (added == 0 and upgraded == 0) {
|
||||
// Nothing changed — leave the file untouched. This is
|
||||
// the common case for repeated Polygon/Tiingo refreshes.
|
||||
return;
|
||||
}
|
||||
// Note: even when nothing was added or upgraded, fall through
|
||||
// and rewrite the file. The on-disk `#!expires=` directive
|
||||
// needs to be refreshed every time we successfully fetched
|
||||
// and merged, otherwise an aged-out file stays aged-out:
|
||||
// every subsequent refresh pays the full provider rate-
|
||||
// limiter cost only to discover no changes and skip the
|
||||
// write, locking the cache into a permanent slow-path. The
|
||||
// write itself is a sub-millisecond atomic rename of a tiny
|
||||
// file, so saving it isn't worth the bookkeeping.
|
||||
|
||||
// Sort descending by date (newest first), matching the
|
||||
// existing on-disk convention.
|
||||
|
|
@ -1779,7 +1810,15 @@ test "writeMerged Dividend: union sorted desc, new entry added" {
|
|||
try std.testing.expect(result.data[2].ex_date.eql(Date.fromYmd(2024, 2, 15)));
|
||||
}
|
||||
|
||||
test "writeMerged Dividend: no-op when nothing new" {
|
||||
test "writeMerged Dividend: no-change merge still rewrites to refresh expires" {
|
||||
// The "nothing changed" path used to skip the rewrite as an
|
||||
// optimization. Problem: once the on-disk `#!expires=` aged
|
||||
// past TTL, every subsequent refresh paid the full provider
|
||||
// rate-limiter cost only to discover no changes and skip the
|
||||
// write again, locking the cache into a permanent slow-path.
|
||||
//
|
||||
// Now we always rewrite. The write is a sub-millisecond atomic
|
||||
// rename of a tiny file; saving it isn't worth the bookkeeping.
|
||||
const allocator = std.testing.allocator;
|
||||
const io = std.testing.io;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
|
|
@ -1788,27 +1827,41 @@ test "writeMerged Dividend: no-op when nothing new" {
|
|||
defer allocator.free(dir_path);
|
||||
|
||||
var s = Store.init(io, allocator, dir_path);
|
||||
var initial = [_]Dividend{
|
||||
|
||||
// Seed a file with expires 30 days in the past — the aged-out
|
||||
// case that motivated this fix. (Pre-fix: the no-change merge
|
||||
// would skip the write and the file would stay aged-out
|
||||
// forever. Post-fix: the file gets rewritten with a fresh
|
||||
// expires.)
|
||||
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
||||
const seed_expires = now_s - 30 * std.time.s_per_day;
|
||||
const divs = [_]Dividend{
|
||||
.{ .ex_date = Date.fromYmd(2024, 5, 15), .amount = 0.50, .type = .regular },
|
||||
};
|
||||
s.write(Dividend, "TEST", initial[0..], .{ .seconds = Ttl.dividends });
|
||||
const seed_bytes = try Store.serializeWithMeta(Dividend, io, allocator, &divs, .{ .expires = seed_expires });
|
||||
defer allocator.free(seed_bytes);
|
||||
try s.writeRaw("TEST", .dividends, seed_bytes);
|
||||
|
||||
// Capture file mtime before second (no-op) write.
|
||||
const path = try std.fs.path.join(allocator, &.{ dir_path, "TEST", "dividends.srf" });
|
||||
defer allocator.free(path);
|
||||
const stat_before = try std.Io.Dir.cwd().statFile(io, path, .{});
|
||||
|
||||
// Sleep briefly so mtime resolution can detect a write if one happens.
|
||||
std.Io.sleep(io, std.Io.Duration.fromMilliseconds(20), .awake) catch {};
|
||||
|
||||
// Same incoming entry — nothing new, should not rewrite.
|
||||
// Same incoming entry — nothing new, but we still expect a rewrite.
|
||||
var repeat = [_]Dividend{
|
||||
.{ .ex_date = Date.fromYmd(2024, 5, 15), .amount = 0.50, .type = .regular },
|
||||
};
|
||||
s.write(Dividend, "TEST", repeat[0..], .{ .seconds = Ttl.dividends });
|
||||
s.writeWithSource(Dividend, "TEST", repeat[0..], .{ .seconds = Ttl.dividends }, "polygon");
|
||||
|
||||
const stat_after = try std.Io.Dir.cwd().statFile(io, path, .{});
|
||||
try std.testing.expectEqual(stat_before.mtime, stat_after.mtime);
|
||||
// Confirm fresh expires landed on disk: read the raw file and
|
||||
// parse out the directive, expecting it to be roughly now+14d.
|
||||
const path = try std.fs.path.join(allocator, &.{ dir_path, "TEST", "dividends.srf" });
|
||||
defer allocator.free(path);
|
||||
const data = try std.Io.Dir.cwd().readFileAlloc(io, path, allocator, .limited(1024 * 1024));
|
||||
defer allocator.free(data);
|
||||
|
||||
var reader = std.Io.Reader.fixed(data);
|
||||
const it = try srf.iterator(&reader, allocator, .{});
|
||||
defer it.deinit();
|
||||
|
||||
const new_expires = it.expires orelse return error.ExpiresMissing;
|
||||
try std.testing.expect(new_expires > now_s);
|
||||
try std.testing.expect(new_expires - now_s > 13 * std.time.s_per_day);
|
||||
}
|
||||
|
||||
test "writeMerged Dividend: field-level upgrade fills nulls (Tiingo-then-Polygon)" {
|
||||
|
|
@ -1860,6 +1913,98 @@ test "writeMerged Dividend: field-level upgrade fills nulls (Tiingo-then-Polygon
|
|||
try std.testing.expectEqual(DividendType.regular, result.data[0].type);
|
||||
}
|
||||
|
||||
test "writeMerged Dividend: currency upgrade does not double-free" {
|
||||
// Tiingo writes a sparse record (no currency). Polygon's later
|
||||
// write supplies a heap-allocated currency string. The merge
|
||||
// path must not let `existing.currency = incoming.currency`
|
||||
// create two records that both believe they own the same
|
||||
// buffer, otherwise std.testing.allocator's double-free
|
||||
// detection trips when the caller's deinit runs later.
|
||||
const allocator = std.testing.allocator;
|
||||
const io = std.testing.io;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
var s = Store.init(io, allocator, dir_path);
|
||||
|
||||
// Tiingo first: sparse — no currency.
|
||||
var tiingo_view = [_]Dividend{
|
||||
.{ .ex_date = Date.fromYmd(2024, 5, 15), .amount = 0.50 },
|
||||
};
|
||||
s.writeWithSource(Dividend, "TEST", tiingo_view[0..], .{ .seconds = Ttl.dividends }, "tiingo");
|
||||
|
||||
// Polygon second: same ex_date, but supplies currency. Caller
|
||||
// owns the heap allocation and frees it after writeMerged
|
||||
// returns (mirrors how Polygon's fetchDividends works in
|
||||
// production: returns slice with heap-allocated currency
|
||||
// strings, caller deinits).
|
||||
var polygon_view = [_]Dividend{
|
||||
.{
|
||||
.ex_date = Date.fromYmd(2024, 5, 15),
|
||||
.amount = 0.50,
|
||||
.currency = try allocator.dupe(u8, "USD"),
|
||||
},
|
||||
};
|
||||
defer for (polygon_view) |d| d.deinit(allocator);
|
||||
s.writeWithSource(Dividend, "TEST", polygon_view[0..], .{ .seconds = Ttl.dividends }, "polygon");
|
||||
|
||||
// Read back and verify the upgrade landed.
|
||||
const result = s.read(Dividend, "TEST", null, .any) orelse return error.NoCache;
|
||||
defer allocator.free(result.data);
|
||||
defer for (result.data) |d| d.deinit(allocator);
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 1), result.data.len);
|
||||
try std.testing.expect(result.data[0].currency != null);
|
||||
try std.testing.expectEqualStrings("USD", result.data[0].currency.?);
|
||||
}
|
||||
|
||||
test "writeMerged Dividend: existing currency preserved on second write with different currency" {
|
||||
// Polygon writes USD first. A later write with a different
|
||||
// currency (CAD) must NOT overwrite — first non-null wins.
|
||||
// This exercises the path where both existing and incoming
|
||||
// have non-null currency strings, which is the trickiest
|
||||
// shape for the merge primitive's lifetime management.
|
||||
const allocator = std.testing.allocator;
|
||||
const io = std.testing.io;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
|
||||
defer allocator.free(dir_path);
|
||||
|
||||
var s = Store.init(io, allocator, dir_path);
|
||||
|
||||
var first = [_]Dividend{
|
||||
.{
|
||||
.ex_date = Date.fromYmd(2024, 5, 15),
|
||||
.amount = 0.50,
|
||||
.currency = try allocator.dupe(u8, "USD"),
|
||||
},
|
||||
};
|
||||
defer for (first) |d| d.deinit(allocator);
|
||||
s.writeWithSource(Dividend, "TEST", first[0..], .{ .seconds = Ttl.dividends }, "polygon");
|
||||
|
||||
var second = [_]Dividend{
|
||||
.{
|
||||
.ex_date = Date.fromYmd(2024, 5, 15),
|
||||
.amount = 0.50,
|
||||
.currency = try allocator.dupe(u8, "CAD"),
|
||||
},
|
||||
};
|
||||
defer for (second) |d| d.deinit(allocator);
|
||||
s.writeWithSource(Dividend, "TEST", second[0..], .{ .seconds = Ttl.dividends }, "polygon");
|
||||
|
||||
const result = s.read(Dividend, "TEST", null, .any) orelse return error.NoCache;
|
||||
defer allocator.free(result.data);
|
||||
defer for (result.data) |d| d.deinit(allocator);
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 1), result.data.len);
|
||||
try std.testing.expect(result.data[0].currency != null);
|
||||
// First write's currency wins.
|
||||
try std.testing.expectEqualStrings("USD", result.data[0].currency.?);
|
||||
}
|
||||
|
||||
test "writeMerged Dividend: type unknown counts as null and gets upgraded" {
|
||||
// Tiingo's dividend records always carry type = .unknown. A
|
||||
// later Polygon write with type = .regular must upgrade the
|
||||
|
|
@ -1939,8 +2084,10 @@ test "writeMerged Dividend: non-null fields are not overwritten" {
|
|||
|
||||
test "writeMerged Dividend: upgrade is no-op when both have same fields" {
|
||||
// Both writes have the same ex_date, amount, pay_date, and
|
||||
// type. There's nothing to upgrade and nothing new — the file
|
||||
// should not be touched on the second write.
|
||||
// type. There's nothing to upgrade and nothing new — but the
|
||||
// file is still rewritten so the on-disk `#!expires=` directive
|
||||
// gets refreshed. (Pre-rewrite-always behavior was to skip;
|
||||
// that locked aged-out files into a permanent slow-path.)
|
||||
const allocator = std.testing.allocator;
|
||||
const io = std.testing.io;
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
|
|
@ -1960,12 +2107,6 @@ test "writeMerged Dividend: upgrade is no-op when both have same fields" {
|
|||
};
|
||||
s.writeWithSource(Dividend, "TEST", initial[0..], .{ .seconds = Ttl.dividends }, "polygon");
|
||||
|
||||
const path = try std.fs.path.join(allocator, &.{ dir_path, "TEST", "dividends.srf" });
|
||||
defer allocator.free(path);
|
||||
const stat_before = try std.Io.Dir.cwd().statFile(io, path, .{});
|
||||
|
||||
std.Io.sleep(io, std.Io.Duration.fromMilliseconds(20), .awake) catch {};
|
||||
|
||||
var repeat = [_]Dividend{
|
||||
.{
|
||||
.ex_date = Date.fromYmd(2024, 5, 15),
|
||||
|
|
@ -1976,8 +2117,11 @@ test "writeMerged Dividend: upgrade is no-op when both have same fields" {
|
|||
};
|
||||
s.writeWithSource(Dividend, "TEST", repeat[0..], .{ .seconds = Ttl.dividends }, "polygon");
|
||||
|
||||
const stat_after = try std.Io.Dir.cwd().statFile(io, path, .{});
|
||||
try std.testing.expectEqual(stat_before.mtime, stat_after.mtime);
|
||||
// The merged result is still just one record (no duplication).
|
||||
const result = s.read(Dividend, "TEST", null, .any) orelse return error.NoCache;
|
||||
defer allocator.free(result.data);
|
||||
defer for (result.data) |d| d.deinit(allocator);
|
||||
try std.testing.expectEqual(@as(usize, 1), result.data.len);
|
||||
}
|
||||
|
||||
test "writeMerged Dividend: near-match dedup catches last-biz-day vs calendar-end" {
|
||||
|
|
@ -2354,22 +2498,48 @@ test "TTL constants are reasonable" {
|
|||
try std.testing.expectEqual(@as(i64, 30 * std.time.s_per_day), Ttl.tickers_companies);
|
||||
}
|
||||
|
||||
test "DataType.ttl returns correct values" {
|
||||
try std.testing.expectEqual(Ttl.dividends, DataType.dividends.ttl());
|
||||
try std.testing.expectEqual(Ttl.splits, DataType.splits.ttl());
|
||||
try std.testing.expectEqual(Ttl.options, DataType.options.ttl());
|
||||
try std.testing.expectEqual(Ttl.earnings, DataType.earnings.ttl());
|
||||
try std.testing.expectEqual(Ttl.etf_profile, DataType.etf_profile.ttl());
|
||||
try std.testing.expectEqual(Ttl.classification, DataType.classification.ttl());
|
||||
try std.testing.expectEqual(Ttl.etf_metrics, DataType.etf_metrics.ttl());
|
||||
try std.testing.expectEqual(Ttl.entity_facts, DataType.entity_facts.ttl());
|
||||
try std.testing.expectEqual(Ttl.tickers_funds, DataType.tickers_funds.ttl());
|
||||
try std.testing.expectEqual(Ttl.tickers_companies, DataType.tickers_companies.ttl());
|
||||
test "DataType.ttl returns correct seconds and jitter policy" {
|
||||
// 11% jitter: dividends and splits.
|
||||
const div = DataType.dividends.ttl();
|
||||
try std.testing.expectEqual(Ttl.dividends, div.seconds);
|
||||
try std.testing.expectEqual(@as(u8, 11), div.jitter_pct);
|
||||
|
||||
// These types have no TTL (0 = managed elsewhere)
|
||||
try std.testing.expectEqual(@as(i64, 0), DataType.candles_daily.ttl());
|
||||
try std.testing.expectEqual(@as(i64, 0), DataType.candles_meta.ttl());
|
||||
try std.testing.expectEqual(@as(i64, 0), DataType.meta.ttl());
|
||||
const spl = DataType.splits.ttl();
|
||||
try std.testing.expectEqual(Ttl.splits, spl.seconds);
|
||||
try std.testing.expectEqual(@as(u8, 11), spl.jitter_pct);
|
||||
|
||||
// 8% jitter: classification, etf_metrics, entity_facts, ticker maps.
|
||||
const cls = DataType.classification.ttl();
|
||||
try std.testing.expectEqual(Ttl.classification, cls.seconds);
|
||||
try std.testing.expectEqual(@as(u8, 8), cls.jitter_pct);
|
||||
|
||||
const em = DataType.etf_metrics.ttl();
|
||||
try std.testing.expectEqual(Ttl.etf_metrics, em.seconds);
|
||||
try std.testing.expectEqual(@as(u8, 8), em.jitter_pct);
|
||||
|
||||
const ef = DataType.entity_facts.ttl();
|
||||
try std.testing.expectEqual(Ttl.entity_facts, ef.seconds);
|
||||
try std.testing.expectEqual(@as(u8, 8), ef.jitter_pct);
|
||||
|
||||
const tf = DataType.tickers_funds.ttl();
|
||||
try std.testing.expectEqual(Ttl.tickers_funds, tf.seconds);
|
||||
try std.testing.expectEqual(@as(u8, 8), tf.jitter_pct);
|
||||
|
||||
const tc = DataType.tickers_companies.ttl();
|
||||
try std.testing.expectEqual(Ttl.tickers_companies, tc.seconds);
|
||||
try std.testing.expectEqual(@as(u8, 8), tc.jitter_pct);
|
||||
|
||||
// No jitter: short-TTL types and etf_profile.
|
||||
try std.testing.expectEqual(Ttl.options, DataType.options.ttl().seconds);
|
||||
try std.testing.expectEqual(@as(u8, 0), DataType.options.ttl().jitter_pct);
|
||||
try std.testing.expectEqual(Ttl.earnings, DataType.earnings.ttl().seconds);
|
||||
try std.testing.expectEqual(@as(u8, 0), DataType.earnings.ttl().jitter_pct);
|
||||
try std.testing.expectEqual(Ttl.etf_profile, DataType.etf_profile.ttl().seconds);
|
||||
try std.testing.expectEqual(@as(u8, 0), DataType.etf_profile.ttl().jitter_pct);
|
||||
|
||||
// candles_daily, candles_meta, and meta have their own writers
|
||||
// (`cacheCandles`, `writeNegative`); calling .ttl() on them is
|
||||
// unreachable and would panic. Not exercised here.
|
||||
}
|
||||
|
||||
test "DataType.fileName returns correct file names" {
|
||||
|
|
|
|||
|
|
@ -142,8 +142,14 @@ pub const LoadProgress = struct {
|
|||
const display_idx = self.index_offset + index + 1;
|
||||
switch (status) {
|
||||
.fetching => {
|
||||
// Show rate-limit wait before the fetch
|
||||
if (self.svc.estimateWaitSeconds()) |w| {
|
||||
// Show rate-limit wait before the fetch.
|
||||
// Prices come from the candle pipeline; ask about
|
||||
// candles_daily so the wait reflects whichever
|
||||
// provider serves candles (currently Tiingo, no
|
||||
// rate limiter -- so this is effectively a no-op,
|
||||
// but stays correct if the provider gains a limiter
|
||||
// or the type's primary changes).
|
||||
if (self.svc.estimateWaitSeconds(.candles_daily)) |w| {
|
||||
if (w > 0) stderrRateLimitWait(self.io, w, self.color);
|
||||
}
|
||||
stderrProgress(self.io, symbol, " (fetching)", display_idx, self.grand_total, self.color);
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ pub const Polygon = struct {
|
|||
return .{
|
||||
.api_key = api_key,
|
||||
.client = http.Client.init(io, allocator),
|
||||
.rate_limiter = RateLimiter.perMinute(io, 5),
|
||||
.rate_limiter = RateLimiter.perMinute(io, 4),
|
||||
.allocator = allocator,
|
||||
};
|
||||
}
|
||||
|
|
@ -72,11 +72,11 @@ pub const Polygon = struct {
|
|||
var to_buf: [10]u8 = undefined;
|
||||
|
||||
if (from) |f| {
|
||||
params[n] = .{ "ex_dividend_date.gte", std.fmt.bufPrint(&from_buf, "{f}", .{f}) catch unreachable };
|
||||
params[n] = .{ "ex_dividend_date.gte", try std.fmt.bufPrint(&from_buf, "{f}", .{f}) };
|
||||
n += 1;
|
||||
}
|
||||
if (to) |t| {
|
||||
params[n] = .{ "ex_dividend_date.lte", std.fmt.bufPrint(&to_buf, "{f}", .{t}) catch unreachable };
|
||||
params[n] = .{ "ex_dividend_date.lte", try std.fmt.bufPrint(&to_buf, "{f}", .{t}) };
|
||||
n += 1;
|
||||
}
|
||||
|
||||
|
|
@ -102,7 +102,14 @@ pub const Polygon = struct {
|
|||
var response = try self.client.get(authed);
|
||||
defer response.deinit();
|
||||
|
||||
// Free the cursor URL we just consumed and clear next_url
|
||||
// BEFORE attempting to parse. If parseDividendsPage errors,
|
||||
// the function-scope defer at the top must not see a
|
||||
// dangling pointer in next_url -- otherwise it double-frees
|
||||
// the buffer we just released. The new next_url (if any)
|
||||
// gets assigned below on success.
|
||||
allocator.free(cursor_url);
|
||||
next_url = null;
|
||||
next_url = try parseDividendsPage(allocator, response.body, &all_dividends);
|
||||
}
|
||||
|
||||
|
|
|
|||
118
src/service.zig
118
src/service.zig
|
|
@ -478,7 +478,7 @@ pub const DataService = struct {
|
|||
const retried = self.fetchFromProvider(T, symbol) catch {
|
||||
return DataError.FetchFailed;
|
||||
};
|
||||
s.writeWithSource(T, symbol, retried, .{ .seconds = data_type.ttl() }, sourceHintFor(T));
|
||||
s.writeWithSource(T, symbol, retried, data_type.ttl(), sourceHintFor(T));
|
||||
return .{ .data = retried, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
}
|
||||
// Only NotFound (provider says "this symbol genuinely has
|
||||
|
|
@ -492,7 +492,7 @@ pub const DataService = struct {
|
|||
return DataError.FetchFailed;
|
||||
};
|
||||
|
||||
s.writeWithSource(T, symbol, fetched, .{ .seconds = data_type.ttl() }, sourceHintFor(T));
|
||||
s.writeWithSource(T, symbol, fetched, data_type.ttl(), sourceHintFor(T));
|
||||
return .{ .data = fetched, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
}
|
||||
|
||||
|
|
@ -586,8 +586,8 @@ pub const DataService = struct {
|
|||
// view supplements rather than replaces existing (typically
|
||||
// Polygon-sourced) records. New entries are logged with
|
||||
// "tiingo" attribution.
|
||||
s.writeWithSource(Dividend, symbol, triple.dividends, .{ .seconds = cache.DataType.dividends.ttl() }, "tiingo");
|
||||
s.writeWithSource(Split, symbol, triple.splits, .{ .seconds = cache.DataType.splits.ttl() }, "tiingo");
|
||||
s.writeWithSource(Dividend, symbol, triple.dividends, cache.DataType.dividends.ttl(), "tiingo");
|
||||
s.writeWithSource(Split, symbol, triple.splits, cache.DataType.splits.ttl(), "tiingo");
|
||||
|
||||
return triple;
|
||||
}
|
||||
|
|
@ -1134,7 +1134,7 @@ pub const DataService = struct {
|
|||
break_blk: {
|
||||
const retried = wd.fetch(self.allocator, &symbols) catch break :break_blk;
|
||||
if (retried.len > 0) {
|
||||
s.write(Wikidata.ClassificationRecord, symbol, retried, .{ .seconds = cache.Ttl.classification, .jitter_pct = 8 });
|
||||
s.write(Wikidata.ClassificationRecord, symbol, retried, cache.DataType.classification.ttl());
|
||||
return .{ .data = retried, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
}
|
||||
self.allocator.free(retried);
|
||||
|
|
@ -1152,7 +1152,7 @@ pub const DataService = struct {
|
|||
return DataError.NotFound;
|
||||
}
|
||||
|
||||
s.write(Wikidata.ClassificationRecord, symbol, fetched, .{ .seconds = cache.Ttl.classification, .jitter_pct = 8 });
|
||||
s.write(Wikidata.ClassificationRecord, symbol, fetched, cache.DataType.classification.ttl());
|
||||
|
||||
return .{ .data = fetched, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
}
|
||||
|
|
@ -1239,7 +1239,7 @@ pub const DataService = struct {
|
|||
// Write each fetched record to its per-symbol cache file.
|
||||
for (fetched) |rec| {
|
||||
const single = [_]Wikidata.ClassificationRecord{rec};
|
||||
s.write(Wikidata.ClassificationRecord, rec.symbol, &single, .{ .seconds = cache.Ttl.classification, .jitter_pct = 8 });
|
||||
s.write(Wikidata.ClassificationRecord, rec.symbol, &single, cache.DataType.classification.ttl());
|
||||
}
|
||||
|
||||
// Combine cached + fetched into the result.
|
||||
|
|
@ -1315,7 +1315,7 @@ pub const DataService = struct {
|
|||
|
||||
const records = try self.allocator.alloc(Edgar.EntityFactRecord, 1);
|
||||
records[0] = .{ .shares_outstanding = shares_record };
|
||||
s.write(Edgar.EntityFactRecord, cik, records, .{ .seconds = cache.Ttl.entity_facts, .jitter_pct = 8 });
|
||||
s.write(Edgar.EntityFactRecord, cik, records, cache.DataType.entity_facts.ttl());
|
||||
|
||||
return .{ .data = records, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
}
|
||||
|
|
@ -1416,7 +1416,7 @@ pub const DataService = struct {
|
|||
}
|
||||
try Edgar.appendEtfMetricRecords(self.allocator, &records, m);
|
||||
const owned = try records.toOwnedSlice(self.allocator);
|
||||
s.write(Edgar.EtfMetricRecord, symbol, owned, .{ .seconds = cache.Ttl.etf_metrics, .jitter_pct = 8 });
|
||||
s.write(Edgar.EtfMetricRecord, symbol, owned, cache.DataType.etf_metrics.ttl());
|
||||
return .{ .data = owned, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
},
|
||||
.profile_only => |m_in| {
|
||||
|
|
@ -1430,7 +1430,7 @@ pub const DataService = struct {
|
|||
}
|
||||
try Edgar.appendEtfMetricRecords(self.allocator, &records, m);
|
||||
const owned = try records.toOwnedSlice(self.allocator);
|
||||
s.write(Edgar.EtfMetricRecord, symbol, owned, .{ .seconds = cache.Ttl.etf_metrics, .jitter_pct = 8 });
|
||||
s.write(Edgar.EtfMetricRecord, symbol, owned, cache.DataType.etf_metrics.ttl());
|
||||
return .{ .data = owned, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||
},
|
||||
.not_a_fund => {
|
||||
|
|
@ -1480,7 +1480,7 @@ pub const DataService = struct {
|
|||
// + rate-limit token), cache the parsed slice, then build
|
||||
// the lookup map (which takes ownership of the slice).
|
||||
const entries = try edgar.fetchMutualFundTickerMap(self.allocator);
|
||||
s.write(Edgar.MutualFundTickerEntry, "_edgar", entries, .{ .seconds = cache.Ttl.tickers_funds, .jitter_pct = 8 });
|
||||
s.write(Edgar.MutualFundTickerEntry, "_edgar", entries, cache.DataType.tickers_funds.ttl());
|
||||
return Edgar.TickerMap(Edgar.MutualFundTickerEntry).fromEntries(self.allocator, entries);
|
||||
}
|
||||
|
||||
|
|
@ -1505,7 +1505,7 @@ pub const DataService = struct {
|
|||
var edgar = try self.getProvider(Edgar);
|
||||
|
||||
const entries = try edgar.fetchCompanyTickerMap(self.allocator);
|
||||
s.write(Edgar.CompanyTickerEntry, "_edgar", entries, .{ .seconds = cache.Ttl.tickers_companies, .jitter_pct = 8 });
|
||||
s.write(Edgar.CompanyTickerEntry, "_edgar", entries, cache.DataType.tickers_companies.ttl());
|
||||
return Edgar.TickerMap(Edgar.CompanyTickerEntry).fromEntries(self.allocator, entries);
|
||||
}
|
||||
|
||||
|
|
@ -1687,14 +1687,35 @@ pub const DataService = struct {
|
|||
return mr.meta.last_date;
|
||||
}
|
||||
|
||||
/// Estimate wait time (in seconds) before the next TwelveData API call can proceed.
|
||||
/// Returns 0 if a request can be made immediately. Returns null if no API key.
|
||||
pub fn estimateWaitSeconds(self: *DataService) ?u64 {
|
||||
if (self.td) |*td| {
|
||||
const ns = td.rate_limiter.estimateWaitNs();
|
||||
return if (ns == 0) 0 else @max(1, ns / std.time.ns_per_s);
|
||||
}
|
||||
return null;
|
||||
/// Estimate wait time (in seconds) before a fetch for `data_type`
|
||||
/// can proceed without blocking on its provider's rate limiter.
|
||||
/// Returns 0 if a request can be made immediately, or if the
|
||||
/// provider for this data type has no rate limiter. Returns null
|
||||
/// if the relevant provider isn't instantiated yet (e.g., no API
|
||||
/// key, or first call hasn't happened to lazy-init it).
|
||||
///
|
||||
/// The caller asks "how long until getX can proceed?" -- the
|
||||
/// service maps data type to provider internally so the caller
|
||||
/// doesn't have to know which provider serves which data.
|
||||
pub fn estimateWaitSeconds(self: *DataService, data_type: cache.DataType) ?u64 {
|
||||
const ns: u64 = switch (data_type) {
|
||||
// Polygon-served: dividends and splits.
|
||||
.dividends, .splits => if (self.pg) |*pg| pg.rate_limiter.estimateWaitNs() else return null,
|
||||
// FMP-served: earnings.
|
||||
.earnings => if (self.fmp) |*fmp| fmp.rate_limiter.estimateWaitNs() else return null,
|
||||
// Cboe-served: options chains.
|
||||
.options => if (self.cboe) |*cboe| cboe.rate_limiter.estimateWaitNs() else return null,
|
||||
// EDGAR-served: ETF metrics, entity facts, ticker maps.
|
||||
.etf_metrics, .entity_facts, .tickers_funds, .tickers_companies => if (self.edgar) |*e| e.rate_limiter.estimateWaitNs() else return null,
|
||||
// No proactive token-bucket limiter for these. Tiingo
|
||||
// (candles) has a 1000/day quota enforced reactively
|
||||
// via 429-then-backoff in `getCandles`; Wikidata
|
||||
// (classification) has no published quota; the legacy
|
||||
// `etf_profile` and `meta` types aren't fetched. Nothing
|
||||
// useful to wait for at the call site, so report 0.
|
||||
.candles_daily, .candles_meta, .classification, .etf_profile, .meta => 0,
|
||||
};
|
||||
return if (ns == 0) 0 else @max(1, ns / std.time.ns_per_s);
|
||||
}
|
||||
|
||||
/// Read candles from cache only (no network fetch). Used by TUI for display.
|
||||
|
|
@ -3087,6 +3108,63 @@ test "DataService getProvider returns NoApiKey for Wikidata without user_email"
|
|||
try std.testing.expectError(DataError.NoApiKey, ed_result);
|
||||
}
|
||||
|
||||
test "estimateWaitSeconds returns null when relevant provider not instantiated" {
|
||||
const allocator = std.testing.allocator;
|
||||
const config = Config{ .cache_dir = "/tmp/zfin-test-cache" };
|
||||
var svc = DataService.init(std.testing.io, allocator, config);
|
||||
defer svc.deinit();
|
||||
|
||||
// No providers initialized yet (lazy). Each rate-limited data
|
||||
// type returns null because its provider is missing.
|
||||
try std.testing.expectEqual(@as(?u64, null), svc.estimateWaitSeconds(.dividends));
|
||||
try std.testing.expectEqual(@as(?u64, null), svc.estimateWaitSeconds(.splits));
|
||||
try std.testing.expectEqual(@as(?u64, null), svc.estimateWaitSeconds(.earnings));
|
||||
try std.testing.expectEqual(@as(?u64, null), svc.estimateWaitSeconds(.options));
|
||||
try std.testing.expectEqual(@as(?u64, null), svc.estimateWaitSeconds(.etf_metrics));
|
||||
try std.testing.expectEqual(@as(?u64, null), svc.estimateWaitSeconds(.entity_facts));
|
||||
}
|
||||
|
||||
test "estimateWaitSeconds returns 0 for types without rate limiters" {
|
||||
// candles_daily, classification, etc. are served by providers
|
||||
// that don't have a rate limiter (Tiingo, Wikidata). The
|
||||
// function returns 0 for these regardless of provider state --
|
||||
// there's nothing to wait for.
|
||||
const allocator = std.testing.allocator;
|
||||
const config = Config{ .cache_dir = "/tmp/zfin-test-cache" };
|
||||
var svc = DataService.init(std.testing.io, allocator, config);
|
||||
defer svc.deinit();
|
||||
|
||||
try std.testing.expectEqual(@as(?u64, 0), svc.estimateWaitSeconds(.candles_daily));
|
||||
try std.testing.expectEqual(@as(?u64, 0), svc.estimateWaitSeconds(.candles_meta));
|
||||
try std.testing.expectEqual(@as(?u64, 0), svc.estimateWaitSeconds(.classification));
|
||||
try std.testing.expectEqual(@as(?u64, 0), svc.estimateWaitSeconds(.etf_profile));
|
||||
try std.testing.expectEqual(@as(?u64, 0), svc.estimateWaitSeconds(.meta));
|
||||
}
|
||||
|
||||
test "estimateWaitSeconds returns 0 for fresh rate-limited providers" {
|
||||
// Once the provider is instantiated, an unused rate limiter
|
||||
// returns 0 (no wait). This is the steady-state happy path
|
||||
// for the call at the top of each refresh iteration.
|
||||
const allocator = std.testing.allocator;
|
||||
const config = Config{
|
||||
.cache_dir = "/tmp/zfin-test-cache",
|
||||
.polygon_key = "test-polygon-key",
|
||||
.fmp_key = "test-fmp-key",
|
||||
};
|
||||
var svc = DataService.init(std.testing.io, allocator, config);
|
||||
defer svc.deinit();
|
||||
|
||||
// Touch each provider to lazy-init it. We don't care about the
|
||||
// returned pointer; just need svc.pg / svc.fmp to be non-null.
|
||||
_ = try svc.getProvider(Polygon);
|
||||
_ = try svc.getProvider(Fmp);
|
||||
|
||||
// Fresh limiters have full token bucket -> 0 wait.
|
||||
try std.testing.expectEqual(@as(?u64, 0), svc.estimateWaitSeconds(.dividends));
|
||||
try std.testing.expectEqual(@as(?u64, 0), svc.estimateWaitSeconds(.splits));
|
||||
try std.testing.expectEqual(@as(?u64, 0), svc.estimateWaitSeconds(.earnings));
|
||||
}
|
||||
|
||||
// ── lookupInTickerMaps ────────────────────────────────────────
|
||||
//
|
||||
// Pure function — no I/O. Consumed by `lookupEdgarFallback`,
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue