market-aware ttl
This commit is contained in:
parent
47462aaab5
commit
c3c990fa68
6 changed files with 662 additions and 60 deletions
|
|
@ -21,6 +21,11 @@ pub fn build(b: *std.Build) void {
|
||||||
.optimize = optimize,
|
.optimize = optimize,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
const zeit_dep = b.dependency("zeit", .{
|
||||||
|
.target = target,
|
||||||
|
.optimize = optimize,
|
||||||
|
});
|
||||||
|
|
||||||
const srf_mod = srf_dep.module("srf");
|
const srf_mod = srf_dep.module("srf");
|
||||||
|
|
||||||
const shiller_mod = b.addModule("shiller_year", .{
|
const shiller_mod = b.addModule("shiller_year", .{
|
||||||
|
|
@ -43,6 +48,7 @@ pub fn build(b: *std.Build) void {
|
||||||
.target = target,
|
.target = target,
|
||||||
.imports = &.{
|
.imports = &.{
|
||||||
.{ .name = "srf", .module = srf_mod },
|
.{ .name = "srf", .module = srf_mod },
|
||||||
|
.{ .name = "zeit", .module = zeit_dep.module("zeit") },
|
||||||
.{ .name = "build_info", .module = build_info },
|
.{ .name = "build_info", .module = build_info },
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
@ -54,6 +60,7 @@ pub fn build(b: *std.Build) void {
|
||||||
.{ .name = "srf", .module = srf_mod },
|
.{ .name = "srf", .module = srf_mod },
|
||||||
.{ .name = "vaxis", .module = vaxis_dep.module("vaxis") },
|
.{ .name = "vaxis", .module = vaxis_dep.module("vaxis") },
|
||||||
.{ .name = "z2d", .module = z2d_dep.module("z2d") },
|
.{ .name = "z2d", .module = z2d_dep.module("z2d") },
|
||||||
|
.{ .name = "zeit", .module = zeit_dep.module("zeit") },
|
||||||
.{ .name = "build_info", .module = build_info },
|
.{ .name = "build_info", .module = build_info },
|
||||||
.{ .name = "shiller_year", .module = shiller_mod },
|
.{ .name = "shiller_year", .module = shiller_mod },
|
||||||
};
|
};
|
||||||
|
|
@ -121,6 +128,7 @@ pub fn build(b: *std.Build) void {
|
||||||
.optimize = optimize,
|
.optimize = optimize,
|
||||||
.imports = &.{
|
.imports = &.{
|
||||||
.{ .name = "srf", .module = srf_mod },
|
.{ .name = "srf", .module = srf_mod },
|
||||||
|
.{ .name = "zeit", .module = zeit_dep.module("zeit") },
|
||||||
.{ .name = "build_info", .module = build_info },
|
.{ .name = "build_info", .module = build_info },
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
|
|
|
||||||
|
|
@ -16,6 +16,10 @@
|
||||||
.url = "git+https://git.lerch.org/lobo/srf#4a3e5f00f15b0e0ba79d06ffe69dbcfa052baa5b",
|
.url = "git+https://git.lerch.org/lobo/srf#4a3e5f00f15b0e0ba79d06ffe69dbcfa052baa5b",
|
||||||
.hash = "srf-0.0.0-qZj572nkAQAAz3zEg6fdD8A7PJnQ9je3zCeAOJS5PoZj",
|
.hash = "srf-0.0.0-qZj572nkAQAAz3zEg6fdD8A7PJnQ9je3zCeAOJS5PoZj",
|
||||||
},
|
},
|
||||||
|
.zeit = .{
|
||||||
|
.url = "git+https://github.com/rockorager/zeit?ref=v0.9.0#b1c1c2fcbc71fd7799a316bbcf0ff88d06d80ccc",
|
||||||
|
.hash = "zeit-0.9.0-5I6bk2m9AgBSMH8-L6rYJkwuQAyhXplnfxnvTSGzVHUR",
|
||||||
|
},
|
||||||
},
|
},
|
||||||
.paths = .{
|
.paths = .{
|
||||||
"build",
|
"build",
|
||||||
|
|
|
||||||
|
|
@ -37,21 +37,66 @@ The `--refresh-data` policy decides which tiers run:
|
||||||
|
|
||||||
Different data ages at different rates, so each type has its own TTL:
|
Different data ages at different rates, so each type has its own TTL:
|
||||||
|
|
||||||
| Data type | TTL | Why |
|
| Data type | TTL | Why |
|
||||||
|---------------|---------------|-------------------------------------------------------------|
|
|---------------|---------------|----------------------------------------------------------------------------------|
|
||||||
| Daily candles | ~24h (23h45m) | One bar per trading day; slightly under 24h for cron jitter |
|
| Daily candles | market-aware | Keyed to the next time a fresh bar is expected, not a rolling window (see below) |
|
||||||
| Dividends | 14 days | Declared well in advance |
|
| Dividends | 14 days | Declared well in advance |
|
||||||
| Splits | 14 days | Rare corporate events |
|
| Splits | 14 days | Rare corporate events |
|
||||||
| Options | 1 hour | Prices move continuously when markets are open |
|
| Options | 1 hour | Prices move continuously when markets are open |
|
||||||
| Earnings | 30 days\* | Quarterly; smart-refreshed around announcements |
|
| Earnings | 30 days\* | Quarterly; smart-refreshed around announcements |
|
||||||
| ETF profiles | ~30 days | Holdings and weights change slowly |
|
| ETF profiles | ~30 days | Holdings and weights change slowly |
|
||||||
| Quotes | never cached | Meant to be a live price check |
|
| Quotes | never cached | Meant to be a live price check |
|
||||||
|
|
||||||
\* **Earnings smart refresh:** even inside the 30-day window, cached
|
\* **Earnings smart refresh:** even inside the 30-day window, cached
|
||||||
earnings re-fetch automatically once an earnings date has passed but
|
earnings re-fetch automatically once an earnings date has passed but
|
||||||
the cache still lacks the actual result -- so numbers appear promptly
|
the cache still lacks the actual result -- so numbers appear promptly
|
||||||
after an announcement without daily polling.
|
after an announcement without daily polling.
|
||||||
|
|
||||||
|
## Market-aware candle freshness
|
||||||
|
|
||||||
|
A daily bar only becomes meaningful once the market settles, so candle
|
||||||
|
freshness is keyed to the market clock rather than a rolling 24-hour
|
||||||
|
window. Each cached candle's expiry is set to the next moment fresh data
|
||||||
|
should be available:
|
||||||
|
|
||||||
|
- **Equities and ETFs** settle shortly after the 16:00 ET close. Their
|
||||||
|
bars expire at **16:55 ET** on the next trading day (weekends and NYSE
|
||||||
|
holidays are skipped). This time allows for providers to become consistent
|
||||||
|
with the market while also allowing a few minutes prior to a scheduled
|
||||||
|
refresh task at the top of the hour.
|
||||||
|
- **Mutual funds** strike a single daily NAV that isn't reliably
|
||||||
|
published until the next morning, so their bars expire at **03:25 ET**
|
||||||
|
the morning after a trading session. Despite Tiingo's claims, NAVs only
|
||||||
|
seem reliably available until about 3am Eastern. This is again timed such
|
||||||
|
that scheduled jobs for the bottom of the hour can run reliably.
|
||||||
|
|
||||||
|
This keeps the expiry boundary out of trading hours, so a refresh fired
|
||||||
|
just after it always sees a finalized bar instead of a half-formed one,
|
||||||
|
and an interactive command run mid-session won't trigger a needless
|
||||||
|
refetch. If a refresh runs but the provider hasn't posted the just-closed
|
||||||
|
bar yet, the entry is retried in ~30 minutes rather than waiting a full
|
||||||
|
day.
|
||||||
|
|
||||||
|
**Un-modeled closures self-correct.** Some market closures aren't on the
|
||||||
|
modeled holiday calendar (Good Friday, which needs the Easter computus,
|
||||||
|
plus ad-hoc closures for national mourning or weather). On such a day the
|
||||||
|
calendar thinks a bar is due, the fetch keeps coming back empty, and the
|
||||||
|
~30-minute retry would otherwise repeat all day - and, for a Friday
|
||||||
|
closure, all weekend. To avoid that thrash, once an expected bar is ~90
|
||||||
|
minutes overdue the cache concludes the market was closed and falls back
|
||||||
|
to the normal next-session boundary - at most three 30-minute retries.
|
||||||
|
That window comfortably covers ordinary provider posting lag, so
|
||||||
|
genuinely-late data is still picked up by the short retry; only a true
|
||||||
|
closure trips the fallback.
|
||||||
|
|
||||||
|
**Warming a shared cache on a schedule.** If you run a cron to warm a
|
||||||
|
[server cache](#server-sync-zfin_server) (or your own local cache), the
|
||||||
|
boundaries above are also the natural cron times: a run shortly after
|
||||||
|
**17:00 ET** picks up the day's equity/ETF closes, and a run shortly
|
||||||
|
after **03:30 ET** picks up the prior session's mutual-fund NAVs. The
|
||||||
|
boundaries sit a couple of minutes before those times so the cron
|
||||||
|
reliably sees the cache already expired.
|
||||||
|
|
||||||
## Quotes are never cached
|
## Quotes are never cached
|
||||||
|
|
||||||
Because quotes exist to give you a live price, they're never served
|
Because quotes exist to give you a live price, they're never served
|
||||||
|
|
|
||||||
36
src/cache/store.zig
vendored
36
src/cache/store.zig
vendored
|
|
@ -32,8 +32,6 @@ pub const Ttl = struct {
|
||||||
const s_per_day = std.time.s_per_day;
|
const s_per_day = std.time.s_per_day;
|
||||||
/// Historical candles older than 1 day never expire
|
/// Historical candles older than 1 day never expire
|
||||||
pub const candles_historical: i64 = -1; // infinite
|
pub const candles_historical: i64 = -1; // infinite
|
||||||
/// Latest day's candle refreshes every 23h45m (15-min buffer for cron jitter)
|
|
||||||
pub const candles_latest: i64 = s_per_day - 15 * std.time.s_per_min;
|
|
||||||
/// Dividend data refreshes biweekly
|
/// Dividend data refreshes biweekly
|
||||||
pub const dividends: i64 = 14 * s_per_day;
|
pub const dividends: i64 = 14 * s_per_day;
|
||||||
/// Split data refreshes biweekly
|
/// Split data refreshes biweekly
|
||||||
|
|
@ -816,9 +814,12 @@ pub const Store = struct {
|
||||||
|
|
||||||
// ── Candle-specific API ──────────────────────────────────────
|
// ── Candle-specific API ──────────────────────────────────────
|
||||||
|
|
||||||
/// Write a full set of candles to cache (no expiry - historical facts don't expire).
|
/// Write a full set of candles to cache (the candle data file itself
|
||||||
/// Also updates candle metadata.
|
/// carries no expiry - historical bars never change). Also refreshes
|
||||||
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8) void {
|
/// candle metadata, stamping its `#!expires=` with `expires_at_s`
|
||||||
|
/// (the caller computes the market-aware freshness boundary; see
|
||||||
|
/// `market.nextCandleExpiry`).
|
||||||
|
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||||
if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
|
if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
|
||||||
defer self.allocator.free(srf_data);
|
defer self.allocator.free(srf_data);
|
||||||
self.writeRaw(symbol, .candles_daily, srf_data) catch |err| {
|
self.writeRaw(symbol, .candles_daily, srf_data) catch |err| {
|
||||||
|
|
@ -830,14 +831,15 @@ pub const Store = struct {
|
||||||
|
|
||||||
if (candles.len > 0) {
|
if (candles.len > 0) {
|
||||||
const last = candles[candles.len - 1];
|
const last = candles[candles.len - 1];
|
||||||
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count);
|
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Append new candle records to the existing cache file.
|
/// Append new candle records to the existing cache file.
|
||||||
/// Falls back to a full rewrite if append fails (e.g. file doesn't exist).
|
/// Falls back to a full rewrite if append fails (e.g. file doesn't exist).
|
||||||
/// Also updates candle metadata.
|
/// Also updates candle metadata, stamping its `#!expires=` with
|
||||||
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8) void {
|
/// `expires_at_s` (caller-computed market-aware boundary).
|
||||||
|
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||||
if (new_candles.len == 0) return;
|
if (new_candles.len == 0) return;
|
||||||
|
|
||||||
if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| {
|
if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| {
|
||||||
|
|
@ -866,19 +868,24 @@ pub const Store = struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
const last = new_candles[new_candles.len - 1];
|
const last = new_candles[new_candles.len - 1];
|
||||||
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count);
|
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Write (or refresh) candle metadata with a specific provider source.
|
/// Write (or refresh) candle metadata with a specific provider source.
|
||||||
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8) void {
|
///
|
||||||
const expires = std.Io.Timestamp.now(self.io, .real).toSeconds() + Ttl.candles_latest;
|
/// `expires_at_s` is the absolute Unix-seconds freshness boundary for
|
||||||
|
/// the `#!expires=` directive, computed by the caller via
|
||||||
|
/// `market.nextCandleExpiry` (close-anchored) or a short retry. The
|
||||||
|
/// cache layer no longer reads the wall clock for this - the boundary
|
||||||
|
/// is market-domain knowledge owned by the caller.
|
||||||
|
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
|
||||||
const meta = CandleMeta{
|
const meta = CandleMeta{
|
||||||
.last_close = last_close,
|
.last_close = last_close,
|
||||||
.last_date = last_date,
|
.last_date = last_date,
|
||||||
.provider = provider,
|
.provider = provider,
|
||||||
.fail_count = fail_count,
|
.fail_count = fail_count,
|
||||||
};
|
};
|
||||||
if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires })) |meta_data| {
|
if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires_at_s })) |meta_data| {
|
||||||
defer self.allocator.free(meta_data);
|
defer self.allocator.free(meta_data);
|
||||||
self.writeRaw(symbol, .candles_meta, meta_data) catch |err| {
|
self.writeRaw(symbol, .candles_meta, meta_data) catch |err| {
|
||||||
log.warn("{s}: failed to write candle metadata: {s}", .{ symbol, @errorName(err) });
|
log.warn("{s}: failed to write candle metadata: {s}", .{ symbol, @errorName(err) });
|
||||||
|
|
@ -2961,9 +2968,8 @@ test "TTL constants are reasonable" {
|
||||||
// Historical candles never expire
|
// Historical candles never expire
|
||||||
try std.testing.expectEqual(@as(i64, -1), Ttl.candles_historical);
|
try std.testing.expectEqual(@as(i64, -1), Ttl.candles_historical);
|
||||||
|
|
||||||
// Latest candles expire just under 24 hours (allowing for cron jitter)
|
// Latest candles use a market-aware expiry computed per-write by
|
||||||
try std.testing.expect(Ttl.candles_latest > 23 * std.time.s_per_hour);
|
// market.nextCandleExpiry (no fixed TTL constant here anymore).
|
||||||
try std.testing.expect(Ttl.candles_latest < 24 * std.time.s_per_hour);
|
|
||||||
|
|
||||||
// Dividends and splits refresh biweekly
|
// Dividends and splits refresh biweekly
|
||||||
try std.testing.expectEqual(@as(i64, 14 * std.time.s_per_day), Ttl.dividends);
|
try std.testing.expectEqual(@as(i64, 14 * std.time.s_per_day), Ttl.dividends);
|
||||||
|
|
|
||||||
538
src/market.zig
Normal file
538
src/market.zig
Normal file
|
|
@ -0,0 +1,538 @@
|
||||||
|
//! Market calendar: trading-day awareness and close-anchored cache
|
||||||
|
//! freshness boundaries for daily candles.
|
||||||
|
//!
|
||||||
|
//! Daily candle data only becomes meaningful once the market settles.
|
||||||
|
//! Two distinct deadlines matter:
|
||||||
|
//!
|
||||||
|
//! - **Equities / ETFs** settle shortly after the 16:00 ET close.
|
||||||
|
//! - **Mutual-fund NAVs** strike once per day and are not reliably
|
||||||
|
//! published until the next morning (~03:30 ET).
|
||||||
|
//!
|
||||||
|
//! The old candle TTL was a rolling 23h45m window, so the expiry
|
||||||
|
//! boundary drifted against the market clock and could fall during
|
||||||
|
//! trading hours - causing mid-session refetches and risking caching a
|
||||||
|
//! not-yet-finalized bar. This module computes the next moment fresh
|
||||||
|
//! data should be available, so the candle cache is keyed to the market
|
||||||
|
//! clock instead. It is an optimization, not a correctness fix: a
|
||||||
|
//! slightly-wrong boundary costs at most one cheap no-op fetch (see the
|
||||||
|
//! direction-of-error note on `isHoliday`).
|
||||||
|
//!
|
||||||
|
//! Timezone handling uses `zeit` with a hardcoded US Eastern POSIX TZ
|
||||||
|
//! spec, so there is no dependency on system zoneinfo files, no
|
||||||
|
//! allocator, and no I/O. The fixed DST rule is correct for current and
|
||||||
|
//! near-future dates, which is all cache-freshness math ever deals with.
|
||||||
|
|
||||||
|
const std = @import("std");
|
||||||
|
const zeit = @import("zeit");
|
||||||
|
const Date = @import("Date.zig");
|
||||||
|
|
||||||
|
/// POSIX TZ spec for US Eastern: EST (UTC-5) / EDT (UTC-4) with the
|
||||||
|
/// current US DST rule (2nd Sunday March -> 1st Sunday November).
|
||||||
|
const eastern_posix_spec = "EST5EDT,M3.2.0,M11.1.0";
|
||||||
|
|
||||||
|
/// US Eastern timezone, parsed at comptime from `eastern_posix_spec`.
|
||||||
|
/// `Posix.parse` allocates nothing - the parsed struct borrows slices
|
||||||
|
/// into the (static) spec string - so this is a zero-cost const.
|
||||||
|
const eastern: zeit.TimeZone = .{
|
||||||
|
.posix = zeit.timezone.Posix.parse(eastern_posix_spec) catch
|
||||||
|
@compileError("invalid eastern POSIX TZ spec: " ++ eastern_posix_spec),
|
||||||
|
};
|
||||||
|
|
||||||
|
/// How candle data for a symbol becomes available.
|
||||||
|
pub const InstrumentKind = enum {
|
||||||
|
/// Continuously-quoted equities and ETFs: the day's bar settles
|
||||||
|
/// shortly after the 16:00 ET close.
|
||||||
|
equity,
|
||||||
|
/// Mutual funds: a single daily NAV that isn't reliably published
|
||||||
|
/// until the next morning.
|
||||||
|
mutual_fund,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Classify a symbol for candle-freshness purposes. Mutual funds use
|
||||||
|
/// 5-letter tickers ending in X (e.g. FDSCX, VSTCX, FAGIX); everything
|
||||||
|
/// else is treated as an equity/ETF. Imperfect, but covers the common
|
||||||
|
/// case.
|
||||||
|
///
|
||||||
|
/// This is the single home for the heuristic: it was consolidated here
|
||||||
|
/// from the former `DataService.isMutualFund` (removed) when candle
|
||||||
|
/// freshness timing moved into this module. It is deliberately distinct
|
||||||
|
/// from `portfolio.isMoneyMarketSymbol`, which answers a different
|
||||||
|
/// question - "is this a fixed-$1-NAV cash equivalent?" - via a curated
|
||||||
|
/// whitelist. (Every symbol on that whitelist also matches this
|
||||||
|
/// heuristic, so freshness callers only need `classify`.)
|
||||||
|
pub fn classify(symbol: []const u8) InstrumentKind {
|
||||||
|
if (symbol.len == 5 and symbol[4] == 'X') return .mutual_fund;
|
||||||
|
return .equity;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Freshness target times (seconds since ET-local midnight) ─────────
|
||||||
|
//
|
||||||
|
// These double as the recommended refresh-cron schedule: each sits a
|
||||||
|
// couple minutes BEFORE its intended cron run so cron-timing jitter
|
||||||
|
// reliably sees the cache already expired (a boundary exactly at the
|
||||||
|
// cron time could be missed by an early tick). The "expected-but-
|
||||||
|
// missing" short retry (see `short_retry_s`) covers the case where the
|
||||||
|
// provider hasn't posted the just-closed bar by then.
|
||||||
|
|
||||||
|
/// Equity/ETF boundary: 16:55 ET (just after the 16:00 close + settle
|
||||||
|
/// margin; pairs with a ~17:00 ET refresh cron).
|
||||||
|
const equity_target_s: i64 = 16 * std.time.s_per_hour + 55 * std.time.s_per_min;
|
||||||
|
|
||||||
|
/// Mutual-fund boundary: 03:25 ET the morning after the trading day
|
||||||
|
/// (pairs with a ~03:30 ET NAV-refresh cron).
|
||||||
|
const mf_target_s: i64 = 3 * std.time.s_per_hour + 25 * std.time.s_per_min;
|
||||||
|
|
||||||
|
/// Retry interval when an expected bar wasn't returned yet (the provider
|
||||||
|
/// hasn't posted the just-closed bar, or a transient fetch failure).
|
||||||
|
/// Short enough to pick the bar up within the same session, long enough
|
||||||
|
/// to avoid hammering a rate-limited provider.
|
||||||
|
pub const short_retry_s: i64 = 30 * std.time.s_per_min;
|
||||||
|
|
||||||
|
/// How long past the moment data is *due* we keep doing `short_retry_s`
|
||||||
|
/// retries before concluding the calendar's "trading day" was actually
|
||||||
|
/// an un-modeled market closure (e.g. Good Friday, which needs the Easter
|
||||||
|
/// computus and is deliberately not modeled - see `isHoliday`) or an
|
||||||
|
/// ad-hoc halt, rather than mere provider lag. Once an expected bar has
|
||||||
|
/// been overdue this long it is almost certainly never coming, so
|
||||||
|
/// `staleCandleExpiry` stops the retry loop and falls back to the normal
|
||||||
|
/// next-boundary expiry. That bounds an un-modeled closure to a few
|
||||||
|
/// no-op fetches that session instead of thrashing every `short_retry_s`
|
||||||
|
/// until the next real session - for a Friday closure, the entire long
|
||||||
|
/// weekend.
|
||||||
|
///
|
||||||
|
/// 90 minutes comfortably covers normal posting lag for both the equity
|
||||||
|
/// close and the overnight mutual-fund NAV (data that is going to appear
|
||||||
|
/// at all is essentially always up well inside that window) while still
|
||||||
|
/// giving up promptly. At the 30-minute `short_retry_s` cadence that is
|
||||||
|
/// three retries (at +30, +60, +90 min); the +90 retry is the one that
|
||||||
|
/// crosses the window and trips the fallback.
|
||||||
|
const provider_lag_grace_s: i64 = 90 * std.time.s_per_min;
|
||||||
|
|
||||||
|
fn targetSeconds(kind: InstrumentKind) i64 {
|
||||||
|
return switch (kind) {
|
||||||
|
.equity => equity_target_s,
|
||||||
|
.mutual_fund => mf_target_s,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Trading-day calendar ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
// Weekday constants in zfin's 0=Mon .. 6=Sun scheme (see Date.dayOfWeek).
|
||||||
|
const mon: u8 = 0;
|
||||||
|
const thu: u8 = 3;
|
||||||
|
const sat: u8 = 5;
|
||||||
|
const sun: u8 = 6;
|
||||||
|
|
||||||
|
/// True if `d` is a US equity-market trading day: a weekday that isn't a
|
||||||
|
/// modeled NYSE holiday.
|
||||||
|
pub fn isTradingDay(d: Date) bool {
|
||||||
|
if (d.dayOfWeek() >= sat) return false; // Saturday or Sunday
|
||||||
|
return !isHoliday(d);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True if `d` is a (modeled) NYSE market holiday.
|
||||||
|
///
|
||||||
|
/// Direction-of-error policy: a *false holiday* (marking a real trading
|
||||||
|
/// day closed) would let the cache stay fresh too long and miss a day's
|
||||||
|
/// data - a staleness bug. A *missed holiday* (treating a closed day as
|
||||||
|
/// open) only costs one harmless no-op fetch. So we model only the
|
||||||
|
/// well-defined, confidently-computable holidays and lean toward "open"
|
||||||
|
/// when unsure. Good Friday is deliberately omitted (it needs the Easter
|
||||||
|
/// computus); an un-modeled Good Friday simply costs one no-op fetch.
|
||||||
|
/// Ad-hoc closures (national mourning, weather) are likewise unmodeled.
|
||||||
|
pub fn isHoliday(d: Date) bool {
|
||||||
|
const y = d.year();
|
||||||
|
|
||||||
|
// Floating Monday/Thursday holidays - these always land on a weekday,
|
||||||
|
// so no weekend-observance adjustment is needed.
|
||||||
|
if (d.eql(nthWeekday(y, 1, mon, 3))) return true; // MLK Day: 3rd Mon Jan
|
||||||
|
if (d.eql(nthWeekday(y, 2, mon, 3))) return true; // Washington's Birthday: 3rd Mon Feb
|
||||||
|
if (d.eql(lastWeekday(y, 5, mon))) return true; // Memorial Day: last Mon May
|
||||||
|
if (d.eql(nthWeekday(y, 9, mon, 1))) return true; // Labor Day: 1st Mon Sep
|
||||||
|
if (d.eql(nthWeekday(y, 11, thu, 4))) return true; // Thanksgiving: 4th Thu Nov
|
||||||
|
|
||||||
|
// New Year's Day: Sunday -> observed Monday. NYSE does NOT close the
|
||||||
|
// preceding Friday when Jan 1 falls on a Saturday (that Friday is in
|
||||||
|
// the prior year and stays a normal trading day).
|
||||||
|
if (observedSundayOnly(d, 1, 1)) return true;
|
||||||
|
|
||||||
|
// Juneteenth: NYSE first observed it in 2022. Sat -> Fri, Sun -> Mon.
|
||||||
|
if (y >= 2022 and observedFixed(d, 6, 19)) return true;
|
||||||
|
|
||||||
|
// Independence Day and Christmas: Sat -> Fri, Sun -> Mon.
|
||||||
|
if (observedFixed(d, 7, 4)) return true;
|
||||||
|
if (observedFixed(d, 12, 25)) return true;
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `n`-th occurrence (1-based) of weekday `wd` (0=Mon..6=Sun) in
|
||||||
|
/// month `mo` of `y`.
|
||||||
|
fn nthWeekday(y: i16, mo: u8, wd: u8, n: u8) Date {
|
||||||
|
const first = Date.fromYmd(y, mo, 1);
|
||||||
|
const offset = @mod(@as(i32, wd) - @as(i32, first.dayOfWeek()), 7);
|
||||||
|
const day: u8 = @intCast(1 + offset + (@as(i32, n - 1) * 7));
|
||||||
|
return Date.fromYmd(y, mo, day);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The last occurrence of weekday `wd` (0=Mon..6=Sun) in month `mo` of `y`.
|
||||||
|
fn lastWeekday(y: i16, mo: u8, wd: u8) Date {
|
||||||
|
const last = Date.lastDayOfMonth(y, mo);
|
||||||
|
const back = @mod(@as(i32, last.dayOfWeek()) - @as(i32, wd), 7);
|
||||||
|
return last.addDays(-back);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// True if `d` is the observed date of the fixed (mo/day) holiday in
|
||||||
|
/// `d`'s year, using the full weekend-observance rule: Saturday holidays
|
||||||
|
/// observed the preceding Friday, Sunday holidays the following Monday.
|
||||||
|
fn observedFixed(d: Date, mo: u8, day: u8) bool {
|
||||||
|
const actual = Date.fromYmd(d.year(), mo, day);
|
||||||
|
return switch (actual.dayOfWeek()) {
|
||||||
|
sat => d.eql(actual.addDays(-1)), // Sat -> preceding Fri
|
||||||
|
sun => d.eql(actual.addDays(1)), // Sun -> following Mon
|
||||||
|
else => d.eql(actual),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Like `observedFixed` but Saturday is NOT observed (used for New
|
||||||
|
/// Year's Day - see `isHoliday`).
|
||||||
|
fn observedSundayOnly(d: Date, mo: u8, day: u8) bool {
|
||||||
|
const actual = Date.fromYmd(d.year(), mo, day);
|
||||||
|
return switch (actual.dayOfWeek()) {
|
||||||
|
sat => false,
|
||||||
|
sun => d.eql(actual.addDays(1)),
|
||||||
|
else => d.eql(actual),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Close-anchored freshness boundaries ──────────────────────────────
|
||||||
|
|
||||||
|
/// True if fresh candle data for `kind` becomes available on calendar
|
||||||
|
/// day `d` (in ET). For equities, that's any trading day (the bar
|
||||||
|
/// settles that afternoon). For mutual funds, it's the morning AFTER a
|
||||||
|
/// trading day (the prior session's NAV posts overnight) - which also
|
||||||
|
/// makes holidays fall out for free: if the prior day wasn't a trading
|
||||||
|
/// day, no NAV is available this morning.
|
||||||
|
fn isAvailabilityDay(d: Date, kind: InstrumentKind) bool {
|
||||||
|
return switch (kind) {
|
||||||
|
.equity => isTradingDay(d),
|
||||||
|
.mutual_fund => isTradingDay(d.addDays(-1)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The trading day whose data an availability day carries.
|
||||||
|
fn dataDateFor(availability_day: Date, kind: InstrumentKind) Date {
|
||||||
|
return switch (kind) {
|
||||||
|
.equity => availability_day,
|
||||||
|
.mutual_fund => availability_day.addDays(-1),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Absolute Unix-seconds expiry for a freshly-written candle cache
|
||||||
|
/// entry: the next moment after `now_s` at which new data for `kind`
|
||||||
|
/// should be available. Pure given `now_s` (the wall clock is read by
|
||||||
|
/// the caller and threaded in), so it is fully deterministic for tests.
|
||||||
|
pub fn nextCandleExpiry(now_s: i64, kind: InstrumentKind) i64 {
|
||||||
|
const target = targetSeconds(kind);
|
||||||
|
const now_local = eastern.adjust(now_s).timestamp;
|
||||||
|
const today_et = Date.fromEpoch(now_local);
|
||||||
|
const now_tod = now_local - today_et.toEpoch(); // seconds since ET midnight
|
||||||
|
|
||||||
|
// Start at today's target; if it has already passed, move to tomorrow.
|
||||||
|
var cand = today_et;
|
||||||
|
if (now_tod >= target) cand = cand.addDays(1);
|
||||||
|
// Skip forward to the next day that actually carries new data.
|
||||||
|
while (!isAvailabilityDay(cand, kind)) cand = cand.addDays(1);
|
||||||
|
|
||||||
|
return etLocalToUtc(cand, target);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The most recent data availability as of `now_s`: which trading day's
|
||||||
|
/// candle should already be published, and the exact instant it went
|
||||||
|
/// live (its target time on the availability day, in Unix seconds).
|
||||||
|
const Availability = struct {
|
||||||
|
/// Trading day whose data the latest availability carries.
|
||||||
|
data_date: Date,
|
||||||
|
/// UTC seconds at which that data became available.
|
||||||
|
instant_s: i64,
|
||||||
|
};
|
||||||
|
|
||||||
|
/// Walk backward from `now_s` to the most recent availability day whose
|
||||||
|
/// target time has already passed, reporting both the trading day it
|
||||||
|
/// carries and the instant its data went live. Pure given `now_s`.
|
||||||
|
fn latestAvailability(now_s: i64, kind: InstrumentKind) Availability {
|
||||||
|
const target = targetSeconds(kind);
|
||||||
|
const now_local = eastern.adjust(now_s).timestamp;
|
||||||
|
const today_et = Date.fromEpoch(now_local);
|
||||||
|
const now_tod = now_local - today_et.toEpoch();
|
||||||
|
|
||||||
|
var cand = today_et;
|
||||||
|
if (now_tod < target) cand = cand.addDays(-1);
|
||||||
|
while (!isAvailabilityDay(cand, kind)) cand = cand.addDays(-1);
|
||||||
|
return .{
|
||||||
|
.data_date = dataDateFor(cand, kind),
|
||||||
|
.instant_s = etLocalToUtc(cand, target),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Expiry to stamp on candle meta after an *incremental* fetch on a stale
|
||||||
|
/// entry returned zero new bars. Three cases:
|
||||||
|
///
|
||||||
|
/// 1. **Nothing newer is due** - a genuine non-trading-day gap
|
||||||
|
/// (weekend/holiday) or the cache is already caught up: use the
|
||||||
|
/// normal next-boundary expiry.
|
||||||
|
/// 2. **Newer data is due, still within the provider-lag grace
|
||||||
|
/// window** (`provider_lag_grace_s`): the provider just hasn't
|
||||||
|
/// posted the just-closed bar yet, so retry soon (`short_retry_s`).
|
||||||
|
/// 3. **Newer data has been due for the full grace window**
|
||||||
|
/// (`provider_lag_grace_s`): the calendar's "trading day" was almost
|
||||||
|
/// certainly an un-modeled closure (e.g. Good Friday) or ad-hoc halt
|
||||||
|
/// - the bar is never going to appear. Stop the short-retry thrash
|
||||||
|
/// and fall back to the normal next-boundary expiry, so we wait for
|
||||||
|
/// the next real session instead of refetching every `short_retry_s`
|
||||||
|
/// all evening (and, for a Friday closure, all weekend).
|
||||||
|
///
|
||||||
|
/// `last_cached` is the newest date already in the candle cache. Pure
|
||||||
|
/// given `now_s`, so it is fully deterministic for tests.
|
||||||
|
pub fn staleCandleExpiry(now_s: i64, kind: InstrumentKind, last_cached: Date) i64 {
|
||||||
|
const avail = latestAvailability(now_s, kind);
|
||||||
|
// Case 1: nothing newer than the cache is due yet.
|
||||||
|
if (!last_cached.lessThan(avail.data_date)) return nextCandleExpiry(now_s, kind);
|
||||||
|
// Case 2: due data is merely late (still inside the grace window) -> short retry.
|
||||||
|
if (now_s - avail.instant_s < provider_lag_grace_s) return now_s + short_retry_s;
|
||||||
|
// Case 3: overdue for the whole grace window -> assume closure, wait for next boundary.
|
||||||
|
return nextCandleExpiry(now_s, kind);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Convert an ET local wall-clock instant (`d` at `tod_s` seconds since
|
||||||
|
/// ET midnight) to Unix seconds, resolving the correct EST/EDT offset
|
||||||
|
/// for that date. Treats the wall clock as a UTC instant for a first
|
||||||
|
/// guess, then corrects by the offset zeit reports at that moment; two
|
||||||
|
/// iterations converge because the offset changes by at most one hour
|
||||||
|
/// and the target times never sit inside the ~02:00 ET DST-transition
|
||||||
|
/// window.
|
||||||
|
fn etLocalToUtc(d: Date, tod_s: i64) i64 {
|
||||||
|
const wall = d.toEpoch() + tod_s;
|
||||||
|
var utc = wall;
|
||||||
|
var i: usize = 0;
|
||||||
|
while (i < 2) : (i += 1) {
|
||||||
|
// adjust(utc).timestamp = utc - west_offset(utc); recover the
|
||||||
|
// westward offset and re-anchor the wall clock to true UTC.
|
||||||
|
const west = utc - eastern.adjust(utc).timestamp;
|
||||||
|
utc = wall + west;
|
||||||
|
}
|
||||||
|
return utc;
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Tests ────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
const testing = std.testing;
|
||||||
|
|
||||||
|
/// Helper: Unix seconds for a UTC wall clock, for constructing test inputs.
|
||||||
|
fn utcSeconds(y: i16, mo: u8, d: u8, h: i64, mi: i64) i64 {
|
||||||
|
return Date.fromYmd(y, mo, d).toEpoch() + h * std.time.s_per_hour + mi * std.time.s_per_min;
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Helper: decompose an expiry back into ET wall-clock components.
|
||||||
|
const EtParts = struct { date: Date, hour: u8, minute: u8 };
|
||||||
|
fn etParts(unix_s: i64) EtParts {
|
||||||
|
const local = eastern.adjust(unix_s).timestamp;
|
||||||
|
const d = Date.fromEpoch(local);
|
||||||
|
const tod = local - d.toEpoch();
|
||||||
|
return .{
|
||||||
|
.date = d,
|
||||||
|
.hour = @intCast(@divFloor(tod, std.time.s_per_hour)),
|
||||||
|
.minute = @intCast(@divFloor(@mod(tod, std.time.s_per_hour), std.time.s_per_min)),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
test "classify: mutual funds are 5-letter X-suffixed tickers" {
|
||||||
|
try testing.expectEqual(InstrumentKind.mutual_fund, classify("FDSCX"));
|
||||||
|
try testing.expectEqual(InstrumentKind.mutual_fund, classify("VSTCX"));
|
||||||
|
try testing.expectEqual(InstrumentKind.mutual_fund, classify("FAGIX"));
|
||||||
|
try testing.expectEqual(InstrumentKind.mutual_fund, classify("VFINX"));
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("AAPL"));
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("VTI"));
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("SPY"));
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("GOOGL"));
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("VOOX")); // 4 chars, not a fund
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("FDSCA")); // 5 chars, not X-suffixed
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("FDSCXA")); // 6 chars ending in A
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("X")); // too short
|
||||||
|
try testing.expectEqual(InstrumentKind.equity, classify("")); // empty
|
||||||
|
}
|
||||||
|
|
||||||
|
test "etLocalToUtc: EST and EDT offsets" {
|
||||||
|
// Winter (EST, UTC-5): 16:55 ET == 21:55 UTC.
|
||||||
|
const est = etLocalToUtc(Date.fromYmd(2025, 1, 15), equity_target_s);
|
||||||
|
try testing.expectEqual(utcSeconds(2025, 1, 15, 21, 55), est);
|
||||||
|
|
||||||
|
// Summer (EDT, UTC-4): 16:55 ET == 20:55 UTC.
|
||||||
|
const edt = etLocalToUtc(Date.fromYmd(2025, 7, 15), equity_target_s);
|
||||||
|
try testing.expectEqual(utcSeconds(2025, 7, 15, 20, 55), edt);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "etLocalToUtc: across a spring-forward boundary" {
|
||||||
|
// 2025 DST begins Sun Mar 9. A Monday Mar 10 target is firmly EDT
|
||||||
|
// even if "now" were the preceding (EST) week. 16:55 EDT == 20:55 UTC.
|
||||||
|
const mon_after = etLocalToUtc(Date.fromYmd(2025, 3, 10), equity_target_s);
|
||||||
|
try testing.expectEqual(utcSeconds(2025, 3, 10, 20, 55), mon_after);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "nextCandleExpiry equity: intraday -> today's close boundary" {
|
||||||
|
// Wed 2025-06-11, 10:00 ET (14:00 UTC, EDT). Expiry is today 16:55 ET.
|
||||||
|
const now = utcSeconds(2025, 6, 11, 14, 0);
|
||||||
|
const exp = nextCandleExpiry(now, .equity);
|
||||||
|
const p = etParts(exp);
|
||||||
|
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 11)));
|
||||||
|
try testing.expectEqual(@as(u8, 16), p.hour);
|
||||||
|
try testing.expectEqual(@as(u8, 55), p.minute);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "nextCandleExpiry equity: after close -> next trading day" {
|
||||||
|
// Wed 2025-06-11, 18:00 ET (22:00 UTC). Past today's 16:55 -> Thu.
|
||||||
|
const now = utcSeconds(2025, 6, 11, 22, 0);
|
||||||
|
const p = etParts(nextCandleExpiry(now, .equity));
|
||||||
|
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 12)));
|
||||||
|
try testing.expectEqual(@as(u8, 16), p.hour);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "nextCandleExpiry equity: Friday evening skips the weekend" {
|
||||||
|
// Fri 2025-06-13, 18:00 ET. Next equity boundary is Mon 2025-06-16.
|
||||||
|
const now = utcSeconds(2025, 6, 13, 22, 0);
|
||||||
|
const p = etParts(nextCandleExpiry(now, .equity));
|
||||||
|
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 16)));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "nextCandleExpiry equity: skips a holiday (Thanksgiving)" {
|
||||||
|
// Wed 2025-11-26 evening. Thu 11-27 is Thanksgiving -> boundary is
|
||||||
|
// Fri 2025-11-28.
|
||||||
|
const now = utcSeconds(2025, 11, 26, 23, 0);
|
||||||
|
const p = etParts(nextCandleExpiry(now, .equity));
|
||||||
|
try testing.expect(p.date.eql(Date.fromYmd(2025, 11, 28)));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "nextCandleExpiry mutual_fund: Friday evening -> Saturday morning" {
|
||||||
|
// Fri 2025-06-13, 20:00 ET (past 03:25). Friday's NAV posts Sat AM.
|
||||||
|
const now = utcSeconds(2025, 6, 14, 0, 0); // 2025-06-13 20:00 EDT
|
||||||
|
const p = etParts(nextCandleExpiry(now, .mutual_fund));
|
||||||
|
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 14))); // Saturday
|
||||||
|
try testing.expectEqual(@as(u8, 3), p.hour);
|
||||||
|
try testing.expectEqual(@as(u8, 25), p.minute);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "nextCandleExpiry mutual_fund: Saturday morning -> Tuesday morning" {
|
||||||
|
// Sat 2025-06-14, 05:00 ET. No new NAV Sun/Mon AM; next is Tue (Mon's NAV).
|
||||||
|
const now = utcSeconds(2025, 6, 14, 9, 0); // 2025-06-14 05:00 EDT
|
||||||
|
const p = etParts(nextCandleExpiry(now, .mutual_fund));
|
||||||
|
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 17))); // Tuesday
|
||||||
|
}
|
||||||
|
|
||||||
|
test "nextCandleExpiry mutual_fund: Monday morning before NAV -> Tuesday" {
|
||||||
|
// Mon 2025-06-16, 01:00 ET (before 03:25). Monday's NAV not out yet,
|
||||||
|
// and there's no new NAV Monday AM either -> Tue 03:25.
|
||||||
|
const now = utcSeconds(2025, 6, 16, 5, 0); // 2025-06-16 01:00 EDT
|
||||||
|
const p = etParts(nextCandleExpiry(now, .mutual_fund));
|
||||||
|
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 17)));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "isHoliday: 2025 NYSE holidays" {
|
||||||
|
// Fixed-date with observance.
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 1, 1))); // New Year (Wed)
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 6, 19))); // Juneteenth (Thu)
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 7, 4))); // Independence (Fri)
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 12, 25))); // Christmas (Thu)
|
||||||
|
// Floating.
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 1, 20))); // MLK (3rd Mon)
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 2, 17))); // Washington (3rd Mon)
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 5, 26))); // Memorial (last Mon)
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 9, 1))); // Labor (1st Mon)
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2025, 11, 27))); // Thanksgiving (4th Thu)
|
||||||
|
// Non-holidays.
|
||||||
|
try testing.expect(!isHoliday(Date.fromYmd(2025, 7, 3)));
|
||||||
|
try testing.expect(!isHoliday(Date.fromYmd(2025, 12, 24)));
|
||||||
|
try testing.expect(!isHoliday(Date.fromYmd(2025, 6, 11)));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "isHoliday: weekend observance rules" {
|
||||||
|
// Independence Day 2026 falls on Saturday -> observed Fri 2026-07-03.
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2026, 7, 3)));
|
||||||
|
try testing.expect(!isHoliday(Date.fromYmd(2026, 7, 4))); // the Saturday itself
|
||||||
|
|
||||||
|
// New Year's Day 2022 was a Saturday: NOT observed on Fri 2021-12-31.
|
||||||
|
try testing.expect(!isHoliday(Date.fromYmd(2021, 12, 31)));
|
||||||
|
|
||||||
|
// New Year's Day 2023 was a Sunday -> observed Mon 2023-01-02.
|
||||||
|
try testing.expect(isHoliday(Date.fromYmd(2023, 1, 2)));
|
||||||
|
|
||||||
|
// Juneteenth not modeled before 2022.
|
||||||
|
try testing.expect(!isHoliday(Date.fromYmd(2021, 6, 18)));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "isTradingDay: weekdays, weekends, holidays" {
|
||||||
|
try testing.expect(isTradingDay(Date.fromYmd(2025, 6, 11))); // Wed
|
||||||
|
try testing.expect(!isTradingDay(Date.fromYmd(2025, 6, 14))); // Sat
|
||||||
|
try testing.expect(!isTradingDay(Date.fromYmd(2025, 6, 15))); // Sun
|
||||||
|
try testing.expect(!isTradingDay(Date.fromYmd(2025, 11, 27))); // Thanksgiving
|
||||||
|
}
|
||||||
|
|
||||||
|
test "staleCandleExpiry equity: due bar merely late retries soon" {
|
||||||
|
// Thu 2025-06-12, 17:30 ET: Thursday's bar is due (16:55 ET passed)
|
||||||
|
// but only ~35m overdue, well inside the lag grace window.
|
||||||
|
const now = etLocalToUtc(Date.fromYmd(2025, 6, 12), 17 * std.time.s_per_hour + 30 * std.time.s_per_min);
|
||||||
|
// Cache last has Wed -> Thursday's bar is due but unposted (lag) -> short retry.
|
||||||
|
try testing.expectEqual(now + short_retry_s, staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 11)));
|
||||||
|
// Cache already has Thu -> nothing newer is due -> normal next boundary.
|
||||||
|
try testing.expectEqual(nextCandleExpiry(now, .equity), staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 12)));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "staleCandleExpiry equity: weekend gap is not missing" {
|
||||||
|
// Sat 2025-06-14, 10:00 ET: latest available equity data is still
|
||||||
|
// Friday, which the cache already has - nothing is overdue.
|
||||||
|
const now = etLocalToUtc(Date.fromYmd(2025, 6, 14), 10 * std.time.s_per_hour);
|
||||||
|
const exp = staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 13));
|
||||||
|
try testing.expectEqual(nextCandleExpiry(now, .equity), exp);
|
||||||
|
// The long boundary (Mon), not a 30-minute retry.
|
||||||
|
try testing.expect(exp > now + short_retry_s);
|
||||||
|
try testing.expect(etParts(exp).date.eql(Date.fromYmd(2025, 6, 16))); // Monday
|
||||||
|
}
|
||||||
|
|
||||||
|
test "staleCandleExpiry equity: un-modeled Good Friday closure gives up after grace" {
|
||||||
|
// Good Friday 2025-04-18 is NOT a modeled holiday (it needs the Easter
|
||||||
|
// computus - see isHoliday), so the calendar treats it as a trading
|
||||||
|
// day and an incremental fetch keeps coming back empty all day. The
|
||||||
|
// Friday bar is due at 16:55 ET; the grace window ends 90m later.
|
||||||
|
try testing.expect(isTradingDay(Date.fromYmd(2025, 4, 18)));
|
||||||
|
const last_cached = Date.fromYmd(2025, 4, 17); // Thursday
|
||||||
|
|
||||||
|
// Just inside the window (+89m): keep retrying in case the bar shows
|
||||||
|
// up late.
|
||||||
|
const inside = etLocalToUtc(Date.fromYmd(2025, 4, 18), 18 * std.time.s_per_hour + 24 * std.time.s_per_min);
|
||||||
|
try testing.expectEqual(inside + short_retry_s, staleCandleExpiry(inside, .equity, last_cached));
|
||||||
|
|
||||||
|
// At the 90-minute window (16:55 + 90m = 18:25 ET) we conclude the
|
||||||
|
// market was closed and wait for the next real session (Mon
|
||||||
|
// 2025-04-21) instead of thrashing every 30 minutes all weekend. The
|
||||||
|
// boundary is strict: the +89m case above still retries, so the
|
||||||
|
// retries land at +30/+60/+90 and the +90 one trips this fallback.
|
||||||
|
const at_window = etLocalToUtc(Date.fromYmd(2025, 4, 18), 18 * std.time.s_per_hour + 25 * std.time.s_per_min);
|
||||||
|
const exp = staleCandleExpiry(at_window, .equity, last_cached);
|
||||||
|
try testing.expectEqual(nextCandleExpiry(at_window, .equity), exp);
|
||||||
|
try testing.expect(exp > at_window + short_retry_s); // not a short retry
|
||||||
|
const p = etParts(exp);
|
||||||
|
try testing.expect(p.date.eql(Date.fromYmd(2025, 4, 21))); // Monday
|
||||||
|
try testing.expectEqual(@as(u8, 16), p.hour);
|
||||||
|
try testing.expectEqual(@as(u8, 55), p.minute);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "staleCandleExpiry mutual_fund: late NAV within grace retries soon" {
|
||||||
|
// Tue 2025-06-17, 04:30 ET: Monday's NAV (data_date 2025-06-16) is due
|
||||||
|
// (posts ~03:25 ET) but the provider is lagging by ~65m, still inside
|
||||||
|
// the grace window. Also exercises the morning -> prior-session mapping
|
||||||
|
// in latestAvailability.
|
||||||
|
const now = etLocalToUtc(Date.fromYmd(2025, 6, 17), 4 * std.time.s_per_hour + 30 * std.time.s_per_min);
|
||||||
|
// Cache has through Friday's NAV -> Monday's is missing -> short retry.
|
||||||
|
try testing.expectEqual(now + short_retry_s, staleCandleExpiry(now, .mutual_fund, Date.fromYmd(2025, 6, 13)));
|
||||||
|
// Cache already has Monday -> nothing newer is due -> normal boundary.
|
||||||
|
try testing.expectEqual(nextCandleExpiry(now, .mutual_fund), staleCandleExpiry(now, .mutual_fund, Date.fromYmd(2025, 6, 16)));
|
||||||
|
}
|
||||||
|
|
@ -40,6 +40,7 @@ const fmt = @import("format.zig");
|
||||||
const performance = @import("analytics/performance.zig");
|
const performance = @import("analytics/performance.zig");
|
||||||
const http = @import("net/http.zig");
|
const http = @import("net/http.zig");
|
||||||
const atomic = @import("atomic.zig");
|
const atomic = @import("atomic.zig");
|
||||||
|
const market = @import("market.zig");
|
||||||
|
|
||||||
// ── Wall-clock policy ────────────────────────────────────────
|
// ── Wall-clock policy ────────────────────────────────────────
|
||||||
//
|
//
|
||||||
|
|
@ -595,7 +596,11 @@ pub const DataService = struct {
|
||||||
// and candles_meta.srf in one shot (last_close, last_date,
|
// and candles_meta.srf in one shot (last_close, last_date,
|
||||||
// provider, fail_count=0).
|
// provider, fail_count=0).
|
||||||
if (triple.candles.len > 0) {
|
if (triple.candles.len > 0) {
|
||||||
s.cacheCandles(symbol, triple.candles, .tiingo, 0);
|
// wall-clock required: stamp the candle-meta freshness boundary
|
||||||
|
// (market-aware next post-close / NAV-availability time).
|
||||||
|
const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds();
|
||||||
|
const expires = market.nextCandleExpiry(now_s, market.classify(symbol));
|
||||||
|
s.cacheCandles(symbol, triple.candles, .tiingo, 0, expires);
|
||||||
}
|
}
|
||||||
// Dividends and splits use the supplement write path: Tiingo's
|
// Dividends and splits use the supplement write path: Tiingo's
|
||||||
// view merges into existing (typically Polygon-sourced) records
|
// view merges into existing (typically Polygon-sourced) records
|
||||||
|
|
@ -745,6 +750,14 @@ pub const DataService = struct {
|
||||||
var s = self.store();
|
var s = self.store();
|
||||||
const today = fmt.todayDate(self.io);
|
const today = fmt.todayDate(self.io);
|
||||||
|
|
||||||
|
// wall-clock required: candle-meta freshness uses a market-aware
|
||||||
|
// boundary (next post-close / NAV-availability time in ET; see
|
||||||
|
// market.nextCandleExpiry). Captured once here and threaded to
|
||||||
|
// every candle-meta write below so a single invocation stamps a
|
||||||
|
// consistent boundary.
|
||||||
|
const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds();
|
||||||
|
const kind = market.classify(symbol);
|
||||||
|
|
||||||
// Check candle metadata for freshness (tiny file, no candle deserialization)
|
// Check candle metadata for freshness (tiny file, no candle deserialization)
|
||||||
const meta_result = s.readCandleMeta(symbol);
|
const meta_result = s.readCandleMeta(symbol);
|
||||||
if (meta_result) |mr| {
|
if (meta_result) |mr| {
|
||||||
|
|
@ -792,9 +805,13 @@ pub const DataService = struct {
|
||||||
// Stale - try incremental update using last_date from meta
|
// Stale - try incremental update using last_date from meta
|
||||||
const fetch_from = m.last_date.addDays(1);
|
const fetch_from = m.last_date.addDays(1);
|
||||||
|
|
||||||
|
// Market-aware freshness boundary for any meta write on
|
||||||
|
// this stale path (next post-close / NAV-availability time).
|
||||||
|
const expires = market.nextCandleExpiry(now_s, kind);
|
||||||
|
|
||||||
// If last cached date is today or later, just refresh the TTL (meta only)
|
// If last cached date is today or later, just refresh the TTL (meta only)
|
||||||
if (!fetch_from.lessThan(today)) {
|
if (!fetch_from.lessThan(today)) {
|
||||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count);
|
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count, expires);
|
||||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
||||||
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||||
} else {
|
} else {
|
||||||
|
|
@ -805,7 +822,7 @@ pub const DataService = struct {
|
||||||
// Increment fail_count for this symbol
|
// Increment fail_count for this symbol
|
||||||
const new_fail_count = m.fail_count +| 1; // saturating add
|
const new_fail_count = m.fail_count +| 1; // saturating add
|
||||||
log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count });
|
log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count });
|
||||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count);
|
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count, now_s + market.short_retry_s);
|
||||||
|
|
||||||
// If degraded (fail_count >= 3), return stale data rather than failing
|
// If degraded (fail_count >= 3), return stale data rather than failing
|
||||||
if (new_fail_count >= 3) {
|
if (new_fail_count >= 3) {
|
||||||
|
|
@ -823,14 +840,23 @@ pub const DataService = struct {
|
||||||
const new_candles = result.candles;
|
const new_candles = result.candles;
|
||||||
|
|
||||||
if (new_candles.len == 0) {
|
if (new_candles.len == 0) {
|
||||||
// No new candles (weekend/holiday) - refresh TTL, reset fail_count
|
// No new candles. Either a genuine non-trading-day
|
||||||
|
// gap (weekend/holiday), the provider hasn't posted
|
||||||
|
// the just-closed bar yet, or the calendar's
|
||||||
|
// "trading day" was an un-modeled closure (e.g.
|
||||||
|
// Good Friday). market.staleCandleExpiry picks the
|
||||||
|
// boundary: a short retry while a due bar is merely
|
||||||
|
// late, falling back to the next close boundary
|
||||||
|
// once it's overdue past the lag grace window - so
|
||||||
|
// an un-modeled closure stops thrashing and waits
|
||||||
|
// for the next real session.
|
||||||
self.allocator.free(new_candles);
|
self.allocator.free(new_candles);
|
||||||
s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0);
|
s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0, market.staleCandleExpiry(now_s, kind, m.last_date));
|
||||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
|
||||||
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||||
} else {
|
} else {
|
||||||
// Append new candles to existing file + update meta, reset fail_count
|
// Append new candles to existing file + update meta, reset fail_count
|
||||||
s.appendCandles(symbol, new_candles, result.provider, 0);
|
s.appendCandles(symbol, new_candles, result.provider, 0, expires);
|
||||||
if (s.read(self.allocator, Candle, symbol, null, .any)) |r| {
|
if (s.read(self.allocator, Candle, symbol, null, .any)) |r| {
|
||||||
self.allocator.free(new_candles);
|
self.allocator.free(new_candles);
|
||||||
return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||||
|
|
@ -874,7 +900,7 @@ pub const DataService = struct {
|
||||||
// we know to back off if this keeps happening.
|
// we know to back off if this keeps happening.
|
||||||
if (meta_result) |mr| {
|
if (meta_result) |mr| {
|
||||||
const new_fail_count = mr.meta.fail_count +| 1;
|
const new_fail_count = mr.meta.fail_count +| 1;
|
||||||
s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count);
|
s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count, now_s + market.short_retry_s);
|
||||||
}
|
}
|
||||||
return DataError.TransientError;
|
return DataError.TransientError;
|
||||||
}
|
}
|
||||||
|
|
@ -918,7 +944,7 @@ pub const DataService = struct {
|
||||||
/// `opts.force_refresh = true` -> treats cache as stale and fetches.
|
/// `opts.force_refresh = true` -> treats cache as stale and fetches.
|
||||||
pub fn getEarnings(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!FetchResult(EarningsEvent) {
|
pub fn getEarnings(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!FetchResult(EarningsEvent) {
|
||||||
// Mutual funds (5-letter tickers ending in X) don't have quarterly earnings.
|
// Mutual funds (5-letter tickers ending in X) don't have quarterly earnings.
|
||||||
if (isMutualFund(symbol)) {
|
if (market.classify(symbol) == .mutual_fund) {
|
||||||
return .{ .data = &.{}, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
return .{ .data = &.{}, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -2928,13 +2954,6 @@ pub const DataService = struct {
|
||||||
return daily and meta;
|
return daily and meta;
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Mutual funds use 5-letter tickers ending in X (e.g. FDSCX, VSTCX, FAGIX).
|
|
||||||
/// These don't have quarterly earnings - skip the fetch rather than
|
|
||||||
/// round-tripping to the provider just to get an empty response.
|
|
||||||
fn isMutualFund(symbol: []const u8) bool {
|
|
||||||
return symbol.len == 5 and symbol[4] == 'X';
|
|
||||||
}
|
|
||||||
|
|
||||||
// ── User config files ─────────────────────────────────────────
|
// ── User config files ─────────────────────────────────────────
|
||||||
|
|
||||||
/// Load and parse accounts.srf from the same directory as the given portfolio path.
|
/// Load and parse accounts.srf from the same directory as the given portfolio path.
|
||||||
|
|
@ -3009,24 +3028,6 @@ test "isPermanentProviderFailure: RateLimited is transient" {
|
||||||
try std.testing.expect(!isPermanentProviderFailure(error.RateLimited));
|
try std.testing.expect(!isPermanentProviderFailure(error.RateLimited));
|
||||||
}
|
}
|
||||||
|
|
||||||
test "isMutualFund identifies mutual funds" {
|
|
||||||
// Standard mutual fund tickers (5 letters ending in X)
|
|
||||||
try std.testing.expect(DataService.isMutualFund("FDSCX"));
|
|
||||||
try std.testing.expect(DataService.isMutualFund("VSTCX"));
|
|
||||||
try std.testing.expect(DataService.isMutualFund("FAGIX"));
|
|
||||||
try std.testing.expect(DataService.isMutualFund("VFINX"));
|
|
||||||
|
|
||||||
// Not mutual funds
|
|
||||||
try std.testing.expect(!DataService.isMutualFund("AAPL"));
|
|
||||||
try std.testing.expect(!DataService.isMutualFund("VTI"));
|
|
||||||
try std.testing.expect(!DataService.isMutualFund("SPY"));
|
|
||||||
try std.testing.expect(!DataService.isMutualFund("GOOGL"));
|
|
||||||
try std.testing.expect(!DataService.isMutualFund("")); // empty
|
|
||||||
try std.testing.expect(!DataService.isMutualFund("X")); // too short
|
|
||||||
try std.testing.expect(!DataService.isMutualFund("FDSCA")); // 5 letters but not ending in X
|
|
||||||
try std.testing.expect(!DataService.isMutualFund("FDSCXA")); // 6 letters ending in A
|
|
||||||
}
|
|
||||||
|
|
||||||
test "DataService init/deinit lifecycle" {
|
test "DataService init/deinit lifecycle" {
|
||||||
const allocator = std.testing.allocator;
|
const allocator = std.testing.allocator;
|
||||||
const config = Config{
|
const config = Config{
|
||||||
|
|
@ -3176,7 +3177,7 @@ test "getCandles offline mode returns cached data without network" {
|
||||||
.{ .date = Date.fromYmd(2026, 5, 19), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
.{ .date = Date.fromYmd(2026, 5, 19), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||||
.{ .date = Date.fromYmd(2026, 5, 20), .open = 104, .high = 106, .low = 103, .close = 105, .adj_close = 105, .volume = 1100 },
|
.{ .date = Date.fromYmd(2026, 5, 20), .open = 104, .high = 106, .low = 103, .close = 105, .adj_close = 105, .volume = 1100 },
|
||||||
};
|
};
|
||||||
store.cacheCandles("TEST", candles[0..], .tiingo, 0);
|
store.cacheCandles("TEST", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||||
|
|
||||||
// Set the test guard: any network call would panic. We expect
|
// Set the test guard: any network call would panic. We expect
|
||||||
// the offline-mode path NOT to touch the network.
|
// the offline-mode path NOT to touch the network.
|
||||||
|
|
@ -3279,7 +3280,7 @@ test "loadAllPrices offline mode skips network and returns cached" {
|
||||||
var fresh_candles = [_]Candle{
|
var fresh_candles = [_]Candle{
|
||||||
.{ .date = Date.fromYmd(2026, 5, 20), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
.{ .date = Date.fromYmd(2026, 5, 20), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||||
};
|
};
|
||||||
store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0);
|
store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||||
|
|
||||||
// Symbol with no cache at all.
|
// Symbol with no cache at all.
|
||||||
// (no setup needed - just passes a symbol that doesn't exist)
|
// (no setup needed - just passes a symbol that doesn't exist)
|
||||||
|
|
@ -3331,7 +3332,7 @@ test "loadAllPrices force_refresh tops up without wiping the candle cache" {
|
||||||
var candles = [_]Candle{
|
var candles = [_]Candle{
|
||||||
.{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
.{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
|
||||||
};
|
};
|
||||||
store.cacheCandles("HELD", candles[0..], .tiingo, 0);
|
store.cacheCandles("HELD", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
|
||||||
|
|
||||||
// Any provider/network attempt now panics. If force_refresh wiped
|
// Any provider/network attempt now panics. If force_refresh wiped
|
||||||
// the cache (old behavior), getCandles would fall through to a full
|
// the cache (old behavior), getCandles would fall through to a full
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue