diff --git a/build.zig b/build.zig index 70788b0..d9e669f 100644 --- a/build.zig +++ b/build.zig @@ -21,6 +21,11 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); + const zeit_dep = b.dependency("zeit", .{ + .target = target, + .optimize = optimize, + }); + const srf_mod = srf_dep.module("srf"); const shiller_mod = b.addModule("shiller_year", .{ @@ -43,6 +48,7 @@ pub fn build(b: *std.Build) void { .target = target, .imports = &.{ .{ .name = "srf", .module = srf_mod }, + .{ .name = "zeit", .module = zeit_dep.module("zeit") }, .{ .name = "build_info", .module = build_info }, }, }); @@ -54,6 +60,7 @@ pub fn build(b: *std.Build) void { .{ .name = "srf", .module = srf_mod }, .{ .name = "vaxis", .module = vaxis_dep.module("vaxis") }, .{ .name = "z2d", .module = z2d_dep.module("z2d") }, + .{ .name = "zeit", .module = zeit_dep.module("zeit") }, .{ .name = "build_info", .module = build_info }, .{ .name = "shiller_year", .module = shiller_mod }, }; @@ -121,6 +128,7 @@ pub fn build(b: *std.Build) void { .optimize = optimize, .imports = &.{ .{ .name = "srf", .module = srf_mod }, + .{ .name = "zeit", .module = zeit_dep.module("zeit") }, .{ .name = "build_info", .module = build_info }, }, }), diff --git a/build.zig.zon b/build.zig.zon index 3718df8..6517060 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -16,6 +16,10 @@ .url = "git+https://git.lerch.org/lobo/srf#4a3e5f00f15b0e0ba79d06ffe69dbcfa052baa5b", .hash = "srf-0.0.0-qZj572nkAQAAz3zEg6fdD8A7PJnQ9je3zCeAOJS5PoZj", }, + .zeit = .{ + .url = "git+https://github.com/rockorager/zeit?ref=v0.9.0#b1c1c2fcbc71fd7799a316bbcf0ff88d06d80ccc", + .hash = "zeit-0.9.0-5I6bk2m9AgBSMH8-L6rYJkwuQAyhXplnfxnvTSGzVHUR", + }, }, .paths = .{ "build", diff --git a/docs/explanation/caching.md b/docs/explanation/caching.md index 1a48af5..3973b83 100644 --- a/docs/explanation/caching.md +++ b/docs/explanation/caching.md @@ -37,21 +37,66 @@ The `--refresh-data` policy decides which tiers run: Different data ages at different rates, so each type has its own TTL: -| Data type | TTL | Why | -|---------------|---------------|-------------------------------------------------------------| -| Daily candles | ~24h (23h45m) | One bar per trading day; slightly under 24h for cron jitter | -| Dividends | 14 days | Declared well in advance | -| Splits | 14 days | Rare corporate events | -| Options | 1 hour | Prices move continuously when markets are open | -| Earnings | 30 days\* | Quarterly; smart-refreshed around announcements | -| ETF profiles | ~30 days | Holdings and weights change slowly | -| Quotes | never cached | Meant to be a live price check | +| Data type | TTL | Why | +|---------------|---------------|----------------------------------------------------------------------------------| +| Daily candles | market-aware | Keyed to the next time a fresh bar is expected, not a rolling window (see below) | +| Dividends | 14 days | Declared well in advance | +| Splits | 14 days | Rare corporate events | +| Options | 1 hour | Prices move continuously when markets are open | +| Earnings | 30 days\* | Quarterly; smart-refreshed around announcements | +| ETF profiles | ~30 days | Holdings and weights change slowly | +| Quotes | never cached | Meant to be a live price check | \* **Earnings smart refresh:** even inside the 30-day window, cached earnings re-fetch automatically once an earnings date has passed but the cache still lacks the actual result -- so numbers appear promptly after an announcement without daily polling. +## Market-aware candle freshness + +A daily bar only becomes meaningful once the market settles, so candle +freshness is keyed to the market clock rather than a rolling 24-hour +window. Each cached candle's expiry is set to the next moment fresh data +should be available: + +- **Equities and ETFs** settle shortly after the 16:00 ET close. Their + bars expire at **16:55 ET** on the next trading day (weekends and NYSE + holidays are skipped). This time allows for providers to become consistent + with the market while also allowing a few minutes prior to a scheduled + refresh task at the top of the hour. +- **Mutual funds** strike a single daily NAV that isn't reliably + published until the next morning, so their bars expire at **03:25 ET** + the morning after a trading session. Despite Tiingo's claims, NAVs only + seem reliably available until about 3am Eastern. This is again timed such + that scheduled jobs for the bottom of the hour can run reliably. + +This keeps the expiry boundary out of trading hours, so a refresh fired +just after it always sees a finalized bar instead of a half-formed one, +and an interactive command run mid-session won't trigger a needless +refetch. If a refresh runs but the provider hasn't posted the just-closed +bar yet, the entry is retried in ~30 minutes rather than waiting a full +day. + +**Un-modeled closures self-correct.** Some market closures aren't on the +modeled holiday calendar (Good Friday, which needs the Easter computus, +plus ad-hoc closures for national mourning or weather). On such a day the +calendar thinks a bar is due, the fetch keeps coming back empty, and the +~30-minute retry would otherwise repeat all day - and, for a Friday +closure, all weekend. To avoid that thrash, once an expected bar is ~90 +minutes overdue the cache concludes the market was closed and falls back +to the normal next-session boundary - at most three 30-minute retries. +That window comfortably covers ordinary provider posting lag, so +genuinely-late data is still picked up by the short retry; only a true +closure trips the fallback. + +**Warming a shared cache on a schedule.** If you run a cron to warm a +[server cache](#server-sync-zfin_server) (or your own local cache), the +boundaries above are also the natural cron times: a run shortly after +**17:00 ET** picks up the day's equity/ETF closes, and a run shortly +after **03:30 ET** picks up the prior session's mutual-fund NAVs. The +boundaries sit a couple of minutes before those times so the cron +reliably sees the cache already expired. + ## Quotes are never cached Because quotes exist to give you a live price, they're never served diff --git a/src/cache/store.zig b/src/cache/store.zig index d4d728f..12d4967 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -32,8 +32,6 @@ pub const Ttl = struct { const s_per_day = std.time.s_per_day; /// Historical candles older than 1 day never expire pub const candles_historical: i64 = -1; // infinite - /// Latest day's candle refreshes every 23h45m (15-min buffer for cron jitter) - pub const candles_latest: i64 = s_per_day - 15 * std.time.s_per_min; /// Dividend data refreshes biweekly pub const dividends: i64 = 14 * s_per_day; /// Split data refreshes biweekly @@ -816,9 +814,12 @@ pub const Store = struct { // ── Candle-specific API ────────────────────────────────────── - /// Write a full set of candles to cache (no expiry - historical facts don't expire). - /// Also updates candle metadata. - pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8) void { + /// Write a full set of candles to cache (the candle data file itself + /// carries no expiry - historical bars never change). Also refreshes + /// candle metadata, stamping its `#!expires=` with `expires_at_s` + /// (the caller computes the market-aware freshness boundary; see + /// `market.nextCandleExpiry`). + pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void { if (serializeCandles(self.allocator, candles, .{})) |srf_data| { defer self.allocator.free(srf_data); self.writeRaw(symbol, .candles_daily, srf_data) catch |err| { @@ -830,14 +831,15 @@ pub const Store = struct { if (candles.len > 0) { const last = candles[candles.len - 1]; - self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count); + self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s); } } /// Append new candle records to the existing cache file. /// Falls back to a full rewrite if append fails (e.g. file doesn't exist). - /// Also updates candle metadata. - pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8) void { + /// Also updates candle metadata, stamping its `#!expires=` with + /// `expires_at_s` (caller-computed market-aware boundary). + pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void { if (new_candles.len == 0) return; if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| { @@ -866,19 +868,24 @@ pub const Store = struct { } const last = new_candles[new_candles.len - 1]; - self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count); + self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s); } /// Write (or refresh) candle metadata with a specific provider source. - pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8) void { - const expires = std.Io.Timestamp.now(self.io, .real).toSeconds() + Ttl.candles_latest; + /// + /// `expires_at_s` is the absolute Unix-seconds freshness boundary for + /// the `#!expires=` directive, computed by the caller via + /// `market.nextCandleExpiry` (close-anchored) or a short retry. The + /// cache layer no longer reads the wall clock for this - the boundary + /// is market-domain knowledge owned by the caller. + pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void { const meta = CandleMeta{ .last_close = last_close, .last_date = last_date, .provider = provider, .fail_count = fail_count, }; - if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires })) |meta_data| { + if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires_at_s })) |meta_data| { defer self.allocator.free(meta_data); self.writeRaw(symbol, .candles_meta, meta_data) catch |err| { log.warn("{s}: failed to write candle metadata: {s}", .{ symbol, @errorName(err) }); @@ -2961,9 +2968,8 @@ test "TTL constants are reasonable" { // Historical candles never expire try std.testing.expectEqual(@as(i64, -1), Ttl.candles_historical); - // Latest candles expire just under 24 hours (allowing for cron jitter) - try std.testing.expect(Ttl.candles_latest > 23 * std.time.s_per_hour); - try std.testing.expect(Ttl.candles_latest < 24 * std.time.s_per_hour); + // Latest candles use a market-aware expiry computed per-write by + // market.nextCandleExpiry (no fixed TTL constant here anymore). // Dividends and splits refresh biweekly try std.testing.expectEqual(@as(i64, 14 * std.time.s_per_day), Ttl.dividends); diff --git a/src/market.zig b/src/market.zig new file mode 100644 index 0000000..e66dcc0 --- /dev/null +++ b/src/market.zig @@ -0,0 +1,538 @@ +//! Market calendar: trading-day awareness and close-anchored cache +//! freshness boundaries for daily candles. +//! +//! Daily candle data only becomes meaningful once the market settles. +//! Two distinct deadlines matter: +//! +//! - **Equities / ETFs** settle shortly after the 16:00 ET close. +//! - **Mutual-fund NAVs** strike once per day and are not reliably +//! published until the next morning (~03:30 ET). +//! +//! The old candle TTL was a rolling 23h45m window, so the expiry +//! boundary drifted against the market clock and could fall during +//! trading hours - causing mid-session refetches and risking caching a +//! not-yet-finalized bar. This module computes the next moment fresh +//! data should be available, so the candle cache is keyed to the market +//! clock instead. It is an optimization, not a correctness fix: a +//! slightly-wrong boundary costs at most one cheap no-op fetch (see the +//! direction-of-error note on `isHoliday`). +//! +//! Timezone handling uses `zeit` with a hardcoded US Eastern POSIX TZ +//! spec, so there is no dependency on system zoneinfo files, no +//! allocator, and no I/O. The fixed DST rule is correct for current and +//! near-future dates, which is all cache-freshness math ever deals with. + +const std = @import("std"); +const zeit = @import("zeit"); +const Date = @import("Date.zig"); + +/// POSIX TZ spec for US Eastern: EST (UTC-5) / EDT (UTC-4) with the +/// current US DST rule (2nd Sunday March -> 1st Sunday November). +const eastern_posix_spec = "EST5EDT,M3.2.0,M11.1.0"; + +/// US Eastern timezone, parsed at comptime from `eastern_posix_spec`. +/// `Posix.parse` allocates nothing - the parsed struct borrows slices +/// into the (static) spec string - so this is a zero-cost const. +const eastern: zeit.TimeZone = .{ + .posix = zeit.timezone.Posix.parse(eastern_posix_spec) catch + @compileError("invalid eastern POSIX TZ spec: " ++ eastern_posix_spec), +}; + +/// How candle data for a symbol becomes available. +pub const InstrumentKind = enum { + /// Continuously-quoted equities and ETFs: the day's bar settles + /// shortly after the 16:00 ET close. + equity, + /// Mutual funds: a single daily NAV that isn't reliably published + /// until the next morning. + mutual_fund, +}; + +/// Classify a symbol for candle-freshness purposes. Mutual funds use +/// 5-letter tickers ending in X (e.g. FDSCX, VSTCX, FAGIX); everything +/// else is treated as an equity/ETF. Imperfect, but covers the common +/// case. +/// +/// This is the single home for the heuristic: it was consolidated here +/// from the former `DataService.isMutualFund` (removed) when candle +/// freshness timing moved into this module. It is deliberately distinct +/// from `portfolio.isMoneyMarketSymbol`, which answers a different +/// question - "is this a fixed-$1-NAV cash equivalent?" - via a curated +/// whitelist. (Every symbol on that whitelist also matches this +/// heuristic, so freshness callers only need `classify`.) +pub fn classify(symbol: []const u8) InstrumentKind { + if (symbol.len == 5 and symbol[4] == 'X') return .mutual_fund; + return .equity; +} + +// ── Freshness target times (seconds since ET-local midnight) ───────── +// +// These double as the recommended refresh-cron schedule: each sits a +// couple minutes BEFORE its intended cron run so cron-timing jitter +// reliably sees the cache already expired (a boundary exactly at the +// cron time could be missed by an early tick). The "expected-but- +// missing" short retry (see `short_retry_s`) covers the case where the +// provider hasn't posted the just-closed bar by then. + +/// Equity/ETF boundary: 16:55 ET (just after the 16:00 close + settle +/// margin; pairs with a ~17:00 ET refresh cron). +const equity_target_s: i64 = 16 * std.time.s_per_hour + 55 * std.time.s_per_min; + +/// Mutual-fund boundary: 03:25 ET the morning after the trading day +/// (pairs with a ~03:30 ET NAV-refresh cron). +const mf_target_s: i64 = 3 * std.time.s_per_hour + 25 * std.time.s_per_min; + +/// Retry interval when an expected bar wasn't returned yet (the provider +/// hasn't posted the just-closed bar, or a transient fetch failure). +/// Short enough to pick the bar up within the same session, long enough +/// to avoid hammering a rate-limited provider. +pub const short_retry_s: i64 = 30 * std.time.s_per_min; + +/// How long past the moment data is *due* we keep doing `short_retry_s` +/// retries before concluding the calendar's "trading day" was actually +/// an un-modeled market closure (e.g. Good Friday, which needs the Easter +/// computus and is deliberately not modeled - see `isHoliday`) or an +/// ad-hoc halt, rather than mere provider lag. Once an expected bar has +/// been overdue this long it is almost certainly never coming, so +/// `staleCandleExpiry` stops the retry loop and falls back to the normal +/// next-boundary expiry. That bounds an un-modeled closure to a few +/// no-op fetches that session instead of thrashing every `short_retry_s` +/// until the next real session - for a Friday closure, the entire long +/// weekend. +/// +/// 90 minutes comfortably covers normal posting lag for both the equity +/// close and the overnight mutual-fund NAV (data that is going to appear +/// at all is essentially always up well inside that window) while still +/// giving up promptly. At the 30-minute `short_retry_s` cadence that is +/// three retries (at +30, +60, +90 min); the +90 retry is the one that +/// crosses the window and trips the fallback. +const provider_lag_grace_s: i64 = 90 * std.time.s_per_min; + +fn targetSeconds(kind: InstrumentKind) i64 { + return switch (kind) { + .equity => equity_target_s, + .mutual_fund => mf_target_s, + }; +} + +// ── Trading-day calendar ───────────────────────────────────────────── + +// Weekday constants in zfin's 0=Mon .. 6=Sun scheme (see Date.dayOfWeek). +const mon: u8 = 0; +const thu: u8 = 3; +const sat: u8 = 5; +const sun: u8 = 6; + +/// True if `d` is a US equity-market trading day: a weekday that isn't a +/// modeled NYSE holiday. +pub fn isTradingDay(d: Date) bool { + if (d.dayOfWeek() >= sat) return false; // Saturday or Sunday + return !isHoliday(d); +} + +/// True if `d` is a (modeled) NYSE market holiday. +/// +/// Direction-of-error policy: a *false holiday* (marking a real trading +/// day closed) would let the cache stay fresh too long and miss a day's +/// data - a staleness bug. A *missed holiday* (treating a closed day as +/// open) only costs one harmless no-op fetch. So we model only the +/// well-defined, confidently-computable holidays and lean toward "open" +/// when unsure. Good Friday is deliberately omitted (it needs the Easter +/// computus); an un-modeled Good Friday simply costs one no-op fetch. +/// Ad-hoc closures (national mourning, weather) are likewise unmodeled. +pub fn isHoliday(d: Date) bool { + const y = d.year(); + + // Floating Monday/Thursday holidays - these always land on a weekday, + // so no weekend-observance adjustment is needed. + if (d.eql(nthWeekday(y, 1, mon, 3))) return true; // MLK Day: 3rd Mon Jan + if (d.eql(nthWeekday(y, 2, mon, 3))) return true; // Washington's Birthday: 3rd Mon Feb + if (d.eql(lastWeekday(y, 5, mon))) return true; // Memorial Day: last Mon May + if (d.eql(nthWeekday(y, 9, mon, 1))) return true; // Labor Day: 1st Mon Sep + if (d.eql(nthWeekday(y, 11, thu, 4))) return true; // Thanksgiving: 4th Thu Nov + + // New Year's Day: Sunday -> observed Monday. NYSE does NOT close the + // preceding Friday when Jan 1 falls on a Saturday (that Friday is in + // the prior year and stays a normal trading day). + if (observedSundayOnly(d, 1, 1)) return true; + + // Juneteenth: NYSE first observed it in 2022. Sat -> Fri, Sun -> Mon. + if (y >= 2022 and observedFixed(d, 6, 19)) return true; + + // Independence Day and Christmas: Sat -> Fri, Sun -> Mon. + if (observedFixed(d, 7, 4)) return true; + if (observedFixed(d, 12, 25)) return true; + + return false; +} + +/// The `n`-th occurrence (1-based) of weekday `wd` (0=Mon..6=Sun) in +/// month `mo` of `y`. +fn nthWeekday(y: i16, mo: u8, wd: u8, n: u8) Date { + const first = Date.fromYmd(y, mo, 1); + const offset = @mod(@as(i32, wd) - @as(i32, first.dayOfWeek()), 7); + const day: u8 = @intCast(1 + offset + (@as(i32, n - 1) * 7)); + return Date.fromYmd(y, mo, day); +} + +/// The last occurrence of weekday `wd` (0=Mon..6=Sun) in month `mo` of `y`. +fn lastWeekday(y: i16, mo: u8, wd: u8) Date { + const last = Date.lastDayOfMonth(y, mo); + const back = @mod(@as(i32, last.dayOfWeek()) - @as(i32, wd), 7); + return last.addDays(-back); +} + +/// True if `d` is the observed date of the fixed (mo/day) holiday in +/// `d`'s year, using the full weekend-observance rule: Saturday holidays +/// observed the preceding Friday, Sunday holidays the following Monday. +fn observedFixed(d: Date, mo: u8, day: u8) bool { + const actual = Date.fromYmd(d.year(), mo, day); + return switch (actual.dayOfWeek()) { + sat => d.eql(actual.addDays(-1)), // Sat -> preceding Fri + sun => d.eql(actual.addDays(1)), // Sun -> following Mon + else => d.eql(actual), + }; +} + +/// Like `observedFixed` but Saturday is NOT observed (used for New +/// Year's Day - see `isHoliday`). +fn observedSundayOnly(d: Date, mo: u8, day: u8) bool { + const actual = Date.fromYmd(d.year(), mo, day); + return switch (actual.dayOfWeek()) { + sat => false, + sun => d.eql(actual.addDays(1)), + else => d.eql(actual), + }; +} + +// ── Close-anchored freshness boundaries ────────────────────────────── + +/// True if fresh candle data for `kind` becomes available on calendar +/// day `d` (in ET). For equities, that's any trading day (the bar +/// settles that afternoon). For mutual funds, it's the morning AFTER a +/// trading day (the prior session's NAV posts overnight) - which also +/// makes holidays fall out for free: if the prior day wasn't a trading +/// day, no NAV is available this morning. +fn isAvailabilityDay(d: Date, kind: InstrumentKind) bool { + return switch (kind) { + .equity => isTradingDay(d), + .mutual_fund => isTradingDay(d.addDays(-1)), + }; +} + +/// The trading day whose data an availability day carries. +fn dataDateFor(availability_day: Date, kind: InstrumentKind) Date { + return switch (kind) { + .equity => availability_day, + .mutual_fund => availability_day.addDays(-1), + }; +} + +/// Absolute Unix-seconds expiry for a freshly-written candle cache +/// entry: the next moment after `now_s` at which new data for `kind` +/// should be available. Pure given `now_s` (the wall clock is read by +/// the caller and threaded in), so it is fully deterministic for tests. +pub fn nextCandleExpiry(now_s: i64, kind: InstrumentKind) i64 { + const target = targetSeconds(kind); + const now_local = eastern.adjust(now_s).timestamp; + const today_et = Date.fromEpoch(now_local); + const now_tod = now_local - today_et.toEpoch(); // seconds since ET midnight + + // Start at today's target; if it has already passed, move to tomorrow. + var cand = today_et; + if (now_tod >= target) cand = cand.addDays(1); + // Skip forward to the next day that actually carries new data. + while (!isAvailabilityDay(cand, kind)) cand = cand.addDays(1); + + return etLocalToUtc(cand, target); +} + +/// The most recent data availability as of `now_s`: which trading day's +/// candle should already be published, and the exact instant it went +/// live (its target time on the availability day, in Unix seconds). +const Availability = struct { + /// Trading day whose data the latest availability carries. + data_date: Date, + /// UTC seconds at which that data became available. + instant_s: i64, +}; + +/// Walk backward from `now_s` to the most recent availability day whose +/// target time has already passed, reporting both the trading day it +/// carries and the instant its data went live. Pure given `now_s`. +fn latestAvailability(now_s: i64, kind: InstrumentKind) Availability { + const target = targetSeconds(kind); + const now_local = eastern.adjust(now_s).timestamp; + const today_et = Date.fromEpoch(now_local); + const now_tod = now_local - today_et.toEpoch(); + + var cand = today_et; + if (now_tod < target) cand = cand.addDays(-1); + while (!isAvailabilityDay(cand, kind)) cand = cand.addDays(-1); + return .{ + .data_date = dataDateFor(cand, kind), + .instant_s = etLocalToUtc(cand, target), + }; +} + +/// Expiry to stamp on candle meta after an *incremental* fetch on a stale +/// entry returned zero new bars. Three cases: +/// +/// 1. **Nothing newer is due** - a genuine non-trading-day gap +/// (weekend/holiday) or the cache is already caught up: use the +/// normal next-boundary expiry. +/// 2. **Newer data is due, still within the provider-lag grace +/// window** (`provider_lag_grace_s`): the provider just hasn't +/// posted the just-closed bar yet, so retry soon (`short_retry_s`). +/// 3. **Newer data has been due for the full grace window** +/// (`provider_lag_grace_s`): the calendar's "trading day" was almost +/// certainly an un-modeled closure (e.g. Good Friday) or ad-hoc halt +/// - the bar is never going to appear. Stop the short-retry thrash +/// and fall back to the normal next-boundary expiry, so we wait for +/// the next real session instead of refetching every `short_retry_s` +/// all evening (and, for a Friday closure, all weekend). +/// +/// `last_cached` is the newest date already in the candle cache. Pure +/// given `now_s`, so it is fully deterministic for tests. +pub fn staleCandleExpiry(now_s: i64, kind: InstrumentKind, last_cached: Date) i64 { + const avail = latestAvailability(now_s, kind); + // Case 1: nothing newer than the cache is due yet. + if (!last_cached.lessThan(avail.data_date)) return nextCandleExpiry(now_s, kind); + // Case 2: due data is merely late (still inside the grace window) -> short retry. + if (now_s - avail.instant_s < provider_lag_grace_s) return now_s + short_retry_s; + // Case 3: overdue for the whole grace window -> assume closure, wait for next boundary. + return nextCandleExpiry(now_s, kind); +} + +/// Convert an ET local wall-clock instant (`d` at `tod_s` seconds since +/// ET midnight) to Unix seconds, resolving the correct EST/EDT offset +/// for that date. Treats the wall clock as a UTC instant for a first +/// guess, then corrects by the offset zeit reports at that moment; two +/// iterations converge because the offset changes by at most one hour +/// and the target times never sit inside the ~02:00 ET DST-transition +/// window. +fn etLocalToUtc(d: Date, tod_s: i64) i64 { + const wall = d.toEpoch() + tod_s; + var utc = wall; + var i: usize = 0; + while (i < 2) : (i += 1) { + // adjust(utc).timestamp = utc - west_offset(utc); recover the + // westward offset and re-anchor the wall clock to true UTC. + const west = utc - eastern.adjust(utc).timestamp; + utc = wall + west; + } + return utc; +} + +// ── Tests ──────────────────────────────────────────────────────────── + +const testing = std.testing; + +/// Helper: Unix seconds for a UTC wall clock, for constructing test inputs. +fn utcSeconds(y: i16, mo: u8, d: u8, h: i64, mi: i64) i64 { + return Date.fromYmd(y, mo, d).toEpoch() + h * std.time.s_per_hour + mi * std.time.s_per_min; +} + +/// Helper: decompose an expiry back into ET wall-clock components. +const EtParts = struct { date: Date, hour: u8, minute: u8 }; +fn etParts(unix_s: i64) EtParts { + const local = eastern.adjust(unix_s).timestamp; + const d = Date.fromEpoch(local); + const tod = local - d.toEpoch(); + return .{ + .date = d, + .hour = @intCast(@divFloor(tod, std.time.s_per_hour)), + .minute = @intCast(@divFloor(@mod(tod, std.time.s_per_hour), std.time.s_per_min)), + }; +} + +test "classify: mutual funds are 5-letter X-suffixed tickers" { + try testing.expectEqual(InstrumentKind.mutual_fund, classify("FDSCX")); + try testing.expectEqual(InstrumentKind.mutual_fund, classify("VSTCX")); + try testing.expectEqual(InstrumentKind.mutual_fund, classify("FAGIX")); + try testing.expectEqual(InstrumentKind.mutual_fund, classify("VFINX")); + try testing.expectEqual(InstrumentKind.equity, classify("AAPL")); + try testing.expectEqual(InstrumentKind.equity, classify("VTI")); + try testing.expectEqual(InstrumentKind.equity, classify("SPY")); + try testing.expectEqual(InstrumentKind.equity, classify("GOOGL")); + try testing.expectEqual(InstrumentKind.equity, classify("VOOX")); // 4 chars, not a fund + try testing.expectEqual(InstrumentKind.equity, classify("FDSCA")); // 5 chars, not X-suffixed + try testing.expectEqual(InstrumentKind.equity, classify("FDSCXA")); // 6 chars ending in A + try testing.expectEqual(InstrumentKind.equity, classify("X")); // too short + try testing.expectEqual(InstrumentKind.equity, classify("")); // empty +} + +test "etLocalToUtc: EST and EDT offsets" { + // Winter (EST, UTC-5): 16:55 ET == 21:55 UTC. + const est = etLocalToUtc(Date.fromYmd(2025, 1, 15), equity_target_s); + try testing.expectEqual(utcSeconds(2025, 1, 15, 21, 55), est); + + // Summer (EDT, UTC-4): 16:55 ET == 20:55 UTC. + const edt = etLocalToUtc(Date.fromYmd(2025, 7, 15), equity_target_s); + try testing.expectEqual(utcSeconds(2025, 7, 15, 20, 55), edt); +} + +test "etLocalToUtc: across a spring-forward boundary" { + // 2025 DST begins Sun Mar 9. A Monday Mar 10 target is firmly EDT + // even if "now" were the preceding (EST) week. 16:55 EDT == 20:55 UTC. + const mon_after = etLocalToUtc(Date.fromYmd(2025, 3, 10), equity_target_s); + try testing.expectEqual(utcSeconds(2025, 3, 10, 20, 55), mon_after); +} + +test "nextCandleExpiry equity: intraday -> today's close boundary" { + // Wed 2025-06-11, 10:00 ET (14:00 UTC, EDT). Expiry is today 16:55 ET. + const now = utcSeconds(2025, 6, 11, 14, 0); + const exp = nextCandleExpiry(now, .equity); + const p = etParts(exp); + try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 11))); + try testing.expectEqual(@as(u8, 16), p.hour); + try testing.expectEqual(@as(u8, 55), p.minute); +} + +test "nextCandleExpiry equity: after close -> next trading day" { + // Wed 2025-06-11, 18:00 ET (22:00 UTC). Past today's 16:55 -> Thu. + const now = utcSeconds(2025, 6, 11, 22, 0); + const p = etParts(nextCandleExpiry(now, .equity)); + try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 12))); + try testing.expectEqual(@as(u8, 16), p.hour); +} + +test "nextCandleExpiry equity: Friday evening skips the weekend" { + // Fri 2025-06-13, 18:00 ET. Next equity boundary is Mon 2025-06-16. + const now = utcSeconds(2025, 6, 13, 22, 0); + const p = etParts(nextCandleExpiry(now, .equity)); + try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 16))); +} + +test "nextCandleExpiry equity: skips a holiday (Thanksgiving)" { + // Wed 2025-11-26 evening. Thu 11-27 is Thanksgiving -> boundary is + // Fri 2025-11-28. + const now = utcSeconds(2025, 11, 26, 23, 0); + const p = etParts(nextCandleExpiry(now, .equity)); + try testing.expect(p.date.eql(Date.fromYmd(2025, 11, 28))); +} + +test "nextCandleExpiry mutual_fund: Friday evening -> Saturday morning" { + // Fri 2025-06-13, 20:00 ET (past 03:25). Friday's NAV posts Sat AM. + const now = utcSeconds(2025, 6, 14, 0, 0); // 2025-06-13 20:00 EDT + const p = etParts(nextCandleExpiry(now, .mutual_fund)); + try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 14))); // Saturday + try testing.expectEqual(@as(u8, 3), p.hour); + try testing.expectEqual(@as(u8, 25), p.minute); +} + +test "nextCandleExpiry mutual_fund: Saturday morning -> Tuesday morning" { + // Sat 2025-06-14, 05:00 ET. No new NAV Sun/Mon AM; next is Tue (Mon's NAV). + const now = utcSeconds(2025, 6, 14, 9, 0); // 2025-06-14 05:00 EDT + const p = etParts(nextCandleExpiry(now, .mutual_fund)); + try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 17))); // Tuesday +} + +test "nextCandleExpiry mutual_fund: Monday morning before NAV -> Tuesday" { + // Mon 2025-06-16, 01:00 ET (before 03:25). Monday's NAV not out yet, + // and there's no new NAV Monday AM either -> Tue 03:25. + const now = utcSeconds(2025, 6, 16, 5, 0); // 2025-06-16 01:00 EDT + const p = etParts(nextCandleExpiry(now, .mutual_fund)); + try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 17))); +} + +test "isHoliday: 2025 NYSE holidays" { + // Fixed-date with observance. + try testing.expect(isHoliday(Date.fromYmd(2025, 1, 1))); // New Year (Wed) + try testing.expect(isHoliday(Date.fromYmd(2025, 6, 19))); // Juneteenth (Thu) + try testing.expect(isHoliday(Date.fromYmd(2025, 7, 4))); // Independence (Fri) + try testing.expect(isHoliday(Date.fromYmd(2025, 12, 25))); // Christmas (Thu) + // Floating. + try testing.expect(isHoliday(Date.fromYmd(2025, 1, 20))); // MLK (3rd Mon) + try testing.expect(isHoliday(Date.fromYmd(2025, 2, 17))); // Washington (3rd Mon) + try testing.expect(isHoliday(Date.fromYmd(2025, 5, 26))); // Memorial (last Mon) + try testing.expect(isHoliday(Date.fromYmd(2025, 9, 1))); // Labor (1st Mon) + try testing.expect(isHoliday(Date.fromYmd(2025, 11, 27))); // Thanksgiving (4th Thu) + // Non-holidays. + try testing.expect(!isHoliday(Date.fromYmd(2025, 7, 3))); + try testing.expect(!isHoliday(Date.fromYmd(2025, 12, 24))); + try testing.expect(!isHoliday(Date.fromYmd(2025, 6, 11))); +} + +test "isHoliday: weekend observance rules" { + // Independence Day 2026 falls on Saturday -> observed Fri 2026-07-03. + try testing.expect(isHoliday(Date.fromYmd(2026, 7, 3))); + try testing.expect(!isHoliday(Date.fromYmd(2026, 7, 4))); // the Saturday itself + + // New Year's Day 2022 was a Saturday: NOT observed on Fri 2021-12-31. + try testing.expect(!isHoliday(Date.fromYmd(2021, 12, 31))); + + // New Year's Day 2023 was a Sunday -> observed Mon 2023-01-02. + try testing.expect(isHoliday(Date.fromYmd(2023, 1, 2))); + + // Juneteenth not modeled before 2022. + try testing.expect(!isHoliday(Date.fromYmd(2021, 6, 18))); +} + +test "isTradingDay: weekdays, weekends, holidays" { + try testing.expect(isTradingDay(Date.fromYmd(2025, 6, 11))); // Wed + try testing.expect(!isTradingDay(Date.fromYmd(2025, 6, 14))); // Sat + try testing.expect(!isTradingDay(Date.fromYmd(2025, 6, 15))); // Sun + try testing.expect(!isTradingDay(Date.fromYmd(2025, 11, 27))); // Thanksgiving +} + +test "staleCandleExpiry equity: due bar merely late retries soon" { + // Thu 2025-06-12, 17:30 ET: Thursday's bar is due (16:55 ET passed) + // but only ~35m overdue, well inside the lag grace window. + const now = etLocalToUtc(Date.fromYmd(2025, 6, 12), 17 * std.time.s_per_hour + 30 * std.time.s_per_min); + // Cache last has Wed -> Thursday's bar is due but unposted (lag) -> short retry. + try testing.expectEqual(now + short_retry_s, staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 11))); + // Cache already has Thu -> nothing newer is due -> normal next boundary. + try testing.expectEqual(nextCandleExpiry(now, .equity), staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 12))); +} + +test "staleCandleExpiry equity: weekend gap is not missing" { + // Sat 2025-06-14, 10:00 ET: latest available equity data is still + // Friday, which the cache already has - nothing is overdue. + const now = etLocalToUtc(Date.fromYmd(2025, 6, 14), 10 * std.time.s_per_hour); + const exp = staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 13)); + try testing.expectEqual(nextCandleExpiry(now, .equity), exp); + // The long boundary (Mon), not a 30-minute retry. + try testing.expect(exp > now + short_retry_s); + try testing.expect(etParts(exp).date.eql(Date.fromYmd(2025, 6, 16))); // Monday +} + +test "staleCandleExpiry equity: un-modeled Good Friday closure gives up after grace" { + // Good Friday 2025-04-18 is NOT a modeled holiday (it needs the Easter + // computus - see isHoliday), so the calendar treats it as a trading + // day and an incremental fetch keeps coming back empty all day. The + // Friday bar is due at 16:55 ET; the grace window ends 90m later. + try testing.expect(isTradingDay(Date.fromYmd(2025, 4, 18))); + const last_cached = Date.fromYmd(2025, 4, 17); // Thursday + + // Just inside the window (+89m): keep retrying in case the bar shows + // up late. + const inside = etLocalToUtc(Date.fromYmd(2025, 4, 18), 18 * std.time.s_per_hour + 24 * std.time.s_per_min); + try testing.expectEqual(inside + short_retry_s, staleCandleExpiry(inside, .equity, last_cached)); + + // At the 90-minute window (16:55 + 90m = 18:25 ET) we conclude the + // market was closed and wait for the next real session (Mon + // 2025-04-21) instead of thrashing every 30 minutes all weekend. The + // boundary is strict: the +89m case above still retries, so the + // retries land at +30/+60/+90 and the +90 one trips this fallback. + const at_window = etLocalToUtc(Date.fromYmd(2025, 4, 18), 18 * std.time.s_per_hour + 25 * std.time.s_per_min); + const exp = staleCandleExpiry(at_window, .equity, last_cached); + try testing.expectEqual(nextCandleExpiry(at_window, .equity), exp); + try testing.expect(exp > at_window + short_retry_s); // not a short retry + const p = etParts(exp); + try testing.expect(p.date.eql(Date.fromYmd(2025, 4, 21))); // Monday + try testing.expectEqual(@as(u8, 16), p.hour); + try testing.expectEqual(@as(u8, 55), p.minute); +} + +test "staleCandleExpiry mutual_fund: late NAV within grace retries soon" { + // Tue 2025-06-17, 04:30 ET: Monday's NAV (data_date 2025-06-16) is due + // (posts ~03:25 ET) but the provider is lagging by ~65m, still inside + // the grace window. Also exercises the morning -> prior-session mapping + // in latestAvailability. + const now = etLocalToUtc(Date.fromYmd(2025, 6, 17), 4 * std.time.s_per_hour + 30 * std.time.s_per_min); + // Cache has through Friday's NAV -> Monday's is missing -> short retry. + try testing.expectEqual(now + short_retry_s, staleCandleExpiry(now, .mutual_fund, Date.fromYmd(2025, 6, 13))); + // Cache already has Monday -> nothing newer is due -> normal boundary. + try testing.expectEqual(nextCandleExpiry(now, .mutual_fund), staleCandleExpiry(now, .mutual_fund, Date.fromYmd(2025, 6, 16))); +} diff --git a/src/service.zig b/src/service.zig index f06a3a0..d8ea4c8 100644 --- a/src/service.zig +++ b/src/service.zig @@ -40,6 +40,7 @@ const fmt = @import("format.zig"); const performance = @import("analytics/performance.zig"); const http = @import("net/http.zig"); const atomic = @import("atomic.zig"); +const market = @import("market.zig"); // ── Wall-clock policy ──────────────────────────────────────── // @@ -595,7 +596,11 @@ pub const DataService = struct { // and candles_meta.srf in one shot (last_close, last_date, // provider, fail_count=0). if (triple.candles.len > 0) { - s.cacheCandles(symbol, triple.candles, .tiingo, 0); + // wall-clock required: stamp the candle-meta freshness boundary + // (market-aware next post-close / NAV-availability time). + const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds(); + const expires = market.nextCandleExpiry(now_s, market.classify(symbol)); + s.cacheCandles(symbol, triple.candles, .tiingo, 0, expires); } // Dividends and splits use the supplement write path: Tiingo's // view merges into existing (typically Polygon-sourced) records @@ -745,6 +750,14 @@ pub const DataService = struct { var s = self.store(); const today = fmt.todayDate(self.io); + // wall-clock required: candle-meta freshness uses a market-aware + // boundary (next post-close / NAV-availability time in ET; see + // market.nextCandleExpiry). Captured once here and threaded to + // every candle-meta write below so a single invocation stamps a + // consistent boundary. + const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds(); + const kind = market.classify(symbol); + // Check candle metadata for freshness (tiny file, no candle deserialization) const meta_result = s.readCandleMeta(symbol); if (meta_result) |mr| { @@ -792,9 +805,13 @@ pub const DataService = struct { // Stale - try incremental update using last_date from meta const fetch_from = m.last_date.addDays(1); + // Market-aware freshness boundary for any meta write on + // this stale path (next post-close / NAV-availability time). + const expires = market.nextCandleExpiry(now_s, kind); + // If last cached date is today or later, just refresh the TTL (meta only) if (!fetch_from.lessThan(today)) { - s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count); + s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count, expires); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } else { @@ -805,7 +822,7 @@ pub const DataService = struct { // Increment fail_count for this symbol const new_fail_count = m.fail_count +| 1; // saturating add log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count }); - s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count); + s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count, now_s + market.short_retry_s); // If degraded (fail_count >= 3), return stale data rather than failing if (new_fail_count >= 3) { @@ -823,14 +840,23 @@ pub const DataService = struct { const new_candles = result.candles; if (new_candles.len == 0) { - // No new candles (weekend/holiday) - refresh TTL, reset fail_count + // No new candles. Either a genuine non-trading-day + // gap (weekend/holiday), the provider hasn't posted + // the just-closed bar yet, or the calendar's + // "trading day" was an un-modeled closure (e.g. + // Good Friday). market.staleCandleExpiry picks the + // boundary: a short retry while a due bar is merely + // late, falling back to the next close boundary + // once it's overdue past the lag grace window - so + // an un-modeled closure stops thrashing and waits + // for the next real session. self.allocator.free(new_candles); - s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0); + s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0, market.staleCandleExpiry(now_s, kind, m.last_date)); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } else { // Append new candles to existing file + update meta, reset fail_count - s.appendCandles(symbol, new_candles, result.provider, 0); + s.appendCandles(symbol, new_candles, result.provider, 0, expires); if (s.read(self.allocator, Candle, symbol, null, .any)) |r| { self.allocator.free(new_candles); return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; @@ -874,7 +900,7 @@ pub const DataService = struct { // we know to back off if this keeps happening. if (meta_result) |mr| { const new_fail_count = mr.meta.fail_count +| 1; - s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count); + s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count, now_s + market.short_retry_s); } return DataError.TransientError; } @@ -918,7 +944,7 @@ pub const DataService = struct { /// `opts.force_refresh = true` -> treats cache as stale and fetches. pub fn getEarnings(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!FetchResult(EarningsEvent) { // Mutual funds (5-letter tickers ending in X) don't have quarterly earnings. - if (isMutualFund(symbol)) { + if (market.classify(symbol) == .mutual_fund) { return .{ .data = &.{}, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } @@ -2928,13 +2954,6 @@ pub const DataService = struct { return daily and meta; } - /// Mutual funds use 5-letter tickers ending in X (e.g. FDSCX, VSTCX, FAGIX). - /// These don't have quarterly earnings - skip the fetch rather than - /// round-tripping to the provider just to get an empty response. - fn isMutualFund(symbol: []const u8) bool { - return symbol.len == 5 and symbol[4] == 'X'; - } - // ── User config files ───────────────────────────────────────── /// Load and parse accounts.srf from the same directory as the given portfolio path. @@ -3009,24 +3028,6 @@ test "isPermanentProviderFailure: RateLimited is transient" { try std.testing.expect(!isPermanentProviderFailure(error.RateLimited)); } -test "isMutualFund identifies mutual funds" { - // Standard mutual fund tickers (5 letters ending in X) - try std.testing.expect(DataService.isMutualFund("FDSCX")); - try std.testing.expect(DataService.isMutualFund("VSTCX")); - try std.testing.expect(DataService.isMutualFund("FAGIX")); - try std.testing.expect(DataService.isMutualFund("VFINX")); - - // Not mutual funds - try std.testing.expect(!DataService.isMutualFund("AAPL")); - try std.testing.expect(!DataService.isMutualFund("VTI")); - try std.testing.expect(!DataService.isMutualFund("SPY")); - try std.testing.expect(!DataService.isMutualFund("GOOGL")); - try std.testing.expect(!DataService.isMutualFund("")); // empty - try std.testing.expect(!DataService.isMutualFund("X")); // too short - try std.testing.expect(!DataService.isMutualFund("FDSCA")); // 5 letters but not ending in X - try std.testing.expect(!DataService.isMutualFund("FDSCXA")); // 6 letters ending in A -} - test "DataService init/deinit lifecycle" { const allocator = std.testing.allocator; const config = Config{ @@ -3176,7 +3177,7 @@ test "getCandles offline mode returns cached data without network" { .{ .date = Date.fromYmd(2026, 5, 19), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 }, .{ .date = Date.fromYmd(2026, 5, 20), .open = 104, .high = 106, .low = 103, .close = 105, .adj_close = 105, .volume = 1100 }, }; - store.cacheCandles("TEST", candles[0..], .tiingo, 0); + store.cacheCandles("TEST", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); // Set the test guard: any network call would panic. We expect // the offline-mode path NOT to touch the network. @@ -3279,7 +3280,7 @@ test "loadAllPrices offline mode skips network and returns cached" { var fresh_candles = [_]Candle{ .{ .date = Date.fromYmd(2026, 5, 20), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 }, }; - store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0); + store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); // Symbol with no cache at all. // (no setup needed - just passes a symbol that doesn't exist) @@ -3331,7 +3332,7 @@ test "loadAllPrices force_refresh tops up without wiping the candle cache" { var candles = [_]Candle{ .{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 }, }; - store.cacheCandles("HELD", candles[0..], .tiingo, 0); + store.cacheCandles("HELD", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity)); // Any provider/network attempt now panics. If force_refresh wiped // the cache (old behavior), getCandles would fall through to a full