lobo/zfin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

market-aware ttl

Browse source
This commit is contained in:
Emil Lerch 2026-06-25 14:42:58 -07:00
parent 47462aaab5
commit c3c990fa68
Signed by: lobo
GPG key ID: A7B62D657EF764F8
6 changed files with 662 additions and 60 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
build.zig
View file

@ -21,6 +21,11 @@ pub fn build(b: *std.Build) void {
.optimize = optimize,
});
const zeit_dep = b.dependency("zeit", .{
.target = target,
.optimize = optimize,
});
const srf_mod = srf_dep.module("srf");
const shiller_mod = b.addModule("shiller_year", .{
@ -43,6 +48,7 @@ pub fn build(b: *std.Build) void {
.target = target,
.imports = &.{
.{ .name = "srf", .module = srf_mod },
.{ .name = "zeit", .module = zeit_dep.module("zeit") },
.{ .name = "build_info", .module = build_info },
},
});
@ -54,6 +60,7 @@ pub fn build(b: *std.Build) void {
.{ .name = "srf", .module = srf_mod },
.{ .name = "vaxis", .module = vaxis_dep.module("vaxis") },
.{ .name = "z2d", .module = z2d_dep.module("z2d") },
.{ .name = "zeit", .module = zeit_dep.module("zeit") },
.{ .name = "build_info", .module = build_info },
.{ .name = "shiller_year", .module = shiller_mod },
};
@ -121,6 +128,7 @@ pub fn build(b: *std.Build) void {
.optimize = optimize,
.imports = &.{
.{ .name = "srf", .module = srf_mod },
.{ .name = "zeit", .module = zeit_dep.module("zeit") },
.{ .name = "build_info", .module = build_info },
},
}),

4
build.zig.zon
View file

@ -16,6 +16,10 @@
.url = "git+https://git.lerch.org/lobo/srf#4a3e5f00f15b0e0ba79d06ffe69dbcfa052baa5b",
.hash = "srf-0.0.0-qZj572nkAQAAz3zEg6fdD8A7PJnQ9je3zCeAOJS5PoZj",
},
.zeit = .{
.url = "git+https://github.com/rockorager/zeit?ref=v0.9.0#b1c1c2fcbc71fd7799a316bbcf0ff88d06d80ccc",
.hash = "zeit-0.9.0-5I6bk2m9AgBSMH8-L6rYJkwuQAyhXplnfxnvTSGzVHUR",
},
},
.paths = .{
"build",

63
docs/explanation/caching.md
View file

@ -37,21 +37,66 @@ The `--refresh-data` policy decides which tiers run:
Different data ages at different rates, so each type has its own TTL:
| Data type | TTL | Why |
|---------------|---------------|-------------------------------------------------------------|
| Daily candles | ~24h (23h45m) | One bar per trading day; slightly under 24h for cron jitter |
| Dividends | 14 days | Declared well in advance |
| Splits | 14 days | Rare corporate events |
| Options | 1 hour | Prices move continuously when markets are open |
| Earnings | 30 days\* | Quarterly; smart-refreshed around announcements |
| ETF profiles | ~30 days | Holdings and weights change slowly |
| Quotes | never cached | Meant to be a live price check |
| Data type | TTL | Why |
|---------------|---------------|----------------------------------------------------------------------------------|
| Daily candles | market-aware | Keyed to the next time a fresh bar is expected, not a rolling window (see below) |
| Dividends | 14 days | Declared well in advance |
| Splits | 14 days | Rare corporate events |
| Options | 1 hour | Prices move continuously when markets are open |
| Earnings | 30 days\* | Quarterly; smart-refreshed around announcements |
| ETF profiles | ~30 days | Holdings and weights change slowly |
| Quotes | never cached | Meant to be a live price check |
\* **Earnings smart refresh:** even inside the 30-day window, cached
earnings re-fetch automatically once an earnings date has passed but
the cache still lacks the actual result -- so numbers appear promptly
after an announcement without daily polling.
## Market-aware candle freshness
A daily bar only becomes meaningful once the market settles, so candle
freshness is keyed to the market clock rather than a rolling 24-hour
window. Each cached candle's expiry is set to the next moment fresh data
should be available:
- **Equities and ETFs** settle shortly after the 16:00 ET close. Their
bars expire at **16:55 ET** on the next trading day (weekends and NYSE
holidays are skipped). This time allows for providers to become consistent
with the market while also allowing a few minutes prior to a scheduled
refresh task at the top of the hour.
- **Mutual funds** strike a single daily NAV that isn't reliably
published until the next morning, so their bars expire at **03:25 ET**
the morning after a trading session. Despite Tiingo's claims, NAVs only
seem reliably available until about 3am Eastern. This is again timed such
that scheduled jobs for the bottom of the hour can run reliably.
This keeps the expiry boundary out of trading hours, so a refresh fired
just after it always sees a finalized bar instead of a half-formed one,
and an interactive command run mid-session won't trigger a needless
refetch. If a refresh runs but the provider hasn't posted the just-closed
bar yet, the entry is retried in ~30 minutes rather than waiting a full
day.
**Un-modeled closures self-correct.** Some market closures aren't on the
modeled holiday calendar (Good Friday, which needs the Easter computus,
plus ad-hoc closures for national mourning or weather). On such a day the
calendar thinks a bar is due, the fetch keeps coming back empty, and the
~30-minute retry would otherwise repeat all day - and, for a Friday
closure, all weekend. To avoid that thrash, once an expected bar is ~90
minutes overdue the cache concludes the market was closed and falls back
to the normal next-session boundary - at most three 30-minute retries.
That window comfortably covers ordinary provider posting lag, so
genuinely-late data is still picked up by the short retry; only a true
closure trips the fallback.
**Warming a shared cache on a schedule.** If you run a cron to warm a
[server cache](#server-sync-zfin_server) (or your own local cache), the
boundaries above are also the natural cron times: a run shortly after
**17:00 ET** picks up the day's equity/ETF closes, and a run shortly
after **03:30 ET** picks up the prior session's mutual-fund NAVs. The
boundaries sit a couple of minutes before those times so the cron
reliably sees the cache already expired.
## Quotes are never cached
Because quotes exist to give you a live price, they're never served

36
src/cache/store.zig vendored
View file

@ -32,8 +32,6 @@ pub const Ttl = struct {
const s_per_day = std.time.s_per_day;
/// Historical candles older than 1 day never expire
pub const candles_historical: i64 = -1; // infinite
/// Latest day's candle refreshes every 23h45m (15-min buffer for cron jitter)
pub const candles_latest: i64 = s_per_day - 15 * std.time.s_per_min;
/// Dividend data refreshes biweekly
pub const dividends: i64 = 14 * s_per_day;
/// Split data refreshes biweekly
@ -816,9 +814,12 @@ pub const Store = struct {
// ── Candle-specific API ──────────────────────────────────────
/// Write a full set of candles to cache (no expiry - historical facts don't expire).
/// Also updates candle metadata.
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8) void {
/// Write a full set of candles to cache (the candle data file itself
/// carries no expiry - historical bars never change). Also refreshes
/// candle metadata, stamping its `#!expires=` with `expires_at_s`
/// (the caller computes the market-aware freshness boundary; see
/// `market.nextCandleExpiry`).
pub fn cacheCandles(self: *Store, symbol: []const u8, candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
if (serializeCandles(self.allocator, candles, .{})) |srf_data| {
defer self.allocator.free(srf_data);
self.writeRaw(symbol, .candles_daily, srf_data) catch |err| {
@ -830,14 +831,15 @@ pub const Store = struct {
if (candles.len > 0) {
const last = candles[candles.len - 1];
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count);
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
}
}
/// Append new candle records to the existing cache file.
/// Falls back to a full rewrite if append fails (e.g. file doesn't exist).
/// Also updates candle metadata.
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8) void {
/// Also updates candle metadata, stamping its `#!expires=` with
/// `expires_at_s` (caller-computed market-aware boundary).
pub fn appendCandles(self: *Store, symbol: []const u8, new_candles: []const Candle, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
if (new_candles.len == 0) return;
if (serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| {
@ -866,19 +868,24 @@ pub const Store = struct {
}
const last = new_candles[new_candles.len - 1];
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count);
self.updateCandleMeta(symbol, last.close, last.date, provider, fail_count, expires_at_s);
}
/// Write (or refresh) candle metadata with a specific provider source.
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8) void {
const expires = std.Io.Timestamp.now(self.io, .real).toSeconds() + Ttl.candles_latest;
///
/// `expires_at_s` is the absolute Unix-seconds freshness boundary for
/// the `#!expires=` directive, computed by the caller via
/// `market.nextCandleExpiry` (close-anchored) or a short retry. The
/// cache layer no longer reads the wall clock for this - the boundary
/// is market-domain knowledge owned by the caller.
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider, fail_count: u8, expires_at_s: i64) void {
const meta = CandleMeta{
.last_close = last_close,
.last_date = last_date,
.provider = provider,
.fail_count = fail_count,
};
if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires })) |meta_data| {
if (serializeCandleMeta(self.io, self.allocator, meta, .{ .expires = expires_at_s })) |meta_data| {
defer self.allocator.free(meta_data);
self.writeRaw(symbol, .candles_meta, meta_data) catch |err| {
log.warn("{s}: failed to write candle metadata: {s}", .{ symbol, @errorName(err) });
@ -2961,9 +2968,8 @@ test "TTL constants are reasonable" {
// Historical candles never expire
try std.testing.expectEqual(@as(i64, -1), Ttl.candles_historical);
// Latest candles expire just under 24 hours (allowing for cron jitter)
try std.testing.expect(Ttl.candles_latest > 23 * std.time.s_per_hour);
try std.testing.expect(Ttl.candles_latest < 24 * std.time.s_per_hour);
// Latest candles use a market-aware expiry computed per-write by
// market.nextCandleExpiry (no fixed TTL constant here anymore).
// Dividends and splits refresh biweekly
try std.testing.expectEqual(@as(i64, 14 * std.time.s_per_day), Ttl.dividends);

538
src/market.zig Normal file
View file

@ -0,0 +1,538 @@
//! Market calendar: trading-day awareness and close-anchored cache
//! freshness boundaries for daily candles.
//!
//! Daily candle data only becomes meaningful once the market settles.
//! Two distinct deadlines matter:
//!
//! - **Equities / ETFs** settle shortly after the 16:00 ET close.
//! - **Mutual-fund NAVs** strike once per day and are not reliably
//! published until the next morning (~03:30 ET).
//!
//! The old candle TTL was a rolling 23h45m window, so the expiry
//! boundary drifted against the market clock and could fall during
//! trading hours - causing mid-session refetches and risking caching a
//! not-yet-finalized bar. This module computes the next moment fresh
//! data should be available, so the candle cache is keyed to the market
//! clock instead. It is an optimization, not a correctness fix: a
//! slightly-wrong boundary costs at most one cheap no-op fetch (see the
//! direction-of-error note on `isHoliday`).
//!
//! Timezone handling uses `zeit` with a hardcoded US Eastern POSIX TZ
//! spec, so there is no dependency on system zoneinfo files, no
//! allocator, and no I/O. The fixed DST rule is correct for current and
//! near-future dates, which is all cache-freshness math ever deals with.
const std = @import("std");
const zeit = @import("zeit");
const Date = @import("Date.zig");
/// POSIX TZ spec for US Eastern: EST (UTC-5) / EDT (UTC-4) with the
/// current US DST rule (2nd Sunday March -> 1st Sunday November).
const eastern_posix_spec = "EST5EDT,M3.2.0,M11.1.0";
/// US Eastern timezone, parsed at comptime from `eastern_posix_spec`.
/// `Posix.parse` allocates nothing - the parsed struct borrows slices
/// into the (static) spec string - so this is a zero-cost const.
const eastern: zeit.TimeZone = .{
.posix = zeit.timezone.Posix.parse(eastern_posix_spec) catch
@compileError("invalid eastern POSIX TZ spec: " ++ eastern_posix_spec),
};
/// How candle data for a symbol becomes available.
pub const InstrumentKind = enum {
/// Continuously-quoted equities and ETFs: the day's bar settles
/// shortly after the 16:00 ET close.
equity,
/// Mutual funds: a single daily NAV that isn't reliably published
/// until the next morning.
mutual_fund,
};
/// Classify a symbol for candle-freshness purposes. Mutual funds use
/// 5-letter tickers ending in X (e.g. FDSCX, VSTCX, FAGIX); everything
/// else is treated as an equity/ETF. Imperfect, but covers the common
/// case.
///
/// This is the single home for the heuristic: it was consolidated here
/// from the former `DataService.isMutualFund` (removed) when candle
/// freshness timing moved into this module. It is deliberately distinct
/// from `portfolio.isMoneyMarketSymbol`, which answers a different
/// question - "is this a fixed-$1-NAV cash equivalent?" - via a curated
/// whitelist. (Every symbol on that whitelist also matches this
/// heuristic, so freshness callers only need `classify`.)
pub fn classify(symbol: []const u8) InstrumentKind {
if (symbol.len == 5 and symbol[4] == 'X') return .mutual_fund;
return .equity;
}
// ── Freshness target times (seconds since ET-local midnight) ─────────
//
// These double as the recommended refresh-cron schedule: each sits a
// couple minutes BEFORE its intended cron run so cron-timing jitter
// reliably sees the cache already expired (a boundary exactly at the
// cron time could be missed by an early tick). The "expected-but-
// missing" short retry (see `short_retry_s`) covers the case where the
// provider hasn't posted the just-closed bar by then.
/// Equity/ETF boundary: 16:55 ET (just after the 16:00 close + settle
/// margin; pairs with a ~17:00 ET refresh cron).
const equity_target_s: i64 = 16 * std.time.s_per_hour + 55 * std.time.s_per_min;
/// Mutual-fund boundary: 03:25 ET the morning after the trading day
/// (pairs with a ~03:30 ET NAV-refresh cron).
const mf_target_s: i64 = 3 * std.time.s_per_hour + 25 * std.time.s_per_min;
/// Retry interval when an expected bar wasn't returned yet (the provider
/// hasn't posted the just-closed bar, or a transient fetch failure).
/// Short enough to pick the bar up within the same session, long enough
/// to avoid hammering a rate-limited provider.
pub const short_retry_s: i64 = 30 * std.time.s_per_min;
/// How long past the moment data is *due* we keep doing `short_retry_s`
/// retries before concluding the calendar's "trading day" was actually
/// an un-modeled market closure (e.g. Good Friday, which needs the Easter
/// computus and is deliberately not modeled - see `isHoliday`) or an
/// ad-hoc halt, rather than mere provider lag. Once an expected bar has
/// been overdue this long it is almost certainly never coming, so
/// `staleCandleExpiry` stops the retry loop and falls back to the normal
/// next-boundary expiry. That bounds an un-modeled closure to a few
/// no-op fetches that session instead of thrashing every `short_retry_s`
/// until the next real session - for a Friday closure, the entire long
/// weekend.
///
/// 90 minutes comfortably covers normal posting lag for both the equity
/// close and the overnight mutual-fund NAV (data that is going to appear
/// at all is essentially always up well inside that window) while still
/// giving up promptly. At the 30-minute `short_retry_s` cadence that is
/// three retries (at +30, +60, +90 min); the +90 retry is the one that
/// crosses the window and trips the fallback.
const provider_lag_grace_s: i64 = 90 * std.time.s_per_min;
fn targetSeconds(kind: InstrumentKind) i64 {
return switch (kind) {
.equity => equity_target_s,
.mutual_fund => mf_target_s,
};
}
// ── Trading-day calendar ─────────────────────────────────────────────
// Weekday constants in zfin's 0=Mon .. 6=Sun scheme (see Date.dayOfWeek).
const mon: u8 = 0;
const thu: u8 = 3;
const sat: u8 = 5;
const sun: u8 = 6;
/// True if `d` is a US equity-market trading day: a weekday that isn't a
/// modeled NYSE holiday.
pub fn isTradingDay(d: Date) bool {
if (d.dayOfWeek() >= sat) return false; // Saturday or Sunday
return !isHoliday(d);
}
/// True if `d` is a (modeled) NYSE market holiday.
///
/// Direction-of-error policy: a *false holiday* (marking a real trading
/// day closed) would let the cache stay fresh too long and miss a day's
/// data - a staleness bug. A *missed holiday* (treating a closed day as
/// open) only costs one harmless no-op fetch. So we model only the
/// well-defined, confidently-computable holidays and lean toward "open"
/// when unsure. Good Friday is deliberately omitted (it needs the Easter
/// computus); an un-modeled Good Friday simply costs one no-op fetch.
/// Ad-hoc closures (national mourning, weather) are likewise unmodeled.
pub fn isHoliday(d: Date) bool {
const y = d.year();
// Floating Monday/Thursday holidays - these always land on a weekday,
// so no weekend-observance adjustment is needed.
if (d.eql(nthWeekday(y, 1, mon, 3))) return true; // MLK Day: 3rd Mon Jan
if (d.eql(nthWeekday(y, 2, mon, 3))) return true; // Washington's Birthday: 3rd Mon Feb
if (d.eql(lastWeekday(y, 5, mon))) return true; // Memorial Day: last Mon May
if (d.eql(nthWeekday(y, 9, mon, 1))) return true; // Labor Day: 1st Mon Sep
if (d.eql(nthWeekday(y, 11, thu, 4))) return true; // Thanksgiving: 4th Thu Nov
// New Year's Day: Sunday -> observed Monday. NYSE does NOT close the
// preceding Friday when Jan 1 falls on a Saturday (that Friday is in
// the prior year and stays a normal trading day).
if (observedSundayOnly(d, 1, 1)) return true;
// Juneteenth: NYSE first observed it in 2022. Sat -> Fri, Sun -> Mon.
if (y >= 2022 and observedFixed(d, 6, 19)) return true;
// Independence Day and Christmas: Sat -> Fri, Sun -> Mon.
if (observedFixed(d, 7, 4)) return true;
if (observedFixed(d, 12, 25)) return true;
return false;
}
/// The `n`-th occurrence (1-based) of weekday `wd` (0=Mon..6=Sun) in
/// month `mo` of `y`.
fn nthWeekday(y: i16, mo: u8, wd: u8, n: u8) Date {
const first = Date.fromYmd(y, mo, 1);
const offset = @mod(@as(i32, wd) - @as(i32, first.dayOfWeek()), 7);
const day: u8 = @intCast(1 + offset + (@as(i32, n - 1) * 7));
return Date.fromYmd(y, mo, day);
}
/// The last occurrence of weekday `wd` (0=Mon..6=Sun) in month `mo` of `y`.
fn lastWeekday(y: i16, mo: u8, wd: u8) Date {
const last = Date.lastDayOfMonth(y, mo);
const back = @mod(@as(i32, last.dayOfWeek()) - @as(i32, wd), 7);
return last.addDays(-back);
}
/// True if `d` is the observed date of the fixed (mo/day) holiday in
/// `d`'s year, using the full weekend-observance rule: Saturday holidays
/// observed the preceding Friday, Sunday holidays the following Monday.
fn observedFixed(d: Date, mo: u8, day: u8) bool {
const actual = Date.fromYmd(d.year(), mo, day);
return switch (actual.dayOfWeek()) {
sat => d.eql(actual.addDays(-1)), // Sat -> preceding Fri
sun => d.eql(actual.addDays(1)), // Sun -> following Mon
else => d.eql(actual),
};
}
/// Like `observedFixed` but Saturday is NOT observed (used for New
/// Year's Day - see `isHoliday`).
fn observedSundayOnly(d: Date, mo: u8, day: u8) bool {
const actual = Date.fromYmd(d.year(), mo, day);
return switch (actual.dayOfWeek()) {
sat => false,
sun => d.eql(actual.addDays(1)),
else => d.eql(actual),
};
}
// ── Close-anchored freshness boundaries ──────────────────────────────
/// True if fresh candle data for `kind` becomes available on calendar
/// day `d` (in ET). For equities, that's any trading day (the bar
/// settles that afternoon). For mutual funds, it's the morning AFTER a
/// trading day (the prior session's NAV posts overnight) - which also
/// makes holidays fall out for free: if the prior day wasn't a trading
/// day, no NAV is available this morning.
fn isAvailabilityDay(d: Date, kind: InstrumentKind) bool {
return switch (kind) {
.equity => isTradingDay(d),
.mutual_fund => isTradingDay(d.addDays(-1)),
};
}
/// The trading day whose data an availability day carries.
fn dataDateFor(availability_day: Date, kind: InstrumentKind) Date {
return switch (kind) {
.equity => availability_day,
.mutual_fund => availability_day.addDays(-1),
};
}
/// Absolute Unix-seconds expiry for a freshly-written candle cache
/// entry: the next moment after `now_s` at which new data for `kind`
/// should be available. Pure given `now_s` (the wall clock is read by
/// the caller and threaded in), so it is fully deterministic for tests.
pub fn nextCandleExpiry(now_s: i64, kind: InstrumentKind) i64 {
const target = targetSeconds(kind);
const now_local = eastern.adjust(now_s).timestamp;
const today_et = Date.fromEpoch(now_local);
const now_tod = now_local - today_et.toEpoch(); // seconds since ET midnight
// Start at today's target; if it has already passed, move to tomorrow.
var cand = today_et;
if (now_tod >= target) cand = cand.addDays(1);
// Skip forward to the next day that actually carries new data.
while (!isAvailabilityDay(cand, kind)) cand = cand.addDays(1);
return etLocalToUtc(cand, target);
}
/// The most recent data availability as of `now_s`: which trading day's
/// candle should already be published, and the exact instant it went
/// live (its target time on the availability day, in Unix seconds).
const Availability = struct {
/// Trading day whose data the latest availability carries.
data_date: Date,
/// UTC seconds at which that data became available.
instant_s: i64,
};
/// Walk backward from `now_s` to the most recent availability day whose
/// target time has already passed, reporting both the trading day it
/// carries and the instant its data went live. Pure given `now_s`.
fn latestAvailability(now_s: i64, kind: InstrumentKind) Availability {
const target = targetSeconds(kind);
const now_local = eastern.adjust(now_s).timestamp;
const today_et = Date.fromEpoch(now_local);
const now_tod = now_local - today_et.toEpoch();
var cand = today_et;
if (now_tod < target) cand = cand.addDays(-1);
while (!isAvailabilityDay(cand, kind)) cand = cand.addDays(-1);
return .{
.data_date = dataDateFor(cand, kind),
.instant_s = etLocalToUtc(cand, target),
};
}
/// Expiry to stamp on candle meta after an *incremental* fetch on a stale
/// entry returned zero new bars. Three cases:
///
/// 1. **Nothing newer is due** - a genuine non-trading-day gap
/// (weekend/holiday) or the cache is already caught up: use the
/// normal next-boundary expiry.
/// 2. **Newer data is due, still within the provider-lag grace
/// window** (`provider_lag_grace_s`): the provider just hasn't
/// posted the just-closed bar yet, so retry soon (`short_retry_s`).
/// 3. **Newer data has been due for the full grace window**
/// (`provider_lag_grace_s`): the calendar's "trading day" was almost
/// certainly an un-modeled closure (e.g. Good Friday) or ad-hoc halt
/// - the bar is never going to appear. Stop the short-retry thrash
/// and fall back to the normal next-boundary expiry, so we wait for
/// the next real session instead of refetching every `short_retry_s`
/// all evening (and, for a Friday closure, all weekend).
///
/// `last_cached` is the newest date already in the candle cache. Pure
/// given `now_s`, so it is fully deterministic for tests.
pub fn staleCandleExpiry(now_s: i64, kind: InstrumentKind, last_cached: Date) i64 {
const avail = latestAvailability(now_s, kind);
// Case 1: nothing newer than the cache is due yet.
if (!last_cached.lessThan(avail.data_date)) return nextCandleExpiry(now_s, kind);
// Case 2: due data is merely late (still inside the grace window) -> short retry.
if (now_s - avail.instant_s < provider_lag_grace_s) return now_s + short_retry_s;
// Case 3: overdue for the whole grace window -> assume closure, wait for next boundary.
return nextCandleExpiry(now_s, kind);
}
/// Convert an ET local wall-clock instant (`d` at `tod_s` seconds since
/// ET midnight) to Unix seconds, resolving the correct EST/EDT offset
/// for that date. Treats the wall clock as a UTC instant for a first
/// guess, then corrects by the offset zeit reports at that moment; two
/// iterations converge because the offset changes by at most one hour
/// and the target times never sit inside the ~02:00 ET DST-transition
/// window.
fn etLocalToUtc(d: Date, tod_s: i64) i64 {
const wall = d.toEpoch() + tod_s;
var utc = wall;
var i: usize = 0;
while (i < 2) : (i += 1) {
// adjust(utc).timestamp = utc - west_offset(utc); recover the
// westward offset and re-anchor the wall clock to true UTC.
const west = utc - eastern.adjust(utc).timestamp;
utc = wall + west;
}
return utc;
}
// ── Tests ────────────────────────────────────────────────────────────
const testing = std.testing;
/// Helper: Unix seconds for a UTC wall clock, for constructing test inputs.
fn utcSeconds(y: i16, mo: u8, d: u8, h: i64, mi: i64) i64 {
return Date.fromYmd(y, mo, d).toEpoch() + h * std.time.s_per_hour + mi * std.time.s_per_min;
}
/// Helper: decompose an expiry back into ET wall-clock components.
const EtParts = struct { date: Date, hour: u8, minute: u8 };
fn etParts(unix_s: i64) EtParts {
const local = eastern.adjust(unix_s).timestamp;
const d = Date.fromEpoch(local);
const tod = local - d.toEpoch();
return .{
.date = d,
.hour = @intCast(@divFloor(tod, std.time.s_per_hour)),
.minute = @intCast(@divFloor(@mod(tod, std.time.s_per_hour), std.time.s_per_min)),
};
}
test "classify: mutual funds are 5-letter X-suffixed tickers" {
try testing.expectEqual(InstrumentKind.mutual_fund, classify("FDSCX"));
try testing.expectEqual(InstrumentKind.mutual_fund, classify("VSTCX"));
try testing.expectEqual(InstrumentKind.mutual_fund, classify("FAGIX"));
try testing.expectEqual(InstrumentKind.mutual_fund, classify("VFINX"));
try testing.expectEqual(InstrumentKind.equity, classify("AAPL"));
try testing.expectEqual(InstrumentKind.equity, classify("VTI"));
try testing.expectEqual(InstrumentKind.equity, classify("SPY"));
try testing.expectEqual(InstrumentKind.equity, classify("GOOGL"));
try testing.expectEqual(InstrumentKind.equity, classify("VOOX")); // 4 chars, not a fund
try testing.expectEqual(InstrumentKind.equity, classify("FDSCA")); // 5 chars, not X-suffixed
try testing.expectEqual(InstrumentKind.equity, classify("FDSCXA")); // 6 chars ending in A
try testing.expectEqual(InstrumentKind.equity, classify("X")); // too short
try testing.expectEqual(InstrumentKind.equity, classify("")); // empty
}
test "etLocalToUtc: EST and EDT offsets" {
// Winter (EST, UTC-5): 16:55 ET == 21:55 UTC.
const est = etLocalToUtc(Date.fromYmd(2025, 1, 15), equity_target_s);
try testing.expectEqual(utcSeconds(2025, 1, 15, 21, 55), est);
// Summer (EDT, UTC-4): 16:55 ET == 20:55 UTC.
const edt = etLocalToUtc(Date.fromYmd(2025, 7, 15), equity_target_s);
try testing.expectEqual(utcSeconds(2025, 7, 15, 20, 55), edt);
}
test "etLocalToUtc: across a spring-forward boundary" {
// 2025 DST begins Sun Mar 9. A Monday Mar 10 target is firmly EDT
// even if "now" were the preceding (EST) week. 16:55 EDT == 20:55 UTC.
const mon_after = etLocalToUtc(Date.fromYmd(2025, 3, 10), equity_target_s);
try testing.expectEqual(utcSeconds(2025, 3, 10, 20, 55), mon_after);
}
test "nextCandleExpiry equity: intraday -> today's close boundary" {
// Wed 2025-06-11, 10:00 ET (14:00 UTC, EDT). Expiry is today 16:55 ET.
const now = utcSeconds(2025, 6, 11, 14, 0);
const exp = nextCandleExpiry(now, .equity);
const p = etParts(exp);
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 11)));
try testing.expectEqual(@as(u8, 16), p.hour);
try testing.expectEqual(@as(u8, 55), p.minute);
}
test "nextCandleExpiry equity: after close -> next trading day" {
// Wed 2025-06-11, 18:00 ET (22:00 UTC). Past today's 16:55 -> Thu.
const now = utcSeconds(2025, 6, 11, 22, 0);
const p = etParts(nextCandleExpiry(now, .equity));
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 12)));
try testing.expectEqual(@as(u8, 16), p.hour);
}
test "nextCandleExpiry equity: Friday evening skips the weekend" {
// Fri 2025-06-13, 18:00 ET. Next equity boundary is Mon 2025-06-16.
const now = utcSeconds(2025, 6, 13, 22, 0);
const p = etParts(nextCandleExpiry(now, .equity));
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 16)));
}
test "nextCandleExpiry equity: skips a holiday (Thanksgiving)" {
// Wed 2025-11-26 evening. Thu 11-27 is Thanksgiving -> boundary is
// Fri 2025-11-28.
const now = utcSeconds(2025, 11, 26, 23, 0);
const p = etParts(nextCandleExpiry(now, .equity));
try testing.expect(p.date.eql(Date.fromYmd(2025, 11, 28)));
}
test "nextCandleExpiry mutual_fund: Friday evening -> Saturday morning" {
// Fri 2025-06-13, 20:00 ET (past 03:25). Friday's NAV posts Sat AM.
const now = utcSeconds(2025, 6, 14, 0, 0); // 2025-06-13 20:00 EDT
const p = etParts(nextCandleExpiry(now, .mutual_fund));
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 14))); // Saturday
try testing.expectEqual(@as(u8, 3), p.hour);
try testing.expectEqual(@as(u8, 25), p.minute);
}
test "nextCandleExpiry mutual_fund: Saturday morning -> Tuesday morning" {
// Sat 2025-06-14, 05:00 ET. No new NAV Sun/Mon AM; next is Tue (Mon's NAV).
const now = utcSeconds(2025, 6, 14, 9, 0); // 2025-06-14 05:00 EDT
const p = etParts(nextCandleExpiry(now, .mutual_fund));
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 17))); // Tuesday
}
test "nextCandleExpiry mutual_fund: Monday morning before NAV -> Tuesday" {
// Mon 2025-06-16, 01:00 ET (before 03:25). Monday's NAV not out yet,
// and there's no new NAV Monday AM either -> Tue 03:25.
const now = utcSeconds(2025, 6, 16, 5, 0); // 2025-06-16 01:00 EDT
const p = etParts(nextCandleExpiry(now, .mutual_fund));
try testing.expect(p.date.eql(Date.fromYmd(2025, 6, 17)));
}
test "isHoliday: 2025 NYSE holidays" {
// Fixed-date with observance.
try testing.expect(isHoliday(Date.fromYmd(2025, 1, 1))); // New Year (Wed)
try testing.expect(isHoliday(Date.fromYmd(2025, 6, 19))); // Juneteenth (Thu)
try testing.expect(isHoliday(Date.fromYmd(2025, 7, 4))); // Independence (Fri)
try testing.expect(isHoliday(Date.fromYmd(2025, 12, 25))); // Christmas (Thu)
// Floating.
try testing.expect(isHoliday(Date.fromYmd(2025, 1, 20))); // MLK (3rd Mon)
try testing.expect(isHoliday(Date.fromYmd(2025, 2, 17))); // Washington (3rd Mon)
try testing.expect(isHoliday(Date.fromYmd(2025, 5, 26))); // Memorial (last Mon)
try testing.expect(isHoliday(Date.fromYmd(2025, 9, 1))); // Labor (1st Mon)
try testing.expect(isHoliday(Date.fromYmd(2025, 11, 27))); // Thanksgiving (4th Thu)
// Non-holidays.
try testing.expect(!isHoliday(Date.fromYmd(2025, 7, 3)));
try testing.expect(!isHoliday(Date.fromYmd(2025, 12, 24)));
try testing.expect(!isHoliday(Date.fromYmd(2025, 6, 11)));
}
test "isHoliday: weekend observance rules" {
// Independence Day 2026 falls on Saturday -> observed Fri 2026-07-03.
try testing.expect(isHoliday(Date.fromYmd(2026, 7, 3)));
try testing.expect(!isHoliday(Date.fromYmd(2026, 7, 4))); // the Saturday itself
// New Year's Day 2022 was a Saturday: NOT observed on Fri 2021-12-31.
try testing.expect(!isHoliday(Date.fromYmd(2021, 12, 31)));
// New Year's Day 2023 was a Sunday -> observed Mon 2023-01-02.
try testing.expect(isHoliday(Date.fromYmd(2023, 1, 2)));
// Juneteenth not modeled before 2022.
try testing.expect(!isHoliday(Date.fromYmd(2021, 6, 18)));
}
test "isTradingDay: weekdays, weekends, holidays" {
try testing.expect(isTradingDay(Date.fromYmd(2025, 6, 11))); // Wed
try testing.expect(!isTradingDay(Date.fromYmd(2025, 6, 14))); // Sat
try testing.expect(!isTradingDay(Date.fromYmd(2025, 6, 15))); // Sun
try testing.expect(!isTradingDay(Date.fromYmd(2025, 11, 27))); // Thanksgiving
}
test "staleCandleExpiry equity: due bar merely late retries soon" {
// Thu 2025-06-12, 17:30 ET: Thursday's bar is due (16:55 ET passed)
// but only ~35m overdue, well inside the lag grace window.
const now = etLocalToUtc(Date.fromYmd(2025, 6, 12), 17 * std.time.s_per_hour + 30 * std.time.s_per_min);
// Cache last has Wed -> Thursday's bar is due but unposted (lag) -> short retry.
try testing.expectEqual(now + short_retry_s, staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 11)));
// Cache already has Thu -> nothing newer is due -> normal next boundary.
try testing.expectEqual(nextCandleExpiry(now, .equity), staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 12)));
}
test "staleCandleExpiry equity: weekend gap is not missing" {
// Sat 2025-06-14, 10:00 ET: latest available equity data is still
// Friday, which the cache already has - nothing is overdue.
const now = etLocalToUtc(Date.fromYmd(2025, 6, 14), 10 * std.time.s_per_hour);
const exp = staleCandleExpiry(now, .equity, Date.fromYmd(2025, 6, 13));
try testing.expectEqual(nextCandleExpiry(now, .equity), exp);
// The long boundary (Mon), not a 30-minute retry.
try testing.expect(exp > now + short_retry_s);
try testing.expect(etParts(exp).date.eql(Date.fromYmd(2025, 6, 16))); // Monday
}
test "staleCandleExpiry equity: un-modeled Good Friday closure gives up after grace" {
// Good Friday 2025-04-18 is NOT a modeled holiday (it needs the Easter
// computus - see isHoliday), so the calendar treats it as a trading
// day and an incremental fetch keeps coming back empty all day. The
// Friday bar is due at 16:55 ET; the grace window ends 90m later.
try testing.expect(isTradingDay(Date.fromYmd(2025, 4, 18)));
const last_cached = Date.fromYmd(2025, 4, 17); // Thursday
// Just inside the window (+89m): keep retrying in case the bar shows
// up late.
const inside = etLocalToUtc(Date.fromYmd(2025, 4, 18), 18 * std.time.s_per_hour + 24 * std.time.s_per_min);
try testing.expectEqual(inside + short_retry_s, staleCandleExpiry(inside, .equity, last_cached));
// At the 90-minute window (16:55 + 90m = 18:25 ET) we conclude the
// market was closed and wait for the next real session (Mon
// 2025-04-21) instead of thrashing every 30 minutes all weekend. The
// boundary is strict: the +89m case above still retries, so the
// retries land at +30/+60/+90 and the +90 one trips this fallback.
const at_window = etLocalToUtc(Date.fromYmd(2025, 4, 18), 18 * std.time.s_per_hour + 25 * std.time.s_per_min);
const exp = staleCandleExpiry(at_window, .equity, last_cached);
try testing.expectEqual(nextCandleExpiry(at_window, .equity), exp);
try testing.expect(exp > at_window + short_retry_s); // not a short retry
const p = etParts(exp);
try testing.expect(p.date.eql(Date.fromYmd(2025, 4, 21))); // Monday
try testing.expectEqual(@as(u8, 16), p.hour);
try testing.expectEqual(@as(u8, 55), p.minute);
}
test "staleCandleExpiry mutual_fund: late NAV within grace retries soon" {
// Tue 2025-06-17, 04:30 ET: Monday's NAV (data_date 2025-06-16) is due
// (posts ~03:25 ET) but the provider is lagging by ~65m, still inside
// the grace window. Also exercises the morning -> prior-session mapping
// in latestAvailability.
const now = etLocalToUtc(Date.fromYmd(2025, 6, 17), 4 * std.time.s_per_hour + 30 * std.time.s_per_min);
// Cache has through Friday's NAV -> Monday's is missing -> short retry.
try testing.expectEqual(now + short_retry_s, staleCandleExpiry(now, .mutual_fund, Date.fromYmd(2025, 6, 13)));
// Cache already has Monday -> nothing newer is due -> normal boundary.
try testing.expectEqual(nextCandleExpiry(now, .mutual_fund), staleCandleExpiry(now, .mutual_fund, Date.fromYmd(2025, 6, 16)));
}

73
src/service.zig
View file

@ -40,6 +40,7 @@ const fmt = @import("format.zig");
const performance = @import("analytics/performance.zig");
const http = @import("net/http.zig");
const atomic = @import("atomic.zig");
const market = @import("market.zig");
// ── Wall-clock policy ────────────────────────────────────────
//
@ -595,7 +596,11 @@ pub const DataService = struct {
// and candles_meta.srf in one shot (last_close, last_date,
// provider, fail_count=0).
if (triple.candles.len > 0) {
s.cacheCandles(symbol, triple.candles, .tiingo, 0);
// wall-clock required: stamp the candle-meta freshness boundary
// (market-aware next post-close / NAV-availability time).
const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds();
const expires = market.nextCandleExpiry(now_s, market.classify(symbol));
s.cacheCandles(symbol, triple.candles, .tiingo, 0, expires);
}
// Dividends and splits use the supplement write path: Tiingo's
// view merges into existing (typically Polygon-sourced) records
@ -745,6 +750,14 @@ pub const DataService = struct {
var s = self.store();
const today = fmt.todayDate(self.io);
// wall-clock required: candle-meta freshness uses a market-aware
// boundary (next post-close / NAV-availability time in ET; see
// market.nextCandleExpiry). Captured once here and threaded to
// every candle-meta write below so a single invocation stamps a
// consistent boundary.
const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds();
const kind = market.classify(symbol);
// Check candle metadata for freshness (tiny file, no candle deserialization)
const meta_result = s.readCandleMeta(symbol);
if (meta_result) |mr| {
@ -792,9 +805,13 @@ pub const DataService = struct {
// Stale - try incremental update using last_date from meta
const fetch_from = m.last_date.addDays(1);
// Market-aware freshness boundary for any meta write on
// this stale path (next post-close / NAV-availability time).
const expires = market.nextCandleExpiry(now_s, kind);
// If last cached date is today or later, just refresh the TTL (meta only)
if (!fetch_from.lessThan(today)) {
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count);
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, m.fail_count, expires);
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
} else {
@ -805,7 +822,7 @@ pub const DataService = struct {
// Increment fail_count for this symbol
const new_fail_count = m.fail_count +| 1; // saturating add
log.warn("{s}: transient failure (fail_count now {d})", .{ symbol, new_fail_count });
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count);
s.updateCandleMeta(symbol, m.last_close, m.last_date, m.provider, new_fail_count, now_s + market.short_retry_s);
// If degraded (fail_count >= 3), return stale data rather than failing
if (new_fail_count >= 3) {
@ -823,14 +840,23 @@ pub const DataService = struct {
const new_candles = result.candles;
if (new_candles.len == 0) {
// No new candles (weekend/holiday) - refresh TTL, reset fail_count
// No new candles. Either a genuine non-trading-day
// gap (weekend/holiday), the provider hasn't posted
// the just-closed bar yet, or the calendar's
// "trading day" was an un-modeled closure (e.g.
// Good Friday). market.staleCandleExpiry picks the
// boundary: a short retry while a due bar is merely
// late, falling back to the next close boundary
// once it's overdue past the lag grace window - so
// an un-modeled closure stops thrashing and waits
// for the next real session.
self.allocator.free(new_candles);
s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0);
s.updateCandleMeta(symbol, m.last_close, m.last_date, result.provider, 0, market.staleCandleExpiry(now_s, kind, m.last_date));
if (s.read(self.allocator, Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
} else {
// Append new candles to existing file + update meta, reset fail_count
s.appendCandles(symbol, new_candles, result.provider, 0);
s.appendCandles(symbol, new_candles, result.provider, 0, expires);
if (s.read(self.allocator, Candle, symbol, null, .any)) |r| {
self.allocator.free(new_candles);
return .{ .data = r.data, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
@ -874,7 +900,7 @@ pub const DataService = struct {
// we know to back off if this keeps happening.
if (meta_result) |mr| {
const new_fail_count = mr.meta.fail_count +| 1;
s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count);
s.updateCandleMeta(symbol, mr.meta.last_close, mr.meta.last_date, mr.meta.provider, new_fail_count, now_s + market.short_retry_s);
}
return DataError.TransientError;
}
@ -918,7 +944,7 @@ pub const DataService = struct {
/// `opts.force_refresh = true` -> treats cache as stale and fetches.
pub fn getEarnings(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!FetchResult(EarningsEvent) {
// Mutual funds (5-letter tickers ending in X) don't have quarterly earnings.
if (isMutualFund(symbol)) {
if (market.classify(symbol) == .mutual_fund) {
return .{ .data = &.{}, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator };
}
@ -2928,13 +2954,6 @@ pub const DataService = struct {
return daily and meta;
}
/// Mutual funds use 5-letter tickers ending in X (e.g. FDSCX, VSTCX, FAGIX).
/// These don't have quarterly earnings - skip the fetch rather than
/// round-tripping to the provider just to get an empty response.
fn isMutualFund(symbol: []const u8) bool {
return symbol.len == 5 and symbol[4] == 'X';
}
// ── User config files ─────────────────────────────────────────
/// Load and parse accounts.srf from the same directory as the given portfolio path.
@ -3009,24 +3028,6 @@ test "isPermanentProviderFailure: RateLimited is transient" {
try std.testing.expect(!isPermanentProviderFailure(error.RateLimited));
}
test "isMutualFund identifies mutual funds" {
// Standard mutual fund tickers (5 letters ending in X)
try std.testing.expect(DataService.isMutualFund("FDSCX"));
try std.testing.expect(DataService.isMutualFund("VSTCX"));
try std.testing.expect(DataService.isMutualFund("FAGIX"));
try std.testing.expect(DataService.isMutualFund("VFINX"));
// Not mutual funds
try std.testing.expect(!DataService.isMutualFund("AAPL"));
try std.testing.expect(!DataService.isMutualFund("VTI"));
try std.testing.expect(!DataService.isMutualFund("SPY"));
try std.testing.expect(!DataService.isMutualFund("GOOGL"));
try std.testing.expect(!DataService.isMutualFund("")); // empty
try std.testing.expect(!DataService.isMutualFund("X")); // too short
try std.testing.expect(!DataService.isMutualFund("FDSCA")); // 5 letters but not ending in X
try std.testing.expect(!DataService.isMutualFund("FDSCXA")); // 6 letters ending in A
}
test "DataService init/deinit lifecycle" {
const allocator = std.testing.allocator;
const config = Config{
@ -3176,7 +3177,7 @@ test "getCandles offline mode returns cached data without network" {
.{ .date = Date.fromYmd(2026, 5, 19), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
.{ .date = Date.fromYmd(2026, 5, 20), .open = 104, .high = 106, .low = 103, .close = 105, .adj_close = 105, .volume = 1100 },
};
store.cacheCandles("TEST", candles[0..], .tiingo, 0);
store.cacheCandles("TEST", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
// Set the test guard: any network call would panic. We expect
// the offline-mode path NOT to touch the network.
@ -3279,7 +3280,7 @@ test "loadAllPrices offline mode skips network and returns cached" {
var fresh_candles = [_]Candle{
.{ .date = Date.fromYmd(2026, 5, 20), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
};
store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0);
store.cacheCandles("FRESH", fresh_candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
// Symbol with no cache at all.
// (no setup needed - just passes a symbol that doesn't exist)
@ -3331,7 +3332,7 @@ test "loadAllPrices force_refresh tops up without wiping the candle cache" {
var candles = [_]Candle{
.{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
};
store.cacheCandles("HELD", candles[0..], .tiingo, 0);
store.cacheCandles("HELD", candles[0..], .tiingo, 0, market.nextCandleExpiry(std.Io.Timestamp.now(io, .real).toSeconds(), .equity));
// Any provider/network attempt now panics. If force_refresh wiped
// the cache (old behavior), getCandles would fall through to a full