From 41dd29064843f1d808e27f0d9ae3b0295951272f Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Sun, 20 Sep 2026 13:12:03 -0700 Subject: [PATCH] update docs with clearer explanation of what is actually happening --- AGENTS.md | 9 +- README.md | 37 +- docs/dev/caching-implementation.md | 287 ++++++++++---- docs/explanation/caching.md | 194 ++++++++-- docs/guides/offline-and-refresh.md | 17 +- docs/reference/providers.md | 37 +- src/cache/store.zig | 99 +++-- src/service.zig | 601 +++++++++++++++++++++++++++-- 8 files changed, 1057 insertions(+), 224 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index a7863a1..06ef503 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -882,12 +882,15 @@ command. - **SRF string lifetimes are handled for you.** `Store.read` parses with a `parse_allocator`, so SRF dupes every owned string into the caller's allocator automatically. **Do not add a `postProcess` callback to dupe strings** - that is not a valid reason for one. `postProcess` exists only for non-trivial post-parse logic, such as recomputing a derived field (see `earningsPostProcess` in `service.zig`, which rebuilds `surprise` from `actual` and `estimate`). -- **A fresh cache entry is not necessarily a complete one.** `fetchCached` takes a second comptime hook, `needsRefresh: ?*const fn ([]const T, Date) bool`, for types where TTL cannot express "is this entry missing something it should have by now". `getDividends` passes `dividendsNeedRefresh`; `getSplits` and `getOptions` pass `null`. Three rules if you add one: - - It is consulted at **both** fresh-read sites (local, and post-server-sync) via `serveFreshOrDiscard`. Only covering the first one means a configured `ZFIN_SERVER` defeats the mechanism, because the sync leaves the still-incomplete entry on disk and the second read serves it. +- **A fresh cache entry is not necessarily a complete one.** `fetchCached` takes three optional comptime hooks: `postProcess` (recompute a derived field), `needsRefresh: ?*const fn ([]const T, Date) bool` ("fresh, but is it complete?"), and `ttlFor: ?*const fn ([]const T) cache.TtlSpec` ("how long should this stay fresh, given these records?"). `getDividends` uses `needsRefresh` + `ttlFor`; `getEarnings` uses `postProcess` + `needsRefresh`; `getSplits` and `getOptions` pass `null` for all three. Four rules if you add a `needsRefresh`: + - It is consulted at **both** fresh-read sites (local, and post-server-sync) via `serveFreshOrDiscard`. Only covering the first one means a configured `ZFIN_SERVER` defeats the mechanism, because the sync leaves the still-incomplete entry on disk and the second read serves it. This was observed live, not theorised. - `serveFreshOrDiscard` owns the free. Don't discard at a call site; a caller that decides and then forgets is a leak, and the tests rely on `std.testing.allocator` to catch exactly that. - It is suppressed under `skip_network`. Offline mode never refetches, so discarding could only turn a served result into a failure. + - It is floored at one ask per symbol per 12 hours (`smart_refresh_recheck_interval` / `askedRecently`). Without that, a hook that stays true because the provider has nothing yet re-fires on **every command**, not once a day. - The branch is comptime-elided for a `null` argument, which is what lets `Split` - a type with no `freeSlice` - keep compiling. See `docs/dev/caching-implementation.md` for the dividend rationale and its three documented blind spots. + **The floor reads `#!expires=`, never `#!created=`.** `serializeWithMeta` re-stamps `created` on every write, and dividend rows arrive as a side effect of Tiingo candle fetches via `writeSupplement` - so `created` means "file last touched" and a candle refresh would fake "we just asked the provider". Only a primary fetch moves `expires` (`.bump`); a supplement restores the old value (`.preserve`). There is a regression test named for this; don't "simplify" it to `created`. + + All three hook branches are comptime-elided for a `null` argument, which is what lets `Split` - a type with no `freeSlice` - keep compiling. See `docs/dev/caching-implementation.md` for the mechanics and `docs/explanation/caching.md` for the worked example. - **Buffered stdout.** CLI output uses a single `std.Io.Writer` with a 4096-byte stack buffer, flushed once at the end of `main()`. Don't write to stdout through other means. diff --git a/README.md b/README.md index 7f4062c..ae2cef4 100644 --- a/README.md +++ b/README.md @@ -79,28 +79,31 @@ around the runnable example portfolios in [`examples/`](examples/). zfin aggregates data from multiple free-tier APIs, using each for what it does best, with aggressive caching to stay within free-tier limits. -| Data type | Provider | Auth | Free-tier limit | Cache TTL | -|-----------------------|------------------|----------------------|----------------------------|--------------| -| Daily candles (OHLCV) | Tiingo | `TIINGO_API_KEY` | 1,000 req/day, 50 req/hour | 23h45m | -| Real-time quotes | Yahoo Finance | None required | Unofficial | Never cached | -| Quote fallback | TwelveData | `TWELVEDATA_API_KEY` | 8 req/min, 800/day | Never cached | -| Dividends | Polygon | `POLYGON_API_KEY` | 5 req/min | 6 days\*\* | -| Splits | Polygon | `POLYGON_API_KEY` | 5 req/min | 14 days | -| Options chains | CBOE | None required | ~30 req/min (self-imposed) | 1 hour | -| Earnings | FMP | `FMP_API_KEY` | 250 req/day | 30 days* | -| ETF profiles | SEC EDGAR | `ZFIN_USER_EMAIL` | 10 req/sec | ~90 days | -| Classification | Wikidata + EDGAR | `ZFIN_USER_EMAIL` | No per-day quota | Long-lived | +| Data type | Provider | Auth | Free-tier limit | Cache TTL | +|-----------------------|------------------|----------------------|----------------------------|------------------| +| Daily candles (OHLCV) | Tiingo | `TIINGO_API_KEY` | 1,000 req/day, 50 req/hour | 23h45m | +| Real-time quotes | Yahoo Finance | None required | Unofficial | Never cached | +| Quote fallback | TwelveData | `TWELVEDATA_API_KEY` | 8 req/min, 800/day | Never cached | +| Dividends | Polygon | `POLYGON_API_KEY` | 5 req/min | 6 or 14 days\*\* | +| Splits | Polygon | `POLYGON_API_KEY` | 5 req/min | 14 days | +| Options chains | CBOE | None required | ~30 req/min (self-imposed) | 1 hour | +| Earnings | FMP | `FMP_API_KEY` | 250 req/day | 30 days* | +| ETF profiles | SEC EDGAR | `ZFIN_USER_EMAIL` | 10 req/sec | ~90 days | +| Classification | Wikidata + EDGAR | `ZFIN_USER_EMAIL` | No per-day quota | Long-lived | \* **Earnings smart refresh** -- cached earnings re-fetch inside the 30-day window once a report date has passed but the actual result is still missing. -\*\* **Dividend smart refresh** -- the TTL is only a backstop. Cached -dividends also re-fetch whenever a distribution the symbol's own -schedule says is due has not arrived. An ETF's amount is not published -until roughly its own ex-date and it pays 1-7 days later, so a long TTL -can be written before the record exists and still be fresh after the -cash has landed. +\*\* **Dividends get three freshness checks, not one.** An ETF's payment +amount does not exist until roughly the day it goes ex, and it pays within +a week, so a fixed expiry time can sail right over the whole event. So +after the expiry clock says an entry is still fresh, a second check asks +whether a payment the symbol's own schedule predicts is missing from it - +and a third stops that from asking the provider more than once every 12 +hours. The expiry clock is 14 days when the schedule is known and 6 days +when it isn't. See +[Caching and data freshness](docs/explanation/caching.md#dividends-a-three-level-freshness-check). Not all keys are required; without a given key, that data type is simply unavailable. For per-provider notes, signup links, the full diff --git a/docs/dev/caching-implementation.md b/docs/dev/caching-implementation.md index 0f70175..9211b9b 100644 --- a/docs/dev/caching-implementation.md +++ b/docs/dev/caching-implementation.md @@ -310,16 +310,51 @@ which is simpler because each type is a single file: 1. `.fresh_only` read; a fresh entry (including a negative one) returns immediately -- **unless** the type supplies a `needsRefresh` hook that - says the fresh entry is incomplete (see below). + says the fresh entry is incomplete (see the freshness levels below). 2. `skip_network`: return any cached entry, even stale; else `FetchFailed`. 3. Server sync (if configured); a fresh synced entry returns, subject to the same `needsRefresh` check. -4. Provider fetch; on success write with the type's TTL; on +4. Provider fetch; on success write with the TTL from `ttlSpecFor`; on `NotFound` write a negative entry; on `NoApiKey` return `NoApiKey`; on any other transient error return `FetchFailed`. Neither of the last two poisons the cache. +#### The three freshness levels + +Dividends and earnings do not get a single freshness verdict; they get +three, in a fixed order. The user-facing explanation is in +[Caching and data freshness](../explanation/caching.md#dividends-a-three-level-freshness-check); +this is where each level lives in code. + +| Level | Name | Question | Implementation | +|-------|--------------------|-----------------------------------------------------------|------------------------------------------------------------| +| **1** | expiry clock | "Is this entry old?" | `#!expires=` vs now, enforced by `Store.read(.fresh_only)` | +| **2** | completeness check | "Is this entry missing something it should already have?" | `needsRefresh` hook, via `serveFreshOrDiscard` | +| **3** | recheck floor | "Did we already ask the provider recently?" | `askedRecently`, 12h | + +**The ordering is the part to get right, and it is the opposite of the +natural reading.** Level 2 is a **second opinion on a "still fresh" verdict +from Level 1**, not an extra gate in front of a refresh: + +- Level 1 says **stale** -> `Store.read(.fresh_only)` returns `null`, the + `if` body in step 1 never executes, and `serveFreshOrDiscard` is never + called. **Levels 2 and 3 do not run at all.** There is nothing for them + to contribute: a refetch is already happening. +- Level 1 says **fresh** -> Level 2 runs, and its whole job is to be able + to overrule that with "no it isn't, a payment has happened that this + entry doesn't contain." +- Level 2 says **incomplete** -> Level 3 runs, and can overrule Level 2 + back to "serve the cache anyway" if we asked too recently. + +So each level can only be reached by the previous one declining to +decide. If you ever find yourself reasoning about Level 2 firing on a +stale entry, the model is inverted. + +`skip_network` short-circuits between Levels 1 and 2: offline mode serves +the fresh entry without consulting either later level, since it can never +refetch and discarding could only turn a served result into a failure. + `NoApiKey` is propagated rather than collapsed into `FetchFailed` because it is the one distinction callers act on differently: `commands/earnings.zig` and `tui/earnings_tab.zig` both name the @@ -333,67 +368,114 @@ implementation: it short-circuits mutual funds (no quarterly earnings exist, so there is nothing to cache or fetch) and then delegates, supplying `earningsPostProcess` and `earningsNeedsRefreshHook`. -#### The two comptime hooks +#### The three comptime hooks -`fetchCached` takes two optional comptime function pointers. Both default -to "absent" by passing `null`, which is the behaviour every type had -before they existed. +`fetchCached` takes three optional comptime function pointers. Passing +`null` for all three gives the behaviour every type had before they +existed, which is what `getSplits` and `getOptions` do. -| Hook | Signature | Purpose | -|-----------------|--------------------------------------------|-----------------------------------------------------| -| `postProcess` | `fn (*T, Allocator) anyerror!void` | Recompute a derived field after parse | -| `needsRefresh` | `fn ([]const T, Date) bool` | "This entry is fresh, but is it *complete*?" | +| Hook | Signature | Question it answers | +|----------------|--------------------------------------|-----------------------------------------------| +| `postProcess` | `fn (*T, Allocator) anyerror!void` | Any derived field to recompute after parsing? | +| `needsRefresh` | `fn ([]const T, Date) bool` | This entry is fresh, but is it *complete*? | +| `ttlFor` | `fn ([]const T) cache.TtlSpec` | How long should this data stay fresh? | + +Who passes what: + +| Getter | postProcess | needsRefresh | ttlFor | +|---------------|------------------------|-----------------------------|----------------| +| `getDividends`| - | `dividendsNeedRefresh` | `dividendTtl` | +| `getEarnings` | `earningsPostProcess` | `earningsNeedsRefreshHook` | - | +| `getSplits` | - | - | - | +| `getOptions` | - | - | - | `postProcess` is **not** for duping strings -- `Store.read` already does -that. Today only `EarningsEvent` uses it (to rebuild `surprise`). +that. Only `EarningsEvent` uses it, to rebuild `surprise`. -`needsRefresh` exists because freshness and completeness are different -questions. It is consulted at **both** fresh-read sites (step 1 and step -3) through `serveFreshOrDiscard`, which owns the free so a caller cannot -decide to discard and then leak. Covering step 3 is not optional: with a -`ZFIN_SERVER` configured, a sync can leave a still-incomplete entry on -disk that step 3 would otherwise serve, defeating the mechanism -entirely. The hook is suppressed under `skip_network`, since offline -mode never refetches and discarding could only turn a served result into -a failure. +`needsRefresh` is consulted at **both** fresh-read sites (steps 1 and 3 +above) through `serveFreshOrDiscard`, which owns the free so a caller +cannot decide to discard and then leak. Covering step 3 is not optional: +with a `ZFIN_SERVER` configured, a sync can leave a still-incomplete entry +on disk that step 3 would otherwise serve, defeating the mechanism +entirely. (This is not theoretical - it was observed against a live +server whose copy was missing the same record the local cache was.) The +hook is suppressed under `skip_network`, since offline mode never +refetches and discarding could only turn a served result into a failure. -The hook's branch is comptime-elided when the argument is `null`, which -is what lets types without a `freeSlice` (such as `Split`) keep -compiling. +`ttlFor` exists because the right expiry time for dividends depends on the +records being written, not just on their type - see `dividendTtl`. It is +applied through `ttlSpecFor` at both write sites (the normal path and the +rate-limit retry) so a retry cannot stamp a different clock than the +first attempt would have. -#### Dividend smart refresh +All three branches are comptime-elided when the argument is `null`, which +is what lets types without a `freeSlice` (such as `Split`) keep compiling. -`dividendsNeedRefresh` is the hook `getDividends` passes. It answers -true when at least one full cadence period has elapsed since the newest -cached ex-date and the distribution that should have landed in it is -still absent. +#### The recheck floor -The problem it solves is that dividend TTL and dividend *availability* -are uncorrelated. An operating company declares weeks ahead of the -ex-date and pays about a month after it. An ETF cannot: its distribution -is the pass-through of whatever the fund earned, so the amount is not -published until roughly its own ex-date, and it pays 1-7 days later. The -window in which the record exists but the cash has not yet landed is -about a week, so any longer TTL can be written before the record exists -and still be fresh after the payment. Quarter-end clustering makes that -systematic rather than unlucky -- a dozen funds go ex within a few days -of each other, so one refresh pass puts all of them on the same expiry -shelf. +A `needsRefresh` hook has a failure mode that is easy to miss. When the +hook says something is overdue and the provider does not have it yet - +the *normal* case, not an error - the cache is rewritten with a fresh +expiry but the same records. The hook therefore still says "overdue", and +the next command asks again. On a quarter-end run touching fifteen funds +that is fifteen provider calls per command, repeated every time anything +runs. -Cadence comes from `dividendCadenceDays`: the **median** of up to the -five newest gaps between cached ex-dates. Median rather than mean (a -special distribution landing days after a regular one drags the mean) and -rather than minimum (the same outlier destroys the minimum outright, -predicting a next ex-date days away and firing for the whole chase window -after every payment). `newestExDates` sorts rather than trusting file -order, because `writeSupplement`'s merge interleaves two providers' -records and a cadence computed from negative gaps would forecast into the -past forever. +`smart_refresh_recheck_interval` (12 hours) is the floor, enforced by +`askedRecently`. It applies to any type with a `needsRefresh` hook. + +**It reads `#!expires=`, not `#!created=`, and that distinction is the +whole trick.** `serializeWithMeta` re-stamps `created` on every write, and +dividend rows also arrive as a side effect of a Tiingo candle fetch via +`writeSupplement`. So `created` means "file last touched", which a candle +refresh changes without anyone asking Polygon anything - keying the floor +off it would suppress a genuinely due refetch for 12 hours. `expires` is +moved only by a primary-provider fetch (`ExpiryPolicy.bump`); a supplement +deliberately puts the old value back (`.preserve`). There is a regression +test for exactly this. + +Recovering the write time from `expires` is *exact*, not approximate, +because `computeExpires` derives its jitter from a hash of the symbol - +the same symbol always gets the same offset. Calling it with a zero "now" +returns precisely the TTL that was applied, so `expires - that` is the +write time to the second. + +Two cases fall back to "ask again", which is the safe direction: a file +with no `#!expires=` at all, and a symbol that crossed from +unknown-schedule to known-schedule since it was written (stored under the +short TTL, recomputed with the long one, so the write looks older than it +was). + +#### Dividend schedule prediction + +`dividendsNeedRefresh` returns true when at least one full cadence period +has elapsed since the newest cached ex-date and the payment that should +have landed in it is still absent. + +It exists because a dividend cache being *fresh* and being *complete* are +unrelated. An operating company declares weeks ahead of the ex-date and +pays about a month later. An ETF cannot: its distribution is the +pass-through of whatever the fund earned, so the amount does not exist +until roughly its own ex-date, and it pays 1-7 days later. The period in +which the record exists but the cash has not landed is about a week, so +any longer expiry can be written before the record exists and still be +fresh after the payment. Quarter-end clustering makes it systematic -- a +dozen funds go ex within a few days, so one refresh pass puts all of them +on the same expiry date with the same blind spot. + +Cadence comes from `dividendCadenceDays`: the **median** of up to the five +newest gaps between cached ex-dates. Median rather than mean (a special +distribution landing days after a regular one drags the mean) and rather +than minimum (the same outlier destroys the minimum outright, predicting a +next ex-date days away and firing for the whole chase window after every +payment). `newestExDates` sorts rather than trusting file order, because +`writeSupplement`'s merge interleaves two providers' records and a cadence +computed from negative gaps would forecast into the past forever. Three properties are load-bearing and each has its own test: -- **A full period must elapse.** Otherwise it refires immediately after - a successful fetch, on the record it just stored. +- **A full period must elapse.** Otherwise it refires immediately after a + successful fetch, on the record it just stored. - **It rolls forward.** The expected date is the latest one at or before today, not the first one after the newest cached record -- so a symbol two periods behind stays recoverable instead of being permanently past @@ -402,17 +484,66 @@ Three properties are load-bearing and each has its own test: unbounded chase refetches forever whenever a sponsor skips a period or the cadence estimate is wrong. -Known blind spots, all three deliberately left to the TTL backstop -rather than papered over with a more eager statistic that would cost -requests on every symbol to protect against a rare event on one: +`dividendTtl` then picks the expiry from the same cadence calculation: +`Ttl.dividends_scheduled` (14 days) when a cadence exists, because +prediction is what finds due payments; `Ttl.dividends` (6 days) when it +does not, because then the clock is the only mechanism and it has to +expire before the next weekly review. Six is the largest value that +guarantees that -- 6 x 1.11 jitter = 6.66 days, under 7, where 7 would +reach 7.77. `"TTL constants are reasonable"` in store.zig asserts that +property rather than just the number. + +Known blind spots, all left to the expiry time rather than papered over +with a more eager statistic that would cost requests on every symbol to +protect against a rare event on one: - A **cadence change** (quarterly to monthly) keeps the old median for several periods, so the forecast runs late. -- An **off-cycle special** is on no schedule, so the predicate never - fires for it. -- A symbol with **fewer than three cached records** has no cadence at - all -- which is the right direction, since a newly-bought holding has - no schedule to be late against. +- An **off-cycle special** is on no schedule, so prediction never fires + for it. +- A symbol with **fewer than three cached records** has no cadence at all + -- which is exactly the case that gets the 6-day expiry. + +#### Why Tiingo can never discover a new dividend + +Tiingo returns dividends and splits alongside candles in a single request +(`populateAllFromTiingo`), written through `writeSupplement` so the merge +lands new records without moving the primary provider's `#!expires=` +clock. That makes it look like a free second source for Level 2, and it is +worth being explicit that it is not - for two independent reasons. + +**Structural.** Tiingo carries a dividend as a `divCash` field on a daily +price bar, so a payment can only appear once a bar exists for it. A +declared-but-not-yet-ex dividend has no bar and therefore cannot be +represented at all. This is exactly why Polygon is the primary source - +see the comment in `fetchFromProvider`, which cites a forward-dated ARCC +ex-date as the case Tiingo's response cannot express. + +**Circular.** `populateAllFromTiingo` is reached from only two places: + +1. A **cold candle cache** - the symbol has no bars at all + (`getCandles`'s full-history path). Here Tiingo genuinely earns its + keep: one request bulk-loads the symbol's entire dividend history. +2. An **adjustment-basis restatement** - cached `adj_close` values predate + a corporate action they should reflect, so the series is refetched in + full. + +Trigger 2 is decided by `freshness.adjustmentBasisStale` fed by +`freshness.newestCorporateAction`, and that function reads the **cached +dividend file** to find the newest action. So zfin only knows to restate +*after* the dividend record is already on disk - which means Polygon must +have supplied it first. By the time Tiingo's rows merge in, they add +nothing about that payment. + +Net: Tiingo bulk-loads history and fills field-level gaps in old records. +Every *new* dividend arrives via Polygon. This is also why the Level 3 +floor reads `#!expires=` and not `#!created=` - a Tiingo supplement +rewrites the file, and therefore `created`, without having asked Polygon +anything. + +For the user-facing version of all this, including a worked quarterly +example, see +[Caching and data freshness](../explanation/caching.md#dividends-a-three-level-freshness-check). ### `loadAllPrices` (portfolio + watchlist price load) @@ -652,26 +783,28 @@ the upstream provider on every request. ## Key code references -| Concern | Location | -|---------------------------------|-----------------------------------------------------| -| Data-access entry point | `DataService` - `src/service.zig` | -| Per-type generic fetch | `fetchCached` - `src/service.zig` | -| Fresh-but-incomplete decision | `serveFreshOrDiscard` - `src/service.zig` | -| Dividend schedule check | `dividendsNeedRefresh`, `dividendCadenceDays` - `src/service.zig` | +| Concern | Location | +|---------------------------------|------------------------------------------------------------------------| +| Data-access entry point | `DataService` - `src/service.zig` | +| Per-type generic fetch | `fetchCached` - `src/service.zig` | +| Fresh-but-incomplete decision | `serveFreshOrDiscard` - `src/service.zig` | +| Recheck floor (12h) | `askedRecently`, `smart_refresh_recheck_interval` - `src/service.zig` | +| Dividend schedule prediction | `dividendsNeedRefresh`, `dividendCadenceDays` - `src/service.zig` | +| Dividend expiry choice | `dividendTtl`, `ttlSpecFor` - `src/service.zig` | | Earnings smart refresh | `earningsNeedsRefresh`, `earningsNeedsRefreshHook` - `src/service.zig` | -| Candle fetch + incremental | `getCandles` - `src/service.zig` | -| Batch price load | `loadAllPrices` - `src/service.zig` | -| Live quotes (uncached) | `loadLiveQuotes`, `getQuote` - `src/service.zig` | -| Cache store, read/write | `Store` - `src/cache/store.zig` | -| Freshness check | `isFresh` (SRF), `isCandleMetaFresh` - `store.zig` | -| TTLs and expiry computation | `Ttl`, `computeExpires` - `src/cache/store.zig` | -| Negative cache | `writeNegative`, `isNegative` - `src/cache/store.zig` | -| Negative-cache veto | `shouldNegativeCache` - `src/service.zig` | -| Candle provenance labels | `CandleProvider` - `src/cache/store.zig` | -| NotFound classification | `isPermanentProviderFailure` - `src/service.zig` | -| Market-aware candle expiry | `nextCandleExpiry`, `shouldRefresh` - `src/market.zig` | -| Price fallback (manual/avg-cost)| `buildFallbackPrices` - `src/analytics/valuation.zig` | -| Server endpoints | `serveSrfFile`, `fetchOnMiss` - `zfin-server/src/main.zig` | +| Candle fetch + incremental | `getCandles` - `src/service.zig` | +| Batch price load | `loadAllPrices` - `src/service.zig` | +| Live quotes (uncached) | `loadLiveQuotes`, `getQuote` - `src/service.zig` | +| Cache store, read/write | `Store` - `src/cache/store.zig` | +| Freshness check | `isFresh` (SRF), `isCandleMetaFresh` - `store.zig` | +| TTLs and expiry computation | `Ttl`, `computeExpires` - `src/cache/store.zig` | +| Negative cache | `writeNegative`, `isNegative` - `src/cache/store.zig` | +| Negative-cache veto | `shouldNegativeCache` - `src/service.zig` | +| Candle provenance labels | `CandleProvider` - `src/cache/store.zig` | +| NotFound classification | `isPermanentProviderFailure` - `src/service.zig` | +| Market-aware candle expiry | `nextCandleExpiry`, `shouldRefresh` - `src/market.zig` | +| Price fallback (manual/avg-cost)| `buildFallbackPrices` - `src/analytics/valuation.zig` | +| Server endpoints | `serveSrfFile`, `fetchOnMiss` - `zfin-server/src/main.zig` | For the user-facing summary and the `--refresh-data` walkthrough, see [Caching and data freshness](../explanation/caching.md). diff --git a/docs/explanation/caching.md b/docs/explanation/caching.md index 7838d10..ceaa424 100644 --- a/docs/explanation/caching.md +++ b/docs/explanation/caching.md @@ -39,44 +39,172 @@ The `--refresh-data` policy decides which tiers run: Different data ages at different rates, so each type has its own TTL: -| Data type | TTL | Why | -|---------------|---------------|----------------------------------------------------------------------------------| -| Daily candles | market-aware | Keyed to the next time a fresh bar is expected, not a rolling window (see below) | -| Dividends | 6 days\* | Short: an ETF distribution is only knowable for a few days before it pays | -| Splits | 14 days | Rare corporate events, announced weeks ahead | -| Options | 1 hour | Prices move continuously when markets are open | -| Earnings | 30 days\*\* | Quarterly; smart-refreshed around announcements | -| ETF profiles | ~30 days | Holdings and weights change slowly | -| Quotes | never cached | Meant to be a live price check | +| Data type | TTL | Why | +|---------------|--------------------|----------------------------------------------------------------------------------| +| Daily candles | market-aware | Keyed to the next time a fresh bar is expected, not a rolling window (see below) | +| Dividends | 6 or 14 days\* | Depends on whether we know the symbol's payment schedule | +| Splits | 14 days | Rare, and announced weeks ahead | +| Options | 1 hour | Prices move continuously when markets are open | +| Earnings | 30 days\*\* | Quarterly, with an early re-check around announcements | +| ETF profiles | ~30 days | Holdings and weights change slowly | +| Quotes | never cached | Meant to be a live price check | -\* **Dividend smart refresh:** even inside the 6-day window, cached -dividends re-fetch automatically once a distribution the symbol's own -schedule says is due has not arrived. This exists because a dividend -cache being *fresh* and being *complete* are different things. +\* The expiry time is only the first of three checks. See +[Dividends: a three-level freshness check](#dividends-a-three-level-freshness-check). -An operating company declares a dividend weeks ahead of the ex-date and -pays about a month after it, so there is plenty of time to notice. An -**ETF does not work that way.** Its distribution is the pass-through of -whatever the fund earned, so the amount is not published until roughly -its own ex-date -- and it pays 1 to 7 days later. The entire window in -which the record exists but the cash has not yet landed is about a week. +\*\* **Earnings** use the same three-level check as dividends, with a +different Level 2: instead of asking "is a payment overdue?", it asks "has +a report date passed with the actual result still missing?". So results +appear promptly after an announcement without daily polling. Level 3 +applies too - one re-check per symbol per 12 hours. -A long TTL straddles that window whole: a cache written a few days -before the ex-date stays "fresh" until well after the payment, so the -distribution is invisible to the run that needed it. Quarter-end makes -it systematic rather than unlucky, because a dozen funds go ex within -the same few days and one refresh pass puts all of them on the same -expiry shelf. +## Dividends: a three-level freshness check -So the schedule is the gate, and the TTL is only the backstop. It still -matters, because the schedule check cannot see three things: a holding -too new to have an established cadence, an off-cycle special -distribution, and a symbol with fewer than three cached records. +For most data a plain expiry time is enough. For dividends it is not, so +dividends (and earnings) go through **three checks instead of one**. The +order matters, and it is the thing most easily got backwards: + +| Level | Name | Question it answers | Runs when | +|-------|------------------------|-----------------------------------------------------------|---------------------------------------------------| +| **1** | **expiry clock** | "Is this cached entry old?" | always | +| **2** | **completeness check** | "Is this entry missing something it should already have?" | **only if Level 1 said the entry is still fresh** | +| **3** | **recheck floor** | "Did we already ask the provider about this recently?" | only if Level 2 said something is missing | + +**Read that middle row carefully, because the intuition runs the other +way.** Level 2 is *not* an extra hurdle before a refresh. It is a +**second opinion on a "no" from Level 1**. If the expiry clock says the +entry is old, zfin just refetches and Level 2 is never consulted at all - +there is nothing for it to add, because a refresh is already happening. +Level 2 only earns its keep in the opposite case: the clock says *"this +is still fine, serve it"*, and Level 2 gets to answer *"no it isn't - +a payment has happened that this entry doesn't contain."* + +Level 3 then stops Level 2 from being too eager, by refusing to ask the +provider about the same symbol more than once every 12 hours. + +### Why one expiry time isn't enough + +A regular company announces a dividend weeks before the ex-date and pays +about a month after it. There is plenty of time to notice. An ETF is +different. Its distribution is simply whatever the fund earned, passed +through to you, so the amount does not exist until roughly the fund's own +ex-date -- and it pays one to seven days later. The whole period between +"the payment becomes knowable" and "the cash is in your account" is about +a week. + +A two-week expiry time sails straight over that. Suppose the cache is +refreshed a few days before an ETF goes ex. It is then considered fresh +for another eleven days, which covers the ex-date, the payment, *and* the +weekly review that was supposed to explain the deposit. The dividend is +invisible exactly when it matters. And it is not bad luck: dozens of +funds go ex within the same few days at each quarter end, so one refresh +pass puts all of them on the same expiry date and every one of them has +the same blind spot. + +Level 2 exists to catch precisely that: it works out the symbol's own +payment schedule from the ex-dates already in the cache, and speaks up +when a payment is due that the cache does not have. + +### Two things that are easy to picture wrongly + +**The expiry clock is a duration, not a date.** When a record is cached, +zfin stamps "expires 14 days from now". It does not store "next check due +on 30 September". Each refetch re-stamps the duration from scratch. + +**The expected payment date is never stored at all.** Level 2 recomputes +it on every single cache read, from whatever ex-dates are currently +cached. Nothing on disk holds it. + +So capturing a payment does not "set the clock to" the next expected +payment. The two mechanisms run independently, on different schedules. + +### A worked example + +Take an ETF that has paid on 1 January, 1 April, 1 July and 1 October for +the last five years. The newest payment in the cache is 1 July 2026. + +Level 2 measures the gaps between the most recent ex-dates -- 91, 90, 92, +92 and 91 days -- and takes the middle value: **91 days**. Added to 1 +July, that makes **30 September** the date the next payment is expected. +(The real date is 1 October. Being a day out does not matter, as you will +see.) + +| When | What the levels say | Result | +|------------------|---------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------| +| 2 July - 29 Sept | L1: fresh, then stale every 14 days. L2: nothing due. | Nothing is expected, so only the clock causes fetches: **about six over the quarter** (91 days, at 14 days a time). | +| 30 Sept - 14 Oct | L1: fresh. L2: **a payment is overdue.** L3: allows one request/12 hours. | The provider is asked, at most twice a day no matter how many commands you run. Stops the moment the new record arrives. | +| 15 Oct onwards | L2: gave up -- two weeks past the expected date. | Back to the clock alone. Chasing a schedule that is clearly wrong would refetch forever. | + +On 30 September the provider probably has nothing yet, because the fund +has not gone ex. That request is wasted, and it is the price of the estimate +being a day early. On 1 October the record appears, gets cached, and +Level 2 immediately goes quiet again -- it now expects the next payment +around 1 January. + +The estimate also corrects itself. Once the 1 October record is cached, +the newest gaps median to 92 days rather than 91, which puts the next +expected date exactly on 1 January 2027. + +### Why Level 1 has two different lengths + +How long the expiry clock runs depends on whether Level 2 can do its job +for that symbol: + +- **14 days when the payment schedule is known.** A symbol with a few + years of history is found by Level 2, so the clock only has to cover + the one thing a schedule cannot predict: a one-off special + distribution. Making it short as well would re-check every symbol + every few days to guard against a problem already solved. + +- **6 days when the schedule is not known.** With fewer than three + cached payments there is no pattern to work from -- typically a holding + you have just bought. Level 2 declines, so the clock is the *only* + thing that will ever make zfin look again, and it has to expire before + a weekly review. Six days guarantees that; seven would not, because + the +/-11% spread applied to expiry times can stretch seven days to 7.8. + +### Why Level 3 exists + +When Level 2 says a payment is due and the provider does not have it yet, +nothing in the cache changes except the expiry time. Level 2 therefore +still says "due", so the next command would ask again -- and the one +after that. On a quarter-end run touching fifteen funds, that is fifteen +provider calls per command, every time you run anything. + +So Level 3 allows a symbol to be asked about at most once every 12 hours. +Twelve hours sits well inside the shortest gap between a fund going ex +and paying (one day, in a real portfolio), while still being far more +frequent than the weekly reconcile that actually uses the answer. + +### What about Tiingo? + +Tiingo returns dividend data as a side effect of fetching prices, which +raises the obvious question of whether it can spot a new payment for +free. It cannot, and it is worth knowing that it never will. + +Tiingo reports a dividend as a field on a daily price bar, so the payment +only shows up once it has already traded -- there is no such thing as a +forward-looking dividend in that data. Polygon is the primary source +precisely because it *does* carry declared-but-not-yet-paid dividends. +And the code path that merges Tiingo's dividend rows in only runs on a +cold cache or when a known corporate action forces a price-history +restatement, which means Polygon has already supplied the record by then. + +So Tiingo is genuinely useful for bulk-loading a new symbol's entire +dividend history in one request, and for filling gaps in old records. It +is simply never the thing that notices a *new* payment. + +### What none of this catches + +Three cases where Level 2 cannot help, all left to Level 1: + +- **A one-off special distribution.** It is on no schedule, so it is + found only when the expiry clock runs out. +- **A fund that changes its schedule** -- quarterly to monthly, say. The + old pattern persists for a few periods, so the estimate runs late. +- **A symbol with fewer than three cached payments.** No pattern to work + from, which is exactly why that case gets the 6-day clock. -\*\* **Earnings smart refresh:** even inside the 30-day window, cached -earnings re-fetch automatically once an earnings date has passed but -the cache still lacks the actual result -- so numbers appear promptly -after an announcement without daily polling. ## Market-aware candle freshness diff --git a/docs/guides/offline-and-refresh.md b/docs/guides/offline-and-refresh.md index 69b4eab..d519bc7 100644 --- a/docs/guides/offline-and-refresh.md +++ b/docs/guides/offline-and-refresh.md @@ -66,7 +66,7 @@ In `auto` mode, each data type has its own time-to-live: | Data | TTL | |--------------------|------------------------------------------------| | Daily candles | ~24 hours | -| Dividends | 6 days (refreshed early once a payout is due) | +| Dividends | 6 or 14 days (see note below) | | Splits | 14 days | | Options | 1 hour | | Earnings | 30 days (refreshed early once a result is due) | @@ -74,8 +74,19 @@ In `auto` mode, each data type has its own time-to-live: | Quotes | never cached | So a second `portfolio` run the same day is instant and network-free -without any flag. For the full rationale, see -[Caching and data freshness](../explanation/caching.md). +without any flag. + +Dividends and earnings are the exception to "the clock decides". For +those, once the clock says an entry is still fresh, a second check asks +whether it is nonetheless *incomplete* -- for dividends, whether a payment +the symbol's own schedule predicts is missing from it. An ETF's amount is +only knowable for a few days before the cash arrives, so a fixed expiry +can miss the whole event. A third check stops the second one asking the +provider more than once every 12 hours. The dividend clock itself is 14 +days when the schedule is known and 6 days when it is not. + +For the full rationale and a worked example, see +[Caching and data freshness](../explanation/caching.md#dividends-a-three-level-freshness-check). ## Inspecting and clearing the cache diff --git a/docs/reference/providers.md b/docs/reference/providers.md index 8ba6d6a..8a7bd11 100644 --- a/docs/reference/providers.md +++ b/docs/reference/providers.md @@ -10,25 +10,26 @@ how keys are configured see ## Summary -| Data | Provider | Auth | Free-tier limit | Cache TTL | -|-----------------------|------------------|-------------------------------|------------------------------------------------|------------| -| Daily candles (OHLCV) | Tiingo | `TIINGO_API_KEY` | 1,000/day, 50/hour (free); 10,000/hour (Power) | ~24h | -| Real-time quotes | Yahoo | none | unofficial | never | -| Real-time IEX quotes | Tiingo | `TIINGO_API_KEY` | any tier (1 req per connect/batch) | never | -| Quote fallback | TwelveData | `TWELVEDATA_API_KEY` | 8/min, 800/day | never | -| Dividends | Polygon | `POLYGON_API_KEY` | 5/min | 6 days\* | -| Splits | Polygon | `POLYGON_API_KEY` | 5/min | 14 days | -| Options chains | CBOE | none | ~30/min (self-imposed) | 1 hour | -| Earnings | FMP | `FMP_API_KEY` | 250 req/day | 30 days | -| ETF profiles | SEC EDGAR | `ZFIN_USER_EMAIL` | 10/sec | ~90 days | -| Classification | Wikidata + EDGAR | `ZFIN_USER_EMAIL` | no daily quota | long-lived | -| CUSIP lookup | OpenFIGI | `OPENFIGI_API_KEY` (optional) | higher with key | indefinite | +| Data | Provider | Auth | Free-tier limit | Cache TTL | +|-----------------------|------------------|-------------------------------|------------------------------------------------|-------------| +| Daily candles (OHLCV) | Tiingo | `TIINGO_API_KEY` | 1,000/day, 50/hour (free); 10,000/hour (Power) | ~24h | +| Real-time quotes | Yahoo | none | unofficial | never | +| Real-time IEX quotes | Tiingo | `TIINGO_API_KEY` | any tier (1 req per connect/batch) | never | +| Quote fallback | TwelveData | `TWELVEDATA_API_KEY` | 8/min, 800/day | never | +| Dividends | Polygon | `POLYGON_API_KEY` | 5/min | 6 or 14 d\* | +| Splits | Polygon | `POLYGON_API_KEY` | 5/min | 14 days | +| Options chains | CBOE | none | ~30/min (self-imposed) | 1 hour | +| Earnings | FMP | `FMP_API_KEY` | 250 req/day | 30 days | +| ETF profiles | SEC EDGAR | `ZFIN_USER_EMAIL` | 10/sec | ~90 days | +| Classification | Wikidata + EDGAR | `ZFIN_USER_EMAIL` | no daily quota | long-lived | +| CUSIP lookup | OpenFIGI | `OPENFIGI_API_KEY` (optional) | higher with key | indefinite | -\* The dividend TTL is only a backstop. Cached dividends also re-fetch -whenever a distribution the symbol's own schedule says is due has not -arrived, because an ETF's amount is not published until roughly its -ex-date and it pays within a week -- see -[Caching and data freshness](../explanation/caching.md). +\* Dividends get three freshness checks rather than one. The expiry time +shown here is only the first; when it says an entry is still fresh, a +second check asks whether a payment the symbol's schedule predicts is +missing from it, and a third limits how often that can hit Polygon. 14 +days when the schedule is known, 6 days when it is not. See +[Caching and data freshness](../explanation/caching.md#dividends-a-three-level-freshness-check). ## Where to get a key diff --git a/src/cache/store.zig b/src/cache/store.zig index 08e6587..7fdc52e 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -35,27 +35,44 @@ pub const Ttl = struct { const s_per_day = std.time.s_per_day; /// Historical candles older than 1 day never expire pub const candles_historical: i64 = -1; // infinite - /// Dividend data. Six days, and the short value is load-bearing: - /// an ETF's distribution is not retrievable until roughly its own - /// ex-date, and it pays 1-7 days later, so the entire window in - /// which a new record exists but the cash has not yet landed is - /// about a week. A 14-day TTL straddles that window whole - a cache - /// written days before the ex-date stays "fresh" until well after - /// the payment, and the distribution is invisible for the run that - /// needed it. + /// How long a dividend cache entry stays fresh when we DON'T know the + /// symbol's payment schedule. Six days. /// - /// Quarter-end clustering is what makes it systematic rather than - /// unlucky: a dozen funds go ex within the same three days, so one - /// refresh pass writes a dozen entries onto the same expiry shelf - /// and every one of them straddles the next quarter's payment. + /// This is the cautious case. With no schedule to predict from, the + /// clock is the only thing that will ever make us look again, so it + /// has to tick faster than the weekly review - otherwise a newly + /// bought holding's first distribution can be paid and spent before + /// anything notices. Six days is the largest value that still + /// guarantees it: the +/-11% jitter tops out at 6.66 days, which is + /// under 7. Seven days would reach 7.77 and could stay "fresh" + /// straight through a review. /// - /// This is the BACKSTOP, not the mechanism. `dividendsNeedRefresh` - /// in service.zig does the real work by asking whether a scheduled - /// distribution has come due; the TTL covers the three cases that - /// predicate cannot see - a holding too new to have a cadence, an - /// off-cycle special, and a symbol with fewer than three cached - /// records. + /// It is also what `DataType.dividends.ttl()` returns, which is what + /// `writeSupplement` falls back to when there is no existing file - + /// i.e. the first time we ever see a symbol. That is exactly the case + /// this value is for. + /// + /// See `dividends_scheduled` for the case where we DO know the + /// schedule, and `dividendsNeedRefresh` in service.zig for how the + /// schedule is worked out. pub const dividends: i64 = 6 * s_per_day; + + /// How long a dividend cache entry stays fresh when we DO know the + /// symbol's payment schedule. Fourteen days. + /// + /// Long on purpose. Once a symbol has enough history to establish a + /// cadence, `dividendsNeedRefresh` predicts when the next payment is + /// due and asks the provider then - so the clock is no longer how we + /// find out. Making it short as well would re-check every symbol + /// every few days to protect against a problem the prediction has + /// already solved. + /// + /// What the clock still covers here is the one thing a schedule + /// cannot predict: an off-cycle special distribution. That can be up + /// to 14 days late being noticed, which is the deliberate price of + /// not re-checking every symbol constantly. + pub const dividends_scheduled: i64 = 14 * s_per_day; + /// Split data refreshes biweekly. Deliberately NOT shortened /// alongside `dividends`: splits are announced weeks to months /// ahead of their effective date, so a forward-looking record is @@ -249,16 +266,16 @@ pub const DataType = enum { /// /// Jitter assignments: /// - /// - 11% on dividends (6d base, ~1.3d total spread) and splits - /// (14d base, ~3d). The two no longer share a base - see - /// `Ttl.dividends` for why only dividends were shortened - but - /// they keep the same percentage because the goal is the same: - /// a daily cron should see a portfolio's symbols expire across - /// several days instead of all in lockstep. Jitter is only a - /// load-spreading measure here, never a correctness one: - /// `dividendsNeedRefresh` is what guarantees a due - /// distribution is seen, and a run that depended on jitter - /// landing favourably would be relying on luck. + /// - 11% on dividends and splits. Dividends have two base values + /// (6d unscheduled / 14d scheduled - see `Ttl.dividends`) and + /// splits one (14d); all three keep the same percentage, because + /// the goal is the same: a daily cron should see a portfolio's + /// symbols expire across several days rather than all at once. + /// + /// Jitter is a load-spreading measure only, never a correctness + /// one. `dividendsNeedRefresh` is what guarantees a due payment + /// is noticed; a run that depended on jitter landing favourably + /// would be relying on luck. /// /// - 8% on the longer-TTL types (classification 90d, /// etf_metrics 90d, entity_facts 30d, earnings 30d, @@ -3408,20 +3425,24 @@ test "TTL constants are reasonable" { // Latest candles use a market-aware expiry computed per-write by // market.nextCandleExpiry (no fixed TTL constant here anymore). - // Dividends refresh every six days, splits biweekly. The two used - // to share a 14-day constant; they were deliberately split apart - // because an ETF distribution is only retrievable for the few days - // between its ex-date and its payment, while a split is announced - // weeks ahead. See `Ttl.dividends`. + // Dividends have two values depending on whether we know the + // symbol's payment schedule; splits have one. See `Ttl.dividends`. try std.testing.expectEqual(@as(i64, 6 * std.time.s_per_day), Ttl.dividends); + try std.testing.expectEqual(@as(i64, 14 * std.time.s_per_day), Ttl.dividends_scheduled); try std.testing.expectEqual(@as(i64, 14 * std.time.s_per_day), Ttl.splits); - // The relationship, not just the values: a dividend TTL longer than - // the ex-to-pay window is what made distributions invisible for the - // run that needed them. Every observed ETF pays within 7 days of - // going ex, so this bound is the property the constant exists to - // satisfy - if someone lengthens it, this fails and says why. - try std.testing.expect(Ttl.dividends <= 7 * std.time.s_per_day); + // THE PROPERTY, not just the value. When we don't know a symbol's + // schedule the clock is the only thing that will make us look again, + // so it has to expire before the next weekly review - including at + // the top of the jitter band. 6 days * 1.11 = 6.66, under 7. At 7 + // days the band reaches 7.77 and an entry could stay fresh straight + // through a review, which is the bug this guards against. + // + // Deliberately NOT asserted for `dividends_scheduled`: there the + // prediction finds due payments, so a long clock is correct. + const jitter = DataType.dividends.ttl().jitter_pct; + const worst_case = Ttl.dividends + @divFloor(Ttl.dividends * @as(i64, jitter), 100); + try std.testing.expect(worst_case < 7 * std.time.s_per_day); // Options refresh hourly try std.testing.expectEqual(@as(i64, std.time.s_per_hour), Ttl.options); diff --git a/src/service.zig b/src/service.zig index e9d0125..c16c8a0 100644 --- a/src/service.zig +++ b/src/service.zig @@ -489,38 +489,165 @@ pub const DataService = struct { return cache.Store.init(self.io, self.allocator, self.config.cache_dir); } - /// Whether a FRESH cache read may be served as-is. Returns false - - /// having already freed `data` - when the smart-refresh hook says the - /// entry is fresh but incomplete. + /// LEVEL 3 of the three-level freshness check: the shortest gap + /// between two provider asks for the same symbol, when the ask is + /// driven by the Level 2 completeness check rather than by the Level 1 + /// expiry clock. Applies to every type with a `needsRefresh` hook - + /// dividends and earnings today. + /// + /// WHY THIS EXISTS. When a hook says something is overdue, we ask the + /// provider. Often the provider does not have it yet - that is the + /// normal case, not a failure, because an ETF publishes its amount + /// around its own ex-date and FMP posts an actual some hours after the + /// report. The answer is "nothing new", and the cache gets rewritten + /// with a fresh clock but the SAME records. So the hook still says + /// "overdue", and the very next command asks again. + /// + /// Without a floor that is one provider call per command, not per day. + /// A run touching fifteen symbols in their due windows makes fifteen + /// calls, and the next run ninety seconds later makes fifteen more. At + /// Polygon's four-per-minute limit that is four minutes of waiting, + /// repeated. + /// + /// WHY TWELVE HOURS. The floor is also the worst case for how late we + /// notice something, so it has to stay well inside the shortest gap + /// between an event and the money moving. For dividends the tightest + /// real ex-to-pay gap is one day (QTUM), so twelve hours leaves half a + /// day of margin. Longer buys almost nothing: asking once a day is + /// already far more often than the weekly reconcile that uses the + /// answer. + const smart_refresh_recheck_interval: i64 = 12 * std.time.s_per_hour; + + /// LEVELS 2 AND 3 of the three-level freshness check. Returns true to + /// serve the cached entry as-is; returns false - having already freed + /// `data` - to discard it and refetch. + /// + /// ── The three levels, and why the order reads backwards ────── + /// + /// Level 1 expiry clock "Is this entry old?" + /// Level 2 completeness check "Is it missing something it should + /// already have?" + /// Level 3 recheck floor "Did we just ask the provider?" + /// + /// LEVEL 2 IS A SECOND OPINION ON A "STILL FRESH" VERDICT, not an + /// extra gate in front of a refresh. That is the thing to get right, + /// because the intuition runs the other way. + /// + /// This function is only ever called on a FRESH entry, because + /// `fetchCached` reaches it through `Store.read(.fresh_only)` - which + /// returns null when Level 1 says the entry is stale. So when Level 1 + /// says "stale", Levels 2 and 3 never run at all; a refetch is already + /// happening and they have nothing to contribute. Level 2 only earns + /// its keep in the opposite case: Level 1 says "this is fine, serve + /// it", and Level 2 gets to answer "no it is not - a payment has + /// happened that this entry does not contain". + /// + /// Level 3 then exists to stop Level 2 being too eager, and can + /// overrule it back to "serve the cache". + /// + /// `skip_network` short-circuits between Levels 1 and 2: offline mode + /// never refetches, so discarding a usable entry could only turn a + /// served result into a failure. /// /// One function rather than two inline blocks because `fetchCached` /// has two fresh-cache read sites (local, and post-server-sync) and /// the rule must be identical at both. It owns the free so a caller /// cannot decide to discard and then leak. - /// - /// Suppressed under `skip_network`: offline mode never refetches, so - /// discarding a usable entry could only turn a served result into a - /// failure. Same rule `getEarnings` applies. fn serveFreshOrDiscard( self: *DataService, comptime T: type, comptime needsRefresh: ?*const fn ([]const T, Date) bool, - data: []const T, + comptime ttlFor: ?*const fn ([]const T) cache.TtlSpec, + symbol: []const u8, + cached: cache.Store.CacheResult(T), skip_network: bool, ) bool { if (needsRefresh) |hook| { + if (skip_network) return true; // wall-clock required: the hook asks whether a scheduled // event has come due, which is a question about the actual // current day. Threading `today` down would put a date // parameter on four public getters for one type's benefit. - if (!skip_network and hook(data, fmt.todayDate(self.io))) { - T.freeSlice(self.allocator, data); - return false; + if (!hook(cached.data, fmt.todayDate(self.io))) return true; + if (self.askedRecently(T, ttlFor, symbol, cached)) { + log.debug("{s}: {s} is overdue but we asked within the last {d}h; serving cache", .{ symbol, @tagName(comptime cache.Store.dataTypeFor(T)), @divFloor(smart_refresh_recheck_interval, std.time.s_per_hour) }); + return true; } + T.freeSlice(self.allocator, cached.data); + return false; } return true; } + /// LEVEL 3 of the three-level freshness check: did we already ask the + /// provider about this symbol inside `smart_refresh_recheck_interval`? + /// + /// Only reached when Level 2 has already said the entry is incomplete, + /// so a true answer here means "incomplete, but asking again this soon + /// would be wasted" - see `serveFreshOrDiscard` for the full ordering. + /// + /// WHY `#!expires=` AND NOT `#!created=`. Both directives sit in the + /// same file and `created` looks like the obvious choice, but it is + /// the wrong one. `serializeWithMeta` re-stamps `created` on EVERY + /// write, and dividend rows also arrive as a side effect of a Tiingo + /// candle fetch (`writeSupplement`). So `created` answers "when was + /// this file last touched", which a candle refresh changes without + /// anyone asking Polygon anything. + /// + /// `expires` answers the question we actually want. Only a fetch from + /// the primary provider moves it (`ExpiryPolicy.bump`); a supplement + /// deliberately puts the old value back (`.preserve`). So it marks the + /// last time we really asked. + /// + /// Recovering the write time from it is exact rather than approximate, + /// because `computeExpires` derives its jitter from a hash of the + /// symbol - the same symbol always gets the same offset. Passing a + /// zero "now" therefore returns precisely the TTL that was applied, + /// and subtracting it from `expires` gives the write time to the + /// second. + /// + /// Two cases fall back to "no, ask again", which is the safe + /// direction: + /// - No `#!expires=` in the file at all. + /// - The symbol crossed from unknown-schedule to known-schedule + /// since it was written, so it was stored under the short TTL and + /// we recompute with the long one. That makes the write look + /// older than it was, so we ask sooner than strictly needed. + fn askedRecently( + self: *DataService, + comptime T: type, + comptime ttlFor: ?*const fn ([]const T) cache.TtlSpec, + symbol: []const u8, + cached: cache.Store.CacheResult(T), + ) bool { + const expires = cached.expires orelse return false; + const spec = if (ttlFor) |f| f(cached.data) else comptime cache.Store.dataTypeFor(T).ttl(); + // `computeExpires` from a zero epoch returns the jittered TTL + // itself, which is what makes this exact - see the doc above. + const applied_ttl = cache.computeExpires(0, spec, symbol); + const written_at = expires - applied_ttl; + // wall-clock required: "recently" is relative to now. + const now_s = std.Io.Timestamp.now(self.io, .real).toSeconds(); + return now_s - written_at < smart_refresh_recheck_interval; + } + + /// The TTL to stamp on a write: from the records when the type + /// supplies a `ttlFor` hook, otherwise from the type alone. + /// + /// Separate from the call sites because `fetchCached` writes in two + /// places (the normal path and the rate-limit retry) and they must + /// agree - a retry that stamped a different clock than the first + /// attempt would be a silently different freshness policy for + /// nobody's benefit. + fn ttlSpecFor( + comptime T: type, + comptime ttlFor: ?*const fn ([]const T) cache.TtlSpec, + items: cache.Store.DataFor(T), + ) cache.TtlSpec { + if (ttlFor) |f| return f(items); + return comptime cache.Store.dataTypeFor(T).ttl(); + } + /// Generic fetch-or-cache for simple data types (dividends, splits, options). /// Checks cache first; on miss, fetches from the appropriate provider, /// writes to cache, and returns. On permanent fetch failure, writes a negative @@ -530,20 +657,35 @@ pub const DataService = struct { /// returns FetchFailed on cache miss without touching the network. /// `opts.force_refresh = true` -> treats cache as stale and fetches. /// - /// `needsRefresh` is the smart-refresh hook: given a FRESH cache - /// entry it answers "is this nonetheless incomplete?". It exists - /// because freshness and completeness are different questions - - /// `dividendsNeedRefresh` documents the case that forced it. Null - /// means TTL is the only gate, which is the behaviour every type had - /// before the hook existed. The hook's branch is comptime-elided for - /// a null argument, so types with no `freeSlice` (Split) stay - /// compilable. + /// `needsRefresh` is LEVEL 2 of the three-level freshness check: given + /// a FRESH cache entry it answers "is this nonetheless incomplete?". + /// It exists because freshness and completeness are different + /// questions - `dividendsNeedRefresh` documents the case that forced + /// it. Null means the expiry clock is the only gate, which is the + /// behaviour every type had before the hook existed. + /// + /// NOTE WHERE IT SITS. Levels 2 and 3 live behind the + /// `Store.read(.fresh_only)` calls below, which return null when Level + /// 1 (the expiry clock) says the entry is stale. So a stale entry + /// bypasses them entirely - they can only ever overrule a "still + /// fresh" verdict, never reinforce a stale one. `serveFreshOrDiscard` + /// has the full ordering. + /// + /// The hook's branch is comptime-elided for a null argument, so types + /// with no `freeSlice` (Split) stay compilable. + /// + /// `ttlFor` sets LEVEL 1's length from the records being written + /// rather than from the type alone. Only dividends need it, because + /// the right clock depends on whether the records establish a payment + /// schedule - see `dividendTtl`. Null means "use `DataType.ttl()`", + /// which is what every other type does. fn fetchCached( self: *DataService, comptime T: type, symbol: []const u8, comptime postProcess: ?*const fn (*T, std.mem.Allocator) anyerror!void, comptime needsRefresh: ?*const fn ([]const T, Date) bool, + comptime ttlFor: ?*const fn ([]const T) cache.TtlSpec, opts_in: FetchOptions, ) DataError!FetchResult(T) { // See `getCandles` - one fold, covering every type routed through here. @@ -556,7 +698,7 @@ pub const DataService = struct { // returns cached even if stale, never touches the network. if (!opts.force_refresh) { if (s.read(self.allocator, T, symbol, postProcess, .fresh_only)) |cached| { - if (self.serveFreshOrDiscard(T, needsRefresh, cached.data, opts.skip_network)) { + if (self.serveFreshOrDiscard(T, needsRefresh, ttlFor, symbol, cached, opts.skip_network)) { log.debug("{s}: {s} fresh in local cache", .{ symbol, @tagName(data_type) }); return .{ .data = cached.data, .source = .cached, .timestamp = cached.timestamp, .allocator = self.allocator }; } @@ -585,7 +727,7 @@ pub const DataService = struct { // server DID have the newer record - that is the tier's // reason for existing, and we return without spending a // provider request. - if (self.serveFreshOrDiscard(T, needsRefresh, cached.data, opts.skip_network)) { + if (self.serveFreshOrDiscard(T, needsRefresh, ttlFor, symbol, cached, opts.skip_network)) { log.debug("{s}: {s} synced from server and fresh", .{ symbol, @tagName(data_type) }); return .{ .data = cached.data, .source = .cached, .timestamp = cached.timestamp, .allocator = self.allocator }; } @@ -603,7 +745,7 @@ pub const DataService = struct { log.warn("{s}: {s} fetch failed after rate-limit retry: {t}", .{ symbol, @tagName(data_type), retry_err }); return DataError.FetchFailed; }; - s.writeWithSource(T, symbol, retried, data_type.ttl(), sourceHintFor(T)); + s.writeWithSource(T, symbol, retried, ttlSpecFor(T, ttlFor, retried), sourceHintFor(T)); return .{ .data = retried, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } // A MISSING KEY IS A CONFIGURATION FAULT, not a data fault, @@ -638,7 +780,7 @@ pub const DataService = struct { return DataError.FetchFailed; }; - s.writeWithSource(T, symbol, fetched, data_type.ttl(), sourceHintFor(T)); + s.writeWithSource(T, symbol, fetched, ttlSpecFor(T, ttlFor, fetched), sourceHintFor(T)); return .{ .data = fetched, .source = .fetched, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } @@ -1464,13 +1606,13 @@ pub const DataService = struct { /// Fetch dividend history for a symbol. /// - /// Carries a schedule-aware refresh hook (`dividendsNeedRefresh`) - /// because a fresh dividend cache can still be *incomplete*: an - /// ETF's distribution is not retrievable until roughly its own - /// ex-date and pays within a week of it, so TTL alone cannot - /// guarantee the record is seen before the cash lands. + /// The only type that uses both extra hooks, and for one reason: a + /// dividend cache can be fresh and still be missing a payment that + /// has already happened. `dividendsNeedRefresh` notices that from the + /// symbol's own payment schedule, and `dividendTtl` sets the clock + /// according to whether that schedule is known. pub fn getDividends(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!FetchResult(Dividend) { - return self.fetchCached(Dividend, symbol, null, dividendsNeedRefresh, opts); + return self.fetchCached(Dividend, symbol, null, dividendsNeedRefresh, dividendTtl, opts); } /// Fetch split history for a symbol. @@ -1478,12 +1620,12 @@ pub const DataService = struct { /// No refresh hook: splits are announced weeks to months ahead, so /// the forward-looking record is cached long before it matters. pub fn getSplits(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!FetchResult(Split) { - return self.fetchCached(Split, symbol, null, null, opts); + return self.fetchCached(Split, symbol, null, null, null, opts); } /// Fetch options chain for a symbol (all expirations, no API key needed). pub fn getOptions(self: *DataService, symbol: []const u8, opts: FetchOptions) DataError!FetchResult(OptionsChain) { - return self.fetchCached(OptionsChain, symbol, null, null, opts); + return self.fetchCached(OptionsChain, symbol, null, null, null, opts); } /// Days after an expected ex-date during which a still-absent @@ -1618,6 +1760,35 @@ pub const DataService = struct { return today.days - expected.days <= dividend_chase_days; } + /// LEVEL 1's length for dividends, chosen from the records themselves + /// rather than fixed per type. + /// + /// Two clocks, because the clock is doing a different job in each + /// case: + /// + /// - **Schedule known** (a cadence can be worked out). The + /// prediction is what finds a due payment, so the clock only has + /// to cover what a schedule cannot predict - an off-cycle special + /// distribution. Fourteen days. + /// + /// - **Schedule unknown** (fewer than three records, or no usable + /// cadence - typically a newly bought holding). There is nothing + /// to predict from, so the clock is the ONLY thing that will ever + /// make us look again. Six days, which guarantees the entry is + /// stale by the next weekly review. + /// + /// Sizing them separately is the whole point. One short clock for + /// everything would re-check long-established payers every few days + /// to guard against a problem they do not have; one long clock would + /// let a new holding's first payment go unnoticed for a fortnight. + fn dividendTtl(divs: []const Dividend) cache.TtlSpec { + const known = dividendCadenceDays(divs) != null; + const base = if (known) cache.Ttl.dividends_scheduled else cache.Ttl.dividends; + // Jitter policy stays with the rest of it in `DataType.ttl()`; + // only the base differs here. + return .{ .seconds = base, .jitter_pct = comptime cache.DataType.dividends.ttl().jitter_pct }; + } + /// Days after an earnings report date during which a still-missing /// `actual` is worth chasing with a re-fetch. Past this window the /// gap is treated as permanent (FMP won't backfill it; earnings has @@ -1677,7 +1848,7 @@ pub const DataService = struct { if (market.classify(symbol) == .mutual_fund) { return .{ .data = &.{}, .source = .cached, .timestamp = std.Io.Timestamp.now(self.io, .real).toSeconds(), .allocator = self.allocator }; } - return self.fetchCached(EarningsEvent, symbol, earningsPostProcess, earningsNeedsRefreshHook, opts); + return self.fetchCached(EarningsEvent, symbol, earningsPostProcess, earningsNeedsRefreshHook, null, opts); } /// Fetch ETF profile for a symbol. Assembles a unified @@ -4752,6 +4923,31 @@ fn corpusAged(today: Date, age_days: i32) [6]Dividend { return quarterlySeries(6, today.addDays(-age_days), 91); } +/// Write a cache entry as though the write had happened `age_s` ago. +/// +/// Needed because `askedRecently` recovers the write time from +/// `#!expires=` minus the TTL that was applied. A freshly written entry +/// therefore always looks like "we just asked", which suppresses the +/// refresh hook - correct in production, useless in a test that wants to +/// exercise the hook. Back-dating the expires by the TTL is the only way +/// to make a test entry look old without sleeping. +/// +/// `spec` must be the TTL the production path would have used for this +/// data, since that is what `askedRecently` will recompute. +fn writeAged( + comptime T: type, + store: *cache.Store, + symbol: []const u8, + items: cache.Store.DataFor(T), + spec: cache.TtlSpec, + age_s: i64, +) void { + // What the production write would have stamped, jitter included. + const applied = cache.computeExpires(0, spec, symbol); + // Exact expires, no jitter of our own, so the recompute lines up. + store.writeWithSource(T, symbol, items, .{ .seconds = applied - age_s }, null); +} + test "fetchCached hook: a null hook leaves the fresh-cache path untouched" { // The regression guard for every other type. Splits pass no hook and // have no `freeSlice`; if the hook branch were not comptime-elided @@ -4855,7 +5051,11 @@ test "fetchCached hook: an overdue distribution discards the fresh entry and ref const today = fmt.todayDate(io); var divs = corpusAged(today, 95); var store = svc.store(); - store.write(Dividend, "TEST", divs[0..], cache.DataType.dividends.ttl()); + // Aged two days: a just-written entry looks like "we already asked" + // to the recheck floor, which would suppress the refresh. The spec + // must match what production would have stamped (`dividendTtl`), since + // that is what the floor recomputes. + writeAged(Dividend, &store, "TEST", divs[0..], DataService.dividendTtl(divs[0..]), 2 * std.time.s_per_day); // Sanity: the entry really is fresh, so TTL alone would have served // it. That is the failure this replaces. @@ -5039,10 +5239,14 @@ test "getEarnings: smart refresh survives the move onto fetchCached" { // A report three days ago with no actual yet -> inside the chase // window -> must refetch, which with no FMP key surfaces as NoApiKey. + // + // Aged deliberately: a just-written entry looks like "we already + // asked" to the recheck floor, which would suppress the very refresh + // this test is about. var pending = [_]EarningsEvent{ .{ .symbol = "TEST", .date = today.addDays(-3), .estimate = 1.5 }, }; - store.write(EarningsEvent, "TEST", pending[0..], cache.DataType.earnings.ttl()); + writeAged(EarningsEvent, &store, "TEST", pending[0..], cache.DataType.earnings.ttl(), 2 * std.time.s_per_day); try std.testing.expectError(DataError.NoApiKey, svc.getEarnings("TEST", .{})); // Same report with the actual posted -> nothing outstanding -> served @@ -5063,6 +5267,335 @@ test "getEarnings: smart refresh survives the move onto fetchCached" { try std.testing.expectApproxEqAbs(@as(f64, 0.12), result.data[0].surprise.?, 1e-9); } +// ── The recheck floor, and the TTL that varies by schedule ──────── + +test "recheck floor: a just-asked symbol is not asked again" { + // The bug this fixes. When the schedule says a payment is due but the + // provider has nothing yet, the cache is rewritten with a fresh clock + // and the SAME records - so the schedule still says "due", and without + // a floor the next command asks again. Here the entry is written and + // immediately read back, which is exactly that situation. + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var svc = DataService.init(io, allocator, Config{ .cache_dir = dir_path }); + defer svc.deinit(); + + const today = fmt.todayDate(io); + var divs = corpusAged(today, 95); + // Confirm the schedule really does say "overdue", so the only thing + // that can stop the refetch is the floor. + try std.testing.expect(DataService.dividendsNeedRefresh(divs[0..], today)); + + var store = svc.store(); + store.write(Dividend, "TEST", divs[0..], DataService.dividendTtl(divs[0..])); + + // Reaching the provider would panic. Being served proves the floor held. + svc.panic_on_network_attempt = true; + const result = try svc.getDividends("TEST", .{}); + defer result.deinit(); + try std.testing.expectEqual(@as(usize, 6), result.data.len); + try std.testing.expectEqual(Source.cached, result.source); +} + +test "recheck floor: once the interval has passed, the symbol is asked again" { + // The other side of the floor. Same overdue corpus, but the entry was + // written longer ago than the interval, so the refetch proceeds. + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var svc = DataService.init(io, allocator, Config{ .cache_dir = dir_path }); + defer svc.deinit(); + + const today = fmt.todayDate(io); + var divs = corpusAged(today, 95); + var store = svc.store(); + const spec = DataService.dividendTtl(divs[0..]); + + // One second inside the interval -> still served. + writeAged(Dividend, &store, "TEST", divs[0..], spec, DataService.smart_refresh_recheck_interval - 1); + { + svc.panic_on_network_attempt = true; + const held = try svc.getDividends("TEST", .{}); + defer held.deinit(); + try std.testing.expectEqual(Source.cached, held.source); + svc.panic_on_network_attempt = false; + } + + // One second past it -> asked again (NoApiKey proves we got to the + // provider rather than being served the cache). + writeAged(Dividend, &store, "TEST", divs[0..], spec, DataService.smart_refresh_recheck_interval + 1); + try std.testing.expectError(DataError.NoApiKey, svc.getDividends("TEST", .{})); +} + +test "recheck floor: a Tiingo supplement does not count as having asked" { + // WHY THE FLOOR READS `#!expires=` AND NOT `#!created=`. + // + // Dividend rows also arrive as a side effect of a Tiingo candle fetch, + // via `writeSupplement`. That rewrites the file - so `#!created=` + // moves - but it deliberately puts the old `#!expires=` back, because + // only the primary provider owns the freshness clock. + // + // If the floor keyed off `created`, any candle refresh would look like + // "we just asked Polygon" and suppress a genuinely due refetch for + // twelve hours. Keying off `expires` is immune. + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var svc = DataService.init(io, allocator, Config{ .cache_dir = dir_path }); + defer svc.deinit(); + + const today = fmt.todayDate(io); + var divs = corpusAged(today, 95); + var store = svc.store(); + + // Asked two days ago, so the floor has expired. + writeAged(Dividend, &store, "TEST", divs[0..], DataService.dividendTtl(divs[0..]), 2 * std.time.s_per_day); + + // Now a candle fetch supplements the same file, moving `created` to + // now and leaving `expires` alone. + store.writeSupplement(Dividend, "TEST", divs[0..], "tiingo"); + + // Still asked. If this returns cached data, the floor is reading the + // wrong directive. + try std.testing.expectError(DataError.NoApiKey, svc.getDividends("TEST", .{})); +} + +test "recheck floor: an entry with no expires directive is asked again" { + // Fails safe. A file carrying no `#!expires=` gives the floor nothing + // to measure from, and guessing "recently" would strand the symbol. + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var svc = DataService.init(io, allocator, Config{ .cache_dir = dir_path }); + defer svc.deinit(); + + const today = fmt.todayDate(io); + var divs = corpusAged(today, 95); + try std.testing.expect(svc.askedRecently(Dividend, DataService.dividendTtl, "TEST", .{ + .data = divs[0..], + .timestamp = 0, + .expires = null, + }) == false); +} + +test "recheck floor: the write time is recovered exactly, jitter and all" { + // The floor subtracts the applied TTL from `#!expires=` to get the + // write time. That is only exact because `computeExpires` derives its + // jitter from a hash of the symbol, so the same symbol always gets the + // same offset and it can be recomputed later. + // + // Checked across several symbols because a jitter bug would show up as + // a per-symbol discrepancy, not a uniform one. + for ([_][]const u8{ "TESTA", "TESTB", "TESTC", "TESTD" }) |sym| { + const spec = cache.DataType.dividends.ttl(); + const applied = cache.computeExpires(0, spec, sym); + // Jitter must actually be doing something, or this proves nothing. + try std.testing.expect(applied != cache.Ttl.dividends or spec.jitter_pct == 0); + + const now_s: i64 = 1_800_000_000; + const expires = cache.computeExpires(now_s, spec, sym); + try std.testing.expectEqual(now_s, expires - applied); + } +} + +test "dividendTtl: six days when the schedule is unknown, fourteen when it is known" { + // The two clocks, and the reason they differ. A symbol with a + // established cadence is found by prediction, so its clock can be + // long; one without is found only by the clock, so it must be short + // enough to expire before the next weekly review. + const today = Date.fromYmd(2026, 9, 19); + + // Five years of quarterly history -> schedule known -> long clock. + const established = quarterlySeries(6, today.addDays(-30), 91); + try std.testing.expectEqual(cache.Ttl.dividends_scheduled, DataService.dividendTtl(&established).seconds); + + // Two records -> no cadence can be inferred -> short clock. + const sparse = divsFromYmd(.{ .{ 2026, 6, 15 }, .{ 2026, 3, 17 } }); + try std.testing.expectEqual(cache.Ttl.dividends, DataService.dividendTtl(&sparse).seconds); + + // Nothing at all -> short clock. + const none: []const Dividend = &.{}; + try std.testing.expectEqual(cache.Ttl.dividends, DataService.dividendTtl(none).seconds); + + // Duplicate ex-dates give a zero cadence, which is not a schedule. + const dup = divsFromYmd(.{ .{ 2026, 6, 15 }, .{ 2026, 6, 15 }, .{ 2026, 6, 15 } }); + try std.testing.expectEqual(cache.Ttl.dividends, DataService.dividendTtl(&dup).seconds); + + // Both keep the shared jitter policy. + const jitter = cache.DataType.dividends.ttl().jitter_pct; + try std.testing.expectEqual(jitter, DataService.dividendTtl(&established).jitter_pct); + try std.testing.expectEqual(jitter, DataService.dividendTtl(&sparse).jitter_pct); +} + +test "ttlSpecFor: the hook wins when present, the type default otherwise" { + // The one-line dispatch that decides which clock a write gets. Only + // reachable in production after a successful provider fetch, so it is + // exercised directly here rather than left to a live-network path no + // test can take. + const today = Date.fromYmd(2026, 9, 19); + var established = quarterlySeries(6, today.addDays(-30), 91); + + // With the dividend hook: the schedule is known, so the long clock. + try std.testing.expectEqual( + cache.Ttl.dividends_scheduled, + DataService.ttlSpecFor(Dividend, DataService.dividendTtl, established[0..]).seconds, + ); + + // Without a hook, the same records get the type default - which is the + // cautious short value. This is what every other type sees. + try std.testing.expectEqual( + cache.Ttl.dividends, + DataService.ttlSpecFor(Dividend, null, established[0..]).seconds, + ); + + // A type that has no hook at all. + var splits = [_]Split{.{ .date = Date.fromYmd(2024, 3, 7), .numerator = 3, .denominator = 1 }}; + try std.testing.expectEqual( + cache.Ttl.splits, + DataService.ttlSpecFor(Split, null, splits[0..]).seconds, + ); +} + +test "dividendTtl: the chosen clock is what actually lands on disk" { + // `dividendTtl` returning the right number is worth nothing if the + // write path ignores it, so this reads `#!expires=` back off the file. + const allocator = std.testing.allocator; + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator); + defer allocator.free(dir_path); + + var store = cache.Store.init(io, allocator, dir_path); + const today = fmt.todayDate(io); + const now_s = std.Io.Timestamp.now(io, .real).toSeconds(); + + // Established payer -> ~14 days. + var established = quarterlySeries(6, today.addDays(-30), 91); + store.write(Dividend, "TESTA", established[0..], DataService.dividendTtl(&established)); + const long_expires = readExpires(io, allocator, dir_path, "TESTA") orelse return error.TestUnexpectedResult; + try std.testing.expectEqual( + cache.computeExpires(0, DataService.dividendTtl(&established), "TESTA"), + long_expires - now_s, + ); + + // Sparse history -> ~6 days. + var sparse = divsFromYmd(.{ .{ 2026, 6, 15 }, .{ 2026, 3, 17 } }); + store.write(Dividend, "TESTB", sparse[0..], DataService.dividendTtl(&sparse)); + const short_expires = readExpires(io, allocator, dir_path, "TESTB") orelse return error.TestUnexpectedResult; + try std.testing.expectEqual( + cache.computeExpires(0, DataService.dividendTtl(&sparse), "TESTB"), + short_expires - now_s, + ); + + // And the long one really is longer, so nobody can "simplify" the two + // constants into one without this failing. + try std.testing.expect(long_expires > short_expires); +} + +test "dividendTtl: a supplement to an unseen symbol gets the cautious clock" { + // `writeSupplement` has no records to reason about when the file does + // not exist yet, so it falls back to `DataType.dividends.ttl()`. That + // must be the SHORT value, because a symbol we have never seen before + // is precisely the unknown-schedule case. + try std.testing.expectEqual(cache.Ttl.dividends, cache.DataType.dividends.ttl().seconds); +} + +test "full cycle: a quarterly ETF through one whole period" { + // The end-to-end story, on the pattern from the discussion that + // produced this code: ex-dates on 1/1, 4/1, 7/1 and 10/1, five years + // of history, newest cached record 2026-07-01. + // + // Median gap is 91 days, so the forecast for the next ex-date is + // 2026-09-30 - one day early, because Jul->Oct is actually 92. Being + // early is harmless; the window is fourteen days wide. + const newest = Date.fromYmd(2026, 7, 1); + var divs: [19]Dividend = undefined; + { + var i: usize = 0; + var y: i16 = 2026; + while (y >= 2022) : (y -= 1) { + for ([_]u8{ 10, 7, 4, 1 }) |m| { + const d = Date.fromYmd(y, m, 1); + if (!newest.lessThan(d) and i < divs.len) { + divs[i] = .{ .ex_date = d, .amount = 1.0, .type = .regular }; + i += 1; + } + } + } + try std.testing.expectEqual(divs.len, i); + } + + try std.testing.expectEqual(@as(i32, 91), DataService.dividendCadenceDays(&divs).?); + // Schedule known, so the clock is the long one. + try std.testing.expectEqual(cache.Ttl.dividends_scheduled, DataService.dividendTtl(&divs).seconds); + + // NOT EXPECTING ANYTHING: the whole quarter between payments. + for ([_]Date{ + Date.fromYmd(2026, 7, 2), + Date.fromYmd(2026, 8, 1), + Date.fromYmd(2026, 9, 1), + Date.fromYmd(2026, 9, 29), + }) |d| { + try std.testing.expect(!DataService.dividendsNeedRefresh(&divs, d)); + } + + // OVERDUE BY PREDICTION: from the forecast date to fourteen days on. + for ([_]Date{ + Date.fromYmd(2026, 9, 30), + Date.fromYmd(2026, 10, 1), + Date.fromYmd(2026, 10, 7), + Date.fromYmd(2026, 10, 14), + }) |d| { + try std.testing.expect(DataService.dividendsNeedRefresh(&divs, d)); + } + + // GAVE UP: past the window the schedule is treated as wrong and the + // clock takes over, so we stop asking every run. + try std.testing.expect(!DataService.dividendsNeedRefresh(&divs, Date.fromYmd(2026, 10, 15))); + + // The 10/1 record arrives. Asking stops immediately... + var with_oct: [20]Dividend = undefined; + with_oct[0] = .{ .ex_date = Date.fromYmd(2026, 10, 1), .amount = 1.0, .type = .regular }; + @memcpy(with_oct[1..], &divs); + try std.testing.expect(!DataService.dividendsNeedRefresh(&with_oct, Date.fromYmd(2026, 10, 1))); + try std.testing.expect(!DataService.dividendsNeedRefresh(&with_oct, Date.fromYmd(2026, 12, 1))); + + // ...and the forecast self-corrects: the newest gaps now median to 92, + // which lands the next expected ex-date exactly on 2027-01-01. + try std.testing.expectEqual(@as(i32, 92), DataService.dividendCadenceDays(&with_oct).?); + try std.testing.expect(DataService.dividendsNeedRefresh(&with_oct, Date.fromYmd(2027, 1, 1))); +} + +/// Read the raw `#!expires=` directive off a cached dividend file. +fn readExpires(io: std.Io, allocator: std.mem.Allocator, dir_path: []const u8, symbol: []const u8) ?i64 { + const path = std.fs.path.join(allocator, &.{ dir_path, symbol, "dividends.srf" }) catch return null; + defer allocator.free(path); + const data = std.Io.Dir.cwd().readFileAlloc(io, path, allocator, .limited(1024 * 1024)) catch return null; + defer allocator.free(data); + var reader = std.Io.Reader.fixed(data); + var it = srf.iterator(&reader, allocator, .{ .parse_allocator = .none }) catch return null; + defer it.deinit(); + return it.expires; +} + test "getEarnings: skip_network suppresses the smart refresh" { // Offline mode must serve the incomplete-but-fresh entry rather than // discard it and fail. Same rule the dividend hook follows, and it now