Compare commits
6 commits
c15ea92b0d
...
18fc52383c
| Author | SHA1 | Date | |
|---|---|---|---|
| 18fc52383c | |||
| 5ae4065d9e | |||
| 0dbac7d416 | |||
| 65fefdb566 | |||
| f39133819f | |||
| ff11940a88 |
18 changed files with 1040 additions and 52 deletions
|
|
@ -13,8 +13,8 @@
|
||||||
.hash = "z2d-0.11.0-j5P_HtLzDwBGyQt49DrT0v4BuVqI_SRs6CXsuj7eBVhR",
|
.hash = "z2d-0.11.0-j5P_HtLzDwBGyQt49DrT0v4BuVqI_SRs6CXsuj7eBVhR",
|
||||||
},
|
},
|
||||||
.srf = .{
|
.srf = .{
|
||||||
.url = "git+https://git.lerch.org/lobo/srf#4a3e5f00f15b0e0ba79d06ffe69dbcfa052baa5b",
|
.url = "git+https://git.lerch.org/lobo/srf#ea2c35825d652691e6a22526d76e2a06f61d70a5",
|
||||||
.hash = "srf-0.0.0-qZj572nkAQAAz3zEg6fdD8A7PJnQ9je3zCeAOJS5PoZj",
|
.hash = "srf-0.0.0-qZj578QeAgCDjih2ii5soqz02fj3g8OhKwUkFh4ReK56",
|
||||||
},
|
},
|
||||||
.zeit = .{
|
.zeit = .{
|
||||||
.url = "git+https://github.com/rockorager/zeit?ref=v0.9.0#b1c1c2fcbc71fd7799a316bbcf0ff88d06d80ccc",
|
.url = "git+https://github.com/rockorager/zeit?ref=v0.9.0#b1c1c2fcbc71fd7799a316bbcf0ff88d06d80ccc",
|
||||||
|
|
|
||||||
|
|
@ -20,8 +20,8 @@ directly. It reads and writes a per-symbol, per-type SRF file cache via
|
||||||
```
|
```
|
||||||
{cache_dir}/ default ~/.cache/zfin, set by ZFIN_CACHE_DIR
|
{cache_dir}/ default ~/.cache/zfin, set by ZFIN_CACHE_DIR
|
||||||
AAPL/
|
AAPL/
|
||||||
candles_daily.srf OHLCV bars (append-only)
|
candles_daily.srf OHLCV bars (appended; replaced on restatement)
|
||||||
candles_meta.srf last_close, last_date, provider + freshness
|
candles_meta.srf last_close, last_date, provider, adj_basis + freshness
|
||||||
dividends.srf
|
dividends.srf
|
||||||
splits.srf
|
splits.srf
|
||||||
options.srf
|
options.srf
|
||||||
|
|
@ -43,12 +43,14 @@ format. Files carry `#!`-prefixed directives (`#!expires=`,
|
||||||
|
|
||||||
Candles are stored as **two** files, and the split is load-bearing:
|
Candles are stored as **two** files, and the split is load-bearing:
|
||||||
|
|
||||||
- `candles_daily.srf` holds the actual OHLCV records and grows
|
- `candles_daily.srf` holds the actual OHLCV records and normally grows
|
||||||
append-only: on a cache miss only bars newer than `last_date` are
|
by appending: on a cache miss only bars newer than `last_date` are
|
||||||
fetched and appended, never the full history.
|
fetched and appended, not the full history. The exception is a
|
||||||
|
**restatement**, which replaces the file wholesale - see
|
||||||
|
[The adjustment basis](#the-adjustment-basis).
|
||||||
- `candles_meta.srf` holds a single small record (`last_close`,
|
- `candles_meta.srf` holds a single small record (`last_close`,
|
||||||
`last_date`, `provider`, `fail_count`) plus the `#!expires=` and
|
`last_date`, `provider`, `fail_count`, `tiingo_retry_after_s`,
|
||||||
`#!created=` directives.
|
`adj_basis`) plus the `#!expires=` and `#!created=` directives.
|
||||||
|
|
||||||
Keeping the metadata separate lets every freshness check and last-price
|
Keeping the metadata separate lets every freshness check and last-price
|
||||||
read touch a ~100-byte file instead of deserializing a multi-megabyte
|
read touch a ~100-byte file instead of deserializing a multi-megabyte
|
||||||
|
|
@ -61,6 +63,62 @@ self-heal clear both together, and a negative cache entry for a
|
||||||
candle-less symbol is keyed off `candles_daily.srf` (see
|
candle-less symbol is keyed off `candles_daily.srf` (see
|
||||||
[Negative caching](#negative-caching)).
|
[Negative caching](#negative-caching)).
|
||||||
|
|
||||||
|
### The adjustment basis
|
||||||
|
|
||||||
|
Appending bars cannot restate the bars already on disk, and sometimes
|
||||||
|
they need it.
|
||||||
|
|
||||||
|
Providers compute `adj_close` by scaling raw `close` by the product of
|
||||||
|
the adjustment factors for every distribution *after* that bar. So a
|
||||||
|
series' adjustment basis is only as current as the fetch that produced
|
||||||
|
it. Newly appended bars arrive with `adj_close == close`, because
|
||||||
|
nothing has gone ex after them yet, while every previously cached bar
|
||||||
|
keeps the basis it was originally fetched with. When the next
|
||||||
|
distribution goes ex, the bars behind it should be marked down by its
|
||||||
|
factor - and an append does not do that. Every total return spanning
|
||||||
|
that ex-date then reads low by roughly the missed yield.
|
||||||
|
|
||||||
|
`CandleMeta.adj_basis` records how current a series' basis is. It is
|
||||||
|
**the date of the newest bar present at the last full fetch**:
|
||||||
|
|
||||||
|
- Not a wall-clock timestamp. A fetch that runs before the day's bar is
|
||||||
|
published gets a basis one bar behind - which is the honest claim,
|
||||||
|
since a provider's adjusted series is only ever as complete as its
|
||||||
|
newest bar.
|
||||||
|
- Not an ex-date. It is compared against ex-dates but is never one.
|
||||||
|
- Never advanced by an append. `Store.appendCandles` takes the existing
|
||||||
|
meta and overrides only `last_close` / `last_date`; only
|
||||||
|
`Store.cacheCandles`, which replaces the whole file, sets a new
|
||||||
|
basis. That asymmetry is the mechanism, so keep it.
|
||||||
|
|
||||||
|
`getCandles` escalates from append to full refetch when
|
||||||
|
`freshness.adjustmentBasisStale` says the basis predates a corporate
|
||||||
|
action that has **already gone ex**:
|
||||||
|
|
||||||
|
```zig
|
||||||
|
newest_ex = freshness.newestCorporateAction(alloc, store, sym, meta.last_date)
|
||||||
|
stale = freshness.adjustmentBasisStale(meta.adj_basis, newest_ex)
|
||||||
|
```
|
||||||
|
|
||||||
|
The already-ex bound lives in `newestCorporateAction`, not in the
|
||||||
|
verdict, and that placement is load-bearing in both directions. A
|
||||||
|
declared-but-not-yet-ex distribution is reflected in no provider's
|
||||||
|
adjusted series, so treating it as something to catch up to would
|
||||||
|
refetch on every pass forever. But bounding the *verdict* instead -
|
||||||
|
"is the newest action of all still in the future? then nothing to do" -
|
||||||
|
lets one forward announcement hide every older unapplied action behind
|
||||||
|
it. That shipped: a quarterly payer announcing a quarter ahead was
|
||||||
|
permanently unable to restate.
|
||||||
|
|
||||||
|
The check runs on the stale path only, not on the fresh-cache
|
||||||
|
early-return. Detection is therefore at most one trading day behind,
|
||||||
|
which the candle TTL guarantees, and the hot portfolio-pricing path
|
||||||
|
pays nothing.
|
||||||
|
|
||||||
|
Steady state for a quarterly payer is four full-history fetches a year,
|
||||||
|
each landing within about a trading day of an ex-date. `zfin diagnose
|
||||||
|
SYMBOL` reports the basis against the newest already-ex action.
|
||||||
|
|
||||||
## Freshness is the `#!expires=` directive, not mtime
|
## Freshness is the `#!expires=` directive, not mtime
|
||||||
|
|
||||||
A cache entry is fresh when the wall clock is earlier than the
|
A cache entry is fresh when the wall clock is earlier than the
|
||||||
|
|
@ -146,7 +204,21 @@ The `--refresh-data` policy maps to `FetchOptions`:
|
||||||
### `getCandles` (single symbol)
|
### `getCandles` (single symbol)
|
||||||
|
|
||||||
This is the most involved path because of the daily/meta split, the
|
This is the most involved path because of the daily/meta split, the
|
||||||
incremental-update logic, and the TwelveData carve-out.
|
incremental-update logic, the adjustment-basis escalation, and the
|
||||||
|
TwelveData carve-out.
|
||||||
|
|
||||||
|
Note also that `force` re-asks the provider but does **not** rebuild
|
||||||
|
candle history: it skips the TTL and the server tier, then takes the
|
||||||
|
same incremental top-up. Replacing the series is the restatement path's
|
||||||
|
job (or `zfin cache clear`).
|
||||||
|
|
||||||
|
Provider routing is keyed off `CandleMeta.tiingo_retry_after_s`, not off
|
||||||
|
`provider`. `provider` is pure provenance - "where did these bars come
|
||||||
|
from" - and using it to route made a single non-transient Tiingo failure
|
||||||
|
permanent: Yahoo got tried first, succeeded, rewrote `provider = .yahoo`,
|
||||||
|
and Tiingo was never consulted again. The backoff is armed only by a
|
||||||
|
genuine 404, expires after `Ttl.tiingo_backoff` with per-symbol jitter,
|
||||||
|
and clears the moment Tiingo serves the symbol again.
|
||||||
|
|
||||||
```mermaid
|
```mermaid
|
||||||
flowchart TD
|
flowchart TD
|
||||||
|
|
@ -154,31 +226,61 @@ flowchart TD
|
||||||
NG -->|yes| FF["return FetchFailed, no network"]
|
NG -->|yes| FF["return FetchFailed, no network"]
|
||||||
NG -->|no| RM{"candles_meta exists?"}
|
NG -->|no| RM{"candles_meta exists?"}
|
||||||
|
|
||||||
RM -->|yes| TW{"provider is twelvedata?"}
|
RM -->|yes| SK{"skip_network?"}
|
||||||
|
SK -->|yes| RETS["return cached even if stale<br/>(FetchFailed if unreadable)"]
|
||||||
|
SK -->|no| TW{"provider is twelvedata?"}
|
||||||
TW -->|yes| FULL
|
TW -->|yes| FULL
|
||||||
TW -->|no| FR{"meta fresh and not force_refresh?"}
|
TW -->|no| FR{"meta fresh and not force_refresh?"}
|
||||||
FR -->|yes| RET["return cached candles"]
|
FR -->|yes| RET["return cached candles"]
|
||||||
FR -->|no| SS1["syncCandlesFromServer"]
|
FR -->|no| SS1["syncCandlesFromServer"]
|
||||||
SS1 --> SF1{"fresh now?"}
|
SS1 --> SF1{"fresh AND adj_basis current?"}
|
||||||
SF1 -->|yes| RET
|
SF1 -->|yes| RET
|
||||||
SF1 -->|no| INC{"shouldRefresh?"}
|
SF1 -->|no| AB{"adj_basis predates an already-ex action?"}
|
||||||
|
AB -->|yes| REST["refetchFullHistory: restate whole series"]
|
||||||
|
REST --> RR{"ok?"}
|
||||||
|
RR -->|yes| RET2["return fetched"]
|
||||||
|
RR -->|no| INC
|
||||||
|
AB -->|no| INC{"shouldRefresh?"}
|
||||||
INC -->|no| BUMP["bump TTL, return cached"]
|
INC -->|no| BUMP["bump TTL, return cached"]
|
||||||
INC -->|yes| INCF["incremental fetch from last_date+1"]
|
INC -->|yes| INCF["incremental fetch from last_date+1, appendCandles"]
|
||||||
|
|
||||||
RM -->|no| SN{"skip_network?"}
|
RM -->|no| SN{"skip_network?"}
|
||||||
SN -->|yes| FF
|
SN -->|yes| FF
|
||||||
SN -->|no| SS2["syncCandlesFromServer"]
|
SN -->|no| SS2["syncCandlesFromServer"]
|
||||||
SS2 --> SF2{"fresh now?"}
|
SS2 --> SF2{"fresh now?"}
|
||||||
SF2 -->|yes| RET
|
SF2 -->|yes| RET
|
||||||
SF2 -->|no| FULL["populateAllFromTiingo, full history"]
|
SF2 -->|no| FULL["refetchFullHistory: Tiingo, then Yahoo"]
|
||||||
FULL --> RES{"result?"}
|
FULL --> RES{"result?"}
|
||||||
RES -->|ok| RET2["return fetched"]
|
RES -->|ok| RET2
|
||||||
RES -->|NotFound| WN["writeNegative candles_daily"]
|
RES -->|"NotFound (EVERY provider disclaims it)"| WN["writeNegative candles_daily"]
|
||||||
WN --> FF
|
WN --> FF
|
||||||
RES -->|transient| TR["bump fail_count, TransientError"]
|
RES -->|transient| TR["bump fail_count, TransientError"]
|
||||||
RES -->|other| FF
|
RES -->|other| FF
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Two things about this shape are easy to get wrong.
|
||||||
|
|
||||||
|
**The basis check precedes `shouldRefresh`.** A symbol that needs both a
|
||||||
|
top-up and a restatement costs one full fetch, not an append followed by
|
||||||
|
a second pass. And a failed restatement falls through to the ordinary
|
||||||
|
top-up rather than erroring: the existing series is untouched and still
|
||||||
|
usable, just understated by the missed adjustment, so it retries on the
|
||||||
|
next stale pass.
|
||||||
|
|
||||||
|
**Only a unanimous `NotFound` earns a negative entry.** `writeNegative`
|
||||||
|
*overwrites* `candles_daily.srf` with a marker, so a verdict of "no such
|
||||||
|
symbol" from Tiingo alone must not reach it - Yahoo gets asked first, and
|
||||||
|
`refetchFullHistory` returns `error.NotFound` only when every provider
|
||||||
|
disclaims the symbol. Anything else (auth trouble, a malformed body, a
|
||||||
|
network blip) fails the call but leaves the cache alone. The restatement
|
||||||
|
path above never writes a negative entry at all, for the same reason: it
|
||||||
|
is reached while holding a working series.
|
||||||
|
|
||||||
|
The no-prior-cache branch does not re-check the basis after a server
|
||||||
|
sync, only freshness. A stale basis inherited from the server is caught
|
||||||
|
on the next invocation, which takes the meta-exists branch. One
|
||||||
|
invocation of understated returns, then it self-corrects.
|
||||||
|
|
||||||
Key invariant: the negative marker for a candle-less symbol lives in
|
Key invariant: the negative marker for a candle-less symbol lives in
|
||||||
`candles_daily.srf`, and **every** candle decision honors it there -
|
`candles_daily.srf`, and **every** candle decision honors it there -
|
||||||
`isCandleMetaFresh` (the price fast-path gate), `getCachedCandles` (the
|
`isCandleMetaFresh` (the price fast-path gate), `getCachedCandles` (the
|
||||||
|
|
@ -255,11 +357,13 @@ re-run the dead lookup on every invocation. The entry is the sentinel:
|
||||||
|
|
||||||
## Candle-less symbols (crypto and friends)
|
## Candle-less symbols (crypto and friends)
|
||||||
|
|
||||||
Some held symbols have **no daily candles available from the candle
|
Some held symbols have **no daily candles available from any candle
|
||||||
provider (Tiingo)** - cryptocurrencies on the Yahoo `DOGE-USD` /
|
provider** - cryptocurrencies on the Yahoo `DOGE-USD` /
|
||||||
`BTC-USD` shape are the common case, and delisted or invalid tickers
|
`BTC-USD` shape are the common case, and delisted or invalid tickers
|
||||||
behave identically. For these symbols `getCandles` writes a negative
|
behave identically. For these symbols `getCandles` writes a negative
|
||||||
entry and never produces a price from history.
|
entry and never produces a price from history. "Any" is literal: Tiingo
|
||||||
|
and Yahoo must both disclaim the symbol, because the marker overwrites
|
||||||
|
`candles_daily.srf`.
|
||||||
|
|
||||||
Such symbols are still priced, through two mechanisms that do **not**
|
Such symbols are still priced, through two mechanisms that do **not**
|
||||||
touch the candle cache:
|
touch the candle cache:
|
||||||
|
|
|
||||||
|
|
@ -115,6 +115,22 @@ using a small `candles_meta.srf` companion file to track the last date
|
||||||
and source provider. A ten-year history costs one big fetch the first
|
and source provider. A ten-year history costs one big fetch the first
|
||||||
time and tiny top-ups thereafter.
|
time and tiny top-ups thereafter.
|
||||||
|
|
||||||
|
With one exception. A provider's *adjusted* close prices bake in every
|
||||||
|
dividend and split that happened after each bar, so when a distribution
|
||||||
|
goes ex, all the bars behind it need marking down - and appending new
|
||||||
|
bars can't do that to bars already on disk. Left alone, total returns
|
||||||
|
spanning that ex-date read low by roughly the missed dividend.
|
||||||
|
|
||||||
|
So zfin tracks how current each series' adjustment basis is, and when a
|
||||||
|
dividend or split has gone ex behind it, re-downloads that symbol's full
|
||||||
|
history once to pick up the corrected values. For a quarterly dividend
|
||||||
|
payer that's about four full fetches a year, each within a day or so of
|
||||||
|
an ex-date. It's why a refresh run occasionally takes noticeably longer
|
||||||
|
than the usual top-up.
|
||||||
|
|
||||||
|
[`zfin diagnose SYMBOL`](../reference/cli/diagnose.md) reports the basis
|
||||||
|
for one symbol and says whether a restatement is pending.
|
||||||
|
|
||||||
## Negative caching
|
## Negative caching
|
||||||
|
|
||||||
When a provider permanently fails for a symbol -- a nonexistent
|
When a provider permanently fails for a symbol -- a nonexistent
|
||||||
|
|
|
||||||
|
|
@ -19,10 +19,25 @@ Two columns show up throughout zfin:
|
||||||
3-Year Return: 20.75% 22.22% ann.
|
3-Year Return: 20.75% 22.22% ann.
|
||||||
```
|
```
|
||||||
|
|
||||||
Total return needs dividend history, which comes from Polygon -- so it
|
Total return is computed two ways, and zfin reports whichever comes out
|
||||||
requires `POLYGON_API_KEY`. Without it, you still get price-only
|
higher per period:
|
||||||
returns. For a dividend payer like SCHD the gap between the two columns
|
|
||||||
is large; for a non-payer it's near zero.
|
- **Dividend reinvestment** -- walk the dividend history, buy more shares
|
||||||
|
at each ex-date's close, compound. Exact, but only as complete as the
|
||||||
|
dividend records, which come from Polygon -- so this path wants
|
||||||
|
`POLYGON_API_KEY`.
|
||||||
|
- **Adjusted close** -- providers publish an `adj_close` that already
|
||||||
|
bakes dividends in. No dividend history needed, but it is only as
|
||||||
|
current as the last full price-history download (see
|
||||||
|
[Incremental candle updates](caching.md#incremental-candle-updates)).
|
||||||
|
|
||||||
|
Both failure modes -- a missing dividend record, a stale adjustment --
|
||||||
|
understate the return, never overstate it, which is why taking the higher
|
||||||
|
of the two is safe rather than arbitrary. So without `POLYGON_API_KEY`
|
||||||
|
you still get a genuine total return, just via the second path; the gap
|
||||||
|
you'd see is at most a distribution or two, not the whole yield. For a
|
||||||
|
dividend payer like SCHD the gap between the price-only and total-return
|
||||||
|
columns is large; for a non-payer it's near zero.
|
||||||
|
|
||||||
## Annualized (CAGR)
|
## Annualized (CAGR)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -17,11 +17,30 @@ zfin --refresh-data=force perf VTI
|
||||||
zfin --refresh-data=never analysis
|
zfin --refresh-data=never analysis
|
||||||
```
|
```
|
||||||
|
|
||||||
| Value | Behavior |
|
| Value | Behavior |
|
||||||
|------------------|---------------------------------------------------------------------------------------------------------------|
|
|------------------|----------------------------------------------------------------------------------------------------------------|
|
||||||
| `auto` (default) | Respect each data type's cache TTL; fetch only what's stale. |
|
| `auto` (default) | Respect each data type's cache TTL; fetch only what's stale. |
|
||||||
| `force` | Re-fetch every symbol regardless of freshness. Use after a market close, or when you suspect bad cached data. |
|
| `force` | Re-ask providers regardless of freshness. Use after a market close, or when a fetch seems to have been missed. |
|
||||||
| `never` | Serve cache contents only; make no network calls. True offline mode. |
|
| `never` | Serve cache contents only; make no network calls. True offline mode. |
|
||||||
|
|
||||||
|
### `force` tops up price history, it does not rebuild it
|
||||||
|
|
||||||
|
Worth knowing before you reach for it: for daily candles, `force` skips
|
||||||
|
the TTL and asks the provider for anything newer than the last cached
|
||||||
|
bar. It does **not** re-download the series. So if you suspect the
|
||||||
|
*existing* bars are wrong rather than merely incomplete, `force` will
|
||||||
|
not help -- it appends and moves on.
|
||||||
|
|
||||||
|
That case is handled automatically now. When a dividend or split goes ex,
|
||||||
|
the bars behind it need their adjusted closes marked down, and zfin
|
||||||
|
re-downloads that symbol's full history to pick up the corrected values
|
||||||
|
on its own -- see
|
||||||
|
[Incremental candle updates](../explanation/caching.md#incremental-candle-updates).
|
||||||
|
[`zfin diagnose SYMBOL`](../reference/cli/diagnose.md) says whether one is
|
||||||
|
pending.
|
||||||
|
|
||||||
|
`zfin cache clear` remains the blunt instrument if you want to discard
|
||||||
|
everything and start over.
|
||||||
|
|
||||||
## Working offline
|
## Working offline
|
||||||
|
|
||||||
|
|
@ -66,7 +85,8 @@ zfin cache clear # delete all cached data
|
||||||
|
|
||||||
`cache clear` is safe -- everything re-fetches on the next run (subject
|
`cache clear` is safe -- everything re-fetches on the next run (subject
|
||||||
to provider rate limits). Reach for it only when you suspect corrupt
|
to provider rate limits). Reach for it only when you suspect corrupt
|
||||||
cached data; normal staleness is handled by `auto`. See
|
cached data; normal staleness is handled by `auto`, and a stale
|
||||||
|
adjustment basis repairs itself. See
|
||||||
[`zfin cache`](../reference/cli/cache.md).
|
[`zfin cache`](../reference/cli/cache.md).
|
||||||
|
|
||||||
## See also
|
## See also
|
||||||
|
|
|
||||||
|
|
@ -3,32 +3,56 @@
|
||||||
Inspect or clear the local provider-data cache.
|
Inspect or clear the local provider-data cache.
|
||||||
|
|
||||||
```
|
```
|
||||||
Usage: zfin cache <stats|clear>
|
Usage: zfin cache <stats|stale|clear> | zfin cache refresh [SYMBOL...]
|
||||||
```
|
```
|
||||||
|
|
||||||
| Subcommand | Does |
|
| Subcommand | Does |
|
||||||
|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|------------|------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
||||||
| `stats` | List every cached symbol with per-data-type size, age, and freshness state. Stale entries (past TTL) are flagged. Includes `cusip_tickers.srf` if present. |
|
| `stats` | List every cached symbol with per-data-type size, age, and freshness state. Stale entries (past TTL) are flagged. Includes `cusip_tickers.srf` if present. |
|
||||||
| `clear` | Delete every file under the cache directory. No confirmation; the next provider call re-fetches everything. |
|
| `stale` | Find symbols whose newest candle is behind their peers'. Read-only; reports, never fetches. |
|
||||||
|
| `refresh` | Force-refresh candle data in the **local** cache, bypassing the TTL and the shared server. No arguments refreshes exactly what `stale` reports. |
|
||||||
|
| `clear` | Delete every file under the cache directory. No confirmation; the next provider call re-fetches everything. |
|
||||||
|
|
||||||
The cache directory is `$ZFIN_CACHE_DIR` if set, otherwise
|
The cache directory is `$ZFIN_CACHE_DIR` if set, otherwise
|
||||||
`~/.cache/zfin`.
|
`~/.cache/zfin`.
|
||||||
|
|
||||||
|
## `stale` compares against peers, not the clock
|
||||||
|
|
||||||
|
A symbol is called stale when other cached symbols **of the same kind**
|
||||||
|
(equity vs mutual fund) hold a newer bar than it does. That is a
|
||||||
|
different question from "is this bar old?", and the peer framing is what
|
||||||
|
makes it trustworthy: an un-modeled market closure moves every symbol
|
||||||
|
together, so it cannot be mistaken for one frozen cache entry.
|
||||||
|
|
||||||
|
## `refresh` deliberately skips the server
|
||||||
|
|
||||||
|
`zfin cache refresh` goes straight to the provider. It does **not** ask
|
||||||
|
`ZFIN_SERVER`, because its job is to repair *your* copy. To refresh the
|
||||||
|
server's copy instead, use
|
||||||
|
[`zfin server refresh`](server.md); to find out which side is behind in
|
||||||
|
the first place, use [`zfin diagnose SYMBOL`](diagnose.md).
|
||||||
|
|
||||||
## Examples
|
## Examples
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
zfin cache stats
|
zfin cache stats # what's cached, sizes, and ages
|
||||||
zfin cache clear # wipe; everything re-fetches on next use
|
zfin cache stale # which symbols are behind their peers
|
||||||
|
zfin cache refresh # re-fetch exactly what `stale` reported
|
||||||
|
zfin cache refresh SPY AGG # re-fetch these two
|
||||||
|
zfin cache clear # wipe; everything re-fetches on next use
|
||||||
```
|
```
|
||||||
|
|
||||||
`clear` is safe -- it only removes cached copies of public market data.
|
`clear` is safe -- it only removes cached copies of public market data.
|
||||||
Reach for it when you suspect corrupt cached data; routine staleness is
|
Reach for it when you suspect corrupt cached data; routine staleness is
|
||||||
handled automatically by the `auto` refresh policy.
|
handled automatically by the `auto` refresh policy, and a stale
|
||||||
|
adjustment basis repairs itself on the next refresh.
|
||||||
|
|
||||||
## See also
|
## See also
|
||||||
|
|
||||||
- [Caching and data freshness](../../explanation/caching.md) -- TTLs and the fetch model.
|
- [Caching and data freshness](../../explanation/caching.md) -- TTLs and the fetch model.
|
||||||
- [Offline use and refreshing data](../../guides/offline-and-refresh.md) -- the `--refresh-data` flag.
|
- [Offline use and refreshing data](../../guides/offline-and-refresh.md) -- the `--refresh-data` flag.
|
||||||
|
- [`diagnose`](diagnose.md) -- trace one symbol through every tier.
|
||||||
|
- [`server`](server.md) -- refresh the shared server's cache instead.
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
|
|
|
||||||
98
docs/reference/cli/diagnose.md
Normal file
98
docs/reference/cli/diagnose.md
Normal file
|
|
@ -0,0 +1,98 @@
|
||||||
|
# `zfin diagnose`
|
||||||
|
|
||||||
|
Trace one symbol's candle data through every tier -- local cache, shared
|
||||||
|
server, upstream provider -- and say where the chain breaks.
|
||||||
|
|
||||||
|
```
|
||||||
|
Usage: zfin diagnose SYMBOL
|
||||||
|
```
|
||||||
|
|
||||||
|
`diagnose` answers one question: **is the provider behind for this
|
||||||
|
symbol, or is it us?** It is the counterpart to
|
||||||
|
[`doctor`](doctor.md), and deliberately its opposite on both axes --
|
||||||
|
`doctor` checks the whole setup and never touches the network, while
|
||||||
|
`diagnose` checks a single symbol and queries every tier including the
|
||||||
|
provider.
|
||||||
|
|
||||||
|
It is **read-only**: no cache writes, and the provider query does not
|
||||||
|
populate the cache. Safe to run while investigating.
|
||||||
|
|
||||||
|
## What it reports
|
||||||
|
|
||||||
|
Seven lines, in the order the data flows:
|
||||||
|
|
||||||
|
| Line | Says |
|
||||||
|
|-------------|-------------------------------------------------------------------------------------------------------|
|
||||||
|
| `local` | Newest cached bar, TTL state, which provider sourced it, consecutive failures, any Tiingo backoff |
|
||||||
|
| `adj basis` | How current the cached adjusted closes are, against the newest dividend or split that has already gone ex |
|
||||||
|
| `peers` | Newest bar held by other cached symbols of the same kind (equity vs mutual fund) |
|
||||||
|
| `tracked` | Whether a normal run fetches this symbol, or it is only pulled on demand |
|
||||||
|
| `server` | What `ZFIN_SERVER` offers, whether it is ahead or behind you, and whether it refreshes this symbol |
|
||||||
|
| `provider` | What the upstream provider actually has right now |
|
||||||
|
| `verdict` | Where the chain breaks, and what to do about it |
|
||||||
|
|
||||||
|
## Example
|
||||||
|
|
||||||
|
```bash
|
||||||
|
zfin diagnose SPY
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
local newest 2026-08-19, TTL still in the future, tiingo
|
||||||
|
adj basis 2026-08-17 - current through the newest corporate action (2026-06-18)
|
||||||
|
peers 23 other equity cached, newest 2026-08-19
|
||||||
|
tracked on demand - a projections benchmark symbol
|
||||||
|
server offers 2026-08-19 - same as local, written 2026-08-19, tracked: yes
|
||||||
|
provider tiingo newest 2026-08-19 (8 bars in the last 10 days)
|
||||||
|
|
||||||
|
verdict up to date with everything available
|
||||||
|
```
|
||||||
|
|
||||||
|
## Reading the `adj basis` line
|
||||||
|
|
||||||
|
This is the least obvious line, so it is worth spelling out. Cached
|
||||||
|
price history grows by appending, which cannot retroactively correct the
|
||||||
|
*adjusted* closes of bars already on disk. When a dividend or split goes
|
||||||
|
ex, those earlier bars need marking down, and until they are, total
|
||||||
|
returns spanning that ex-date read low by roughly the missed yield. See
|
||||||
|
[Incremental candle updates](../../explanation/caching.md#incremental-candle-updates).
|
||||||
|
|
||||||
|
Three forms:
|
||||||
|
|
||||||
|
```
|
||||||
|
adj basis 2026-08-17 - current through the newest corporate action (2026-06-18)
|
||||||
|
adj basis 1970-01-01 - STALE, 2026-06-01 went ex behind it; total returns read low until restated
|
||||||
|
adj basis 2026-08-19 - no dividends or splits cached, nothing to restate
|
||||||
|
```
|
||||||
|
|
||||||
|
`STALE` is not something to act on -- the next refresh re-downloads that
|
||||||
|
symbol's full history on its own. It explains a total return that looks
|
||||||
|
slightly low in the meantime.
|
||||||
|
|
||||||
|
Two things the date is **not**: it is not when the restatement ran (it is
|
||||||
|
the newest bar present at the time, which lags when a session's bar has
|
||||||
|
not posted yet), and it is not an ex-date. `1970-01-01` means "never
|
||||||
|
restated", which is what a cache predating the feature reads as.
|
||||||
|
|
||||||
|
The compared-against date is the newest ex-date that has **already
|
||||||
|
passed**. A declared-but-not-yet-ex dividend is deliberately ignored:
|
||||||
|
no provider has applied it either, so waiting on it would refetch
|
||||||
|
forever.
|
||||||
|
|
||||||
|
## Rate limit caveat
|
||||||
|
|
||||||
|
The provider query uses its own rate limiter, separate from the budget
|
||||||
|
the rest of zfin accounts for -- so it is invisible to that budget. On a
|
||||||
|
free tier close to its cap, `diagnose` can be the request that trips it.
|
||||||
|
One request per invocation.
|
||||||
|
|
||||||
|
## See also
|
||||||
|
|
||||||
|
- [`doctor`](doctor.md) -- whole-setup health check, no network.
|
||||||
|
- [`cache`](cache.md) -- `stale` finds symbols behind their peers; `refresh` forces a local re-fetch.
|
||||||
|
- [`server`](server.md) -- force a refresh in the shared server's cache instead.
|
||||||
|
- [Caching and data freshness](../../explanation/caching.md) -- the tiers and TTLs.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
[CLI command reference](index.md)
|
||||||
|
|
@ -55,12 +55,14 @@ Get help at any time with `zfin help` or per command with
|
||||||
|
|
||||||
## Infrastructure
|
## Infrastructure
|
||||||
|
|
||||||
| Command | Does |
|
| Command | Does |
|
||||||
|---------------------------------|------------------------------------------------|
|
|---------------------------------|--------------------------------------------------|
|
||||||
| [`cache`](cache.md) | Inspect or clear the local data cache |
|
| [`cache`](cache.md) | Inspect or clear the local data cache |
|
||||||
| [`doctor`](doctor.md) | Health-check files and environment (read-only) |
|
| [`diagnose`](diagnose.md) | Trace one symbol through cache, server, provider |
|
||||||
| [`version`](version.md) | Show version and build info |
|
| [`server`](server.md) | Ask the shared server to refresh its own cache |
|
||||||
| [`interactive`](interactive.md) | Launch the interactive TUI (alias `i`) |
|
| [`doctor`](doctor.md) | Health-check files and environment (read-only) |
|
||||||
|
| [`version`](version.md) | Show version and build info |
|
||||||
|
| [`interactive`](interactive.md) | Launch the interactive TUI (alias `i`) |
|
||||||
|
|
||||||
## Global options
|
## Global options
|
||||||
|
|
||||||
|
|
|
||||||
65
docs/reference/cli/server.md
Normal file
65
docs/reference/cli/server.md
Normal file
|
|
@ -0,0 +1,65 @@
|
||||||
|
# `zfin server`
|
||||||
|
|
||||||
|
Ask the shared server (`ZFIN_SERVER`) to refresh its own cache.
|
||||||
|
|
||||||
|
```
|
||||||
|
Usage: zfin server refresh SYMBOL [SYMBOL...]
|
||||||
|
```
|
||||||
|
|
||||||
|
Force-refreshes candle data in the **server's** cache, bypassing its
|
||||||
|
TTL. Nothing local changes -- your next normal run picks up the server's
|
||||||
|
new copy through the usual sync.
|
||||||
|
|
||||||
|
## Which side are you refreshing?
|
||||||
|
|
||||||
|
Two commands look similar and do opposite things:
|
||||||
|
|
||||||
|
| Command | Refreshes | Goes through the server? |
|
||||||
|
|-------------------------------|----------------------|--------------------------|
|
||||||
|
| `zfin cache refresh SYMBOL` | your **local** cache | No, deliberately |
|
||||||
|
| `zfin server refresh SYMBOL` | the **server's** cache | That is the point |
|
||||||
|
|
||||||
|
Run [`zfin diagnose SYMBOL`](diagnose.md) first -- its `local` and
|
||||||
|
`server` lines say which side is actually behind, so you do not refresh
|
||||||
|
the wrong one.
|
||||||
|
|
||||||
|
## `moved` vs `unchanged`
|
||||||
|
|
||||||
|
The per-symbol result reports whether the newest bar actually **moved**,
|
||||||
|
not merely whether the fetch succeeded. A successful refresh that changes
|
||||||
|
nothing is the signature of a provider with no newer data -- which is
|
||||||
|
usually the finding you came for, so it is not reported as success.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
zfin server refresh SPY AGG
|
||||||
|
```
|
||||||
|
|
||||||
|
```
|
||||||
|
Asking https://zfin.example.org to refresh 2 symbol(s)...
|
||||||
|
SPY moved 2026-08-19
|
||||||
|
AGG unchanged 2026-08-18
|
||||||
|
|
||||||
|
1 moved, 1 unchanged, 0 failed
|
||||||
|
Your local cache is untouched - a normal run will sync the new copy.
|
||||||
|
```
|
||||||
|
|
||||||
|
## Requirements and limits
|
||||||
|
|
||||||
|
- **`ZFIN_SERVER`** must be set, and **`ZFIN_SERVER_API_KEY`** when the
|
||||||
|
server enforces one. Without a server configured there is nothing to
|
||||||
|
ask, and the command says so rather than falling back to a local fetch.
|
||||||
|
- The server caps a single request at **25 symbols**.
|
||||||
|
- It **refuses a second concurrent refresh** rather than queueing it, so
|
||||||
|
a refresh already in flight (a cron run, say) returns a conflict rather
|
||||||
|
than doubling the provider load.
|
||||||
|
|
||||||
|
## See also
|
||||||
|
|
||||||
|
- [`diagnose`](diagnose.md) -- which tier is behind, before you pick a side.
|
||||||
|
- [`cache`](cache.md) -- `refresh` for the local cache instead.
|
||||||
|
- [Caching and data freshness](../../explanation/caching.md) -- the server as an optional second tier.
|
||||||
|
- [Environment variables](../config/environment.md) -- `ZFIN_SERVER`, `ZFIN_SERVER_API_KEY`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
[CLI command reference](index.md)
|
||||||
|
|
@ -547,7 +547,17 @@ pub fn parseAccountsFile(allocator: std.mem.Allocator, data: []const u8) !Accoun
|
||||||
defer it.deinit();
|
defer it.deinit();
|
||||||
|
|
||||||
while (try it.next()) |fields| {
|
while (try it.next()) |fields| {
|
||||||
const entry = fields.to(AccountTaxEntry, srf_opts.user_edited) catch continue;
|
const entry = fields.to(AccountTaxEntry, srf_opts.user_edited) catch |err| {
|
||||||
|
// Skip the account rather than losing the whole file, but
|
||||||
|
// name the error: a dropped account loses its tax type,
|
||||||
|
// cadence and carve-outs, and every consumer then silently
|
||||||
|
// falls back to defaults. Quiet under `zig build test`,
|
||||||
|
// where fixtures feed malformed records on purpose.
|
||||||
|
if (!builtin.is_test) {
|
||||||
|
log.warn("accounts.srf: skipping malformed record: {s}", .{@errorName(err)});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
};
|
||||||
|
|
||||||
// A zero/negative large-lot threshold is nonsensical (zero
|
// A zero/negative large-lot threshold is nonsensical (zero
|
||||||
// flags every new lot; negative is meaningless). Reject it and
|
// flags every new lot; negative is meaningless). Reject it and
|
||||||
|
|
|
||||||
|
|
@ -763,7 +763,17 @@ pub fn parseProjectionsConfig(data: ?[]const u8) UserConfig {
|
||||||
var annotation_count: u8 = 0;
|
var annotation_count: u8 = 0;
|
||||||
|
|
||||||
while (it.next() catch null) |field_it| {
|
while (it.next() catch null) |field_it| {
|
||||||
const rec = field_it.to(SrfProjection, srf_opts.user_edited) catch continue;
|
const rec = field_it.to(SrfProjection, srf_opts.user_edited) catch |err| {
|
||||||
|
// Skip the record rather than losing the whole file, but
|
||||||
|
// name the error: a dropped record reverts that setting to
|
||||||
|
// its default without saying so. Quiet under
|
||||||
|
// `zig build test`, where fixtures feed malformed records
|
||||||
|
// on purpose.
|
||||||
|
if (!builtin.is_test) {
|
||||||
|
log.warn("projections.srf: skipping malformed record: {s}", .{@errorName(err)});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
};
|
||||||
switch (rec) {
|
switch (rec) {
|
||||||
.config => |c| {
|
.config => |c| {
|
||||||
config.target_stock_pct = c.target_stock_pct orelse config.target_stock_pct;
|
config.target_stock_pct = c.target_stock_pct orelse config.target_stock_pct;
|
||||||
|
|
|
||||||
254
src/cache/store.zig
vendored
254
src/cache/store.zig
vendored
|
|
@ -1,7 +1,9 @@
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
|
const builtin = @import("builtin");
|
||||||
const log = std.log.scoped(.cache);
|
const log = std.log.scoped(.cache);
|
||||||
const srf = @import("srf");
|
const srf = @import("srf");
|
||||||
const srf_opts = @import("../srf_opts.zig");
|
const srf_opts = @import("../srf_opts.zig");
|
||||||
|
const format = @import("../format.zig");
|
||||||
const atomic = @import("../atomic.zig");
|
const atomic = @import("../atomic.zig");
|
||||||
const version = @import("../version.zig");
|
const version = @import("../version.zig");
|
||||||
const Date = @import("../Date.zig");
|
const Date = @import("../Date.zig");
|
||||||
|
|
@ -2090,8 +2092,79 @@ pub fn serializePortfolio(allocator: std.mem.Allocator, lots: []const Lot) ![]co
|
||||||
return aw.toOwnedSlice();
|
return aw.toOwnedSlice();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Collected diagnostics for records that could not be parsed. Messages
|
||||||
|
/// are allocator-owned; the caller frees each one and the list.
|
||||||
|
///
|
||||||
|
/// One message per skipped record, so `items.len` is the skip count.
|
||||||
|
pub const ParseDiagnostics = std.ArrayList([]const u8);
|
||||||
|
|
||||||
|
/// Longest stretch of a malformed record echoed back to the user.
|
||||||
|
///
|
||||||
|
/// SRF returns which error occurred but not which field caused it, so
|
||||||
|
/// showing the record is how the user finds the culprit. 120 columns is
|
||||||
|
/// wide enough for a typical lot to appear whole and narrow enough not
|
||||||
|
/// to wrap and bury the message it is attached to.
|
||||||
|
const diag_record_cols: usize = 120;
|
||||||
|
|
||||||
|
/// Append a "could not parse this record" diagnostic naming the error
|
||||||
|
/// and echoing the record.
|
||||||
|
///
|
||||||
|
/// `data` is the whole file and `line` is 1-based, so the record text is
|
||||||
|
/// recovered by counting newlines - `srf`'s `state.current_line` is
|
||||||
|
/// consumed as fields are read and would only yield the unparsed
|
||||||
|
/// remainder. Multi-line (`#!long`) records show their first line, which
|
||||||
|
/// is enough to locate them.
|
||||||
|
fn appendParseDiag(
|
||||||
|
allocator: std.mem.Allocator,
|
||||||
|
diags: *ParseDiagnostics,
|
||||||
|
data: []const u8,
|
||||||
|
line: usize,
|
||||||
|
detail: []const u8,
|
||||||
|
) !void {
|
||||||
|
const raw = nthLine(data, line);
|
||||||
|
const shown = format.truncateToCols(raw, diag_record_cols);
|
||||||
|
const msg = if (shown.len < raw.len)
|
||||||
|
try std.fmt.allocPrint(allocator, "line {d}: {s}\n {s}...", .{ line, detail, shown })
|
||||||
|
else
|
||||||
|
try std.fmt.allocPrint(allocator, "line {d}: {s}\n {s}", .{ line, detail, shown });
|
||||||
|
errdefer allocator.free(msg);
|
||||||
|
try diags.append(allocator, msg);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The `line`-th line of `data`, 1-based, without its terminator.
|
||||||
|
/// Returns an empty slice when `line` is out of range. Borrowed.
|
||||||
|
fn nthLine(data: []const u8, line: usize) []const u8 {
|
||||||
|
if (line == 0) return data[0..0];
|
||||||
|
var n: usize = 1;
|
||||||
|
var it = std.mem.splitScalar(u8, data, '\n');
|
||||||
|
while (it.next()) |l| : (n += 1) {
|
||||||
|
if (n == line) return std.mem.trimEnd(u8, l, "\r");
|
||||||
|
}
|
||||||
|
return data[0..0];
|
||||||
|
}
|
||||||
|
|
||||||
/// Deserialize a portfolio from SRF data. Caller owns the returned Portfolio.
|
/// Deserialize a portfolio from SRF data. Caller owns the returned Portfolio.
|
||||||
pub fn deserializePortfolio(allocator: std.mem.Allocator, data: []const u8) !Portfolio {
|
pub fn deserializePortfolio(allocator: std.mem.Allocator, data: []const u8) !Portfolio {
|
||||||
|
return deserializePortfolioDiag(allocator, data, null);
|
||||||
|
}
|
||||||
|
|
||||||
|
/// `deserializePortfolio`, optionally collecting a diagnostic per
|
||||||
|
/// skipped record into `diags`.
|
||||||
|
///
|
||||||
|
/// A skipped lot silently changes every figure zfin prints - net worth,
|
||||||
|
/// allocation, contributions, compare - so the caller needs to be able
|
||||||
|
/// to say so next to those figures rather than hoping a `std.log.warn`
|
||||||
|
/// on stderr was noticed. `portfolio_loader` collects these into
|
||||||
|
/// `LoadedPortfolio.warnings`, and `cli.loadPortfolio` prints them.
|
||||||
|
///
|
||||||
|
/// Passing null keeps the previous behaviour exactly: warn to the log
|
||||||
|
/// and carry on. That is what the git-historical and import paths want,
|
||||||
|
/// where a warning about an old revision would be noise.
|
||||||
|
pub fn deserializePortfolioDiag(
|
||||||
|
allocator: std.mem.Allocator,
|
||||||
|
data: []const u8,
|
||||||
|
diags: ?*ParseDiagnostics,
|
||||||
|
) !Portfolio {
|
||||||
var lots: std.ArrayList(Lot) = .empty;
|
var lots: std.ArrayList(Lot) = .empty;
|
||||||
errdefer {
|
errdefer {
|
||||||
for (lots.items) |lot| {
|
for (lots.items) |lot| {
|
||||||
|
|
@ -2115,8 +2188,15 @@ pub fn deserializePortfolio(allocator: std.mem.Allocator, data: []const u8) !Por
|
||||||
// `user_edited` coercion: see `srf_opts.zig` for why hand-edited
|
// `user_edited` coercion: see `srf_opts.zig` for why hand-edited
|
||||||
// files get different options from cache files. The `catch`
|
// files get different options from cache files. The `catch`
|
||||||
// below still handles genuinely unparseable values.
|
// below still handles genuinely unparseable values.
|
||||||
var lot = fields.to(Lot, srf_opts.user_edited) catch {
|
var lot = fields.to(Lot, srf_opts.user_edited) catch |err| {
|
||||||
std.log.warn("portfolio: could not parse record at line {d}", .{line});
|
if (diags) |d| {
|
||||||
|
try appendParseDiag(allocator, d, data, line, @errorName(err));
|
||||||
|
} else if (!builtin.is_test) {
|
||||||
|
// Quiet under `zig build test`: fixtures feed malformed
|
||||||
|
// records on purpose to pin the skip, and the warn spam
|
||||||
|
// pollutes every run's output.
|
||||||
|
std.log.warn("portfolio: could not parse record at line {d}: {s}", .{ line, @errorName(err) });
|
||||||
|
}
|
||||||
skipped += 1;
|
skipped += 1;
|
||||||
continue;
|
continue;
|
||||||
};
|
};
|
||||||
|
|
@ -2136,7 +2216,11 @@ pub fn deserializePortfolio(allocator: std.mem.Allocator, data: []const u8) !Por
|
||||||
.cash => try allocator.dupe(u8, "CASH"),
|
.cash => try allocator.dupe(u8, "CASH"),
|
||||||
.illiquid => try allocator.dupe(u8, "ILLIQUID"),
|
.illiquid => try allocator.dupe(u8, "ILLIQUID"),
|
||||||
else => {
|
else => {
|
||||||
std.log.warn("portfolio: record at line {d} has no symbol, skipping", .{line});
|
if (diags) |d| {
|
||||||
|
try appendParseDiag(allocator, d, data, line, "no symbol");
|
||||||
|
} else if (!builtin.is_test) {
|
||||||
|
std.log.warn("portfolio: record at line {d} has no symbol, skipping", .{line});
|
||||||
|
}
|
||||||
if (lot.note) |n| allocator.free(n);
|
if (lot.note) |n| allocator.free(n);
|
||||||
if (lot.label) |l| allocator.free(l);
|
if (lot.label) |l| allocator.free(l);
|
||||||
if (lot.account) |a| allocator.free(a);
|
if (lot.account) |a| allocator.free(a);
|
||||||
|
|
@ -2151,7 +2235,10 @@ pub fn deserializePortfolio(allocator: std.mem.Allocator, data: []const u8) !Por
|
||||||
try lots.append(allocator, lot);
|
try lots.append(allocator, lot);
|
||||||
}
|
}
|
||||||
|
|
||||||
if (skipped > 0) {
|
// Only log the rollup when nobody is collecting: a caller with
|
||||||
|
// `diags` reports the count itself, in band, and would otherwise say
|
||||||
|
// it twice.
|
||||||
|
if (skipped > 0 and diags == null and !builtin.is_test) {
|
||||||
std.log.warn("portfolio: {d} record(s) could not be parsed and were skipped", .{skipped});
|
std.log.warn("portfolio: {d} record(s) could not be parsed and were skipped", .{skipped});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -4171,6 +4258,165 @@ test "deserializePortfolio: underscore digit separators parse (they always did)"
|
||||||
try std.testing.expectApproxEqAbs(@as(f64, 1_234_567), p.lots[0].shares, 0.5);
|
try std.testing.expectApproxEqAbs(@as(f64, 1_234_567), p.lots[0].shares, 0.5);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "deserializePortfolioDiag: a bad enum names the error and echoes the record" {
|
||||||
|
// The shape `strings_to_numbers` cannot help with: enum coercion
|
||||||
|
// ignores that option entirely, so a typo'd security_type is the
|
||||||
|
// realistic way a lot gets dropped. srf reports WHICH error but not
|
||||||
|
// which field, so the record text is how the user finds `stok`.
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
const data =
|
||||||
|
"#!srfv1\n" ++
|
||||||
|
"symbol::AAPL,shares:num:100,open_date::2024-01-15,open_price:num:140.00\n" ++
|
||||||
|
"symbol::MSFT,security_type::stok,shares:num:50,open_date::2024-02-01,open_price:num:400.00\n";
|
||||||
|
|
||||||
|
var diags: ParseDiagnostics = .empty;
|
||||||
|
defer {
|
||||||
|
for (diags.items) |w| allocator.free(w);
|
||||||
|
diags.deinit(allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
var p = try deserializePortfolioDiag(allocator, data, &diags);
|
||||||
|
defer p.deinit();
|
||||||
|
|
||||||
|
// The good lot survives; the bad one is dropped, not fatal.
|
||||||
|
try std.testing.expectEqual(@as(usize, 1), p.lots.len);
|
||||||
|
try std.testing.expectEqualStrings("AAPL", p.lots[0].symbol);
|
||||||
|
|
||||||
|
// One message per skipped record, so len is the count.
|
||||||
|
try std.testing.expectEqual(@as(usize, 1), diags.items.len);
|
||||||
|
const msg = diags.items[0];
|
||||||
|
try std.testing.expect(std.mem.indexOf(u8, msg, "line 3") != null);
|
||||||
|
try std.testing.expect(std.mem.indexOf(u8, msg, "StringValueNotValidEnumMember") != null);
|
||||||
|
// The record is echoed so the culprit is visible without opening the file.
|
||||||
|
try std.testing.expect(std.mem.indexOf(u8, msg, "security_type::stok") != null);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "deserializePortfolioDiag: a symbol-less non-cash record is reported too" {
|
||||||
|
// The second skip path. A record with no symbol that is not cash or
|
||||||
|
// illiquid has nothing to key on, and previously vanished with only
|
||||||
|
// a log line.
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
const data =
|
||||||
|
"#!srfv1\n" ++
|
||||||
|
"shares:num:100,open_date::2024-01-15,open_price:num:140.00\n";
|
||||||
|
|
||||||
|
var diags: ParseDiagnostics = .empty;
|
||||||
|
defer {
|
||||||
|
for (diags.items) |w| allocator.free(w);
|
||||||
|
diags.deinit(allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
var p = try deserializePortfolioDiag(allocator, data, &diags);
|
||||||
|
defer p.deinit();
|
||||||
|
|
||||||
|
try std.testing.expectEqual(@as(usize, 0), p.lots.len);
|
||||||
|
try std.testing.expectEqual(@as(usize, 1), diags.items.len);
|
||||||
|
try std.testing.expect(std.mem.indexOf(u8, diags.items[0], "no symbol") != null);
|
||||||
|
try std.testing.expect(std.mem.indexOf(u8, diags.items[0], "line 2") != null);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "deserializePortfolioDiag: an over-long record is truncated with a marker" {
|
||||||
|
// A wide direct-indexing or option line would wrap and bury the
|
||||||
|
// message it is attached to, so the echo is clipped to
|
||||||
|
// `diag_record_cols` display columns.
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
var long: std.ArrayList(u8) = .empty;
|
||||||
|
defer long.deinit(allocator);
|
||||||
|
try long.appendSlice(allocator, "#!srfv1\nsymbol::MSFT,security_type::stok,shares:num:50,open_date::2024-02-01,open_price:num:400.00,note::");
|
||||||
|
try long.appendSlice(allocator, "x" ** 200);
|
||||||
|
try long.append(allocator, '\n');
|
||||||
|
|
||||||
|
var diags: ParseDiagnostics = .empty;
|
||||||
|
defer {
|
||||||
|
for (diags.items) |w| allocator.free(w);
|
||||||
|
diags.deinit(allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
var p = try deserializePortfolioDiag(allocator, long.items, &diags);
|
||||||
|
defer p.deinit();
|
||||||
|
|
||||||
|
try std.testing.expectEqual(@as(usize, 1), diags.items.len);
|
||||||
|
const msg = diags.items[0];
|
||||||
|
try std.testing.expect(std.mem.endsWith(u8, msg, "..."));
|
||||||
|
// Clipped well short of the 200-char note.
|
||||||
|
try std.testing.expect(msg.len < 200);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "deserializePortfolioDiag: a clean file produces no diagnostics" {
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
const data =
|
||||||
|
"#!srfv1\n" ++
|
||||||
|
"symbol::AAPL,shares:num:100,open_date::2024-01-15,open_price:num:140.00\n";
|
||||||
|
|
||||||
|
var diags: ParseDiagnostics = .empty;
|
||||||
|
defer diags.deinit(allocator);
|
||||||
|
|
||||||
|
var p = try deserializePortfolioDiag(allocator, data, &diags);
|
||||||
|
defer p.deinit();
|
||||||
|
|
||||||
|
try std.testing.expectEqual(@as(usize, 1), p.lots.len);
|
||||||
|
try std.testing.expectEqual(@as(usize, 0), diags.items.len);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "deserializePortfolio: null diags keeps the old skip-and-carry-on behaviour" {
|
||||||
|
// The 16 call sites that did not opt in must be unaffected: the bad
|
||||||
|
// record is skipped, the good one survives, nothing is returned
|
||||||
|
// about it.
|
||||||
|
const data =
|
||||||
|
"#!srfv1\n" ++
|
||||||
|
"symbol::AAPL,shares:num:100,open_date::2024-01-15,open_price:num:140.00\n" ++
|
||||||
|
"symbol::MSFT,security_type::stok,shares:num:50,open_date::2024-02-01,open_price:num:400.00\n";
|
||||||
|
var p = try deserializePortfolio(std.testing.allocator, data);
|
||||||
|
defer p.deinit();
|
||||||
|
try std.testing.expectEqual(@as(usize, 1), p.lots.len);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "deserializePortfolio: null diags still logs the symbol-less skip" {
|
||||||
|
// Covers the non-collecting branch of the second skip path: a record
|
||||||
|
// with no symbol that is neither cash nor illiquid. The 16 call sites
|
||||||
|
// that never opted in must keep skipping it rather than aborting.
|
||||||
|
const prev_level = std.testing.log_level;
|
||||||
|
std.testing.log_level = .err;
|
||||||
|
defer std.testing.log_level = prev_level;
|
||||||
|
|
||||||
|
const data =
|
||||||
|
"#!srfv1\n" ++
|
||||||
|
"shares:num:100,open_date::2024-01-15,open_price:num:140.00\n" ++
|
||||||
|
"symbol::AAPL,shares:num:10,open_date::2024-01-15,open_price:num:150.00\n";
|
||||||
|
var p = try deserializePortfolio(std.testing.allocator, data);
|
||||||
|
defer p.deinit();
|
||||||
|
try std.testing.expectEqual(@as(usize, 1), p.lots.len);
|
||||||
|
try std.testing.expectEqualStrings("AAPL", p.lots[0].symbol);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "deserializePortfolioDiag: an illiquid record with no symbol gets a placeholder" {
|
||||||
|
// Cash and illiquid are the two types allowed to omit `symbol` -
|
||||||
|
// they get a placeholder rather than being skipped.
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
const data =
|
||||||
|
"#!srfv1\n" ++
|
||||||
|
"security_type::illiquid,shares:num:450000,open_date::2020-06-01,open_price:num:350000\n";
|
||||||
|
|
||||||
|
var diags: ParseDiagnostics = .empty;
|
||||||
|
defer diags.deinit(allocator);
|
||||||
|
|
||||||
|
var p = try deserializePortfolioDiag(allocator, data, &diags);
|
||||||
|
defer p.deinit();
|
||||||
|
|
||||||
|
try std.testing.expectEqual(@as(usize, 1), p.lots.len);
|
||||||
|
try std.testing.expectEqualStrings("ILLIQUID", p.lots[0].symbol);
|
||||||
|
try std.testing.expectEqual(@as(usize, 0), diags.items.len);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "nthLine: 1-based, terminator-free, out of range is empty" {
|
||||||
|
const data = "alpha\nbeta\r\ngamma";
|
||||||
|
try std.testing.expectEqualStrings("alpha", nthLine(data, 1));
|
||||||
|
try std.testing.expectEqualStrings("beta", nthLine(data, 2)); // \r trimmed
|
||||||
|
try std.testing.expectEqualStrings("gamma", nthLine(data, 3)); // no trailing \n
|
||||||
|
try std.testing.expectEqualStrings("", nthLine(data, 4));
|
||||||
|
try std.testing.expectEqualStrings("", nthLine(data, 0));
|
||||||
|
}
|
||||||
|
|
||||||
test "cacheKeys: directory names only, sorted, store-internal keys excluded" {
|
test "cacheKeys: directory names only, sorted, store-internal keys excluded" {
|
||||||
const allocator = std.testing.allocator;
|
const allocator = std.testing.allocator;
|
||||||
const io = std.testing.io;
|
const io = std.testing.io;
|
||||||
|
|
|
||||||
|
|
@ -499,9 +499,126 @@ pub fn loadPortfolio(ctx: *framework.RunCtx, as_of: zfin.Date) ?LoadedPortfolio
|
||||||
portfolio_loader.applySplitAdjustment(svc, ctx.allocator, &loaded, as_of, fetchOptionsFromPolicy(ctx.globals.refresh_policy));
|
portfolio_loader.applySplitAdjustment(svc, ctx.allocator, &loaded, as_of, fetchOptionsFromPolicy(ctx.globals.refresh_policy));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// A failed write here means stdout is broken, so the command's own
|
||||||
|
// output is about to fail too. Refuse to hand back a portfolio we
|
||||||
|
// could not attach the caveat to rather than letting it be reported
|
||||||
|
// as if it were complete.
|
||||||
|
printPortfolioWarnings(ctx.out, ctx.color, loaded.warnings) catch |err| {
|
||||||
|
stderrPrint(ctx.io, "Error reporting skipped portfolio records: ");
|
||||||
|
stderrPrint(ctx.io, @errorName(err));
|
||||||
|
stderrPrint(ctx.io, "\n");
|
||||||
|
loaded.deinit(ctx.allocator);
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
return loaded;
|
return loaded;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Report records that were dropped during the load, before whatever the
|
||||||
|
/// command is about to print.
|
||||||
|
///
|
||||||
|
/// This is the one place it happens. All 16 CLI entry points reach a live
|
||||||
|
/// portfolio through `loadPortfolio`, so a single call there covers them
|
||||||
|
/// without each command remembering to. It deliberately is NOT called
|
||||||
|
/// from `portfolio_loader`: `contributions` loads two historical
|
||||||
|
/// revisions through the same loader, and warnings about a months-old
|
||||||
|
/// commit would be noise rather than news.
|
||||||
|
///
|
||||||
|
/// Written to the command's writer rather than `std.log`, because the
|
||||||
|
/// point is to be read. A dropped lot silently lowers net worth,
|
||||||
|
/// allocation percentages and every derived figure; a stderr line is easy
|
||||||
|
/// to miss with stdout piped or a screen of tables following it. Hence
|
||||||
|
/// the closing sentence naming the consequence rather than just a count.
|
||||||
|
///
|
||||||
|
/// Takes the writer and color flag rather than a `RunCtx` so the exact
|
||||||
|
/// wording can be asserted in a test - this output is the whole point of
|
||||||
|
/// collecting the warnings, so it should not be the untested part.
|
||||||
|
fn printPortfolioWarnings(out: *std.Io.Writer, color: bool, warnings: []const []const u8) !void {
|
||||||
|
if (warnings.len == 0) return;
|
||||||
|
|
||||||
|
try setFg(out, color, CLR_WARNING);
|
||||||
|
try out.print("warning: {d} record(s) skipped while reading the portfolio\n", .{warnings.len});
|
||||||
|
try reset(out, color);
|
||||||
|
for (warnings) |w| {
|
||||||
|
try setFg(out, color, CLR_MUTED);
|
||||||
|
try out.print(" {s}\n", .{w});
|
||||||
|
try reset(out, color);
|
||||||
|
}
|
||||||
|
try setFg(out, color, CLR_WARNING);
|
||||||
|
try out.writeAll(" Figures below exclude them.\n\n");
|
||||||
|
try reset(out, color);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "printPortfolioWarnings: nothing to report writes nothing" {
|
||||||
|
var aw: std.Io.Writer.Allocating = .init(std.testing.allocator);
|
||||||
|
defer aw.deinit();
|
||||||
|
try printPortfolioWarnings(&aw.writer, false, &.{});
|
||||||
|
try std.testing.expectEqualStrings("", aw.written());
|
||||||
|
}
|
||||||
|
|
||||||
|
test "printPortfolioWarnings: names the count and states the consequence" {
|
||||||
|
// The consequence line is the reason this is in band at all: the
|
||||||
|
// count alone does not tell the user their totals are short.
|
||||||
|
var aw: std.Io.Writer.Allocating = .init(std.testing.allocator);
|
||||||
|
defer aw.deinit();
|
||||||
|
const warnings = [_][]const u8{
|
||||||
|
"portfolio.srf: line 3: StringValueNotValidEnumMember\n symbol::MSFT,security_type::stok",
|
||||||
|
"portfolio_closed.srf: line 9: no symbol\n shares:num:5",
|
||||||
|
};
|
||||||
|
try printPortfolioWarnings(&aw.writer, false, &warnings);
|
||||||
|
const text = aw.written();
|
||||||
|
|
||||||
|
try std.testing.expect(std.mem.startsWith(u8, text, "warning: 2 record(s) skipped while reading the portfolio\n"));
|
||||||
|
try std.testing.expect(std.mem.indexOf(u8, text, "portfolio.srf: line 3: StringValueNotValidEnumMember") != null);
|
||||||
|
try std.testing.expect(std.mem.indexOf(u8, text, "portfolio_closed.srf: line 9: no symbol") != null);
|
||||||
|
try std.testing.expect(std.mem.endsWith(u8, text, " Figures below exclude them.\n\n"));
|
||||||
|
}
|
||||||
|
|
||||||
|
test "loadWatchlist: valid symbols load, a malformed record is skipped" {
|
||||||
|
// `loadWatchlist` had no coverage at all. A record missing `symbol`
|
||||||
|
// fails coercion and must be skipped rather than costing the rest of
|
||||||
|
// the watchlist.
|
||||||
|
const prev_level = std.testing.log_level;
|
||||||
|
std.testing.log_level = .err;
|
||||||
|
defer std.testing.log_level = prev_level;
|
||||||
|
|
||||||
|
const io = std.testing.io;
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
|
||||||
|
var tmp = std.testing.tmpDir(.{});
|
||||||
|
defer tmp.cleanup();
|
||||||
|
try tmp.dir.writeFile(io, .{ .sub_path = "watchlist.srf", .data =
|
||||||
|
\\#!srfv1
|
||||||
|
\\symbol::AAPL
|
||||||
|
\\note::no symbol here
|
||||||
|
\\symbol::MSFT
|
||||||
|
\\
|
||||||
|
});
|
||||||
|
|
||||||
|
var path_buf: [std.fs.max_path_bytes]u8 = undefined;
|
||||||
|
const dir_len = try tmp.dir.realPathFile(io, ".", &path_buf);
|
||||||
|
const path = try std.fs.path.join(allocator, &.{ path_buf[0..dir_len], "watchlist.srf" });
|
||||||
|
defer allocator.free(path);
|
||||||
|
|
||||||
|
const syms = loadWatchlist(io, allocator, path) orelse {
|
||||||
|
try std.testing.expect(false);
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
defer {
|
||||||
|
for (syms) |sym| allocator.free(sym);
|
||||||
|
allocator.free(syms);
|
||||||
|
}
|
||||||
|
|
||||||
|
try std.testing.expectEqual(@as(usize, 2), syms.len);
|
||||||
|
try std.testing.expectEqualStrings("AAPL", syms[0]);
|
||||||
|
try std.testing.expectEqualStrings("MSFT", syms[1]);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "loadWatchlist: a missing file is null, not an error" {
|
||||||
|
const allocator = std.testing.allocator;
|
||||||
|
try std.testing.expect(loadWatchlist(std.testing.io, allocator, "does_not_exist_watchlist.srf") == null);
|
||||||
|
}
|
||||||
|
|
||||||
// ── As-of date parsing (shared by CLI --as-of and TUI date popup) ──
|
// ── As-of date parsing (shared by CLI --as-of and TUI date popup) ──
|
||||||
|
|
||||||
pub const AsOfParseError = error{
|
pub const AsOfParseError = error{
|
||||||
|
|
@ -1008,7 +1125,16 @@ pub fn loadWatchlist(io: std.Io, allocator: std.mem.Allocator, path: []const u8)
|
||||||
|
|
||||||
var syms: std.ArrayList([]const u8) = .empty;
|
var syms: std.ArrayList([]const u8) = .empty;
|
||||||
while (it.next() catch null) |fields| {
|
while (it.next() catch null) |fields| {
|
||||||
const entry = fields.to(WatchEntry, srf_opts.user_edited) catch continue;
|
const entry = fields.to(WatchEntry, srf_opts.user_edited) catch |err| {
|
||||||
|
// Skip the entry rather than losing the whole watchlist, but
|
||||||
|
// name the error - otherwise a symbol just stops appearing.
|
||||||
|
// Quiet under `zig build test`, where fixtures feed
|
||||||
|
// malformed records on purpose.
|
||||||
|
if (!builtin.is_test) {
|
||||||
|
std.log.warn("watchlist.srf: skipping malformed record: {s}", .{@errorName(err)});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
};
|
||||||
const duped = allocator.dupe(u8, entry.symbol) catch continue;
|
const duped = allocator.dupe(u8, entry.symbol) catch continue;
|
||||||
syms.append(allocator, duped) catch {
|
syms.append(allocator, duped) catch {
|
||||||
allocator.free(duped);
|
allocator.free(duped);
|
||||||
|
|
|
||||||
|
|
@ -45,8 +45,13 @@ pub const meta: framework.Meta = .{
|
||||||
\\Usage: zfin diagnose SYMBOL
|
\\Usage: zfin diagnose SYMBOL
|
||||||
\\
|
\\
|
||||||
\\Reports, in order:
|
\\Reports, in order:
|
||||||
\\ local newest cached bar, TTL state, provider, failure count
|
\\ local newest cached bar, TTL state, provider, failure count,
|
||||||
|
\\ and any active Tiingo backoff
|
||||||
|
\\ adj basis how current the cached adj_close values are, against the
|
||||||
|
\\ newest dividend or split that has already gone ex
|
||||||
\\ peers newest bar held by other symbols of the same kind
|
\\ peers newest bar held by other symbols of the same kind
|
||||||
|
\\ tracked whether a normal run fetches this symbol at all, or it is
|
||||||
|
\\ only pulled on demand (a projections benchmark, say)
|
||||||
\\ server what ZFIN_SERVER offers, whether it is ahead or behind, and
|
\\ server what ZFIN_SERVER offers, whether it is ahead or behind, and
|
||||||
\\ whether it refreshes this symbol at all
|
\\ whether it refreshes this symbol at all
|
||||||
\\ provider what the upstream provider actually has right now
|
\\ provider what the upstream provider actually has right now
|
||||||
|
|
|
||||||
|
|
@ -34,10 +34,13 @@
|
||||||
//! "model said you're already there", or absent.
|
//! "model said you're already there", or absent.
|
||||||
|
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
|
const builtin = @import("builtin");
|
||||||
const srf = @import("srf");
|
const srf = @import("srf");
|
||||||
const srf_opts = @import("../srf_opts.zig");
|
const srf_opts = @import("../srf_opts.zig");
|
||||||
const Date = @import("../Date.zig");
|
const Date = @import("../Date.zig");
|
||||||
|
|
||||||
|
const log = std.log.scoped(.imported_values);
|
||||||
|
|
||||||
// ── Types ────────────────────────────────────────────────────
|
// ── Types ────────────────────────────────────────────────────
|
||||||
|
|
||||||
/// Projection of "when can the user retire," as captured at a
|
/// Projection of "when can the user retire," as captured at a
|
||||||
|
|
@ -177,7 +180,27 @@ pub fn parseImportedValues(
|
||||||
errdefer points.deinit(allocator);
|
errdefer points.deinit(allocator);
|
||||||
|
|
||||||
while (it.next() catch return error.InvalidSrf) |fields| {
|
while (it.next() catch return error.InvalidSrf) |fields| {
|
||||||
const point = fields.to(HistoryPoint, srf_opts.user_edited) catch return error.InvalidSrf;
|
// Aborts rather than skipping: this is a small curated series,
|
||||||
|
// so one unparseable row means the transcription is suspect and
|
||||||
|
// a partial series would be worse than none.
|
||||||
|
//
|
||||||
|
// Collapses to `InvalidSrf` on purpose, and logs the real error
|
||||||
|
// instead of returning it. `history.resolveAsOfDate` enumerates
|
||||||
|
// this function's parse errors to degrade gracefully to
|
||||||
|
// "no data at or before" - and srf's coercion errors are an
|
||||||
|
// open-ended set, so propagating them raw would silently fall
|
||||||
|
// through that switch and surface an unexplained error instead
|
||||||
|
// of the actionable "run `zfin snapshot` or populate
|
||||||
|
// imported_values.srf" message. The log carries the cause; the
|
||||||
|
// error set stays enumerable.
|
||||||
|
const point = fields.to(HistoryPoint, srf_opts.user_edited) catch |err| {
|
||||||
|
// Quiet under `zig build test`, where a fixture feeds a
|
||||||
|
// malformed row on purpose to pin the collapse above.
|
||||||
|
if (!builtin.is_test) {
|
||||||
|
log.warn("imported_values.srf: record {d}: {s}", .{ points.items.len + 1, @errorName(err) });
|
||||||
|
}
|
||||||
|
return error.InvalidSrf;
|
||||||
|
};
|
||||||
try points.append(allocator, point);
|
try points.append(allocator, point);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -284,6 +307,45 @@ test "parseImportedValues: missing optional fields" {
|
||||||
try std.testing.expectEqual(@as(?ProjectedRetirement, null), iv.points[0].projected_retirement);
|
try std.testing.expectEqual(@as(?ProjectedRetirement, null), iv.points[0].projected_retirement);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "parseImportedValues: an unparseable record collapses to InvalidSrf" {
|
||||||
|
// REGRESSION GUARD. `history.resolveAsOfDate` enumerates this
|
||||||
|
// function's parse errors to degrade gracefully to
|
||||||
|
// `NoDataAtOrBefore`, which is what produces the actionable "run
|
||||||
|
// `zfin snapshot` or populate imported_values.srf" message. srf's
|
||||||
|
// coercion errors are an open-ended set, so letting one through raw
|
||||||
|
// fell past that switch and surfaced an unexplained error instead.
|
||||||
|
//
|
||||||
|
// `liquid` is declared numeric; a string separator makes it a
|
||||||
|
// coercion failure that `strings_to_numbers` cannot rescue, since
|
||||||
|
// "abc" is not a number either.
|
||||||
|
const data =
|
||||||
|
\\#!srfv1
|
||||||
|
\\date::2014-07-03,liquid::abc
|
||||||
|
\\
|
||||||
|
;
|
||||||
|
try std.testing.expectError(
|
||||||
|
error.InvalidSrf,
|
||||||
|
parseImportedValues(std.testing.allocator, data),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "parseImportedValues: a missing required field also collapses to InvalidSrf" {
|
||||||
|
// A second, structurally different srf error. `date` and `liquid`
|
||||||
|
// have no defaults, so omitting one fails coercion via a different
|
||||||
|
// path than a bad value does - and it must collapse identically,
|
||||||
|
// because the point is that the caller's error set stays enumerable
|
||||||
|
// no matter which way coercion fails.
|
||||||
|
const data =
|
||||||
|
\\#!srfv1
|
||||||
|
\\liquid:num:1280036.42
|
||||||
|
\\
|
||||||
|
;
|
||||||
|
try std.testing.expectError(
|
||||||
|
error.InvalidSrf,
|
||||||
|
parseImportedValues(std.testing.allocator, data),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
test "parseImportedValues: empty file (header only)" {
|
test "parseImportedValues: empty file (header only)" {
|
||||||
const data = "#!srfv1\n";
|
const data = "#!srfv1\n";
|
||||||
var iv = try parseImportedValues(std.testing.allocator, data);
|
var iv = try parseImportedValues(std.testing.allocator, data);
|
||||||
|
|
|
||||||
|
|
@ -1433,6 +1433,46 @@ test "resolveAsOfDate: imported-only falls back to imported_values" {
|
||||||
try testing.expectEqual(@as(f64, 1_510_000), r.liquid);
|
try testing.expectEqual(@as(f64, 1_510_000), r.liquid);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "resolveAsOfDate: a malformed imported_values.srf degrades to NoDataAtOrBefore" {
|
||||||
|
// THE REGRESSION THIS GUARDS. The switch below enumerates
|
||||||
|
// `parseImportedValues`' parse errors to degrade gracefully, which is
|
||||||
|
// what lets `resolveAsOfOrExplain` print the actionable "run
|
||||||
|
// `zfin snapshot` or populate imported_values.srf" message.
|
||||||
|
//
|
||||||
|
// srf's coercion errors are an open-ended set, so when
|
||||||
|
// `parseImportedValues` was briefly changed to propagate them raw
|
||||||
|
// they fell straight through this switch and the user got an
|
||||||
|
// unexplained `IntNotNumberType` instead. The collapse to
|
||||||
|
// `InvalidSrf` is load-bearing, not laziness - see the comment in
|
||||||
|
// `parseImportedValues`.
|
||||||
|
const io = std.testing.io;
|
||||||
|
var tmp = std.testing.tmpDir(.{});
|
||||||
|
defer tmp.cleanup();
|
||||||
|
|
||||||
|
// `liquid` is declared numeric; a string separator with a
|
||||||
|
// non-numeric value is a coercion failure `strings_to_numbers`
|
||||||
|
// cannot rescue.
|
||||||
|
try tmp.dir.writeFile(io, .{ .sub_path = "imported_values.srf", .data =
|
||||||
|
\\#!srfv1
|
||||||
|
\\date::2016-01-03,liquid::abc
|
||||||
|
\\
|
||||||
|
});
|
||||||
|
|
||||||
|
const hist_dir = try tmp.dir.realPathFileAlloc(io, ".", testing.allocator);
|
||||||
|
defer testing.allocator.free(hist_dir);
|
||||||
|
|
||||||
|
var arena = std.heap.ArenaAllocator.init(testing.allocator);
|
||||||
|
defer arena.deinit();
|
||||||
|
|
||||||
|
// No snapshots either, so the imported path is the only candidate -
|
||||||
|
// and an unusable file must read as "no data", not as a raw parse
|
||||||
|
// error escaping to the command.
|
||||||
|
try testing.expectError(
|
||||||
|
error.NoDataAtOrBefore,
|
||||||
|
resolveAsOfDate(io, arena.allocator(), hist_dir, Date.fromYmd(2016, 6, 1)),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
test "resolveAsOfDate: snapshot wins over imported when both present" {
|
test "resolveAsOfDate: snapshot wins over imported when both present" {
|
||||||
const io = std.testing.io;
|
const io = std.testing.io;
|
||||||
var tmp = std.testing.tmpDir(.{});
|
var tmp = std.testing.tmpDir(.{});
|
||||||
|
|
|
||||||
|
|
@ -10,10 +10,13 @@
|
||||||
/// symbol::02315N600,asset_class::International Developed,pct:num:20
|
/// symbol::02315N600,asset_class::International Developed,pct:num:20
|
||||||
/// symbol::02315N600,asset_class::Bonds,pct:num:15
|
/// symbol::02315N600,asset_class::Bonds,pct:num:15
|
||||||
const std = @import("std");
|
const std = @import("std");
|
||||||
|
const builtin = @import("builtin");
|
||||||
const srf = @import("srf");
|
const srf = @import("srf");
|
||||||
const srf_opts = @import("../srf_opts.zig");
|
const srf_opts = @import("../srf_opts.zig");
|
||||||
const Date = @import("../Date.zig");
|
const Date = @import("../Date.zig");
|
||||||
|
|
||||||
|
const log = std.log.scoped(.metadata);
|
||||||
|
|
||||||
/// A single classification entry for a symbol.
|
/// A single classification entry for a symbol.
|
||||||
pub const ClassificationEntry = struct {
|
pub const ClassificationEntry = struct {
|
||||||
symbol: []const u8,
|
symbol: []const u8,
|
||||||
|
|
@ -90,7 +93,16 @@ pub fn parseClassificationFile(allocator: std.mem.Allocator, data: []const u8) !
|
||||||
defer it.deinit();
|
defer it.deinit();
|
||||||
|
|
||||||
while (try it.next()) |fields| {
|
while (try it.next()) |fields| {
|
||||||
const entry = fields.to(ClassificationEntry, srf_opts.user_edited) catch continue;
|
const entry = fields.to(ClassificationEntry, srf_opts.user_edited) catch |err| {
|
||||||
|
// Skip the row rather than losing the whole file, but name
|
||||||
|
// the error: a silently dropped row quietly changes a
|
||||||
|
// breakdown's percentages. Quiet under `zig build test`,
|
||||||
|
// where fixtures feed malformed rows on purpose.
|
||||||
|
if (!builtin.is_test) {
|
||||||
|
log.warn("metadata.srf: skipping malformed record: {s}", .{@errorName(err)});
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
};
|
||||||
// Pre-fill `bucket` if the user didn't curate one. This
|
// Pre-fill `bucket` if the user didn't curate one. This
|
||||||
// shifts the cost of `deriveBucket` to parse time and
|
// shifts the cost of `deriveBucket` to parse time and
|
||||||
// makes downstream code free to read `entry.bucket`
|
// makes downstream code free to read `entry.bucket`
|
||||||
|
|
@ -288,6 +300,28 @@ test "parse classification file: a hand-typed string separator on pct still pars
|
||||||
try std.testing.expectApproxEqAbs(@as(f64, 40), map.entries[1].pct, 0.001);
|
try std.testing.expectApproxEqAbs(@as(f64, 40), map.entries[1].pct, 0.001);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "parse classification file: a record missing symbol is skipped, not fatal" {
|
||||||
|
// `symbol` has no default, so a record without it fails coercion.
|
||||||
|
// One bad row must not cost the whole file - every other symbol's
|
||||||
|
// sector and geo would silently vanish from the breakdowns.
|
||||||
|
const prev_level = std.testing.log_level;
|
||||||
|
std.testing.log_level = .err;
|
||||||
|
defer std.testing.log_level = prev_level;
|
||||||
|
|
||||||
|
const data =
|
||||||
|
\\#!srfv1
|
||||||
|
\\symbol::AAPL,sector::Technology
|
||||||
|
\\sector::Healthcare,pct:num:100
|
||||||
|
\\symbol::MSFT,sector::Technology
|
||||||
|
;
|
||||||
|
var map = try parseClassificationFile(std.testing.allocator, data);
|
||||||
|
defer map.deinit();
|
||||||
|
|
||||||
|
try std.testing.expectEqual(@as(usize, 2), map.entries.len);
|
||||||
|
try std.testing.expectEqualStrings("AAPL", map.entries[0].symbol);
|
||||||
|
try std.testing.expectEqualStrings("MSFT", map.entries[1].symbol);
|
||||||
|
}
|
||||||
|
|
||||||
test "parse classification file: bucket round-trips" {
|
test "parse classification file: bucket round-trips" {
|
||||||
const data =
|
const data =
|
||||||
\\#!srfv1
|
\\#!srfv1
|
||||||
|
|
|
||||||
|
|
@ -90,6 +90,16 @@ pub const LoadedPortfolio = struct {
|
||||||
portfolio: zfin.Portfolio,
|
portfolio: zfin.Portfolio,
|
||||||
positions: []const zfin.Position,
|
positions: []const zfin.Position,
|
||||||
syms: []const []const u8,
|
syms: []const []const u8,
|
||||||
|
/// One message per record that could not be parsed and was
|
||||||
|
/// therefore left out of `portfolio`. Empty on a clean load.
|
||||||
|
///
|
||||||
|
/// A dropped lot silently changes every figure derived from this
|
||||||
|
/// portfolio, so these travel with the load rather than going only
|
||||||
|
/// to `std.log`: whoever prints the figures can print the caveat
|
||||||
|
/// beside them. `commands.common.loadPortfolio` does exactly that.
|
||||||
|
/// Each message is prefixed with the file it came from, since the
|
||||||
|
/// load is a union over the whole `portfolio*.srf` glob. Owned.
|
||||||
|
warnings: []const []const u8 = &.{},
|
||||||
|
|
||||||
pub fn deinit(self: *LoadedPortfolio, allocator: std.mem.Allocator) void {
|
pub fn deinit(self: *LoadedPortfolio, allocator: std.mem.Allocator) void {
|
||||||
allocator.free(self.syms);
|
allocator.free(self.syms);
|
||||||
|
|
@ -97,6 +107,8 @@ pub const LoadedPortfolio = struct {
|
||||||
self.portfolio.deinit();
|
self.portfolio.deinit();
|
||||||
for (self.file_datas) |d| allocator.free(d);
|
for (self.file_datas) |d| allocator.free(d);
|
||||||
allocator.free(self.file_datas);
|
allocator.free(self.file_datas);
|
||||||
|
for (self.warnings) |w| allocator.free(w);
|
||||||
|
allocator.free(self.warnings);
|
||||||
// Path-string ownership: `resolved_paths` (if present) owns
|
// Path-string ownership: `resolved_paths` (if present) owns
|
||||||
// the underlying path strings. The `paths` slice is the
|
// the underlying path strings. The `paths` slice is the
|
||||||
// borrowed view; free only its outer storage.
|
// borrowed view; free only its outer storage.
|
||||||
|
|
@ -387,6 +399,11 @@ fn loadFromBytes(
|
||||||
var lots_owner: LotsOwner = .merged_list;
|
var lots_owner: LotsOwner = .merged_list;
|
||||||
var success = false;
|
var success = false;
|
||||||
|
|
||||||
|
// Per-record parse diagnostics, accumulated across every file in the
|
||||||
|
// glob. Handed to the caller on success so it can report alongside
|
||||||
|
// the figures the missing lots have silently changed.
|
||||||
|
var warnings: zfin.cache.ParseDiagnostics = .empty;
|
||||||
|
|
||||||
defer if (!success) {
|
defer if (!success) {
|
||||||
switch (lots_owner) {
|
switch (lots_owner) {
|
||||||
.merged_list => {
|
.merged_list => {
|
||||||
|
|
@ -403,6 +420,8 @@ fn loadFromBytes(
|
||||||
.combined_struct => combined.deinit(),
|
.combined_struct => combined.deinit(),
|
||||||
.none => {},
|
.none => {},
|
||||||
}
|
}
|
||||||
|
for (warnings.items) |w| allocator.free(w);
|
||||||
|
warnings.deinit(allocator);
|
||||||
for (file_datas_owned) |d| allocator.free(d);
|
for (file_datas_owned) |d| allocator.free(d);
|
||||||
allocator.free(file_datas_owned);
|
allocator.free(file_datas_owned);
|
||||||
allocator.free(paths_owned);
|
allocator.free(paths_owned);
|
||||||
|
|
@ -422,12 +441,36 @@ fn loadFromBytes(
|
||||||
// without trying to parse.
|
// without trying to parse.
|
||||||
if (data.len == 0) continue;
|
if (data.len == 0) continue;
|
||||||
|
|
||||||
var portfolio = zfin.cache.deserializePortfolio(allocator, data) catch {
|
// Diagnostics are gathered per file so each message can name
|
||||||
|
// the file it came from - the load is a union over the glob, and
|
||||||
|
// "line 183" is ambiguous across several portfolio files.
|
||||||
|
var file_diags: zfin.cache.ParseDiagnostics = .empty;
|
||||||
|
defer {
|
||||||
|
for (file_diags.items) |w| allocator.free(w);
|
||||||
|
file_diags.deinit(allocator);
|
||||||
|
}
|
||||||
|
|
||||||
|
var portfolio = zfin.cache.deserializePortfolioDiag(allocator, data, &file_diags) catch {
|
||||||
var msg_buf: [512]u8 = undefined;
|
var msg_buf: [512]u8 = undefined;
|
||||||
const msg = std.fmt.bufPrint(&msg_buf, "Error: Cannot parse portfolio file: {s}\n", .{paths_owned[idx]}) catch "Error: Cannot parse portfolio file\n";
|
const msg = std.fmt.bufPrint(&msg_buf, "Error: Cannot parse portfolio file: {s}\n", .{paths_owned[idx]}) catch "Error: Cannot parse portfolio file\n";
|
||||||
stderr.print(io, msg);
|
stderr.print(io, msg);
|
||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
for (file_diags.items) |raw_msg| {
|
||||||
|
const named = std.fmt.allocPrint(allocator, "{s}: {s}", .{
|
||||||
|
std.fs.path.basename(paths_owned[idx]),
|
||||||
|
raw_msg,
|
||||||
|
}) catch {
|
||||||
|
portfolio.deinit();
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
warnings.append(allocator, named) catch {
|
||||||
|
allocator.free(named);
|
||||||
|
portfolio.deinit();
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
}
|
||||||
for (portfolio.lots) |lot| {
|
for (portfolio.lots) |lot| {
|
||||||
merged.append(allocator, lot) catch {
|
merged.append(allocator, lot) catch {
|
||||||
portfolio.deinit();
|
portfolio.deinit();
|
||||||
|
|
@ -457,6 +500,12 @@ fn loadFromBytes(
|
||||||
return null;
|
return null;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
const warnings_owned = warnings.toOwnedSlice(allocator) catch {
|
||||||
|
allocator.free(syms);
|
||||||
|
allocator.free(positions);
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
success = true;
|
success = true;
|
||||||
return .{
|
return .{
|
||||||
.paths = paths_owned,
|
.paths = paths_owned,
|
||||||
|
|
@ -465,6 +514,7 @@ fn loadFromBytes(
|
||||||
.portfolio = combined,
|
.portfolio = combined,
|
||||||
.positions = positions,
|
.positions = positions,
|
||||||
.syms = syms,
|
.syms = syms,
|
||||||
|
.warnings = warnings_owned,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -778,6 +828,67 @@ test "loadFromBytes: union of two synthetic SRF files" {
|
||||||
try testing.expectEqualStrings("MSFT", loaded.portfolio.lots[1].symbol);
|
try testing.expectEqualStrings("MSFT", loaded.portfolio.lots[1].symbol);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
test "loadFromBytes: a dropped record is reported and names its file" {
|
||||||
|
// The union spans several files, so "line 3" alone is ambiguous -
|
||||||
|
// each message carries the basename it came from. Only the second
|
||||||
|
// file has a bad record here, and only it should be named.
|
||||||
|
const allocator = testing.allocator;
|
||||||
|
|
||||||
|
const file_a =
|
||||||
|
\\#!srfv1
|
||||||
|
\\symbol::AAPL,shares:num:10,open_date::2024-01-15,open_price:num:150,account::Sample IRA
|
||||||
|
\\
|
||||||
|
;
|
||||||
|
const file_b =
|
||||||
|
\\#!srfv1
|
||||||
|
\\symbol::MSFT,shares:num:5,open_date::2024-02-01,open_price:num:300,account::Sample Roth
|
||||||
|
\\symbol::NVDA,security_type::stok,shares:num:1,open_date::2024-03-01,open_price:num:900
|
||||||
|
\\
|
||||||
|
;
|
||||||
|
|
||||||
|
const paths = try allocator.dupe([]const u8, &.{ "portfolio.srf", "portfolio_closed.srf" });
|
||||||
|
const datas = try dupeBytes(allocator, &.{ file_a, file_b });
|
||||||
|
|
||||||
|
var loaded = loadFromBytes(testing.io, allocator, paths, null, datas, zfin.Date.fromYmd(2026, 5, 23)) orelse {
|
||||||
|
try testing.expect(false);
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
defer loaded.deinit(allocator);
|
||||||
|
|
||||||
|
// Two good lots merged; the bad one dropped rather than fatal.
|
||||||
|
try testing.expectEqual(@as(usize, 2), loaded.portfolio.lots.len);
|
||||||
|
|
||||||
|
try testing.expectEqual(@as(usize, 1), loaded.warnings.len);
|
||||||
|
const w = loaded.warnings[0];
|
||||||
|
try testing.expect(std.mem.startsWith(u8, w, "portfolio_closed.srf: "));
|
||||||
|
try testing.expect(std.mem.indexOf(u8, w, "line 3") != null);
|
||||||
|
try testing.expect(std.mem.indexOf(u8, w, "StringValueNotValidEnumMember") != null);
|
||||||
|
try testing.expect(std.mem.indexOf(u8, w, "security_type::stok") != null);
|
||||||
|
}
|
||||||
|
|
||||||
|
test "loadFromBytes: a clean load reports nothing" {
|
||||||
|
// Guards against the warnings slice being non-empty (or unfreed) on
|
||||||
|
// the happy path - `testing.allocator` catches the leak either way.
|
||||||
|
const allocator = testing.allocator;
|
||||||
|
|
||||||
|
const file_a =
|
||||||
|
\\#!srfv1
|
||||||
|
\\symbol::AAPL,shares:num:10,open_date::2024-01-15,open_price:num:150,account::Sample IRA
|
||||||
|
\\
|
||||||
|
;
|
||||||
|
|
||||||
|
const paths = try allocator.dupe([]const u8, &.{"portfolio.srf"});
|
||||||
|
const datas = try dupeBytes(allocator, &.{file_a});
|
||||||
|
|
||||||
|
var loaded = loadFromBytes(testing.io, allocator, paths, null, datas, zfin.Date.fromYmd(2026, 5, 23)) orelse {
|
||||||
|
try testing.expect(false);
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
defer loaded.deinit(allocator);
|
||||||
|
|
||||||
|
try testing.expectEqual(@as(usize, 0), loaded.warnings.len);
|
||||||
|
}
|
||||||
|
|
||||||
test "loadFromBytes: single file with valid contents" {
|
test "loadFromBytes: single file with valid contents" {
|
||||||
const allocator = testing.allocator;
|
const allocator = testing.allocator;
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Reference in a new issue