diff --git a/TODO.md b/TODO.md index f56871a..11103ad 100644 --- a/TODO.md +++ b/TODO.md @@ -489,6 +489,59 @@ populate. This could be solved on the server by spawning a thread to fetch the data, then returning 202 Accepted, which could then be polled client side. Maybe this is a better long term approach? +## Configurable live-quote provider (Tiingo IEX) - priority LOW + +The TUI refresh key (`r`) values the portfolio with live intraday +quotes via `DataService.loadLiveQuotes`, which is Yahoo-only: Yahoo is +keyless, consolidated, and stays off every rate-limit budget, so bursty +refresh traffic costs nothing. The tradeoffs are that Yahoo's +unofficial feed is ~15-minute delayed and "can break without notice." + +Tiingo's IEX endpoint (`/iex/?tickers=A,B,C`) is a strong opt-in +alternative: it's genuinely real-time (IEX last-sale, no 15-min delay), +official/keyed, and bills per HTTP request - one call returns the whole +portfolio (confirmed empirically: a 2-ticker batch decrements the daily +quota by 1, not 2). Fields map cleanly: `tngoLast` to price, `prevClose` +to day-change. Caveats: IEX is a single venue (~2-3% of volume), so +`tngoLast` can sit stale between prints on illiquid names, and IEX +doesn't trade mutual funds, so those fall back to the candle close. + +Proposal: a config knob (env var, e.g. `ZFIN_LIVE_QUOTE_PROVIDER` = +`yahoo` (default) | `tiingo`) that switches `loadLiveQuotes` to a new +`Tiingo.fetchQuotes(tickers)` batched call. Someone on Tiingo's Power +tier ($30/mo, higher limits) who wants real-time and mashes `r` a lot +(or once we add streaming) reuses their existing `TIINGO_API_KEY` and +gets real-time coverage; everyone else keeps the keyless Yahoo default. + +Implementation notes: + +- `Tiingo.fetchQuotes` returns an array whose order is NOT guaranteed to + match the request order, so key results by the returned `ticker` + field, not by position. +- Depends on honoring Tiingo's 50 req/hour limit (see below): routing + bursty live traffic through Tiingo without that enforcement risks + starving the candle/EOD path. +- Tiingo websocket streaming would be the natural follow-on for true + push-based real-time, replacing poll-on-`r` entirely. + +## Rate limiter doesn't honor Tiingo's 50 req/hour limit - priority MEDIUM + +Tiingo's free tier caps at both 1,000 req/day AND 50 req/hour, but the +`net/RateLimiter.zig` token bucket (and the `tiingo.zig` header note, +"no per-minute restriction") only accounts for the daily ceiling. The +hourly limit is unenforced today. + +This bites the candle path hardest: candle fetches are per-symbol +(`/tiingo/daily/{sym}/prices`), so a cold-cache load of a ~40-name +portfolio fires ~40 requests in one burst - most of the hourly budget +in a single app open. Nothing throttles that; we'd just start taking +429s mid-load. + +Fix: extend the limiter to enforce a rolling 50/hour window for Tiingo +(in addition to the daily count), backing off or queueing when the +window is full. Directly relevant to the live-quote-provider knob +above: any Tiingo-sourced live quotes would share this same bucket. + ## Audit: em-dash sentinel usage across all tables — priority LOW The codebase uses `—` (em-dash) as the canonical "no data" sentinel diff --git a/src/PortfolioData.zig b/src/PortfolioData.zig index ceaa06e..b273e94 100644 --- a/src/PortfolioData.zig +++ b/src/PortfolioData.zig @@ -182,6 +182,15 @@ pub const LoadOptions = struct { /// these with portfolio's own `.watch` lots and dedups. /// Borrowed; pd does not take ownership of the slice. watchlist_syms: []const []const u8 = &.{}, + /// Optional live-quote overlay: symbol → live last price. When + /// present, these prices override the candle-derived last close + /// for matching symbols as the portfolio summary is built (held + /// positions overlay the summary prices; watchlist symbols overlay + /// `pd.watchlist_prices`). Symbols absent from the map keep their + /// candle last close. Used by the TUI refresh key to value the + /// portfolio with current intraday quotes rather than the prior + /// daily close. Borrowed; pd does not take ownership. + live_quotes: ?*const std.StringHashMap(f64) = null, /// Per-worker start delays. Each background worker sleeps for /// its delay before doing any work, letting the caller /// deprioritize a specific worker (e.g. push it later so a @@ -429,6 +438,38 @@ fn awaitWorker(self: *PortfolioData, fut: *?std.Io.Future(void)) void { // ── Loading ────────────────────────────────────────────────── +/// Overlay live quotes onto the candle-derived price maps before the +/// summary is built. Held symbols (in `portfolio_set`) win in +/// `prices`; watchlist symbols (in `watchlist_set`) win in `wp` (keys +/// duped into `arena` so they outlive the load). Symbols in neither +/// set are ignored. Held puts may insert a new entry borrowing the +/// live map's key, which the caller guarantees outlives the summary +/// build. Factored out of `load` so the overlay logic is unit-testable +/// without the full load machinery. +fn applyLiveQuoteOverlay( + arena: Allocator, + prices: *std.StringHashMap(f64), + wp: *std.StringHashMap(f64), + portfolio_set: *const std.StringHashMap(void), + watchlist_set: *const std.StringHashMap(void), + live: *const std.StringHashMap(f64), +) error{OutOfMemory}!void { + var lit = live.iterator(); + while (lit.next()) |e| { + const sym = e.key_ptr.*; + const px = e.value_ptr.*; + if (portfolio_set.contains(sym)) { + // put() updates the value, keeping the existing + // (load_all-borrowed) key when present; on insert it + // borrows the live map's key, which outlives this call. + prices.put(sym, px) catch return error.OutOfMemory; + } else if (watchlist_set.contains(sym)) { + const owned = arena.dupe(u8, sym) catch continue; + wp.put(owned, px) catch |err| log.warn("live watchlist price put({s}): {t}", .{ sym, err }); + } + } +} + /// Kick off a portfolio load. Synchronous: parses the portfolio /// files, fetches prices, computes the summary, populates /// `latest_quote_date` / `watchlist_prices`, then spawns four @@ -593,6 +634,16 @@ pub fn load( } } self.latest_quote_date = load_all.latest_date; + + // Live-quote overlay: when the caller supplies current intraday + // quotes (TUI refresh), they win over the candle-derived last + // close for matching symbols. Held symbols overlay `prices` (fed + // to the summary below); watchlist symbols overlay `wp`. Symbols + // absent from the overlay keep their candle close — the candle + // layer (built above) is always the fallback. + if (opts.live_quotes) |live| { + try applyLiveQuoteOverlay(arena_alloc, &prices, &wp, &portfolio_set, &watchlist_set, live); + } self.watchlist_prices = wp; // ── Build summary ───────────────────────────────────────── @@ -1053,6 +1104,46 @@ test "PortfolioData.WorkerDelays: defaults are all zero" { // real DataService — pre-populating the map directly and // inspecting state. +test "applyLiveQuoteOverlay: held wins in prices, watchlist in wp, others ignored" { + var arena = std.heap.ArenaAllocator.init(testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + // Candle-derived base layer. + var prices = std.StringHashMap(f64).init(testing.allocator); + defer prices.deinit(); + try prices.put("AAPL", 100.0); // held; live should override + var wp = std.StringHashMap(f64).init(testing.allocator); + defer wp.deinit(); + try wp.put("TSLA", 200.0); // watchlist; live should override + + var portfolio_set = std.StringHashMap(void).init(testing.allocator); + defer portfolio_set.deinit(); + try portfolio_set.put("AAPL", {}); + try portfolio_set.put("MSFT", {}); // held, no base price yet + + var watchlist_set = std.StringHashMap(void).init(testing.allocator); + defer watchlist_set.deinit(); + try watchlist_set.put("TSLA", {}); + + var live = std.StringHashMap(f64).init(testing.allocator); + defer live.deinit(); + try live.put("AAPL", 111.0); // held override + try live.put("MSFT", 222.0); // held insert (no prior base) + try live.put("TSLA", 333.0); // watchlist override + try live.put("NVDA", 999.0); // in neither set → ignored + + try applyLiveQuoteOverlay(a, &prices, &wp, &portfolio_set, &watchlist_set, &live); + + try testing.expectEqual(@as(f64, 111.0), prices.get("AAPL").?); + try testing.expectEqual(@as(f64, 222.0), prices.get("MSFT").?); + try testing.expectEqual(@as(f64, 333.0), wp.get("TSLA").?); + // NVDA is neither held nor watchlisted — it must not leak into + // either map (it would have no shares/row to attach to). + try testing.expect(!prices.contains("NVDA")); + try testing.expect(!wp.contains("NVDA")); +} + test "PortfolioData.candles_data: deinit frees candles_arena cleanly" { var svc: DataService = .{ .allocator = testing.allocator, diff --git a/src/service.zig b/src/service.zig index e7f6f49..a008b2b 100644 --- a/src/service.zig +++ b/src/service.zig @@ -2177,6 +2177,63 @@ pub const DataService = struct { return result; } + /// Fetch live intraday quotes for `symbols` in parallel, returning + /// a map of symbol → live last price. Symbols whose quote fetch + /// fails (or that the provider can't price) are simply absent; the + /// caller falls back to the last cached close. + /// + /// This is a pure live-price fetch: quotes are never cached, so it + /// neither reads nor writes the candle cache. It exists for the + /// TUI refresh key (`r`), whose job is "give me current prices," + /// distinct from candle-history maintenance (TTL/startup) and from + /// `--refresh-data=force` (incremental candle top-up). + /// + /// Unlike `getQuote` (single-symbol, Yahoo→TwelveData fallback), + /// this is Yahoo-only: Yahoo is keyless with no shared rate + /// limiter, so each worker can safely own its HTTP client. + /// TwelveData's shared rate limiter makes it unsafe to fan out, and + /// its fallback role isn't worth the complexity for a bulk refresh. + /// + /// Concurrency mirrors `parallelServerSync`: one task per symbol in + /// a single `std.Io.Group`, each with its own `Yahoo` client (a + /// shared `std.http.Client` is not safe across threads — see + /// `tryOneSync`). Relies on a thread-safe `allocator`/`io`, the + /// same assumption the server-sync fan-out already makes. + /// + /// The returned map's keys borrow `symbols`: keep `symbols` alive + /// while using the map, and `deinit()` the map when done. + pub fn loadLiveQuotes(self: *DataService, symbols: []const []const u8) std.StringHashMap(f64) { + var prices = std.StringHashMap(f64).init(self.allocator); + if (symbols.len == 0) return prices; + + self.assertNetworkAllowed("loadLiveQuotes"); + + const QuoteSlot = struct { symbol: []const u8, price: ?f64 = null }; + const slots = self.allocator.alloc(QuoteSlot, symbols.len) catch return prices; + defer self.allocator.free(slots); + for (slots, 0..) |*slot, i| slot.* = .{ .symbol = symbols[i] }; + + const worker = struct { + fn run(io: std.Io, allocator: std.mem.Allocator, slot: *QuoteSlot) std.Io.Cancelable!void { + try io.checkCancel(); + var yh = Yahoo.init(io, allocator); + defer yh.deinit(); + // Quote borrows `symbol` and carries no owned memory, + // so the f64 close is all we keep — nothing to free. + slot.price = if (yh.fetchQuote(allocator, slot.symbol)) |q| q.close else |_| null; + } + }; + + var group: std.Io.Group = .init; + for (slots) |*slot| group.async(self.io, worker.run, .{ self.io, self.allocator, slot }); + group.await(self.io) catch |err| log.debug("loadLiveQuotes group await: {t}", .{err}); + + for (slots) |slot| { + if (slot.price) |p| prices.put(slot.symbol, p) catch |err| log.warn("loadLiveQuotes put({s}): {t}", .{ slot.symbol, err }); + } + return prices; + } + /// Parallel server sync via `std.Io.Group`. /// /// Concurrency shape: one task per symbol, spawned into a diff --git a/src/tui/performance_tab.zig b/src/tui/performance_tab.zig index cdc4997..2c968e7 100644 --- a/src/tui/performance_tab.zig +++ b/src/tui/performance_tab.zig @@ -59,21 +59,21 @@ pub const tab = struct { pub const deactivate = framework.noopDeactivate(State); - /// Manual refresh: invalidate the shared svc cache for candles - /// and dividends so the next `loadData` re-fetches from - /// network, then drop in-memory copies and the chart cache - /// shared with the quote tab. Quote and performance share - /// `app.symbol_data`; quote piggybacks on this reload via its - /// own delegating reload. + /// Manual refresh (r/F5): the live header quote is re-fetched by + /// the App's refresh handler (see `tui.zig`); here we just re-run + /// `loadData` so trailing returns / the chart reflect any candle + /// top-up. We do NOT invalidate candles or dividends — `r` is for + /// live prices, not candle re-downloads. Candle history is + /// maintained by the TTL/startup path, so `loadData`'s cache- + /// respecting fetch only tops up when actually stale. + /// + /// Quote and performance share `app.symbol_data`; the quote tab + /// piggybacks on this reload via its own delegating reload. pub fn reload(state: *State, app: *App) !void { - if (app.symbol.len > 0) { - app.svc.invalidate(app.symbol, .candles_daily); - app.svc.invalidate(app.symbol, .dividends); - } - // The chart is rendered by the quote tab but is fed from - // `app.symbol_data.candles` which performance owns. After - // a refresh the next quote draw must re-render and the - // indicator overlay cache (SMA/Bollinger/etc) must drop. + // The chart is rendered by the quote tab but fed from + // `app.symbol_data.candles` which performance owns. Drop the + // indicator overlay cache (SMA/Bollinger/etc) and mark the + // chart dirty so the next quote draw re-renders. app.states.quote.chart.dirty = true; app.states.quote.chart.freeCache(app.allocator); diff --git a/src/tui/portfolio_tab.zig b/src/tui/portfolio_tab.zig index f0c7cf4..14c8230 100644 --- a/src/tui/portfolio_tab.zig +++ b/src/tui/portfolio_tab.zig @@ -324,24 +324,57 @@ pub const tab = struct { pub const deactivate = framework.noopDeactivate(State); - /// Manual refresh (r/F5): re-fetch live prices and rebuild - /// the summary. Distinct from `reloadPortfolioFile` (R), - /// which also re-reads the portfolio file from disk. Refresh - /// keeps the same captured paths and just re-runs the load. + /// Manual refresh (r/F5): re-value the portfolio with live + /// intraday quotes, then rebuild the summary. `r` means "give me + /// current prices now" — it does NOT force candle work. Candle + /// history is maintained by the TTL/startup path; the reload below + /// passes `force_refresh = false`, so candles only top up if their + /// TTL is stale. (A full candle re-download is `cache clear`'s + /// job; an incremental TTL-ignoring top-up is `--refresh-data=force`.) + /// Distinct from `reloadPortfolioFile` (R), which re-reads the + /// portfolio file from disk. pub fn reload(state: *State, app: *App) !void { - // Collect watchlist symbols from app.watchlist (the - // separate `watchlist.srf` file). Portfolio's own - // `watch` lots are picked up by pd.load via the parsed - // file. Allocate against app.allocator; `pd.load` - // borrows during the call. + // Arena for the symbol strings we hand to the live-quote fetch + // and the price overlay. Held symbols MUST be duped: the reload + // below deinits the old portfolio before re-parsing, and the + // live overlay runs after that — borrowing the old portfolio's + // strings would dangle. Freed once reload + render complete. + var arena = std.heap.ArenaAllocator.init(app.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + // Watchlist symbols (separate `watchlist.srf`) are owned by + // `app` and stable across the reload, so we borrow them for the + // `watchlist_syms` arg. Portfolio's own `watch` lots are picked + // up by pd.load via the parsed file. var watch_syms: std.ArrayList([]const u8) = .empty; - defer watch_syms.deinit(app.allocator); if (app.watchlist) |wl| { - for (wl) |sym| watch_syms.append(app.allocator, sym) catch |err| std.log.debug("watch_syms append failed: {t}", .{err}); + for (wl) |sym| watch_syms.append(a, sym) catch |err| std.log.debug("watch_syms append: {t}", .{err}); } + + // Symbols to quote live = watchlist + held stock symbols. Held + // symbols are duped into the arena (see above). + var quote_syms: std.ArrayList([]const u8) = .empty; + for (watch_syms.items) |sym| quote_syms.append(a, sym) catch |err| std.log.debug("quote_syms append: {t}", .{err}); + if (app.portfolio.file) |pf| { + if (pf.stockSymbols(app.allocator)) |hs| { + defer app.allocator.free(hs); // outer slice; strings duped into arena + for (hs) |sym| { + const dup = a.dupe(u8, sym) catch continue; + quote_syms.append(a, dup) catch |err| std.log.debug("quote_syms append: {t}", .{err}); + } + } else |err| std.log.debug("stockSymbols for live quotes: {t}", .{err}); + } + + // Parallel live-quote fetch (never cached). Symbols that fail + // (or can't be quoted) are absent from the map → the summary + // falls back to the candle last close for those. + var live = app.svc.loadLiveQuotes(quote_syms.items); + defer live.deinit(); + _ = app.portfolio.reload(app.today, .{ - .force_refresh = true, .watchlist_syms = watch_syms.items, + .live_quotes = &live, }) catch |err| { app.setStatus("Error refreshing portfolio data"); std.log.scoped(.tui).warn("portfolio.reload: {t}", .{err});