//! DataService -- unified data access layer for zfin. //! //! Encapsulates the "check cache -> fresh? return -> else fetch from provider -> cache -> return" //! pattern that was previously duplicated between CLI and TUI. Both frontends should use this //! as their sole data source. //! //! Provider selection is internal: each data type routes to the appropriate provider //! based on available API keys. Callers never need to know which provider was used. const std = @import("std"); const Date = @import("models/date.zig").Date; const Candle = @import("models/candle.zig").Candle; const Dividend = @import("models/dividend.zig").Dividend; const Split = @import("models/split.zig").Split; const OptionsChain = @import("models/option.zig").OptionsChain; const EarningsEvent = @import("models/earnings.zig").EarningsEvent; const Quote = @import("models/quote.zig").Quote; const EtfProfile = @import("models/etf_profile.zig").EtfProfile; const Config = @import("config.zig").Config; const cache = @import("cache/store.zig"); const TwelveData = @import("providers/twelvedata.zig").TwelveData; const Polygon = @import("providers/polygon.zig").Polygon; const Finnhub = @import("providers/finnhub.zig").Finnhub; const Cboe = @import("providers/cboe.zig").Cboe; const AlphaVantage = @import("providers/alphavantage.zig").AlphaVantage; const alphavantage = @import("providers/alphavantage.zig"); const OpenFigi = @import("providers/openfigi.zig"); const performance = @import("analytics/performance.zig"); pub const DataError = error{ NoApiKey, FetchFailed, CacheError, ParseError, OutOfMemory, }; /// Re-exported provider types needed by commands via DataService. pub const CompanyOverview = alphavantage.CompanyOverview; /// Result of a CUSIP-to-ticker lookup (provider-agnostic). pub const CusipResult = OpenFigi.FigiResult; /// Indicates whether the returned data came from cache or was freshly fetched. pub const Source = enum { cached, fetched, }; pub const DataService = struct { allocator: std.mem.Allocator, config: Config, // Lazily initialized providers (null until first use) td: ?TwelveData = null, pg: ?Polygon = null, fh: ?Finnhub = null, cboe: ?Cboe = null, av: ?AlphaVantage = null, pub fn init(allocator: std.mem.Allocator, config: Config) DataService { return .{ .allocator = allocator, .config = config, }; } pub fn deinit(self: *DataService) void { if (self.td) |*td| td.deinit(); if (self.pg) |*pg| pg.deinit(); if (self.fh) |*fh| fh.deinit(); if (self.cboe) |*c| c.deinit(); if (self.av) |*av| av.deinit(); } // ── Provider accessors ─────────────────────────────────────── fn getTwelveData(self: *DataService) DataError!*TwelveData { if (self.td) |*td| return td; const key = self.config.twelvedata_key orelse return DataError.NoApiKey; self.td = TwelveData.init(self.allocator, key); return &self.td.?; } fn getPolygon(self: *DataService) DataError!*Polygon { if (self.pg) |*pg| return pg; const key = self.config.polygon_key orelse return DataError.NoApiKey; self.pg = Polygon.init(self.allocator, key); return &self.pg.?; } fn getFinnhub(self: *DataService) DataError!*Finnhub { if (self.fh) |*fh| return fh; const key = self.config.finnhub_key orelse return DataError.NoApiKey; self.fh = Finnhub.init(self.allocator, key); return &self.fh.?; } fn getCboe(self: *DataService) *Cboe { if (self.cboe) |*c| return c; self.cboe = Cboe.init(self.allocator); return &self.cboe.?; } fn getAlphaVantage(self: *DataService) DataError!*AlphaVantage { if (self.av) |*av| return av; const key = self.config.alphavantage_key orelse return DataError.NoApiKey; self.av = AlphaVantage.init(self.allocator, key); return &self.av.?; } // ── Cache helper ───────────────────────────────────────────── fn store(self: *DataService) cache.Store { return cache.Store.init(self.allocator, self.config.cache_dir); } /// Invalidate cached data for a symbol so the next get* call forces a fresh fetch. pub fn invalidate(self: *DataService, symbol: []const u8, data_type: cache.DataType) void { var s = self.store(); s.clearData(symbol, data_type); } // ── Public data methods ────────────────────────────────────── /// Fetch daily candles for a symbol (10+ years for trailing returns). /// Checks cache first; fetches from TwelveData if stale/missing. pub fn getCandles(self: *DataService, symbol: []const u8) DataError!struct { data: []Candle, source: Source, timestamp: i64 } { var s = self.store(); // Try cache const cached_raw = s.readRaw(symbol, .candles_daily) catch return DataError.CacheError; if (cached_raw) |data| { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const candles = cache.Store.deserializeCandles(self.allocator, data) catch null; if (candles) |c| return .{ .data = c, .source = .cached, .timestamp = s.getMtime(symbol, .candles_daily) orelse std.time.timestamp() }; } } // Fetch from provider var td = try self.getTwelveData(); const today = todayDate(); const from = today.addDays(-365 * 10 - 60); const fetched = td.fetchCandles(self.allocator, symbol, from, today) catch { return DataError.FetchFailed; }; // Cache the result if (fetched.len > 0) { const expires = std.time.timestamp() + cache.Ttl.candles_latest; if (cache.Store.serializeCandles(self.allocator, fetched, .{ .expires = expires })) |srf_data| { defer self.allocator.free(srf_data); s.writeRaw(symbol, .candles_daily, srf_data) catch {}; } else |_| {} } return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; } /// Fetch dividend history for a symbol. /// Checks cache first; fetches from Polygon if stale/missing. pub fn getDividends(self: *DataService, symbol: []const u8) DataError!struct { data: []Dividend, source: Source, timestamp: i64 } { var s = self.store(); const cached_raw = s.readRaw(symbol, .dividends) catch return DataError.CacheError; if (cached_raw) |data| { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const divs = cache.Store.deserializeDividends(self.allocator, data) catch null; if (divs) |d| return .{ .data = d, .source = .cached, .timestamp = s.getMtime(symbol, .dividends) orelse std.time.timestamp() }; } } var pg = try self.getPolygon(); const fetched = pg.fetchDividends(self.allocator, symbol, null, null) catch { return DataError.FetchFailed; }; if (fetched.len > 0) { const expires = std.time.timestamp() + cache.Ttl.dividends; if (cache.Store.serializeDividends(self.allocator, fetched, .{ .expires = expires })) |srf_data| { defer self.allocator.free(srf_data); s.writeRaw(symbol, .dividends, srf_data) catch {}; } else |_| {} } return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; } /// Fetch split history for a symbol. /// Checks cache first; fetches from Polygon if stale/missing. pub fn getSplits(self: *DataService, symbol: []const u8) DataError!struct { data: []Split, source: Source, timestamp: i64 } { var s = self.store(); const cached_raw = s.readRaw(symbol, .splits) catch return DataError.CacheError; if (cached_raw) |data| { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const splits = cache.Store.deserializeSplits(self.allocator, data) catch null; if (splits) |sp| return .{ .data = sp, .source = .cached, .timestamp = s.getMtime(symbol, .splits) orelse std.time.timestamp() }; } } var pg = try self.getPolygon(); const fetched = pg.fetchSplits(self.allocator, symbol) catch { return DataError.FetchFailed; }; if (cache.Store.serializeSplits(self.allocator, fetched, .{ .expires = std.time.timestamp() + cache.Ttl.splits })) |srf_data| { defer self.allocator.free(srf_data); s.writeRaw(symbol, .splits, srf_data) catch {}; } else |_| {} return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; } /// Fetch options chain for a symbol (all expirations). /// Checks cache first; fetches from CBOE if stale/missing (no API key needed). pub fn getOptions(self: *DataService, symbol: []const u8) DataError!struct { data: []OptionsChain, source: Source, timestamp: i64 } { var s = self.store(); const cached_raw = s.readRaw(symbol, .options) catch return DataError.CacheError; if (cached_raw) |data| { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const chains = cache.Store.deserializeOptions(self.allocator, data) catch null; if (chains) |c| return .{ .data = c, .source = .cached, .timestamp = s.getMtime(symbol, .options) orelse std.time.timestamp() }; } } var cboe = self.getCboe(); const fetched = cboe.fetchOptionsChain(self.allocator, symbol) catch { return DataError.FetchFailed; }; if (fetched.len > 0) { const expires = std.time.timestamp() + cache.Ttl.options; if (cache.Store.serializeOptions(self.allocator, fetched, .{ .expires = expires })) |srf_data| { defer self.allocator.free(srf_data); s.writeRaw(symbol, .options, srf_data) catch {}; } else |_| {} } return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; } /// Fetch earnings history for a symbol (5 years back, 1 year forward). /// Checks cache first; fetches from Finnhub if stale/missing. /// Smart refresh: even if cache is fresh, re-fetches when a past earnings /// date has no actual results yet (i.e. results just came out). pub fn getEarnings(self: *DataService, symbol: []const u8) DataError!struct { data: []EarningsEvent, source: Source, timestamp: i64 } { var s = self.store(); const today = todayDate(); const cached_raw = s.readRaw(symbol, .earnings) catch return DataError.CacheError; if (cached_raw) |data| { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const events = cache.Store.deserializeEarnings(self.allocator, data) catch null; if (events) |e| { // Check if any past/today earnings event is still missing actual results. // If so, the announcement likely just happened — force a refresh. const needs_refresh = for (e) |ev| { if (ev.actual == null and !today.lessThan(ev.date)) break true; } else false; if (!needs_refresh) { return .{ .data = e, .source = .cached, .timestamp = s.getMtime(symbol, .earnings) orelse std.time.timestamp() }; } // Stale: free cached events and re-fetch below self.allocator.free(e); } } } var fh = try self.getFinnhub(); const from = today.subtractYears(5); const to = today.addDays(365); const fetched = fh.fetchEarnings(self.allocator, symbol, from, to) catch { return DataError.FetchFailed; }; if (fetched.len > 0) { const expires = std.time.timestamp() + cache.Ttl.earnings; if (cache.Store.serializeEarnings(self.allocator, fetched, .{ .expires = expires })) |srf_data| { defer self.allocator.free(srf_data); s.writeRaw(symbol, .earnings, srf_data) catch {}; } else |_| {} } return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; } /// Fetch ETF profile for a symbol. /// Checks cache first; fetches from Alpha Vantage if stale/missing. pub fn getEtfProfile(self: *DataService, symbol: []const u8) DataError!struct { data: EtfProfile, source: Source, timestamp: i64 } { var s = self.store(); const cached_raw = s.readRaw(symbol, .etf_profile) catch return DataError.CacheError; if (cached_raw) |data| { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const profile = cache.Store.deserializeEtfProfile(self.allocator, data) catch null; if (profile) |p| return .{ .data = p, .source = .cached, .timestamp = s.getMtime(symbol, .etf_profile) orelse std.time.timestamp() }; } } var av = try self.getAlphaVantage(); const fetched = av.fetchEtfProfile(self.allocator, symbol) catch { return DataError.FetchFailed; }; if (cache.Store.serializeEtfProfile(self.allocator, fetched, .{ .expires = std.time.timestamp() + cache.Ttl.etf_profile })) |srf_data| { defer self.allocator.free(srf_data); s.writeRaw(symbol, .etf_profile, srf_data) catch {}; } else |_| {} return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; } /// Fetch a real-time (or 15-min delayed) quote for a symbol. /// No cache -- always fetches fresh from TwelveData. pub fn getQuote(self: *DataService, symbol: []const u8) DataError!Quote { var td = try self.getTwelveData(); return td.fetchQuote(self.allocator, symbol) catch return DataError.FetchFailed; } /// Fetch company overview (sector, industry, country, market cap) from Alpha Vantage. /// No cache -- always fetches fresh. Caller must free the returned string fields. pub fn getCompanyOverview(self: *DataService, symbol: []const u8) DataError!CompanyOverview { var av = try self.getAlphaVantage(); return av.fetchCompanyOverview(self.allocator, symbol) catch return DataError.FetchFailed; } /// Compute trailing returns for a symbol (fetches candles + dividends). /// Returns both as-of-date and month-end trailing returns. /// As-of-date: end = latest close. Matches Morningstar "Trailing Returns" page. /// Month-end: end = last business day of prior month. Matches Morningstar "Performance" page. pub fn getTrailingReturns(self: *DataService, symbol: []const u8) DataError!struct { asof_price: performance.TrailingReturns, asof_total: ?performance.TrailingReturns, me_price: performance.TrailingReturns, me_total: ?performance.TrailingReturns, candles: []Candle, dividends: ?[]Dividend, source: Source, timestamp: i64, } { const candle_result = try self.getCandles(symbol); const c = candle_result.data; if (c.len == 0) return DataError.FetchFailed; const today = todayDate(); // As-of-date (end = last candle) const asof_price = performance.trailingReturns(c); // Month-end (end = last business day of prior month) const me_price = performance.trailingReturnsMonthEnd(c, today); // Try to get dividends (non-fatal if unavailable) var divs: ?[]Dividend = null; var asof_total: ?performance.TrailingReturns = null; var me_total: ?performance.TrailingReturns = null; if (self.getDividends(symbol)) |div_result| { divs = div_result.data; asof_total = performance.trailingReturnsWithDividends(c, div_result.data); me_total = performance.trailingReturnsMonthEndWithDividends(c, div_result.data, today); } else |_| {} return .{ .asof_price = asof_price, .asof_total = asof_total, .me_price = me_price, .me_total = me_total, .candles = c, .dividends = divs, .source = candle_result.source, .timestamp = candle_result.timestamp, }; } /// Check if candle data is fresh in cache without full deserialization. pub fn isCandleCacheFresh(self: *DataService, symbol: []const u8) bool { var s = self.store(); const data = s.readRaw(symbol, .candles_daily) catch return false; if (data) |d| { defer self.allocator.free(d); return cache.Store.isFreshData(d, self.allocator); } return false; } /// Read only the latest close price from cached candles (no full deserialization). /// Returns null if no cached data exists. pub fn getCachedLastClose(self: *DataService, symbol: []const u8) ?f64 { var s = self.store(); return s.readLastClose(symbol); } /// Estimate wait time (in seconds) before the next TwelveData API call can proceed. /// Returns 0 if a request can be made immediately. Returns null if no API key. pub fn estimateWaitSeconds(self: *DataService) ?u64 { if (self.td) |*td| { const ns = td.rate_limiter.estimateWaitNs(); return if (ns == 0) 0 else @max(1, ns / std.time.ns_per_s); } return null; } /// Read candles from cache only (no network fetch). Used by TUI for display. /// Returns null if no cached data exists or if the entry is a negative cache (fetch_failed). pub fn getCachedCandles(self: *DataService, symbol: []const u8) ?[]Candle { var s = self.store(); if (s.isNegative(symbol, .candles_daily)) return null; const data = s.readRaw(symbol, .candles_daily) catch return null; if (data) |d| { defer self.allocator.free(d); return cache.Store.deserializeCandles(self.allocator, d) catch null; } return null; } /// Read dividends from cache only (no network fetch). pub fn getCachedDividends(self: *DataService, symbol: []const u8) ?[]Dividend { var s = self.store(); const data = s.readRaw(symbol, .dividends) catch return null; if (data) |d| { defer self.allocator.free(d); return cache.Store.deserializeDividends(self.allocator, d) catch null; } return null; } /// Read earnings from cache only (no network fetch). pub fn getCachedEarnings(self: *DataService, symbol: []const u8) ?[]EarningsEvent { var s = self.store(); const data = s.readRaw(symbol, .earnings) catch return null; if (data) |d| { defer self.allocator.free(d); return cache.Store.deserializeEarnings(self.allocator, d) catch null; } return null; } /// Read options from cache only (no network fetch). pub fn getCachedOptions(self: *DataService, symbol: []const u8) ?[]OptionsChain { var s = self.store(); const data = s.readRaw(symbol, .options) catch return null; if (data) |d| { defer self.allocator.free(d); return cache.Store.deserializeOptions(self.allocator, d) catch null; } return null; } // ── Portfolio price loading ────────────────────────────────── /// Result of loading prices for a set of symbols. pub const PriceLoadResult = struct { /// Number of symbols whose price came from fresh cache. cached_count: usize, /// Number of symbols successfully fetched from API. fetched_count: usize, /// Number of symbols where API fetch failed. fail_count: usize, /// Number of failed symbols that fell back to stale cache. stale_count: usize, /// Latest candle date seen across all symbols. latest_date: ?Date, }; /// Status emitted for each symbol during price loading. pub const SymbolStatus = enum { /// Price resolved from fresh cache. cached, /// About to attempt an API fetch (emitted before the network call). fetching, /// Price fetched successfully from API. fetched, /// API fetch failed but stale cached price was used. failed_used_stale, /// API fetch failed and no cached price exists. failed, }; /// Callback for progress reporting during price loading. /// `context` is an opaque pointer to caller-owned state. pub const ProgressCallback = struct { context: *anyopaque, on_progress: *const fn (ctx: *anyopaque, index: usize, total: usize, symbol: []const u8, status: SymbolStatus) void, fn emit(self: ProgressCallback, index: usize, total: usize, symbol: []const u8, status: SymbolStatus) void { self.on_progress(self.context, index, total, symbol, status); } }; /// Load current prices for a list of symbols into `prices`. /// /// For each symbol the resolution order is: /// 1. Fresh cache -> use cached last close (fast, no deserialization) /// 2. API fetch -> use latest candle close /// 3. Stale cache -> use last close from expired cache entry /// /// If `force_refresh` is true, cache is invalidated before checking freshness. /// If `progress` is provided, it is called for each symbol with the outcome. pub fn loadPrices( self: *DataService, symbols: []const []const u8, prices: *std.StringHashMap(f64), force_refresh: bool, progress: ?ProgressCallback, ) PriceLoadResult { var result = PriceLoadResult{ .cached_count = 0, .fetched_count = 0, .fail_count = 0, .stale_count = 0, .latest_date = null, }; const total = symbols.len; for (symbols, 0..) |sym, i| { if (force_refresh) { self.invalidate(sym, .candles_daily); } // 1. Fresh cache — fast path (no full deserialization) if (!force_refresh and self.isCandleCacheFresh(sym)) { if (self.getCachedLastClose(sym)) |close| { prices.put(sym, close) catch {}; } result.cached_count += 1; if (progress) |p| p.emit(i, total, sym, .cached); continue; } // About to fetch — notify caller (so it can show rate-limit waits etc.) if (progress) |p| p.emit(i, total, sym, .fetching); // 2. Try API fetch if (self.getCandles(sym)) |candle_result| { defer self.allocator.free(candle_result.data); if (candle_result.data.len > 0) { const last = candle_result.data[candle_result.data.len - 1]; prices.put(sym, last.close) catch {}; if (result.latest_date == null or last.date.days > result.latest_date.?.days) { result.latest_date = last.date; } } result.fetched_count += 1; if (progress) |p| p.emit(i, total, sym, .fetched); continue; } else |_| {} // 3. Fetch failed — fall back to stale cache result.fail_count += 1; if (self.getCachedLastClose(sym)) |close| { prices.put(sym, close) catch {}; result.stale_count += 1; if (progress) |p| p.emit(i, total, sym, .failed_used_stale); } else { if (progress) |p| p.emit(i, total, sym, .failed); } } return result; } // ── CUSIP Resolution ────────────────────────────────────────── /// Look up multiple CUSIPs in a single batch request via OpenFIGI. /// Results array is parallel to the input cusips array (same length, same order). /// Caller owns the returned slice and all strings within each CusipResult. pub fn lookupCusips(self: *DataService, cusips: []const []const u8) DataError![]CusipResult { return OpenFigi.lookupCusips(self.allocator, cusips, self.config.openfigi_key) catch return DataError.FetchFailed; } /// Look up a CUSIP via OpenFIGI API. Returns the ticker if found, null otherwise. /// Results are cached in {cache_dir}/cusip_tickers.srf. /// Caller owns the returned string. pub fn lookupCusip(self: *DataService, cusip: []const u8) ?[]const u8 { // Check local cache first if (self.getCachedCusipTicker(cusip)) |t| return t; // Try OpenFIGI const result = OpenFigi.lookupCusip(self.allocator, cusip, self.config.openfigi_key) catch return null; defer { if (result.name) |n| self.allocator.free(n); if (result.security_type) |s| self.allocator.free(s); } if (result.ticker) |ticker| { // Cache the mapping self.cacheCusipTicker(cusip, ticker); return ticker; // caller takes ownership } return null; } /// Read a cached CUSIP->ticker mapping. Returns null if not cached. /// Caller owns the returned string. fn getCachedCusipTicker(self: *DataService, cusip: []const u8) ?[]const u8 { const path = std.fs.path.join(self.allocator, &.{ self.config.cache_dir, "cusip_tickers.srf" }) catch return null; defer self.allocator.free(path); const data = std.fs.cwd().readFileAlloc(self.allocator, path, 64 * 1024) catch return null; defer self.allocator.free(data); // Simple line-based format: cusip::XXXXX,ticker::YYYYY var lines = std.mem.splitScalar(u8, data, '\n'); while (lines.next()) |line| { const trimmed = std.mem.trim(u8, line, &std.ascii.whitespace); if (trimmed.len == 0 or trimmed[0] == '#') continue; // Parse cusip:: field const cusip_prefix = "cusip::"; if (!std.mem.startsWith(u8, trimmed, cusip_prefix)) continue; const after_cusip = trimmed[cusip_prefix.len..]; const comma_idx = std.mem.indexOfScalar(u8, after_cusip, ',') orelse continue; const cached_cusip = after_cusip[0..comma_idx]; if (!std.mem.eql(u8, cached_cusip, cusip)) continue; // Parse ticker:: field const rest = after_cusip[comma_idx + 1 ..]; const ticker_prefix = "ticker::"; if (!std.mem.startsWith(u8, rest, ticker_prefix)) continue; const ticker_val = rest[ticker_prefix.len..]; // Trim any trailing comma/fields const ticker_end = std.mem.indexOfScalar(u8, ticker_val, ',') orelse ticker_val.len; const ticker = ticker_val[0..ticker_end]; if (ticker.len > 0) { return self.allocator.dupe(u8, ticker) catch null; } } return null; } /// Append a CUSIP->ticker mapping to the cache file. pub fn cacheCusipTicker(self: *DataService, cusip: []const u8, ticker: []const u8) void { const path = std.fs.path.join(self.allocator, &.{ self.config.cache_dir, "cusip_tickers.srf" }) catch return; defer self.allocator.free(path); // Ensure cache dir exists if (std.fs.path.dirnamePosix(path)) |dir| { std.fs.cwd().makePath(dir) catch {}; } // Append the mapping const file = std.fs.cwd().openFile(path, .{ .mode = .write_only }) catch blk: { // File doesn't exist, create it break :blk std.fs.cwd().createFile(path, .{}) catch return; }; defer file.close(); file.seekFromEnd(0) catch {}; var buf: [256]u8 = undefined; const line = std.fmt.bufPrint(&buf, "cusip::{s},ticker::{s}\n", .{ cusip, ticker }) catch return; _ = file.write(line) catch {}; } // ── Utility ────────────────────────────────────────────────── fn todayDate() Date { const ts = std.time.timestamp(); const days: i32 = @intCast(@divFloor(ts, 86400)); return .{ .days = days }; } };