From f8a9607bc952c511c9b9c83e5422e28a68bb52c4 Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Wed, 11 Mar 2026 15:31:17 -0700 Subject: [PATCH] add Yahoo finance into the mix for candles and quotes --- TODO.md | 7 - src/cache/store.zig | 19 ++ src/models/date.zig | 10 ++ src/providers/yahoo.zig | 373 ++++++++++++++++++++++++++++++++++++++++ src/service.zig | 110 ++++++++---- 5 files changed, 480 insertions(+), 39 deletions(-) create mode 100644 src/providers/yahoo.zig diff --git a/TODO.md b/TODO.md index e408542..3576efa 100644 --- a/TODO.md +++ b/TODO.md @@ -1,12 +1,5 @@ # Future Work -## Yahoo Finance as primary quote source - -Consider adding Yahoo Finance as the primary provider for real-time quotes, -with a silent fallback to TwelveData. Yahoo is free and has no API key -requirement, but the unofficial API is brittle and can break without notice. -TwelveData would serve as the reliable backup when Yahoo is unavailable. - ## Covered call portfolio valuation Portfolio value should account for sold call options. Shares covered by diff --git a/src/cache/store.zig b/src/cache/store.zig index 722bed2..0d65712 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -241,10 +241,16 @@ pub const Store = struct { /// Write (or refresh) candle metadata without touching the candle data file. pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date) void { + self.updateCandleMetaWithProvider(symbol, last_close, last_date, .twelvedata); + } + + /// Write candle metadata with a specific provider source. + pub fn updateCandleMetaWithProvider(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider) void { const expires = std.time.timestamp() + Ttl.candles_latest; const meta = CandleMeta{ .last_close = last_close, .last_date = last_date, + .provider = provider, }; if (serializeCandleMeta(self.allocator, meta, .{ .expires = expires })) |meta_data| { defer self.allocator.free(meta_data); @@ -360,6 +366,19 @@ pub const Store = struct { pub const CandleMeta = struct { last_close: f64, last_date: Date, + /// Which provider sourced the candle data. Used during incremental refresh + /// to go directly to the right provider instead of trying TwelveData first. + provider: CandleProvider = .twelvedata, + }; + + pub const CandleProvider = enum { + twelvedata, + yahoo, + + pub fn fromString(s: []const u8) CandleProvider { + if (std.mem.eql(u8, s, "yahoo")) return .yahoo; + return .twelvedata; + } }; // ── Private I/O ────────────────────────────────────────────── diff --git a/src/models/date.zig b/src/models/date.zig index abd4110..f8a88f4 100644 --- a/src/models/date.zig +++ b/src/models/date.zig @@ -80,6 +80,16 @@ pub const Date = struct { return .{ .days = self.days + n }; } + /// Convert to Unix epoch seconds (midnight UTC on this date). + pub fn toEpoch(self: Date) i64 { + return @as(i64, self.days) * std.time.s_per_day; + } + + /// Create a Date from a Unix epoch timestamp (seconds since 1970-01-01). + pub fn fromEpoch(epoch_secs: i64) Date { + return .{ .days = @intCast(@divFloor(epoch_secs, std.time.s_per_day)) }; + } + /// Subtract N calendar years. Clamps Feb 29 -> Feb 28 if target is not a leap year. pub fn subtractYears(self: Date, n: u16) Date { const ymd = epochDaysToYmd(self.days); diff --git a/src/providers/yahoo.zig b/src/providers/yahoo.zig new file mode 100644 index 0000000..a755fb0 --- /dev/null +++ b/src/providers/yahoo.zig @@ -0,0 +1,373 @@ +//! Yahoo Finance provider -- free, no API key required. +//! +//! Uses the unofficial v8 chart API for both candles and quotes. +//! No rate limiting is applied since there's no documented limit, +//! but this API is unofficial and can break without notice. +//! +//! Yahoo has broader mutual fund coverage than TwelveData/Polygon, +//! making it useful as a fallback for symbols like VTTHX. + +const std = @import("std"); +const http = @import("../net/http.zig"); +const Date = @import("../models/date.zig").Date; +const Candle = @import("../models/candle.zig").Candle; +const Quote = @import("../models/quote.zig").Quote; +const parseJsonFloat = @import("json_utils.zig").parseJsonFloat; + +const base_url = "https://query1.finance.yahoo.com/v8/finance/chart"; + +pub const Yahoo = struct { + client: http.Client, + allocator: std.mem.Allocator, + + pub fn init(allocator: std.mem.Allocator) Yahoo { + return .{ + .client = http.Client.init(allocator), + .allocator = allocator, + }; + } + + pub fn deinit(self: *Yahoo) void { + self.client.deinit(); + } + + /// Fetch daily candles for a symbol between two dates. + /// Returns candles sorted oldest-first. + pub fn fetchCandles( + self: *Yahoo, + allocator: std.mem.Allocator, + symbol: []const u8, + from: Date, + to: Date, + ) ![]Candle { + const period1 = from.toEpoch(); + const period2 = to.toEpoch() + std.time.s_per_day; // inclusive end + + var p1_buf: [20]u8 = undefined; + var p2_buf: [20]u8 = undefined; + const p1_str = std.fmt.bufPrint(&p1_buf, "{d}", .{period1}) catch return error.ParseError; + const p2_str = std.fmt.bufPrint(&p2_buf, "{d}", .{period2}) catch return error.ParseError; + + const symbol_url = try std.fmt.allocPrint(allocator, base_url ++ "/{s}", .{symbol}); + defer allocator.free(symbol_url); + + const url = try http.buildUrl(allocator, symbol_url, &.{ + .{ "interval", "1d" }, + .{ "period1", p1_str }, + .{ "period2", p2_str }, + }); + defer allocator.free(url); + + var response = try self.client.get(url); + defer response.deinit(); + + return parseChartCandles(allocator, response.body); + } + + /// Fetch a quote snapshot for a symbol. + /// Uses the chart API meta fields (regularMarketPrice, etc.). + pub fn fetchQuote( + self: *Yahoo, + allocator: std.mem.Allocator, + symbol: []const u8, + ) !Quote { + // Fetch just 1 day of data to get the meta block with quote info + const symbol_url = try std.fmt.allocPrint(allocator, base_url ++ "/{s}", .{symbol}); + defer allocator.free(symbol_url); + + const url = try http.buildUrl(allocator, symbol_url, &.{ + .{ "interval", "1d" }, + .{ "range", "5d" }, + }); + defer allocator.free(url); + + var response = try self.client.get(url); + defer response.deinit(); + + return parseChartQuote(allocator, response.body, symbol); + } +}; + +// -- JSON parsing -- + +/// Parse the chart API result array, returning the first result's chart data. +fn getChartResult(allocator: std.mem.Allocator, body: []const u8) !struct { parsed: std.json.Parsed(std.json.Value), result: std.json.ObjectMap } { + const parsed = std.json.parseFromSlice(std.json.Value, allocator, body, .{}) catch + return error.ParseError; + errdefer parsed.deinit(); + + const chart = parsed.value.object.get("chart") orelse return error.ParseError; + const chart_obj = switch (chart) { + .object => |o| o, + else => return error.ParseError, + }; + + // Check for error + if (chart_obj.get("error")) |err_val| { + if (err_val != .null) return error.RequestFailed; + } + + const result_arr = chart_obj.get("result") orelse return error.ParseError; + const results = switch (result_arr) { + .array => |a| a.items, + else => return error.ParseError, + }; + + if (results.len == 0) return error.ParseError; + + const result = switch (results[0]) { + .object => |o| o, + else => return error.ParseError, + }; + + return .{ .parsed = parsed, .result = result }; +} + +fn parseChartCandles(allocator: std.mem.Allocator, body: []const u8) ![]Candle { + const chart = try getChartResult(allocator, body); + const parsed = chart.parsed; + defer parsed.deinit(); + const result = chart.result; + + // Get timestamps + const ts_json = result.get("timestamp") orelse return error.ParseError; + const timestamps = switch (ts_json) { + .array => |a| a.items, + else => return error.ParseError, + }; + + // Get quote data (open/high/low/close/volume arrays) + const indicators = result.get("indicators") orelse return error.ParseError; + const indicators_obj = switch (indicators) { + .object => |o| o, + else => return error.ParseError, + }; + const quote_arr = indicators_obj.get("quote") orelse return error.ParseError; + const quotes = switch (quote_arr) { + .array => |a| a.items, + else => return error.ParseError, + }; + if (quotes.len == 0) return error.ParseError; + const q = switch (quotes[0]) { + .object => |o| o, + else => return error.ParseError, + }; + + const opens = getFloatArray(q.get("open")) orelse return error.ParseError; + const highs = getFloatArray(q.get("high")) orelse return error.ParseError; + const lows = getFloatArray(q.get("low")) orelse return error.ParseError; + const closes = getFloatArray(q.get("close")) orelse return error.ParseError; + const volumes = getFloatArray(q.get("volume")) orelse return error.ParseError; + + // Get adjclose if available + const adjcloses = blk: { + const adjclose_section = indicators_obj.get("adjclose") orelse break :blk null; + const adj_arr = switch (adjclose_section) { + .array => |a| a.items, + else => break :blk null, + }; + if (adj_arr.len == 0) break :blk null; + const adj_obj = switch (adj_arr[0]) { + .object => |o| o, + else => break :blk null, + }; + break :blk getFloatArray(adj_obj.get("adjclose")); + }; + + var candles: std.ArrayList(Candle) = .empty; + errdefer candles.deinit(allocator); + + // Yahoo returns oldest-first, which is what we want + for (timestamps, 0..) |ts, i| { + if (i >= opens.len or i >= closes.len) break; + + // Skip null entries (incomplete trading day) + const close_val = closes[i]; + if (close_val == .null) continue; + + const epoch: i64 = switch (ts) { + .integer => |v| v, + else => continue, + }; + + const date = Date.fromEpoch(epoch); + + const close = parseJsonFloat(close_val); + const adj = if (adjcloses) |ac| (if (i < ac.len) parseJsonFloat(ac[i]) else close) else close; + + try candles.append(allocator, .{ + .date = date, + .open = parseJsonFloat(opens[i]), + .high = parseJsonFloat(highs[i]), + .low = parseJsonFloat(lows[i]), + .close = close, + .adj_close = adj, + .volume = @intFromFloat(@max(0, parseJsonFloat(volumes[i]))), + }); + } + + return candles.toOwnedSlice(allocator); +} + +fn parseChartQuote(allocator: std.mem.Allocator, body: []const u8, symbol: []const u8) !Quote { + const chart = try getChartResult(allocator, body); + const parsed = chart.parsed; + defer parsed.deinit(); + const result = chart.result; + + const meta = result.get("meta") orelse return error.ParseError; + const m = switch (meta) { + .object => |o| o, + else => return error.ParseError, + }; + + const price = parseJsonFloat(m.get("regularMarketPrice")); + const prev_close = parseJsonFloat(m.get("chartPreviousClose")); + const change = price - prev_close; + const pct = if (prev_close != 0) (change / prev_close) * 100.0 else 0; + + return .{ + .symbol = symbol, + .name = symbol, + .exchange = "", + .datetime = "", + .close = price, + .open = price, // meta doesn't have open + .high = parseJsonFloat(m.get("fiftyTwoWeekHigh")), + .low = parseJsonFloat(m.get("fiftyTwoWeekLow")), + .volume = 0, + .previous_close = prev_close, + .change = change, + .percent_change = pct, + .average_volume = 0, + .fifty_two_week_low = parseJsonFloat(m.get("fiftyTwoWeekLow")), + .fifty_two_week_high = parseJsonFloat(m.get("fiftyTwoWeekHigh")), + }; +} + +fn getFloatArray(val: ?std.json.Value) ?[]const std.json.Value { + const v = val orelse return null; + return switch (v) { + .array => |a| a.items, + else => null, + }; +} + +// -- Tests -- + +test "parseChartCandles basic" { + const body = + \\{ + \\ "chart": { + \\ "result": [{ + \\ "timestamp": [1704067800, 1704154200], + \\ "indicators": { + \\ "quote": [{ + \\ "open": [185.0, 187.15], + \\ "high": [186.1, 188.44], + \\ "low": [184.0, 183.89], + \\ "close": [185.5, 184.25], + \\ "volume": [42000000, 58414460] + \\ }], + \\ "adjclose": [{ + \\ "adjclose": [185.5, 184.25] + \\ }] + \\ } + \\ }], + \\ "error": null + \\ } + \\} + ; + + const allocator = std.testing.allocator; + const candles = try parseChartCandles(allocator, body); + defer allocator.free(candles); + + try std.testing.expectEqual(@as(usize, 2), candles.len); + + // Oldest first (Yahoo returns chronological order) + try std.testing.expectApproxEqAbs(@as(f64, 185.0), candles[0].open, 0.01); + try std.testing.expectApproxEqAbs(@as(f64, 185.5), candles[0].close, 0.01); + try std.testing.expectEqual(@as(u64, 42000000), candles[0].volume); + + try std.testing.expectApproxEqAbs(@as(f64, 187.15), candles[1].open, 0.01); + try std.testing.expectApproxEqAbs(@as(f64, 184.25), candles[1].close, 0.01); +} + +test "parseChartCandles skips null entries" { + const body = + \\{ + \\ "chart": { + \\ "result": [{ + \\ "timestamp": [1704067800, 1704154200], + \\ "indicators": { + \\ "quote": [{ + \\ "open": [185.0, null], + \\ "high": [186.1, null], + \\ "low": [184.0, null], + \\ "close": [185.5, null], + \\ "volume": [42000000, null] + \\ }] + \\ } + \\ }], + \\ "error": null + \\ } + \\} + ; + + const allocator = std.testing.allocator; + const candles = try parseChartCandles(allocator, body); + defer allocator.free(candles); + + // Should only have 1 candle (null entry skipped) + try std.testing.expectEqual(@as(usize, 1), candles.len); + try std.testing.expectApproxEqAbs(@as(f64, 185.5), candles[0].close, 0.01); +} + +test "parseChartCandles error response" { + const body = + \\{ + \\ "chart": { + \\ "result": null, + \\ "error": {"code": "Not Found", "description": "No data found"} + \\ } + \\} + ; + + const allocator = std.testing.allocator; + const result = parseChartCandles(allocator, body); + try std.testing.expectError(error.RequestFailed, result); +} + +test "parseChartQuote basic" { + const body = + \\{ + \\ "chart": { + \\ "result": [{ + \\ "meta": { + \\ "symbol": "VTTHX", + \\ "regularMarketPrice": 27.78, + \\ "chartPreviousClose": 28.06, + \\ "fiftyTwoWeekHigh": 28.59, + \\ "fiftyTwoWeekLow": 22.21 + \\ }, + \\ "timestamp": [1704067800], + \\ "indicators": {"quote": [{"open": [27.78], "high": [27.78], "low": [27.78], "close": [27.78], "volume": [0]}]} + \\ }], + \\ "error": null + \\ } + \\} + ; + + const allocator = std.testing.allocator; + const quote = try parseChartQuote(allocator, body, "VTTHX"); + + try std.testing.expectApproxEqAbs(@as(f64, 27.78), quote.close, 0.01); + try std.testing.expectApproxEqAbs(@as(f64, 28.06), quote.previous_close, 0.01); + try std.testing.expectApproxEqAbs(@as(f64, 28.59), quote.fifty_two_week_high, 0.01); + try std.testing.expectApproxEqAbs(@as(f64, 22.21), quote.fifty_two_week_low, 0.01); + // change = 27.78 - 28.06 = -0.28 + try std.testing.expectApproxEqAbs(@as(f64, -0.28), quote.change, 0.01); + // percent_change = (-0.28 / 28.06) * 100 ≈ -0.998 + try std.testing.expectApproxEqAbs(@as(f64, -0.998), quote.percent_change, 0.01); +} diff --git a/src/service.zig b/src/service.zig index 9cf2d9d..610922d 100644 --- a/src/service.zig +++ b/src/service.zig @@ -27,6 +27,7 @@ const Cboe = @import("providers/cboe.zig").Cboe; const AlphaVantage = @import("providers/alphavantage.zig").AlphaVantage; const alphavantage = @import("providers/alphavantage.zig"); const OpenFigi = @import("providers/openfigi.zig"); +const Yahoo = @import("providers/yahoo.zig").Yahoo; const fmt = @import("format.zig"); const performance = @import("analytics/performance.zig"); const http = @import("net/http.zig"); @@ -92,6 +93,7 @@ pub const DataService = struct { fh: ?Finnhub = null, cboe: ?Cboe = null, av: ?AlphaVantage = null, + yh: ?Yahoo = null, pub fn init(allocator: std.mem.Allocator, config: Config) DataService { return .{ @@ -106,6 +108,7 @@ pub const DataService = struct { if (self.fh) |*fh| fh.deinit(); if (self.cboe) |*c| c.deinit(); if (self.av) |*av| av.deinit(); + if (self.yh) |*yh| yh.deinit(); } // ── Provider accessor ────────────────────────────────────────── @@ -113,8 +116,8 @@ pub const DataService = struct { fn getProvider(self: *DataService, comptime T: type) DataError!*T { const field_name = comptime providerField(T); if (@field(self, field_name)) |*p| return p; - if (T == Cboe) { - // CBOE has no key + if (T == Cboe or T == Yahoo) { + // CBOE and Yahoo have no API key @field(self, field_name) = T.init(self.allocator); } else { // All we're doing here is lower casing the type name, then @@ -227,6 +230,55 @@ pub const DataService = struct { // ── Public data methods ────────────────────────────────────── + /// Fetch candles from providers with fallback logic. + /// Tries the provider recorded in meta (if any), then TwelveData, then Yahoo. + /// Returns the candles and which provider succeeded. + fn fetchCandlesFromProviders( + self: *DataService, + symbol: []const u8, + from: Date, + to: Date, + preferred: cache.Store.CandleProvider, + ) !struct { candles: []Candle, provider: cache.Store.CandleProvider } { + // If preferred is Yahoo, try it first + if (preferred == .yahoo) { + if (self.getProvider(Yahoo)) |yh| { + if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| { + log.debug("{s}: candles from Yahoo (preferred)", .{symbol}); + return .{ .candles = candles, .provider = .yahoo }; + } else |_| {} + } else |_| {} + } + + // Try TwelveData + if (self.getProvider(TwelveData)) |td| { + if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| { + log.debug("{s}: candles from TwelveData", .{symbol}); + return .{ .candles = candles, .provider = .twelvedata }; + } else |err| { + if (err == error.RateLimited) { + self.rateLimitBackoff(); + if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| { + log.debug("{s}: candles from TwelveData (after rate limit retry)", .{symbol}); + return .{ .candles = candles, .provider = .twelvedata }; + } else |_| {} + } + } + } else |_| {} + + // Fallback: Yahoo (if not already tried as preferred) + if (preferred != .yahoo) { + if (self.getProvider(Yahoo)) |yh| { + if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| { + log.debug("{s}: candles from Yahoo (fallback)", .{symbol}); + return .{ .candles = candles, .provider = .yahoo }; + } else |_| {} + } else |_| {} + } + + return error.FetchFailed; + } + /// Fetch daily candles for a symbol (10+ years for trailing returns). /// Checks cache first; fetches from TwelveData if stale/missing. /// Uses incremental updates: when the cache is stale, only fetches @@ -268,31 +320,18 @@ pub const DataService = struct { return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() }; } else { // Incremental fetch from day after last cached candle - var td = self.getProvider(TwelveData) catch { - // No API key — return stale data - if (s.read(Candle, symbol, null, .any)) |r| - return .{ .data = r.data, .source = .cached, .timestamp = mr.created }; - return DataError.NoApiKey; - }; - const new_candles = td.fetchCandles(self.allocator, symbol, fetch_from, today) catch |err| blk: { - if (err == error.RateLimited) { - self.rateLimitBackoff(); - break :blk td.fetchCandles(self.allocator, symbol, fetch_from, today) catch { - if (s.read(Candle, symbol, null, .any)) |r| - return .{ .data = r.data, .source = .cached, .timestamp = mr.created }; - return DataError.FetchFailed; - }; - } - // Non-rate-limit failure — return stale data + const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, m.provider) catch { + // All providers failed — return stale data if (s.read(Candle, symbol, null, .any)) |r| return .{ .data = r.data, .source = .cached, .timestamp = mr.created }; return DataError.FetchFailed; }; + const new_candles = result.candles; if (new_candles.len == 0) { // No new candles (weekend/holiday) — refresh TTL only (meta rewrite) self.allocator.free(new_candles); - s.updateCandleMeta(symbol, m.last_close, m.last_date); + s.updateCandleMetaWithProvider(symbol, m.last_close, m.last_date, result.provider); if (s.read(Candle, symbol, null, .any)) |r| return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() }; } else { @@ -322,25 +361,21 @@ pub const DataService = struct { // No usable cache — full fetch (~10 years, plus buffer for leap years) log.debug("{s}: fetching full candle history from provider", .{symbol}); - var td = try self.getProvider(TwelveData); const from = today.addDays(-3700); - const fetched = td.fetchCandles(self.allocator, symbol, from, today) catch |err| blk: { - if (err == error.RateLimited) { - self.rateLimitBackoff(); - break :blk td.fetchCandles(self.allocator, symbol, from, today) catch { - return DataError.FetchFailed; - }; - } + const result = self.fetchCandlesFromProviders(symbol, from, today, .twelvedata) catch { s.writeNegative(symbol, .candles_daily); return DataError.FetchFailed; }; - if (fetched.len > 0) { - s.cacheCandles(symbol, fetched); + if (result.candles.len > 0) { + s.cacheCandles(symbol, result.candles); + // Record which provider sourced this data + const last = result.candles[result.candles.len - 1]; + s.updateCandleMetaWithProvider(symbol, last.close, last.date, result.provider); } - return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; + return .{ .data = result.candles, .source = .fetched, .timestamp = std.time.timestamp() }; } /// Fetch dividend history for a symbol. @@ -441,10 +476,21 @@ pub const DataService = struct { return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; } - /// Fetch a real-time (or 15-min delayed) quote for a symbol. - /// No cache -- always fetches fresh from TwelveData. + /// Fetch a real-time quote for a symbol. + /// Yahoo Finance is primary (free, no API key, no 15-min delay). + /// Falls back to TwelveData if Yahoo fails. pub fn getQuote(self: *DataService, symbol: []const u8) DataError!Quote { + // Primary: Yahoo Finance (free, real-time) + if (self.getProvider(Yahoo)) |yh| { + if (yh.fetchQuote(self.allocator, symbol)) |quote| { + log.debug("{s}: quote from Yahoo", .{symbol}); + return quote; + } else |_| {} + } else |_| {} + + // Fallback: TwelveData (requires API key, may be 15-min delayed) var td = try self.getProvider(TwelveData); + log.debug("{s}: quote fallback to TwelveData", .{symbol}); return td.fetchQuote(self.allocator, symbol) catch return DataError.FetchFailed; }