add Yahoo finance into the mix for candles and quotes

This commit is contained in:
Emil Lerch 2026-03-11 15:31:17 -07:00
parent f637740c13
commit f8a9607bc9
Signed by: lobo
GPG key ID: A7B62D657EF764F8
5 changed files with 480 additions and 39 deletions

View file

@ -1,12 +1,5 @@
# Future Work
## Yahoo Finance as primary quote source
Consider adding Yahoo Finance as the primary provider for real-time quotes,
with a silent fallback to TwelveData. Yahoo is free and has no API key
requirement, but the unofficial API is brittle and can break without notice.
TwelveData would serve as the reliable backup when Yahoo is unavailable.
## Covered call portfolio valuation
Portfolio value should account for sold call options. Shares covered by

19
src/cache/store.zig vendored
View file

@ -241,10 +241,16 @@ pub const Store = struct {
/// Write (or refresh) candle metadata without touching the candle data file.
pub fn updateCandleMeta(self: *Store, symbol: []const u8, last_close: f64, last_date: Date) void {
self.updateCandleMetaWithProvider(symbol, last_close, last_date, .twelvedata);
}
/// Write candle metadata with a specific provider source.
pub fn updateCandleMetaWithProvider(self: *Store, symbol: []const u8, last_close: f64, last_date: Date, provider: CandleProvider) void {
const expires = std.time.timestamp() + Ttl.candles_latest;
const meta = CandleMeta{
.last_close = last_close,
.last_date = last_date,
.provider = provider,
};
if (serializeCandleMeta(self.allocator, meta, .{ .expires = expires })) |meta_data| {
defer self.allocator.free(meta_data);
@ -360,6 +366,19 @@ pub const Store = struct {
pub const CandleMeta = struct {
last_close: f64,
last_date: Date,
/// Which provider sourced the candle data. Used during incremental refresh
/// to go directly to the right provider instead of trying TwelveData first.
provider: CandleProvider = .twelvedata,
};
pub const CandleProvider = enum {
twelvedata,
yahoo,
pub fn fromString(s: []const u8) CandleProvider {
if (std.mem.eql(u8, s, "yahoo")) return .yahoo;
return .twelvedata;
}
};
// Private I/O

View file

@ -80,6 +80,16 @@ pub const Date = struct {
return .{ .days = self.days + n };
}
/// Convert to Unix epoch seconds (midnight UTC on this date).
pub fn toEpoch(self: Date) i64 {
return @as(i64, self.days) * std.time.s_per_day;
}
/// Create a Date from a Unix epoch timestamp (seconds since 1970-01-01).
pub fn fromEpoch(epoch_secs: i64) Date {
return .{ .days = @intCast(@divFloor(epoch_secs, std.time.s_per_day)) };
}
/// Subtract N calendar years. Clamps Feb 29 -> Feb 28 if target is not a leap year.
pub fn subtractYears(self: Date, n: u16) Date {
const ymd = epochDaysToYmd(self.days);

373
src/providers/yahoo.zig Normal file
View file

@ -0,0 +1,373 @@
//! Yahoo Finance provider -- free, no API key required.
//!
//! Uses the unofficial v8 chart API for both candles and quotes.
//! No rate limiting is applied since there's no documented limit,
//! but this API is unofficial and can break without notice.
//!
//! Yahoo has broader mutual fund coverage than TwelveData/Polygon,
//! making it useful as a fallback for symbols like VTTHX.
const std = @import("std");
const http = @import("../net/http.zig");
const Date = @import("../models/date.zig").Date;
const Candle = @import("../models/candle.zig").Candle;
const Quote = @import("../models/quote.zig").Quote;
const parseJsonFloat = @import("json_utils.zig").parseJsonFloat;
const base_url = "https://query1.finance.yahoo.com/v8/finance/chart";
pub const Yahoo = struct {
client: http.Client,
allocator: std.mem.Allocator,
pub fn init(allocator: std.mem.Allocator) Yahoo {
return .{
.client = http.Client.init(allocator),
.allocator = allocator,
};
}
pub fn deinit(self: *Yahoo) void {
self.client.deinit();
}
/// Fetch daily candles for a symbol between two dates.
/// Returns candles sorted oldest-first.
pub fn fetchCandles(
self: *Yahoo,
allocator: std.mem.Allocator,
symbol: []const u8,
from: Date,
to: Date,
) ![]Candle {
const period1 = from.toEpoch();
const period2 = to.toEpoch() + std.time.s_per_day; // inclusive end
var p1_buf: [20]u8 = undefined;
var p2_buf: [20]u8 = undefined;
const p1_str = std.fmt.bufPrint(&p1_buf, "{d}", .{period1}) catch return error.ParseError;
const p2_str = std.fmt.bufPrint(&p2_buf, "{d}", .{period2}) catch return error.ParseError;
const symbol_url = try std.fmt.allocPrint(allocator, base_url ++ "/{s}", .{symbol});
defer allocator.free(symbol_url);
const url = try http.buildUrl(allocator, symbol_url, &.{
.{ "interval", "1d" },
.{ "period1", p1_str },
.{ "period2", p2_str },
});
defer allocator.free(url);
var response = try self.client.get(url);
defer response.deinit();
return parseChartCandles(allocator, response.body);
}
/// Fetch a quote snapshot for a symbol.
/// Uses the chart API meta fields (regularMarketPrice, etc.).
pub fn fetchQuote(
self: *Yahoo,
allocator: std.mem.Allocator,
symbol: []const u8,
) !Quote {
// Fetch just 1 day of data to get the meta block with quote info
const symbol_url = try std.fmt.allocPrint(allocator, base_url ++ "/{s}", .{symbol});
defer allocator.free(symbol_url);
const url = try http.buildUrl(allocator, symbol_url, &.{
.{ "interval", "1d" },
.{ "range", "5d" },
});
defer allocator.free(url);
var response = try self.client.get(url);
defer response.deinit();
return parseChartQuote(allocator, response.body, symbol);
}
};
// -- JSON parsing --
/// Parse the chart API result array, returning the first result's chart data.
fn getChartResult(allocator: std.mem.Allocator, body: []const u8) !struct { parsed: std.json.Parsed(std.json.Value), result: std.json.ObjectMap } {
const parsed = std.json.parseFromSlice(std.json.Value, allocator, body, .{}) catch
return error.ParseError;
errdefer parsed.deinit();
const chart = parsed.value.object.get("chart") orelse return error.ParseError;
const chart_obj = switch (chart) {
.object => |o| o,
else => return error.ParseError,
};
// Check for error
if (chart_obj.get("error")) |err_val| {
if (err_val != .null) return error.RequestFailed;
}
const result_arr = chart_obj.get("result") orelse return error.ParseError;
const results = switch (result_arr) {
.array => |a| a.items,
else => return error.ParseError,
};
if (results.len == 0) return error.ParseError;
const result = switch (results[0]) {
.object => |o| o,
else => return error.ParseError,
};
return .{ .parsed = parsed, .result = result };
}
fn parseChartCandles(allocator: std.mem.Allocator, body: []const u8) ![]Candle {
const chart = try getChartResult(allocator, body);
const parsed = chart.parsed;
defer parsed.deinit();
const result = chart.result;
// Get timestamps
const ts_json = result.get("timestamp") orelse return error.ParseError;
const timestamps = switch (ts_json) {
.array => |a| a.items,
else => return error.ParseError,
};
// Get quote data (open/high/low/close/volume arrays)
const indicators = result.get("indicators") orelse return error.ParseError;
const indicators_obj = switch (indicators) {
.object => |o| o,
else => return error.ParseError,
};
const quote_arr = indicators_obj.get("quote") orelse return error.ParseError;
const quotes = switch (quote_arr) {
.array => |a| a.items,
else => return error.ParseError,
};
if (quotes.len == 0) return error.ParseError;
const q = switch (quotes[0]) {
.object => |o| o,
else => return error.ParseError,
};
const opens = getFloatArray(q.get("open")) orelse return error.ParseError;
const highs = getFloatArray(q.get("high")) orelse return error.ParseError;
const lows = getFloatArray(q.get("low")) orelse return error.ParseError;
const closes = getFloatArray(q.get("close")) orelse return error.ParseError;
const volumes = getFloatArray(q.get("volume")) orelse return error.ParseError;
// Get adjclose if available
const adjcloses = blk: {
const adjclose_section = indicators_obj.get("adjclose") orelse break :blk null;
const adj_arr = switch (adjclose_section) {
.array => |a| a.items,
else => break :blk null,
};
if (adj_arr.len == 0) break :blk null;
const adj_obj = switch (adj_arr[0]) {
.object => |o| o,
else => break :blk null,
};
break :blk getFloatArray(adj_obj.get("adjclose"));
};
var candles: std.ArrayList(Candle) = .empty;
errdefer candles.deinit(allocator);
// Yahoo returns oldest-first, which is what we want
for (timestamps, 0..) |ts, i| {
if (i >= opens.len or i >= closes.len) break;
// Skip null entries (incomplete trading day)
const close_val = closes[i];
if (close_val == .null) continue;
const epoch: i64 = switch (ts) {
.integer => |v| v,
else => continue,
};
const date = Date.fromEpoch(epoch);
const close = parseJsonFloat(close_val);
const adj = if (adjcloses) |ac| (if (i < ac.len) parseJsonFloat(ac[i]) else close) else close;
try candles.append(allocator, .{
.date = date,
.open = parseJsonFloat(opens[i]),
.high = parseJsonFloat(highs[i]),
.low = parseJsonFloat(lows[i]),
.close = close,
.adj_close = adj,
.volume = @intFromFloat(@max(0, parseJsonFloat(volumes[i]))),
});
}
return candles.toOwnedSlice(allocator);
}
fn parseChartQuote(allocator: std.mem.Allocator, body: []const u8, symbol: []const u8) !Quote {
const chart = try getChartResult(allocator, body);
const parsed = chart.parsed;
defer parsed.deinit();
const result = chart.result;
const meta = result.get("meta") orelse return error.ParseError;
const m = switch (meta) {
.object => |o| o,
else => return error.ParseError,
};
const price = parseJsonFloat(m.get("regularMarketPrice"));
const prev_close = parseJsonFloat(m.get("chartPreviousClose"));
const change = price - prev_close;
const pct = if (prev_close != 0) (change / prev_close) * 100.0 else 0;
return .{
.symbol = symbol,
.name = symbol,
.exchange = "",
.datetime = "",
.close = price,
.open = price, // meta doesn't have open
.high = parseJsonFloat(m.get("fiftyTwoWeekHigh")),
.low = parseJsonFloat(m.get("fiftyTwoWeekLow")),
.volume = 0,
.previous_close = prev_close,
.change = change,
.percent_change = pct,
.average_volume = 0,
.fifty_two_week_low = parseJsonFloat(m.get("fiftyTwoWeekLow")),
.fifty_two_week_high = parseJsonFloat(m.get("fiftyTwoWeekHigh")),
};
}
fn getFloatArray(val: ?std.json.Value) ?[]const std.json.Value {
const v = val orelse return null;
return switch (v) {
.array => |a| a.items,
else => null,
};
}
// -- Tests --
test "parseChartCandles basic" {
const body =
\\{
\\ "chart": {
\\ "result": [{
\\ "timestamp": [1704067800, 1704154200],
\\ "indicators": {
\\ "quote": [{
\\ "open": [185.0, 187.15],
\\ "high": [186.1, 188.44],
\\ "low": [184.0, 183.89],
\\ "close": [185.5, 184.25],
\\ "volume": [42000000, 58414460]
\\ }],
\\ "adjclose": [{
\\ "adjclose": [185.5, 184.25]
\\ }]
\\ }
\\ }],
\\ "error": null
\\ }
\\}
;
const allocator = std.testing.allocator;
const candles = try parseChartCandles(allocator, body);
defer allocator.free(candles);
try std.testing.expectEqual(@as(usize, 2), candles.len);
// Oldest first (Yahoo returns chronological order)
try std.testing.expectApproxEqAbs(@as(f64, 185.0), candles[0].open, 0.01);
try std.testing.expectApproxEqAbs(@as(f64, 185.5), candles[0].close, 0.01);
try std.testing.expectEqual(@as(u64, 42000000), candles[0].volume);
try std.testing.expectApproxEqAbs(@as(f64, 187.15), candles[1].open, 0.01);
try std.testing.expectApproxEqAbs(@as(f64, 184.25), candles[1].close, 0.01);
}
test "parseChartCandles skips null entries" {
const body =
\\{
\\ "chart": {
\\ "result": [{
\\ "timestamp": [1704067800, 1704154200],
\\ "indicators": {
\\ "quote": [{
\\ "open": [185.0, null],
\\ "high": [186.1, null],
\\ "low": [184.0, null],
\\ "close": [185.5, null],
\\ "volume": [42000000, null]
\\ }]
\\ }
\\ }],
\\ "error": null
\\ }
\\}
;
const allocator = std.testing.allocator;
const candles = try parseChartCandles(allocator, body);
defer allocator.free(candles);
// Should only have 1 candle (null entry skipped)
try std.testing.expectEqual(@as(usize, 1), candles.len);
try std.testing.expectApproxEqAbs(@as(f64, 185.5), candles[0].close, 0.01);
}
test "parseChartCandles error response" {
const body =
\\{
\\ "chart": {
\\ "result": null,
\\ "error": {"code": "Not Found", "description": "No data found"}
\\ }
\\}
;
const allocator = std.testing.allocator;
const result = parseChartCandles(allocator, body);
try std.testing.expectError(error.RequestFailed, result);
}
test "parseChartQuote basic" {
const body =
\\{
\\ "chart": {
\\ "result": [{
\\ "meta": {
\\ "symbol": "VTTHX",
\\ "regularMarketPrice": 27.78,
\\ "chartPreviousClose": 28.06,
\\ "fiftyTwoWeekHigh": 28.59,
\\ "fiftyTwoWeekLow": 22.21
\\ },
\\ "timestamp": [1704067800],
\\ "indicators": {"quote": [{"open": [27.78], "high": [27.78], "low": [27.78], "close": [27.78], "volume": [0]}]}
\\ }],
\\ "error": null
\\ }
\\}
;
const allocator = std.testing.allocator;
const quote = try parseChartQuote(allocator, body, "VTTHX");
try std.testing.expectApproxEqAbs(@as(f64, 27.78), quote.close, 0.01);
try std.testing.expectApproxEqAbs(@as(f64, 28.06), quote.previous_close, 0.01);
try std.testing.expectApproxEqAbs(@as(f64, 28.59), quote.fifty_two_week_high, 0.01);
try std.testing.expectApproxEqAbs(@as(f64, 22.21), quote.fifty_two_week_low, 0.01);
// change = 27.78 - 28.06 = -0.28
try std.testing.expectApproxEqAbs(@as(f64, -0.28), quote.change, 0.01);
// percent_change = (-0.28 / 28.06) * 100 -0.998
try std.testing.expectApproxEqAbs(@as(f64, -0.998), quote.percent_change, 0.01);
}

View file

@ -27,6 +27,7 @@ const Cboe = @import("providers/cboe.zig").Cboe;
const AlphaVantage = @import("providers/alphavantage.zig").AlphaVantage;
const alphavantage = @import("providers/alphavantage.zig");
const OpenFigi = @import("providers/openfigi.zig");
const Yahoo = @import("providers/yahoo.zig").Yahoo;
const fmt = @import("format.zig");
const performance = @import("analytics/performance.zig");
const http = @import("net/http.zig");
@ -92,6 +93,7 @@ pub const DataService = struct {
fh: ?Finnhub = null,
cboe: ?Cboe = null,
av: ?AlphaVantage = null,
yh: ?Yahoo = null,
pub fn init(allocator: std.mem.Allocator, config: Config) DataService {
return .{
@ -106,6 +108,7 @@ pub const DataService = struct {
if (self.fh) |*fh| fh.deinit();
if (self.cboe) |*c| c.deinit();
if (self.av) |*av| av.deinit();
if (self.yh) |*yh| yh.deinit();
}
// Provider accessor
@ -113,8 +116,8 @@ pub const DataService = struct {
fn getProvider(self: *DataService, comptime T: type) DataError!*T {
const field_name = comptime providerField(T);
if (@field(self, field_name)) |*p| return p;
if (T == Cboe) {
// CBOE has no key
if (T == Cboe or T == Yahoo) {
// CBOE and Yahoo have no API key
@field(self, field_name) = T.init(self.allocator);
} else {
// All we're doing here is lower casing the type name, then
@ -227,6 +230,55 @@ pub const DataService = struct {
// Public data methods
/// Fetch candles from providers with fallback logic.
/// Tries the provider recorded in meta (if any), then TwelveData, then Yahoo.
/// Returns the candles and which provider succeeded.
fn fetchCandlesFromProviders(
self: *DataService,
symbol: []const u8,
from: Date,
to: Date,
preferred: cache.Store.CandleProvider,
) !struct { candles: []Candle, provider: cache.Store.CandleProvider } {
// If preferred is Yahoo, try it first
if (preferred == .yahoo) {
if (self.getProvider(Yahoo)) |yh| {
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from Yahoo (preferred)", .{symbol});
return .{ .candles = candles, .provider = .yahoo };
} else |_| {}
} else |_| {}
}
// Try TwelveData
if (self.getProvider(TwelveData)) |td| {
if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from TwelveData", .{symbol});
return .{ .candles = candles, .provider = .twelvedata };
} else |err| {
if (err == error.RateLimited) {
self.rateLimitBackoff();
if (td.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from TwelveData (after rate limit retry)", .{symbol});
return .{ .candles = candles, .provider = .twelvedata };
} else |_| {}
}
}
} else |_| {}
// Fallback: Yahoo (if not already tried as preferred)
if (preferred != .yahoo) {
if (self.getProvider(Yahoo)) |yh| {
if (yh.fetchCandles(self.allocator, symbol, from, to)) |candles| {
log.debug("{s}: candles from Yahoo (fallback)", .{symbol});
return .{ .candles = candles, .provider = .yahoo };
} else |_| {}
} else |_| {}
}
return error.FetchFailed;
}
/// Fetch daily candles for a symbol (10+ years for trailing returns).
/// Checks cache first; fetches from TwelveData if stale/missing.
/// Uses incremental updates: when the cache is stale, only fetches
@ -268,31 +320,18 @@ pub const DataService = struct {
return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() };
} else {
// Incremental fetch from day after last cached candle
var td = self.getProvider(TwelveData) catch {
// No API key return stale data
if (s.read(Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = mr.created };
return DataError.NoApiKey;
};
const new_candles = td.fetchCandles(self.allocator, symbol, fetch_from, today) catch |err| blk: {
if (err == error.RateLimited) {
self.rateLimitBackoff();
break :blk td.fetchCandles(self.allocator, symbol, fetch_from, today) catch {
if (s.read(Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = mr.created };
return DataError.FetchFailed;
};
}
// Non-rate-limit failure return stale data
const result = self.fetchCandlesFromProviders(symbol, fetch_from, today, m.provider) catch {
// All providers failed return stale data
if (s.read(Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = mr.created };
return DataError.FetchFailed;
};
const new_candles = result.candles;
if (new_candles.len == 0) {
// No new candles (weekend/holiday) refresh TTL only (meta rewrite)
self.allocator.free(new_candles);
s.updateCandleMeta(symbol, m.last_close, m.last_date);
s.updateCandleMetaWithProvider(symbol, m.last_close, m.last_date, result.provider);
if (s.read(Candle, symbol, null, .any)) |r|
return .{ .data = r.data, .source = .cached, .timestamp = std.time.timestamp() };
} else {
@ -322,25 +361,21 @@ pub const DataService = struct {
// No usable cache full fetch (~10 years, plus buffer for leap years)
log.debug("{s}: fetching full candle history from provider", .{symbol});
var td = try self.getProvider(TwelveData);
const from = today.addDays(-3700);
const fetched = td.fetchCandles(self.allocator, symbol, from, today) catch |err| blk: {
if (err == error.RateLimited) {
self.rateLimitBackoff();
break :blk td.fetchCandles(self.allocator, symbol, from, today) catch {
return DataError.FetchFailed;
};
}
const result = self.fetchCandlesFromProviders(symbol, from, today, .twelvedata) catch {
s.writeNegative(symbol, .candles_daily);
return DataError.FetchFailed;
};
if (fetched.len > 0) {
s.cacheCandles(symbol, fetched);
if (result.candles.len > 0) {
s.cacheCandles(symbol, result.candles);
// Record which provider sourced this data
const last = result.candles[result.candles.len - 1];
s.updateCandleMetaWithProvider(symbol, last.close, last.date, result.provider);
}
return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() };
return .{ .data = result.candles, .source = .fetched, .timestamp = std.time.timestamp() };
}
/// Fetch dividend history for a symbol.
@ -441,10 +476,21 @@ pub const DataService = struct {
return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() };
}
/// Fetch a real-time (or 15-min delayed) quote for a symbol.
/// No cache -- always fetches fresh from TwelveData.
/// Fetch a real-time quote for a symbol.
/// Yahoo Finance is primary (free, no API key, no 15-min delay).
/// Falls back to TwelveData if Yahoo fails.
pub fn getQuote(self: *DataService, symbol: []const u8) DataError!Quote {
// Primary: Yahoo Finance (free, real-time)
if (self.getProvider(Yahoo)) |yh| {
if (yh.fetchQuote(self.allocator, symbol)) |quote| {
log.debug("{s}: quote from Yahoo", .{symbol});
return quote;
} else |_| {}
} else |_| {}
// Fallback: TwelveData (requires API key, may be 15-min delayed)
var td = try self.getProvider(TwelveData);
log.debug("{s}: quote fallback to TwelveData", .{symbol});
return td.fetchQuote(self.allocator, symbol) catch
return DataError.FetchFailed;
}