475 lines
19 KiB
Zig
475 lines
19 KiB
Zig
//! Tiingo provider -- official REST API for end-of-day prices and corporate actions.
|
|
//!
|
|
//! Free tier: 50 requests/hour and 1,000 requests/day. We enforce the
|
|
//! hourly cap with a 50/hour token bucket (`RateLimiter.perHour`,
|
|
//! which starts full so a nightly refresh burst runs unthrottled);
|
|
//! the daily ceiling is far from binding for zfin's bursty usage.
|
|
//! Covers stocks, ETFs, and mutual funds with same-day NAV updates
|
|
//! (mutual fund NAVs available after midnight ET).
|
|
//!
|
|
//! API docs: https://www.tiingo.com/documentation/end-of-day
|
|
//!
|
|
//! ## Role in the data pipeline
|
|
//!
|
|
//! Tiingo is the **primary candle provider**. Yahoo is the fallback
|
|
//! when Tiingo can't serve a symbol. Tiingo's `/daily/<sym>/prices`
|
|
//! response also carries per-row `divCash` and `splitFactor`, which
|
|
//! we extract during candle parsing as a free side benefit — the
|
|
//! candle, dividend, and split data all come from a single HTTP call.
|
|
//!
|
|
//! For dividends and splits the **primary source is Polygon**, not
|
|
//! Tiingo. Polygon's dedicated corporate-actions endpoints carry
|
|
//! forward-looking declared events (e.g. ARCC's next ex-dividend
|
|
//! date several months out) that Tiingo's price-series response
|
|
//! cannot provide — Tiingo only reports events that have already
|
|
//! affected a price bar. Polygon also carries richer metadata per
|
|
//! dividend (`pay_date`, `record_date`, `type`, `currency`).
|
|
//!
|
|
//! Tiingo's dividend/split contribution is **supplementary**. The
|
|
//! `populateAllFromTiingo` orchestration in `service.zig` writes
|
|
//! Tiingo's view through `cache.Store.writeWithSource(..., "tiingo")`,
|
|
//! which dispatches to the sorted-union merge primitive. Polygon's
|
|
//! existing entries in `dividends.srf` / `splits.srf` are preserved
|
|
//! on key collision; Tiingo entries that name new ex_dates / split
|
|
//! dates are merged in and logged at `info(cache)` level.
|
|
//!
|
|
//! The canonical case where Tiingo's supplementary view rescues the
|
|
//! cache is SPYM's 2017-10-16 4:1 split — present in Tiingo's
|
|
//! historical bars but absent from Polygon's splits endpoint. Without
|
|
//! the merge, SPYM's 10Y price-only return would be off by ~14pp.
|
|
//!
|
|
//! ## Tiingo dividend records carry less metadata
|
|
//!
|
|
//! Tiingo only emits `divCash` (the cash amount) per dividend event.
|
|
//! When Tiingo merges a previously-unseen ex_date into the cache,
|
|
//! `pay_date`, `record_date`, `type`, and `currency` will be `null`
|
|
//! / `.unknown`. The total-return calculation only needs `ex_date`
|
|
//! and `amount`, both of which Tiingo provides; `divs.zig`
|
|
//! gracefully handles missing display fields.
|
|
|
|
const std = @import("std");
|
|
const http = @import("../net/http.zig");
|
|
const RateLimiter = @import("../net/RateLimiter.zig");
|
|
const Date = @import("../Date.zig");
|
|
const Candle = @import("../models/candle.zig").Candle;
|
|
const Dividend = @import("../models/dividend.zig").Dividend;
|
|
const Split = @import("../models/split.zig").Split;
|
|
const json_utils = @import("json_utils.zig");
|
|
const optFloat = json_utils.optFloat;
|
|
const jsonStr = json_utils.jsonStr;
|
|
|
|
const base_url = "https://api.tiingo.com/tiingo/daily";
|
|
|
|
/// Combined fetch result: candles, dividends, and splits parsed from
|
|
/// a single `/daily/<sym>/prices` response. Caller owns all three
|
|
/// slices and must free them (and `Dividend.deinit` each entry for
|
|
/// the currency string).
|
|
pub const CandleAndCorporateActions = struct {
|
|
candles: []Candle,
|
|
dividends: []Dividend,
|
|
splits: []Split,
|
|
};
|
|
|
|
pub const Tiingo = struct {
|
|
client: http.Client,
|
|
allocator: std.mem.Allocator,
|
|
api_key: []const u8,
|
|
/// Free-tier hourly cap (50/hour). Starts full so a nightly
|
|
/// refresh burst (one candle file per held symbol) isn't paced;
|
|
/// sustained usage beyond 50/hour blocks in `acquire`.
|
|
rate_limiter: RateLimiter,
|
|
|
|
pub fn init(io: std.Io, allocator: std.mem.Allocator, api_key: []const u8) Tiingo {
|
|
return .{
|
|
.client = http.Client.init(io, allocator),
|
|
.allocator = allocator,
|
|
.api_key = api_key,
|
|
.rate_limiter = RateLimiter.perHour(io, 50),
|
|
};
|
|
}
|
|
|
|
pub fn deinit(self: *Tiingo) void {
|
|
self.client.deinit();
|
|
}
|
|
|
|
/// Fetch candles, dividends, and splits in one HTTP call. This is
|
|
/// the primary provider entry point — the three convenience
|
|
/// methods below all call this and free the slices they don't
|
|
/// need.
|
|
pub fn fetchCandlesAndCorporateActions(
|
|
self: *Tiingo,
|
|
allocator: std.mem.Allocator,
|
|
symbol: []const u8,
|
|
from: Date,
|
|
to: Date,
|
|
) !CandleAndCorporateActions {
|
|
var from_buf: [10]u8 = undefined;
|
|
var to_buf: [10]u8 = undefined;
|
|
// Date's `{f}` output is exactly 10 bytes (YYYY-MM-DD), so
|
|
// these cannot fail in practice; `try` instead of
|
|
// `catch unreachable` to keep the error path honest.
|
|
const from_str = try std.fmt.bufPrint(&from_buf, "{f}", .{from});
|
|
const to_str = try std.fmt.bufPrint(&to_buf, "{f}", .{to});
|
|
|
|
const symbol_url = try std.fmt.allocPrint(allocator, base_url ++ "/{s}/prices", .{symbol});
|
|
defer allocator.free(symbol_url);
|
|
|
|
const url = try http.buildUrl(allocator, symbol_url, &.{
|
|
.{ "startDate", from_str },
|
|
.{ "endDate", to_str },
|
|
.{ "token", self.api_key },
|
|
});
|
|
defer allocator.free(url);
|
|
|
|
// Honor Tiingo's 50/hour free-tier cap. Blocks only once a
|
|
// single run has spent its 50-token burst within the hour.
|
|
self.rate_limiter.acquire();
|
|
|
|
var response = try self.client.get(url);
|
|
defer response.deinit();
|
|
|
|
return parseAll(allocator, response.body);
|
|
}
|
|
|
|
/// Fetch daily candles for a symbol between two dates (inclusive).
|
|
/// Convenience wrapper around `fetchCandlesAndCorporateActions`
|
|
/// for callers that only want the candle slice.
|
|
pub fn fetchCandles(
|
|
self: *Tiingo,
|
|
allocator: std.mem.Allocator,
|
|
symbol: []const u8,
|
|
from: Date,
|
|
to: Date,
|
|
) ![]Candle {
|
|
const triple = try self.fetchCandlesAndCorporateActions(allocator, symbol, from, to);
|
|
Dividend.freeSlice(allocator, triple.dividends);
|
|
allocator.free(triple.splits);
|
|
return triple.candles;
|
|
}
|
|
|
|
/// Fetch dividends for a symbol between two dates (inclusive).
|
|
/// Convenience wrapper around `fetchCandlesAndCorporateActions`.
|
|
pub fn fetchDividends(
|
|
self: *Tiingo,
|
|
allocator: std.mem.Allocator,
|
|
symbol: []const u8,
|
|
from: Date,
|
|
to: Date,
|
|
) ![]Dividend {
|
|
const triple = try self.fetchCandlesAndCorporateActions(allocator, symbol, from, to);
|
|
allocator.free(triple.candles);
|
|
allocator.free(triple.splits);
|
|
return triple.dividends;
|
|
}
|
|
|
|
/// Fetch splits for a symbol between two dates (inclusive).
|
|
/// Convenience wrapper around `fetchCandlesAndCorporateActions`.
|
|
pub fn fetchSplits(
|
|
self: *Tiingo,
|
|
allocator: std.mem.Allocator,
|
|
symbol: []const u8,
|
|
from: Date,
|
|
to: Date,
|
|
) ![]Split {
|
|
const triple = try self.fetchCandlesAndCorporateActions(allocator, symbol, from, to);
|
|
allocator.free(triple.candles);
|
|
Dividend.freeSlice(allocator, triple.dividends);
|
|
return triple.splits;
|
|
}
|
|
};
|
|
|
|
/// Walk Tiingo's JSON array of price rows once, emitting candles,
|
|
/// dividends (where `divCash != 0`), and splits (where `splitFactor != 1`).
|
|
fn parseAll(allocator: std.mem.Allocator, body: []const u8) !CandleAndCorporateActions {
|
|
const parsed = std.json.parseFromSlice(std.json.Value, allocator, body, .{}) catch
|
|
return error.ParseError;
|
|
defer parsed.deinit();
|
|
|
|
const items = switch (parsed.value) {
|
|
.array => |a| a.items,
|
|
// Tiingo returns an object with "detail" on errors (bad symbol, auth, etc.)
|
|
else => return error.RequestFailed,
|
|
};
|
|
|
|
var candles: std.ArrayList(Candle) = .empty;
|
|
errdefer candles.deinit(allocator);
|
|
|
|
var dividends: std.ArrayList(Dividend) = .empty;
|
|
errdefer {
|
|
for (dividends.items) |d| d.deinit(allocator);
|
|
dividends.deinit(allocator);
|
|
}
|
|
|
|
var splits: std.ArrayList(Split) = .empty;
|
|
errdefer splits.deinit(allocator);
|
|
|
|
for (items) |item| {
|
|
const obj = switch (item) {
|
|
.object => |o| o,
|
|
else => continue,
|
|
};
|
|
|
|
const date = parseDate(obj.get("date")) orelse continue;
|
|
const close = optFloat(obj.get("close")) orelse continue;
|
|
|
|
try candles.append(allocator, .{
|
|
.date = date,
|
|
.open = optFloat(obj.get("open")) orelse close,
|
|
.high = optFloat(obj.get("high")) orelse close,
|
|
.low = optFloat(obj.get("low")) orelse close,
|
|
.close = close,
|
|
.adj_close = optFloat(obj.get("adjClose")) orelse close,
|
|
.volume = blk: {
|
|
const v = optFloat(obj.get("volume")) orelse break :blk 0;
|
|
break :blk @intFromFloat(@max(0, v));
|
|
},
|
|
});
|
|
|
|
// Dividend event on this row (if any)
|
|
const div_cash = optFloat(obj.get("divCash")) orelse 0;
|
|
if (div_cash != 0) {
|
|
try dividends.append(allocator, .{
|
|
.ex_date = date,
|
|
.amount = div_cash,
|
|
// Tiingo doesn't carry pay_date / record_date /
|
|
// type. Display-only fields stay null / .unknown;
|
|
// total-return math only needs ex_date and amount.
|
|
});
|
|
}
|
|
|
|
// Split event on this row (if any). Tiingo represents a 4:1
|
|
// split as splitFactor = 4.0 and a 1:10 reverse split as
|
|
// splitFactor = 0.1. Both shapes are stored as
|
|
// numerator=splitFactor, denominator=1.0; `Split.ratio()`
|
|
// returns splitFactor in either case.
|
|
const split_factor = optFloat(obj.get("splitFactor")) orelse 1.0;
|
|
if (split_factor != 1.0 and split_factor != 0) {
|
|
try splits.append(allocator, .{
|
|
.date = date,
|
|
.numerator = split_factor,
|
|
.denominator = 1.0,
|
|
});
|
|
}
|
|
}
|
|
|
|
return .{
|
|
.candles = try candles.toOwnedSlice(allocator),
|
|
.dividends = try dividends.toOwnedSlice(allocator),
|
|
.splits = try splits.toOwnedSlice(allocator),
|
|
};
|
|
}
|
|
|
|
/// Parse a Tiingo date string (e.g. "2026-03-16T00:00:00.000Z") into a Date.
|
|
fn parseDate(val: ?std.json.Value) ?Date {
|
|
const s = jsonStr(val) orelse return null;
|
|
if (s.len < 10) return null;
|
|
return Date.parse(s[0..10]) catch null;
|
|
}
|
|
|
|
// -- Tests --
|
|
|
|
test "parseAll basic candles, no events" {
|
|
const body =
|
|
\\[
|
|
\\ {
|
|
\\ "date": "2026-03-13T00:00:00.000Z",
|
|
\\ "close": 42.41, "high": 42.41, "low": 42.41, "open": 42.41,
|
|
\\ "volume": 0, "adjClose": 42.41, "adjHigh": 42.41,
|
|
\\ "adjLow": 42.41, "adjOpen": 42.41, "adjVolume": 0,
|
|
\\ "divCash": 0.0, "splitFactor": 1.0
|
|
\\ },
|
|
\\ {
|
|
\\ "date": "2026-03-16T00:00:00.000Z",
|
|
\\ "close": 42.74, "high": 42.74, "low": 42.74, "open": 42.74,
|
|
\\ "volume": 0, "adjClose": 42.74, "adjHigh": 42.74,
|
|
\\ "adjLow": 42.74, "adjOpen": 42.74, "adjVolume": 0,
|
|
\\ "divCash": 0.0, "splitFactor": 1.0
|
|
\\ }
|
|
\\]
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const triple = try parseAll(allocator, body);
|
|
defer allocator.free(triple.candles);
|
|
defer Dividend.freeSlice(allocator, triple.dividends);
|
|
defer allocator.free(triple.splits);
|
|
|
|
try std.testing.expectEqual(@as(usize, 2), triple.candles.len);
|
|
try std.testing.expectEqual(@as(usize, 0), triple.dividends.len);
|
|
try std.testing.expectEqual(@as(usize, 0), triple.splits.len);
|
|
|
|
try std.testing.expectEqual(@as(i16, 2026), triple.candles[0].date.year());
|
|
try std.testing.expectApproxEqAbs(@as(f64, 42.41), triple.candles[0].close, 0.01);
|
|
try std.testing.expectApproxEqAbs(@as(f64, 42.74), triple.candles[1].close, 0.01);
|
|
}
|
|
|
|
test "parseAll extracts a dividend from a divCash row" {
|
|
// NKE 2024-03-01 dividend of $0.37 (real Tiingo response shape)
|
|
const body =
|
|
\\[
|
|
\\ {
|
|
\\ "date": "2024-03-01T00:00:00.000Z",
|
|
\\ "close": 101.88, "high": 103.94, "low": 101.83, "open": 103.87,
|
|
\\ "volume": 7349270, "adjClose": 97.5550917628,
|
|
\\ "divCash": 0.37, "splitFactor": 1.0
|
|
\\ }
|
|
\\]
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const triple = try parseAll(allocator, body);
|
|
defer allocator.free(triple.candles);
|
|
defer Dividend.freeSlice(allocator, triple.dividends);
|
|
defer allocator.free(triple.splits);
|
|
|
|
try std.testing.expectEqual(@as(usize, 1), triple.candles.len);
|
|
try std.testing.expectEqual(@as(usize, 1), triple.dividends.len);
|
|
try std.testing.expectEqual(@as(usize, 0), triple.splits.len);
|
|
|
|
const div = triple.dividends[0];
|
|
try std.testing.expect(div.ex_date.eql(Date.fromYmd(2024, 3, 1)));
|
|
try std.testing.expectApproxEqAbs(@as(f64, 0.37), div.amount, 0.001);
|
|
// Metadata fields are absent for Tiingo-sourced dividends
|
|
try std.testing.expect(div.pay_date == null);
|
|
try std.testing.expect(div.record_date == null);
|
|
try std.testing.expectEqual(@import("../models/dividend.zig").DividendType.unknown, div.type);
|
|
}
|
|
|
|
test "parseAll extracts forward 4:1 split (SPYM 2017 fixture)" {
|
|
// SPYM's actual 2017-10-16 split — verbatim Tiingo response shape.
|
|
// Polygon and FMP both miss this split; Tiingo has it via
|
|
// splitFactor: 4.0.
|
|
const body =
|
|
\\[
|
|
\\ {
|
|
\\ "date": "2017-10-13T00:00:00.000Z",
|
|
\\ "close": 119.7493, "open": 119.7667, "high": 120.26, "low": 119.7396,
|
|
\\ "volume": 7638, "adjClose": 26.0674934371,
|
|
\\ "divCash": 0.0, "splitFactor": 1.0
|
|
\\ },
|
|
\\ {
|
|
\\ "date": "2017-10-16T00:00:00.000Z",
|
|
\\ "close": 29.9556, "open": 30.01, "high": 30.01, "low": 29.9399,
|
|
\\ "volume": 8804, "adjClose": 26.0810001294,
|
|
\\ "divCash": 0.0, "splitFactor": 4.0
|
|
\\ },
|
|
\\ {
|
|
\\ "date": "2017-10-17T00:00:00.000Z",
|
|
\\ "close": 29.92, "open": 29.95, "high": 30.05, "low": 29.92,
|
|
\\ "volume": 21456, "adjClose": 26.0524079435,
|
|
\\ "divCash": 0.0, "splitFactor": 1.0
|
|
\\ }
|
|
\\]
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const triple = try parseAll(allocator, body);
|
|
defer allocator.free(triple.candles);
|
|
defer Dividend.freeSlice(allocator, triple.dividends);
|
|
defer allocator.free(triple.splits);
|
|
|
|
try std.testing.expectEqual(@as(usize, 3), triple.candles.len);
|
|
try std.testing.expectEqual(@as(usize, 0), triple.dividends.len);
|
|
try std.testing.expectEqual(@as(usize, 1), triple.splits.len);
|
|
|
|
const split = triple.splits[0];
|
|
try std.testing.expect(split.date.eql(Date.fromYmd(2017, 10, 16)));
|
|
try std.testing.expectApproxEqAbs(@as(f64, 4.0), split.numerator, 0.001);
|
|
try std.testing.expectApproxEqAbs(@as(f64, 1.0), split.denominator, 0.001);
|
|
try std.testing.expectApproxEqAbs(@as(f64, 4.0), split.ratio(), 0.001);
|
|
}
|
|
|
|
test "parseAll extracts reverse 1:10 split (splitFactor < 1)" {
|
|
// Reverse split: 1:10 means splitFactor = 0.1
|
|
const body =
|
|
\\[
|
|
\\ {
|
|
\\ "date": "2024-06-10T00:00:00.000Z",
|
|
\\ "close": 50.0, "open": 5.0, "high": 50.0, "low": 5.0,
|
|
\\ "volume": 1000, "adjClose": 50.0,
|
|
\\ "divCash": 0.0, "splitFactor": 0.1
|
|
\\ }
|
|
\\]
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const triple = try parseAll(allocator, body);
|
|
defer allocator.free(triple.candles);
|
|
defer Dividend.freeSlice(allocator, triple.dividends);
|
|
defer allocator.free(triple.splits);
|
|
|
|
try std.testing.expectEqual(@as(usize, 1), triple.splits.len);
|
|
const split = triple.splits[0];
|
|
try std.testing.expectApproxEqAbs(@as(f64, 0.1), split.ratio(), 0.001);
|
|
}
|
|
|
|
test "parseAll: combined dividend + split in same response" {
|
|
const body =
|
|
\\[
|
|
\\ {"date": "2024-01-15T00:00:00.000Z", "close": 100.0, "open": 100.0, "high": 100.0, "low": 100.0,
|
|
\\ "volume": 0, "adjClose": 100.0, "divCash": 0.5, "splitFactor": 1.0},
|
|
\\ {"date": "2024-06-10T00:00:00.000Z", "close": 25.0, "open": 100.0, "high": 100.0, "low": 25.0,
|
|
\\ "volume": 0, "adjClose": 25.0, "divCash": 0.0, "splitFactor": 4.0},
|
|
\\ {"date": "2024-12-15T00:00:00.000Z", "close": 30.0, "open": 30.0, "high": 30.0, "low": 30.0,
|
|
\\ "volume": 0, "adjClose": 30.0, "divCash": 0.15, "splitFactor": 1.0}
|
|
\\]
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const triple = try parseAll(allocator, body);
|
|
defer allocator.free(triple.candles);
|
|
defer Dividend.freeSlice(allocator, triple.dividends);
|
|
defer allocator.free(triple.splits);
|
|
|
|
try std.testing.expectEqual(@as(usize, 3), triple.candles.len);
|
|
try std.testing.expectEqual(@as(usize, 2), triple.dividends.len);
|
|
try std.testing.expectEqual(@as(usize, 1), triple.splits.len);
|
|
|
|
try std.testing.expectApproxEqAbs(@as(f64, 0.5), triple.dividends[0].amount, 0.001);
|
|
try std.testing.expectApproxEqAbs(@as(f64, 0.15), triple.dividends[1].amount, 0.001);
|
|
try std.testing.expectApproxEqAbs(@as(f64, 4.0), triple.splits[0].ratio(), 0.001);
|
|
}
|
|
|
|
test "parseAll: large dividend (VPMAX-style cap-gains distribution)" {
|
|
// VPMAX's 2025-12-17 distribution of $30.43 — chunky year-end
|
|
// cap-gains payout that inflates 1Y total return because Tiingo
|
|
// (and Polygon) lump it under regular dividends.
|
|
const body =
|
|
\\[
|
|
\\ {"date": "2025-12-17T00:00:00.000Z", "close": 214.0, "open": 244.43, "high": 244.43, "low": 213.5,
|
|
\\ "volume": 0, "adjClose": 214.0, "divCash": 30.429903, "splitFactor": 1.0}
|
|
\\]
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const triple = try parseAll(allocator, body);
|
|
defer allocator.free(triple.candles);
|
|
defer Dividend.freeSlice(allocator, triple.dividends);
|
|
defer allocator.free(triple.splits);
|
|
|
|
try std.testing.expectEqual(@as(usize, 1), triple.dividends.len);
|
|
try std.testing.expectApproxEqAbs(@as(f64, 30.429903), triple.dividends[0].amount, 0.000001);
|
|
}
|
|
|
|
test "parseAll error response" {
|
|
const body =
|
|
\\{"detail": "Not found."}
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
const result = parseAll(allocator, body);
|
|
try std.testing.expectError(error.RequestFailed, result);
|
|
}
|
|
|
|
test "parseAll empty array" {
|
|
const body = "[]";
|
|
const allocator = std.testing.allocator;
|
|
const triple = try parseAll(allocator, body);
|
|
defer allocator.free(triple.candles);
|
|
defer Dividend.freeSlice(allocator, triple.dividends);
|
|
defer allocator.free(triple.splits);
|
|
|
|
try std.testing.expectEqual(@as(usize, 0), triple.candles.len);
|
|
try std.testing.expectEqual(@as(usize, 0), triple.dividends.len);
|
|
try std.testing.expectEqual(@as(usize, 0), triple.splits.len);
|
|
}
|