960 lines
39 KiB
Zig
960 lines
39 KiB
Zig
//! zfin-server — HTTP data service backed by zfin's provider infrastructure.
|
|
//!
|
|
//! Two modes:
|
|
//! zfin-server serve [--port=8080] Start the HTTP server
|
|
//! zfin-server refresh Refresh cache for all tracked symbols (for cron)
|
|
//!
|
|
//! See GET /help for endpoint documentation.
|
|
|
|
const std = @import("std");
|
|
const zfin = @import("zfin");
|
|
const httpz = @import("httpz");
|
|
const build_options = @import("build_options");
|
|
|
|
const version = build_options.version;
|
|
const log = std.log.scoped(.@"zfin-server");
|
|
|
|
// ── App ──────────────────────────────────────────────────────
|
|
|
|
const App = struct {
|
|
io: std.Io,
|
|
environ: *const std.process.Environ.Map,
|
|
allocator: std.mem.Allocator,
|
|
config: zfin.Config,
|
|
svc: zfin.DataService,
|
|
/// Threshold in milliseconds above which a request is logged
|
|
/// as slow. Tunable via `ZFIN_SERVER_SLOW_MS` env var; defaults
|
|
/// to 500ms. Captured once at App.init so the dispatch hot path
|
|
/// doesn't re-parse on every request.
|
|
slow_threshold_ms: u64,
|
|
|
|
fn init(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) App {
|
|
const config = zfin.Config.fromEnv(io, allocator, environ);
|
|
const svc = zfin.DataService.init(io, allocator, config);
|
|
const slow_threshold_ms = if (environ.get("ZFIN_SERVER_SLOW_MS")) |s|
|
|
std.fmt.parseInt(u64, s, 10) catch 500
|
|
else
|
|
500;
|
|
return .{
|
|
.io = io,
|
|
.environ = environ,
|
|
.allocator = allocator,
|
|
.config = config,
|
|
.svc = svc,
|
|
.slow_threshold_ms = slow_threshold_ms,
|
|
};
|
|
}
|
|
|
|
fn deinit(self: *App) void {
|
|
self.svc.deinit();
|
|
self.config.deinit();
|
|
}
|
|
|
|
/// httpz dispatch hook: every request flows through here so we
|
|
/// have a single place to wrap timing and error logging without
|
|
/// modifying every handler. Slow requests (above
|
|
/// `slow_threshold_ms`) and error responses (status >= 400)
|
|
/// emit a structured stderr line; everything else stays silent.
|
|
pub fn dispatch(self: *App, action: httpz.Action(*App), req: *httpz.Request, res: *httpz.Response) !void {
|
|
// wall-clock required: per-request elapsed for slow-request
|
|
// logging. `.awake` (monotonic) avoids spurious negatives
|
|
// on system clock skew.
|
|
const start_ns = std.Io.Timestamp.now(self.io, .awake).nanoseconds;
|
|
try action(self, req, res);
|
|
|
|
// using defer here so we execute unconditionally
|
|
defer {
|
|
const elapsed_ns = std.Io.Timestamp.now(self.io, .awake).nanoseconds - start_ns;
|
|
const elapsed_ms: u64 = @intCast(@divTrunc(elapsed_ns, std.time.ns_per_ms));
|
|
if (shouldLogRequest(elapsed_ms, res.status, self.slow_threshold_ms)) {
|
|
// wall-clock required: ts in stderr line lets the
|
|
// operator correlate slow requests with cron / system
|
|
// events using `date -d @<ts>`.
|
|
const ts = std.Io.Timestamp.now(self.io, .real).toSeconds();
|
|
log.warn("ts={d} elapsed_ms={d} status={d} method={s} path={s}", .{
|
|
ts,
|
|
elapsed_ms,
|
|
res.status,
|
|
@tagName(req.method),
|
|
req.url.path,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
};
|
|
|
|
/// Pure predicate: should this request emit a stderr log line?
|
|
/// Logs slow successes (above threshold) and any error response.
|
|
fn shouldLogRequest(elapsed_ms: u64, status: u16, threshold_ms: u64) bool {
|
|
return elapsed_ms > threshold_ms or status >= 400;
|
|
}
|
|
|
|
// ── Route handlers ───────────────────────────────────────────
|
|
|
|
fn handleIndex(_: *App, _: *httpz.Request, res: *httpz.Response) !void {
|
|
res.content_type = httpz.ContentType.HTML;
|
|
res.body =
|
|
\\<!DOCTYPE html>
|
|
\\<html><head><title>zfin-server</title></head>
|
|
\\<body>
|
|
\\<h1>zfin-server</h1>
|
|
\\<p>This is a financial data API server. Not intended for browser use.</p>
|
|
\\<p>See <a href="/help">/help</a> for endpoint documentation.</p>
|
|
\\</body></html>
|
|
;
|
|
}
|
|
|
|
fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void {
|
|
res.content_type = httpz.ContentType.TEXT;
|
|
res.body = "zfin-server " ++ version ++ " - financial data API" ++
|
|
\\
|
|
\\
|
|
\\Endpoints:
|
|
\\ GET /{SYMBOL}/returns Trailing 1/3/5/10yr returns (JSON)
|
|
\\ GET /{SYMBOL}/returns?fmt=xml Trailing returns (XML, for LibreCalc)
|
|
\\ GET /{SYMBOL}/quote Latest quote (JSON)
|
|
\\ GET /{SYMBOL}/candles Raw SRF cache file
|
|
\\ GET /{SYMBOL}/candles_meta Candle freshness metadata (SRF)
|
|
\\ GET /{SYMBOL}/dividends Raw SRF cache file
|
|
\\ GET /{SYMBOL}/splits Raw SRF cache file
|
|
\\ GET /{SYMBOL}/earnings Raw SRF cache file
|
|
\\ GET /{SYMBOL}/options Raw SRF cache file
|
|
\\ GET /{SYMBOL}/classification Wikidata classification (SRF)
|
|
\\ GET /{SYMBOL}/etf_metrics EDGAR NPORT-P fund metrics (SRF; 404 for non-funds)
|
|
\\ GET /{CIK}/entity_facts EDGAR XBRL entity facts (SRF; CIK-keyed)
|
|
\\ GET /_edgar/tickers_funds EDGAR mutual-fund ticker map (SRF)
|
|
\\ GET /_edgar/tickers_companies EDGAR company ticker map (SRF)
|
|
\\ GET /symbols List of tracked symbols
|
|
\\
|
|
\\Returns fields:
|
|
\\ lastClose Last closing price
|
|
\\ trailing{1,3,5,10}YearReturn Total return with dividend reinvestment
|
|
\\ price{1,3,5,10}YearReturn Price-only return (from adjusted close)
|
|
\\ volatility Longest-term available annualized volatility
|
|
\\ volatilityTerm Period (years) of the volatility field
|
|
\\ volatility{1,3,5,10}Year Per-period annualized volatility
|
|
\\
|
|
\\XML example (LibreCalc):
|
|
\\ =FILTERXML(WEBSERVICE("http://host/AAPL/returns?fmt=xml"),"//total10YearReturn")
|
|
\\
|
|
;
|
|
}
|
|
|
|
fn handleSymbols(app: *App, _: *httpz.Request, res: *httpz.Response) !void {
|
|
const arena = res.arena;
|
|
const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf";
|
|
|
|
const file_data = std.Io.Dir.cwd().readFileAlloc(app.io, portfolio_path, arena, .limited(10 * 1024 * 1024)) catch {
|
|
res.content_type = httpz.ContentType.JSON;
|
|
res.body = "[]";
|
|
return;
|
|
};
|
|
|
|
var portfolio = zfin.cache.deserializePortfolio(arena, file_data) catch {
|
|
res.content_type = httpz.ContentType.JSON;
|
|
res.body = "[]";
|
|
return;
|
|
};
|
|
defer portfolio.deinit();
|
|
|
|
// Collect unique symbols
|
|
var seen = std.StringHashMap(void).init(arena);
|
|
var symbols = std.ArrayList([]const u8).empty;
|
|
for (portfolio.lots) |lot| {
|
|
if (lot.symbol.len == 0) continue;
|
|
if (seen.contains(lot.symbol)) continue;
|
|
try seen.put(lot.symbol, {});
|
|
try symbols.append(arena, lot.symbol);
|
|
}
|
|
|
|
// Build JSON array
|
|
var aw: std.Io.Writer.Allocating = .init(arena);
|
|
try aw.writer.writeByte('[');
|
|
for (symbols.items, 0..) |sym, i| {
|
|
if (i > 0) try aw.writer.writeByte(',');
|
|
try aw.writer.print("\"{s}\"", .{sym});
|
|
}
|
|
try aw.writer.writeByte(']');
|
|
|
|
res.content_type = httpz.ContentType.JSON;
|
|
res.body = try aw.toOwnedSlice();
|
|
}
|
|
|
|
fn handleReturns(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
const raw_symbol = req.param("symbol") orelse {
|
|
res.status = 404;
|
|
res.body = "Missing symbol";
|
|
return;
|
|
};
|
|
const arena = res.arena;
|
|
const symbol = try upperDupe(arena, raw_symbol);
|
|
|
|
// Auto-add to watchlist if requested
|
|
const q = try req.query();
|
|
if (q.get("watch")) |w| {
|
|
if (std.ascii.eqlIgnoreCase(w, "true")) {
|
|
appendWatchSymbol(app, symbol) catch |err| {
|
|
log.warn("failed to append watch symbol {s}: {}", .{ symbol, err });
|
|
};
|
|
}
|
|
}
|
|
|
|
const result = app.svc.getTrailingReturns(symbol, .{}) catch {
|
|
res.status = 404;
|
|
res.body = "Symbol not found or fetch failed";
|
|
return;
|
|
};
|
|
defer app.allocator.free(result.candles);
|
|
if (result.dividends) |divs| {
|
|
defer zfin.Dividend.freeSlice(app.allocator, divs);
|
|
}
|
|
|
|
const candles = result.candles;
|
|
if (candles.len == 0) {
|
|
res.status = 404;
|
|
res.body = "No candle data";
|
|
return;
|
|
}
|
|
|
|
const last_close = candles[candles.len - 1].close;
|
|
var date_buf: [10]u8 = undefined;
|
|
const date_str = try std.fmt.bufPrint(&date_buf, "{f}", .{candles[candles.len - 1].date});
|
|
|
|
// Price-only returns (split-adjusted, NOT dividend-adjusted —
|
|
// see analytics/performance.zig:trailingReturnsPriceOnly).
|
|
// Matches the "price return" numbers public sources publish
|
|
// (Yahoo chart-bar, FMP, Barchart, Fidelity stock pages).
|
|
const p1y = if (result.asof_price.one_year) |r| r.annualized_return else null;
|
|
const p3y = if (result.asof_price.three_year) |r| r.annualized_return else null;
|
|
const p5y = if (result.asof_price.five_year) |r| r.annualized_return else null;
|
|
const p10y = if (result.asof_price.ten_year) |r| r.annualized_return else null;
|
|
|
|
// Total returns (dividend reinvestment when dividends are
|
|
// available; falls back to adj_close-based total return when
|
|
// dividend records are missing). Matches Morningstar
|
|
// "Trailing Returns" / Yahoo "Performance Overview" / Koyfin
|
|
// "Total Return".
|
|
const total = result.asof_total orelse result.asof_price;
|
|
const t1y = if (total.one_year) |r| r.annualized_return else null;
|
|
const t3y = if (total.three_year) |r| r.annualized_return else null;
|
|
const t5y = if (total.five_year) |r| r.annualized_return else null;
|
|
const t10y = if (total.ten_year) |r| r.annualized_return else null;
|
|
|
|
// Per-period volatility
|
|
const risk = zfin.risk.trailingRisk(candles);
|
|
const v1y = if (risk.one_year) |r| r.volatility else null;
|
|
const v3y = if (risk.three_year) |r| r.volatility else null;
|
|
const v5y = if (risk.five_year) |r| r.volatility else null;
|
|
const v10y = if (risk.ten_year) |r| r.volatility else null;
|
|
|
|
// Longest-term volatility convenience fields
|
|
const vol_best = v10y orelse v5y orelse v3y orelse v1y;
|
|
const vol_term: ?u8 = if (v10y != null) 10 else if (v5y != null) 5 else if (v3y != null) 3 else if (v1y != null) 1 else null;
|
|
|
|
// Check if XML requested
|
|
if (q.get("fmt")) |fmt| {
|
|
if (std.ascii.eqlIgnoreCase(fmt, "xml")) {
|
|
res.content_type = httpz.ContentType.XML;
|
|
res.body = try std.fmt.allocPrint(arena,
|
|
\\<returns>
|
|
\\ <ticker>{s}</ticker>
|
|
\\ <returnDate>{s}</returnDate>
|
|
\\ <lastClose>{d:.2}</lastClose>
|
|
\\ <trailing1YearReturn>{s}</trailing1YearReturn>
|
|
\\ <trailing3YearReturn>{s}</trailing3YearReturn>
|
|
\\ <trailing5YearReturn>{s}</trailing5YearReturn>
|
|
\\ <trailing10YearReturn>{s}</trailing10YearReturn>
|
|
\\ <price1YearReturn>{s}</price1YearReturn>
|
|
\\ <price3YearReturn>{s}</price3YearReturn>
|
|
\\ <price5YearReturn>{s}</price5YearReturn>
|
|
\\ <price10YearReturn>{s}</price10YearReturn>
|
|
\\ <volatility>{s}</volatility>
|
|
\\ <volatilityTerm>{s}</volatilityTerm>
|
|
\\ <volatility1Year>{s}</volatility1Year>
|
|
\\ <volatility3Year>{s}</volatility3Year>
|
|
\\ <volatility5Year>{s}</volatility5Year>
|
|
\\ <volatility10Year>{s}</volatility10Year>
|
|
\\</returns>
|
|
\\
|
|
, .{
|
|
symbol,
|
|
date_str,
|
|
last_close,
|
|
fmtPct(arena, t1y),
|
|
fmtPct(arena, t3y),
|
|
fmtPct(arena, t5y),
|
|
fmtPct(arena, t10y),
|
|
fmtPct(arena, p1y),
|
|
fmtPct(arena, p3y),
|
|
fmtPct(arena, p5y),
|
|
fmtPct(arena, p10y),
|
|
fmtPct(arena, vol_best),
|
|
fmtInt(arena, vol_term),
|
|
fmtPct(arena, v1y),
|
|
fmtPct(arena, v3y),
|
|
fmtPct(arena, v5y),
|
|
fmtPct(arena, v10y),
|
|
});
|
|
return;
|
|
}
|
|
}
|
|
|
|
res.content_type = httpz.ContentType.JSON;
|
|
res.body = try std.fmt.allocPrint(arena,
|
|
\\{{"ticker":"{s}","returnDate":"{s}","lastClose":{d:.2},"trailing1YearReturn":{s},"trailing3YearReturn":{s},"trailing5YearReturn":{s},"trailing10YearReturn":{s},"price1YearReturn":{s},"price3YearReturn":{s},"price5YearReturn":{s},"price10YearReturn":{s},"volatility":{s},"volatilityTerm":{s},"volatility1Year":{s},"volatility3Year":{s},"volatility5Year":{s},"volatility10Year":{s}}}
|
|
, .{
|
|
symbol,
|
|
date_str,
|
|
last_close,
|
|
fmtPct(arena, t1y),
|
|
fmtPct(arena, t3y),
|
|
fmtPct(arena, t5y),
|
|
fmtPct(arena, t10y),
|
|
fmtPct(arena, p1y),
|
|
fmtPct(arena, p3y),
|
|
fmtPct(arena, p5y),
|
|
fmtPct(arena, p10y),
|
|
fmtPct(arena, vol_best),
|
|
fmtInt(arena, vol_term),
|
|
fmtPct(arena, v1y),
|
|
fmtPct(arena, v3y),
|
|
fmtPct(arena, v5y),
|
|
fmtPct(arena, v10y),
|
|
});
|
|
}
|
|
|
|
fn handleQuote(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
const raw_symbol = req.param("symbol") orelse {
|
|
res.status = 400;
|
|
res.body = "Missing symbol";
|
|
return;
|
|
};
|
|
const arena = res.arena;
|
|
const symbol = try upperDupe(arena, raw_symbol);
|
|
|
|
const q = app.svc.getQuote(symbol, .{}) catch {
|
|
res.status = 404;
|
|
res.body = "Quote not available";
|
|
return;
|
|
};
|
|
|
|
res.content_type = httpz.ContentType.JSON;
|
|
res.body = try std.fmt.allocPrint(arena,
|
|
\\{{"symbol":"{s}","close":{d:.2},"open":{d:.2},"high":{d:.2},"low":{d:.2},"volume":{d},"previous_close":{d:.2}}}
|
|
, .{ symbol, q.close, q.open, q.high, q.low, q.volume, q.previous_close });
|
|
}
|
|
|
|
fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8) !void {
|
|
return handleSrfFileByKey(app, req, res, "symbol", filename);
|
|
}
|
|
|
|
/// Generalized SRF cache-file passthrough: reads
|
|
/// `<cache_dir>/<key>/<filename>` where `<key>` is whatever URL
|
|
/// parameter `key_param` resolves to. The default `handleSrfFile`
|
|
/// uses `"symbol"`; CIK-keyed routes (e.g. `/:cik/entity_facts`)
|
|
/// pass `"cik"` instead. The cache-key segment is uppercased
|
|
/// (safe for both symbols and zero-padded CIK digit strings).
|
|
fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8) !void {
|
|
const raw_key = req.param(key_param) orelse {
|
|
res.status = 400;
|
|
res.body = "Missing key";
|
|
return;
|
|
};
|
|
const arena = res.arena;
|
|
const key = try upperDupe(arena, raw_key);
|
|
return serveSrfFile(app, res, key, filename);
|
|
}
|
|
|
|
/// Static-key SRF cache-file passthrough for routes that don't
|
|
/// take a path parameter (e.g. `/_edgar/tickers_funds` reads
|
|
/// `<cache_dir>/_edgar/tickers_funds.srf` directly). The `key`
|
|
/// is a literal directory name; not uppercased because the
|
|
/// cache uses `_edgar` as-is.
|
|
fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8) !void {
|
|
return serveSrfFile(app, res, key, filename);
|
|
}
|
|
|
|
/// Inner shared helper. Reads the file, computes etag, sets
|
|
/// headers, sends. Caller has already resolved the cache-key
|
|
/// segment (per-request param or static literal).
|
|
fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8) !void {
|
|
const arena = res.arena;
|
|
const path = try std.fs.path.join(arena, &.{ app.config.cache_dir, key, filename });
|
|
const content = std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch {
|
|
res.status = 404;
|
|
res.body = "Cache file not found";
|
|
return;
|
|
};
|
|
|
|
// Body integrity header: sha256 of the bytes we're about to send.
|
|
// Clients can use this to detect mid-stream truncation that Zig's
|
|
// std.http.Client.fetch silently accepts on the Content-Length path
|
|
// (a premature EOF from the transport bubbles up as EndOfStream and
|
|
// is swallowed as a normal end-of-body). Shaped as a standard
|
|
// `ETag` value so future conditional-request work gets it for free.
|
|
var hash: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined;
|
|
std.crypto.hash.sha2.Sha256.hash(content, &hash, .{});
|
|
var etag_buf: [std.crypto.hash.sha2.Sha256.digest_length * 2 + "\"sha256:\"".len]u8 = undefined;
|
|
const etag = try std.fmt.bufPrint(&etag_buf, "\"sha256:{x}\"", .{&hash});
|
|
// httpz.Response.header borrows the value — duplicate into the
|
|
// per-request arena so the slice outlives `etag_buf`.
|
|
const etag_owned = try arena.dupe(u8, etag);
|
|
|
|
res.content_type = httpz.ContentType.BINARY;
|
|
res.header("content-type", "application/x-srf");
|
|
res.header("etag", etag_owned);
|
|
res.body = content;
|
|
}
|
|
|
|
fn handleCandles(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleSrfFile(app, req, res, "candles_daily.srf");
|
|
}
|
|
|
|
fn handleCandlesMeta(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleSrfFile(app, req, res, "candles_meta.srf");
|
|
}
|
|
|
|
fn handleDividends(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleSrfFile(app, req, res, "dividends.srf");
|
|
}
|
|
|
|
fn handleSplits(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleSrfFile(app, req, res, "splits.srf");
|
|
}
|
|
|
|
fn handleEarnings(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleSrfFile(app, req, res, "earnings.srf");
|
|
}
|
|
|
|
fn handleOptions(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleSrfFile(app, req, res, "options.srf");
|
|
}
|
|
|
|
fn handleClassification(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleSrfFile(app, req, res, "classification.srf");
|
|
}
|
|
|
|
fn handleEtfMetrics(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleSrfFile(app, req, res, "etf_metrics.srf");
|
|
}
|
|
|
|
fn handleEntityFacts(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
|
|
// CIK-keyed route: cache layout is
|
|
// `<cache_dir>/<CIK>/entity_facts.srf` (the CIK is the
|
|
// zero-padded 10-digit string Wikidata's P5531 emits).
|
|
return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf");
|
|
}
|
|
|
|
fn handleTickersFunds(app: *App, _: *httpz.Request, res: *httpz.Response) !void {
|
|
// Static-key route: `<cache_dir>/_edgar/tickers_funds.srf`
|
|
// is a single file shared across all symbol lookups, not a
|
|
// per-symbol cache.
|
|
return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf");
|
|
}
|
|
|
|
fn handleTickersCompanies(app: *App, _: *httpz.Request, res: *httpz.Response) !void {
|
|
return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf");
|
|
}
|
|
|
|
// ── Helpers ──────────────────────────────────────────────────
|
|
|
|
fn upperDupe(allocator: std.mem.Allocator, s: []const u8) ![]u8 {
|
|
const d = try allocator.dupe(u8, s);
|
|
for (d) |*c| c.* = std.ascii.toUpper(c.*);
|
|
return d;
|
|
}
|
|
|
|
fn printRateLimitWait(svc: *zfin.DataService, data_type: zfin.cache.DataType, stdout: *std.Io.Writer) !void {
|
|
if (svc.estimateWaitSeconds(data_type)) |wait| {
|
|
if (wait > 0) {
|
|
try stdout.print("\n (rate limit -- waiting {d}s)\n ", .{wait});
|
|
try stdout.flush();
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Format as percentage (e.g., 0.1234 -> "12.34000"), or "null" if absent.
|
|
fn fmtPct(arena: std.mem.Allocator, value: ?f64) []const u8 {
|
|
if (value) |v| return std.fmt.allocPrint(arena, "{d:.5}", .{v * 100.0}) catch "null";
|
|
return "null";
|
|
}
|
|
|
|
/// Format an optional integer, or "null" if absent.
|
|
fn fmtInt(arena: std.mem.Allocator, value: ?u8) []const u8 {
|
|
if (value) |v| return std.fmt.allocPrint(arena, "{d}", .{v}) catch "null";
|
|
return "null";
|
|
}
|
|
|
|
/// Append a watch lot for the given symbol to the portfolio SRF file,
|
|
/// unless it already exists. Best-effort — errors are logged, not fatal.
|
|
fn appendWatchSymbol(app: *App, symbol: []const u8) !void {
|
|
const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf";
|
|
const allocator = app.allocator;
|
|
const io = app.io;
|
|
|
|
// Read and deserialize existing portfolio (or start empty)
|
|
const file_data = std.Io.Dir.cwd().readFileAlloc(io, portfolio_path, allocator, .limited(10 * 1024 * 1024)) catch |err| {
|
|
if (err == error.FileNotFound) return writeNewPortfolio(io, allocator, portfolio_path, symbol);
|
|
return err;
|
|
};
|
|
defer allocator.free(file_data);
|
|
|
|
var portfolio = zfin.cache.deserializePortfolio(allocator, file_data) catch return;
|
|
defer portfolio.deinit();
|
|
|
|
// Check if symbol already tracked
|
|
for (portfolio.lots) |lot| {
|
|
if (std.ascii.eqlIgnoreCase(lot.symbol, symbol)) return;
|
|
}
|
|
|
|
// Build new lot list with the watch entry appended
|
|
var new_lots = try allocator.alloc(zfin.Lot, portfolio.lots.len + 1);
|
|
defer allocator.free(new_lots);
|
|
@memcpy(new_lots[0..portfolio.lots.len], portfolio.lots);
|
|
new_lots[portfolio.lots.len] = .{
|
|
.symbol = symbol,
|
|
.shares = 0,
|
|
.open_date = zfin.Date.fromYmd(2026, 1, 1),
|
|
.open_price = 0,
|
|
.security_type = .watch,
|
|
};
|
|
|
|
// Serialize and write
|
|
const output = try zfin.cache.serializePortfolio(allocator, new_lots);
|
|
defer allocator.free(output);
|
|
|
|
const file = try std.Io.Dir.cwd().createFile(io, portfolio_path, .{});
|
|
defer file.close(io);
|
|
var write_buf: [4096]u8 = undefined;
|
|
var fw = file.writer(io, &write_buf);
|
|
try fw.interface.writeAll(output);
|
|
try fw.interface.flush();
|
|
|
|
log.info("added watch symbol {s} to {s}", .{ symbol, portfolio_path });
|
|
}
|
|
|
|
fn writeNewPortfolio(io: std.Io, allocator: std.mem.Allocator, path: []const u8, symbol: []const u8) !void {
|
|
const lot = [_]zfin.Lot{.{
|
|
.symbol = symbol,
|
|
.shares = 0,
|
|
.open_date = zfin.Date.fromYmd(2026, 1, 1),
|
|
.open_price = 0,
|
|
.security_type = .watch,
|
|
}};
|
|
const output = try zfin.cache.serializePortfolio(allocator, &lot);
|
|
defer allocator.free(output);
|
|
|
|
const file = try std.Io.Dir.cwd().createFile(io, path, .{});
|
|
defer file.close(io);
|
|
var write_buf: [4096]u8 = undefined;
|
|
var fw = file.writer(io, &write_buf);
|
|
try fw.interface.writeAll(output);
|
|
try fw.interface.flush();
|
|
|
|
log.info("created {s} with watch symbol {s}", .{ path, symbol });
|
|
}
|
|
|
|
// ── Refresh command ──────────────────────────────────────────
|
|
|
|
fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) !u8 {
|
|
var config = zfin.Config.fromEnv(io, allocator, environ);
|
|
defer config.deinit();
|
|
var svc = zfin.DataService.init(io, allocator, config);
|
|
defer svc.deinit();
|
|
|
|
// wall-clock required: the provider-lag check compares each symbol's
|
|
// newest cached bar against the most recent session the market should
|
|
// already have data for (see zfin.market.candleFreshness). Captured
|
|
// once so the whole run shares a consistent "now".
|
|
const now_s = std.Io.Timestamp.now(io, .real).toSeconds();
|
|
|
|
const portfolio_path = environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf";
|
|
|
|
const data = std.Io.Dir.cwd().readFileAlloc(io, portfolio_path, allocator, .limited(10 * 1024 * 1024)) catch {
|
|
log.err("failed to read portfolio: {s}", .{portfolio_path});
|
|
return error.ReadFailed;
|
|
};
|
|
defer allocator.free(data);
|
|
|
|
var portfolio = zfin.cache.deserializePortfolio(allocator, data) catch {
|
|
log.err("failed to parse portfolio", .{});
|
|
return error.ParseFailed;
|
|
};
|
|
defer portfolio.deinit();
|
|
|
|
var symbols = std.StringHashMap(void).init(allocator);
|
|
defer symbols.deinit();
|
|
for (portfolio.lots) |lot| {
|
|
if (lot.security_type != .stock and lot.security_type != .watch) continue;
|
|
if (lot.symbol.len == 0) continue;
|
|
const sym = lot.priceSymbol();
|
|
if (!symbols.contains(sym)) {
|
|
try symbols.put(sym, {});
|
|
}
|
|
}
|
|
|
|
const stdout_file = std.Io.File.stdout();
|
|
var buf: [4096]u8 = undefined;
|
|
var writer = stdout_file.writer(io, &buf);
|
|
const stdout = &writer.interface;
|
|
|
|
try stdout.print("zfin-server {s}\n", .{version});
|
|
try stdout.print("Refreshing {d} symbols from {s}\n", .{ symbols.count(), portfolio_path });
|
|
try stdout.flush();
|
|
|
|
var success_count: u32 = 0;
|
|
var fail_count: u32 = 0;
|
|
var lag_count: u32 = 0;
|
|
|
|
// Warm the EDGAR ticker maps once per refresh run. They're
|
|
// ~3-5 MB each, cached for 30 days; warming guarantees the
|
|
// shared `<cache>/_edgar/tickers_funds.srf` and
|
|
// `tickers_companies.srf` files exist for the static-route
|
|
// handlers to serve. Per-symbol `getEtfMetrics` calls below
|
|
// also rely on these maps being loaded.
|
|
{
|
|
try printRateLimitWait(&svc, .tickers_funds, stdout);
|
|
if (svc.loadMutualFundTickerMap(.{})) |mut_map| {
|
|
var m = mut_map;
|
|
m.deinit();
|
|
try stdout.print("EDGAR mutual-fund ticker map ok\n", .{});
|
|
} else |err| {
|
|
try stdout.print("EDGAR mutual-fund ticker map FAILED ({t})\n", .{err});
|
|
}
|
|
try printRateLimitWait(&svc, .tickers_companies, stdout);
|
|
if (svc.loadCompanyTickerMap(.{})) |co_map| {
|
|
var m = co_map;
|
|
m.deinit();
|
|
try stdout.print("EDGAR company ticker map ok\n", .{});
|
|
} else |err| {
|
|
try stdout.print("EDGAR company ticker map FAILED ({t})\n", .{err});
|
|
}
|
|
try stdout.flush();
|
|
}
|
|
|
|
var it = symbols.iterator();
|
|
while (it.next()) |entry| {
|
|
const sym = entry.key_ptr.*;
|
|
try stdout.print("{s}: ", .{sym});
|
|
try stdout.flush();
|
|
|
|
var sym_ok = true;
|
|
|
|
// Candles
|
|
try printRateLimitWait(&svc, .candles_daily, stdout);
|
|
if (svc.getCandles(sym, .{})) |result| {
|
|
defer result.deinit();
|
|
try stdout.print("candles ok ({s})", .{@tagName(result.source)});
|
|
|
|
// Provider-data-lag check: did we end up with the latest bar
|
|
// the market should have posted by now? A `.lagging` bar is
|
|
// merely unposted -> flag it so the run exits EX_TEMPFAIL and
|
|
// cron retries. An `.overdue` bar is almost certainly an
|
|
// un-modeled closure (e.g. Good Friday) -> note it, no retry.
|
|
if (result.data.len > 0) {
|
|
const last = result.data[result.data.len - 1].date;
|
|
var date_buf: [10]u8 = undefined;
|
|
const ds = std.fmt.bufPrint(&date_buf, "{f}", .{last}) catch "?";
|
|
switch (zfin.market.candleFreshness(now_s, zfin.market.classify(sym), last)) {
|
|
.lagging => {
|
|
lag_count += 1;
|
|
try stdout.print(" LAGGING (latest {s})", .{ds});
|
|
log.info("provider data lag: {s} latest bar {s}; newer session due but unposted", .{ sym, ds });
|
|
},
|
|
.overdue => log.info("{s}: latest bar {s} overdue past grace window; assuming market closure (no retry)", .{ sym, ds }),
|
|
.current => {},
|
|
}
|
|
}
|
|
} else |err| {
|
|
try stdout.print("candles FAILED ({s})", .{@errorName(err)});
|
|
sym_ok = false;
|
|
if (err == zfin.DataError.TransientError or err == zfin.DataError.AuthError) {
|
|
const reason = if (err == zfin.DataError.AuthError) "auth failure" else "transient provider failure";
|
|
try stdout.print("\n", .{});
|
|
try stdout.print("\nStopping refresh: {s}\n", .{reason});
|
|
try stdout.print("Refresh aborted: {d} ok, {d} failed\n", .{ success_count, fail_count + 1 });
|
|
try stdout.flush();
|
|
return error.RefreshFailed;
|
|
}
|
|
}
|
|
|
|
// Dividends
|
|
try printRateLimitWait(&svc, .dividends, stdout);
|
|
if (svc.getDividends(sym, .{})) |result| {
|
|
defer result.deinit();
|
|
try stdout.print(", dividends ok ({s})", .{@tagName(result.source)});
|
|
} else |err| {
|
|
try stdout.print(", dividends FAILED ({s})", .{@errorName(err)});
|
|
sym_ok = false;
|
|
}
|
|
|
|
// Splits
|
|
try printRateLimitWait(&svc, .splits, stdout);
|
|
if (svc.getSplits(sym, .{})) |result| {
|
|
defer result.deinit();
|
|
try stdout.print(", splits ok ({s})", .{@tagName(result.source)});
|
|
} else |err| {
|
|
try stdout.print(", splits FAILED ({s})", .{@errorName(err)});
|
|
sym_ok = false;
|
|
}
|
|
|
|
// Earnings
|
|
try printRateLimitWait(&svc, .earnings, stdout);
|
|
if (svc.getEarnings(sym, .{})) |result| {
|
|
defer result.deinit();
|
|
try stdout.print(", earnings ok ({s})", .{@tagName(result.source)});
|
|
} else |err| {
|
|
try stdout.print(", earnings FAILED ({s})", .{@errorName(err)});
|
|
sym_ok = false;
|
|
}
|
|
|
|
// Classification (Wikidata + EDGAR fallback). Captures
|
|
// CIK and is_etf — used to chain into entity_facts below.
|
|
// NotFound is logged as `n/a` (symbol genuinely has no
|
|
// Wikidata or EDGAR entry) and doesn't flip sym_ok.
|
|
var cik_buf: ?[]u8 = null;
|
|
defer if (cik_buf) |b| allocator.free(b);
|
|
var is_etf = false;
|
|
try printRateLimitWait(&svc, .classification, stdout);
|
|
if (svc.getClassification(sym, .{})) |result| {
|
|
defer result.deinit();
|
|
if (result.data.len > 0) {
|
|
if (result.data[0].cik) |cik| {
|
|
cik_buf = allocator.dupe(u8, cik) catch null;
|
|
}
|
|
is_etf = result.data[0].is_etf;
|
|
}
|
|
try stdout.print(", classification ok ({s})", .{@tagName(result.source)});
|
|
} else |err| switch (err) {
|
|
zfin.DataError.NotFound => try stdout.print(", classification n/a", .{}),
|
|
else => {
|
|
try stdout.print(", classification FAILED ({t})", .{err});
|
|
sym_ok = false;
|
|
},
|
|
}
|
|
|
|
// ETF metrics. NotFound is the expected outcome for
|
|
// non-funds (NPORT-P only exists for funds + UITs); a
|
|
// negative-cache entry suppresses retries. Logged as
|
|
// `n/a` and doesn't flip sym_ok.
|
|
try printRateLimitWait(&svc, .etf_metrics, stdout);
|
|
if (svc.getEtfMetrics(sym, .{})) |result| {
|
|
defer result.deinit();
|
|
try stdout.print(", etf_metrics ok ({s})", .{@tagName(result.source)});
|
|
} else |err| switch (err) {
|
|
zfin.DataError.NotFound => try stdout.print(", etf_metrics n/a", .{}),
|
|
else => {
|
|
try stdout.print(", etf_metrics FAILED ({t})", .{err});
|
|
sym_ok = false;
|
|
},
|
|
}
|
|
|
|
// Entity facts (XBRL). Only attempted when the
|
|
// classification step yielded a CIK from a non-fund
|
|
// record. ETFs/funds CIKs (iShares Trust, Fidelity series
|
|
// CIKs, etc.) don't file the operating-company XBRL
|
|
// concepts entity_facts looks for; calling EDGAR for
|
|
// them is guaranteed-404 noise. Skip them up front.
|
|
if (cik_buf) |cik| {
|
|
if (is_etf) {
|
|
try stdout.print(", entity_facts n/a (ETF)", .{});
|
|
} else {
|
|
try printRateLimitWait(&svc, .entity_facts, stdout);
|
|
if (svc.getEntityFacts(cik, .{})) |result| {
|
|
defer result.deinit();
|
|
try stdout.print(", entity_facts ok ({s})", .{@tagName(result.source)});
|
|
} else |err| switch (err) {
|
|
zfin.DataError.NotFound => try stdout.print(", entity_facts n/a", .{}),
|
|
else => {
|
|
try stdout.print(", entity_facts FAILED ({t})", .{err});
|
|
sym_ok = false;
|
|
},
|
|
}
|
|
}
|
|
}
|
|
|
|
try stdout.print("\n", .{});
|
|
try stdout.flush();
|
|
|
|
if (sym_ok) success_count += 1 else fail_count += 1;
|
|
}
|
|
|
|
try stdout.print("\nRefresh complete: {d} ok, {d} failed, {d} lagging\n", .{ success_count, fail_count, lag_count });
|
|
try stdout.flush();
|
|
|
|
return refreshExit(fail_count, lag_count);
|
|
}
|
|
|
|
/// Map a refresh run's failure/lag counts to a process exit code:
|
|
/// 0 - every symbol current and fetched cleanly
|
|
/// 75 - EX_TEMPFAIL: no hard failures, but at least one symbol's
|
|
/// just-closed bar hadn't posted yet (provider lag); cron should
|
|
/// retry shortly
|
|
/// 1 - at least one hard failure (fetch error)
|
|
/// Hard failure dominates lag - if anything failed outright that's the
|
|
/// code the operator needs to act on.
|
|
fn refreshExit(fail_count: u32, lag_count: u32) u8 {
|
|
if (fail_count > 0) return 1;
|
|
if (lag_count > 0) return 75;
|
|
return 0;
|
|
}
|
|
|
|
// ── Main ─────────────────────────────────────────────────────
|
|
|
|
pub fn main(init: std.process.Init) !u8 {
|
|
const allocator = init.gpa;
|
|
const io = init.io;
|
|
const environ = init.environ_map;
|
|
|
|
const args = try init.minimal.args.toSlice(allocator);
|
|
defer allocator.free(args);
|
|
|
|
if (args.len < 2) {
|
|
try printUsage(io);
|
|
return 1;
|
|
}
|
|
|
|
const command = args[1];
|
|
|
|
if (std.mem.eql(u8, command, "serve")) {
|
|
var port: u16 = 8080;
|
|
for (args[2..]) |arg| {
|
|
if (std.mem.startsWith(u8, arg, "--port=")) {
|
|
port = std.fmt.parseInt(u16, arg["--port=".len..], 10) catch 8080;
|
|
}
|
|
}
|
|
|
|
var app = App.init(io, allocator, environ);
|
|
defer app.deinit();
|
|
|
|
var server = try httpz.Server(*App).init(io, allocator, .{
|
|
.address = .all(port),
|
|
}, &app);
|
|
defer {
|
|
server.stop();
|
|
server.deinit();
|
|
}
|
|
|
|
var router = try server.router(.{});
|
|
|
|
// Static routes
|
|
router.get("/", handleIndex, .{});
|
|
router.get("/help", handleHelp, .{});
|
|
router.get("/symbols", handleSymbols, .{});
|
|
|
|
// Symbol routes
|
|
router.get("/:symbol/returns", handleReturns, .{});
|
|
router.get("/:symbol/quote", handleQuote, .{});
|
|
router.get("/:symbol/candles", handleCandles, .{});
|
|
router.get("/:symbol/candles_meta", handleCandlesMeta, .{});
|
|
router.get("/:symbol/dividends", handleDividends, .{});
|
|
router.get("/:symbol/splits", handleSplits, .{});
|
|
router.get("/:symbol/earnings", handleEarnings, .{});
|
|
router.get("/:symbol/options", handleOptions, .{});
|
|
|
|
// Wikidata + EDGAR derived data — populated by `refresh`.
|
|
router.get("/:symbol/classification", handleClassification, .{});
|
|
router.get("/:symbol/etf_metrics", handleEtfMetrics, .{});
|
|
router.get("/:cik/entity_facts", handleEntityFacts, .{});
|
|
|
|
// EDGAR shared ticker maps (~3-5 MB each, refreshed
|
|
// every 30 days). Static-key routes — single file
|
|
// shared across every symbol lookup.
|
|
router.get("/_edgar/tickers_funds", handleTickersFunds, .{});
|
|
router.get("/_edgar/tickers_companies", handleTickersCompanies, .{});
|
|
|
|
log.info("zfin-server {s}", .{version});
|
|
log.info("listening on port {d}", .{port});
|
|
try server.listen();
|
|
} else if (std.mem.eql(u8, command, "refresh")) {
|
|
return try refresh(io, init.arena.allocator(), environ);
|
|
} else {
|
|
try printUsage(io);
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
fn printUsage(io: std.Io) !void {
|
|
var buf: [2048]u8 = undefined;
|
|
var fw = std.Io.File.stderr().writer(io, &buf);
|
|
const w = &fw.interface;
|
|
try w.print("zfin-server {s}\n", .{version});
|
|
try w.writeAll(
|
|
\\Usage: zfin-server <command>
|
|
\\
|
|
\\Commands:
|
|
\\ serve [--port=8080] Start the HTTP server
|
|
\\ refresh Refresh cache for all tracked symbols
|
|
\\
|
|
\\Environment:
|
|
\\ ZFIN_PORTFOLIO Path to portfolio SRF file (default: portfolio.srf)
|
|
\\ TWELVEDATA_API_KEY TwelveData API key
|
|
\\ POLYGON_API_KEY Polygon API key
|
|
\\ FINNHUB_API_KEY Finnhub API key
|
|
\\ ALPHAVANTAGE_API_KEY Alpha Vantage API key
|
|
\\
|
|
\\refresh exit codes:
|
|
\\ 0 all tracked symbols current
|
|
\\ 75 provider data lag (a just-closed bar not yet posted) - retry soon
|
|
\\ 1 one or more hard failures
|
|
\\
|
|
);
|
|
try w.flush();
|
|
}
|
|
|
|
// ── Tests ────────────────────────────────────────────────────
|
|
|
|
test "fmtPct" {
|
|
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
|
|
defer arena_state.deinit();
|
|
const arena = arena_state.allocator();
|
|
|
|
try std.testing.expectEqualStrings("null", fmtPct(arena, null));
|
|
const result = fmtPct(arena, 0.1234);
|
|
try std.testing.expect(std.mem.startsWith(u8, result, "12.34"));
|
|
}
|
|
|
|
test "upperDupe" {
|
|
const result = try upperDupe(std.testing.allocator, "aapl");
|
|
defer std.testing.allocator.free(result);
|
|
try std.testing.expectEqualStrings("AAPL", result);
|
|
}
|
|
|
|
test "shouldLogRequest: fast 2xx is silent" {
|
|
try std.testing.expect(!shouldLogRequest(10, 200, 500));
|
|
try std.testing.expect(!shouldLogRequest(499, 200, 500));
|
|
try std.testing.expect(!shouldLogRequest(0, 204, 500));
|
|
}
|
|
|
|
test "shouldLogRequest: slow 2xx logs" {
|
|
try std.testing.expect(shouldLogRequest(501, 200, 500));
|
|
try std.testing.expect(shouldLogRequest(2000, 200, 500));
|
|
// Boundary: == threshold is NOT logged (strict >).
|
|
try std.testing.expect(!shouldLogRequest(500, 200, 500));
|
|
}
|
|
|
|
test "shouldLogRequest: any error response logs regardless of timing" {
|
|
try std.testing.expect(shouldLogRequest(1, 400, 500));
|
|
try std.testing.expect(shouldLogRequest(1, 404, 500));
|
|
try std.testing.expect(shouldLogRequest(1, 500, 500));
|
|
try std.testing.expect(shouldLogRequest(1, 503, 500));
|
|
// 3xx is not flagged as error.
|
|
try std.testing.expect(!shouldLogRequest(1, 301, 500));
|
|
try std.testing.expect(!shouldLogRequest(1, 304, 500));
|
|
}
|
|
|
|
test "shouldLogRequest: custom threshold respected" {
|
|
try std.testing.expect(!shouldLogRequest(50, 200, 100));
|
|
try std.testing.expect(shouldLogRequest(150, 200, 100));
|
|
// Higher threshold (e.g. user sets ZFIN_SERVER_SLOW_MS=2000).
|
|
try std.testing.expect(!shouldLogRequest(1500, 200, 2000));
|
|
try std.testing.expect(shouldLogRequest(2500, 200, 2000));
|
|
}
|
|
|
|
test "refreshExit: hard failure dominates, then lag, else clean" {
|
|
try std.testing.expectEqual(@as(u8, 0), refreshExit(0, 0));
|
|
try std.testing.expectEqual(@as(u8, 75), refreshExit(0, 3));
|
|
try std.testing.expectEqual(@as(u8, 1), refreshExit(2, 0));
|
|
// A hard failure outranks lag.
|
|
try std.testing.expectEqual(@as(u8, 1), refreshExit(1, 5));
|
|
}
|