From e9826c06edb5f2b31b5ed985ade9e89a6a442c3c Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Thu, 13 Aug 2026 10:20:22 -0700 Subject: [PATCH] split logic into multiple files --- src/App.zig | 239 +++++ src/handlers.zig | 940 ++++++++++++++++++ src/main.zig | 2404 +--------------------------------------------- src/refresh.zig | 1221 +++++++++++++++++++++++ 4 files changed, 2431 insertions(+), 2373 deletions(-) create mode 100644 src/App.zig create mode 100644 src/handlers.zig create mode 100644 src/refresh.zig diff --git a/src/App.zig b/src/App.zig new file mode 100644 index 0000000..c357b52 --- /dev/null +++ b/src/App.zig @@ -0,0 +1,239 @@ +//! The HTTP application: shared state, the request gate, and per-request +//! bookkeeping. +//! +//! Split from `main.zig` so the route handlers and the refresh command can each +//! live in their own module without a cycle: handlers need `App`, `main` needs +//! both, and this file needs neither. + +const std = @import("std"); +const zfin = @import("zfin"); +const httpz = @import("httpz"); + +const log = std.log.scoped(.@"zfin-server"); + +pub const App = struct { + io: std.Io, + environ: *const std.process.Environ.Map, + allocator: std.mem.Allocator, + config: zfin.Config, + svc: zfin.DataService, + /// Threshold in milliseconds above which a request is logged + /// as slow. Tunable via `ZFIN_SERVER_SLOW_MS` env var; defaults + /// to 500ms. Captured once at App.init so the dispatch hot path + /// doesn't re-parse on every request. + slow_threshold_ms: u64, + /// Optional shared API key. When set (via `ZFIN_SERVER_API_KEY`), + /// every endpoint except the public surface (`/`, `/help`, and + /// `/:symbol/returns`) requires a matching key, supplied either as + /// an `X-API-Key` header or an `api_key` query parameter. When null + /// (env var unset or empty) the server is fully open - this + /// soft cutover lets the key roll out to clients before enforcement + /// is switched on. Captured once at init. + api_key: ?[]const u8, + /// Serializes the portfolio read-modify-write across concurrent + /// watchlist adds (httpz dispatches requests on multiple threads). + /// Without it two simultaneous adds could both read the old file and + /// the second writer would clobber the first's new symbol. + watch_mutex: std.Io.Mutex = .init, + + pub fn init(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) App { + const config = zfin.Config.fromEnv(io, allocator, environ); + const svc = zfin.DataService.init(io, allocator, config); + const slow_threshold_ms = if (environ.get("ZFIN_SERVER_SLOW_MS")) |s| + std.fmt.parseInt(u64, s, 10) catch 500 + else + 500; + // Treat an empty value as unset so `ZFIN_SERVER_API_KEY=` can't + // accidentally enable enforcement with an empty (never-matching) key. + const api_key: ?[]const u8 = if (environ.get("ZFIN_SERVER_API_KEY")) |v| + (if (v.len == 0) null else v) + else + null; + return .{ + .io = io, + .environ = environ, + .allocator = allocator, + .config = config, + .svc = svc, + .slow_threshold_ms = slow_threshold_ms, + .api_key = api_key, + }; + } + + pub fn deinit(self: *App) void { + self.svc.deinit(); + self.config.deinit(); + } + + /// httpz dispatch hook: every request flows through here so we + /// have a single place to wrap timing and error logging without + /// modifying every handler. Slow requests (above + /// `slow_threshold_ms`) and error responses (status >= 400) + /// emit a structured stderr line; everything else stays silent. + pub fn dispatch(self: *App, action: httpz.Action(*App), req: *httpz.Request, res: *httpz.Response) !void { + // wall-clock required: per-request elapsed for slow-request + // logging. `.awake` (monotonic) avoids spurious negatives + // on system clock skew. + const start_ns = std.Io.Timestamp.now(self.io, .awake).nanoseconds; + + // Registered before the gate and the action so every exit path - + // a 401 from the gate, an error from the action, or a normal + // response - emits the slow/error log line. + defer { + const elapsed_ns = std.Io.Timestamp.now(self.io, .awake).nanoseconds - start_ns; + const elapsed_ms: u64 = @intCast(@divTrunc(elapsed_ns, std.time.ns_per_ms)); + if (shouldLogRequest(elapsed_ms, res.status, self.slow_threshold_ms)) { + // wall-clock required: ts in stderr line lets the + // operator correlate slow requests with cron / system + // events using `date -d @`. + const ts = std.Io.Timestamp.now(self.io, .real).toSeconds(); + log.warn("ts={d} elapsed_ms={d} status={d} method={s} path={s}", .{ + ts, + elapsed_ms, + res.status, + @tagName(req.method), + req.url.path, + }); + } + } + + // API-key gate. Soft cutover: only enforced when a key is + // configured. The public surface (`/`, `/help`, and the + // LibreOffice `/:symbol/returns` endpoint) is never gated. + if (self.api_key) |expected| { + if (!pathIsPublic(req.url.path) and !providedKeyMatches(req, expected)) { + res.status = 401; + res.content_type = httpz.ContentType.TEXT; + res.body = "Unauthorized: missing or invalid API key\n"; + return; + } + } + + try action(self, req, res); + } +}; + +/// Pure predicate: should this request emit a stderr log line? +/// Logs slow successes (above threshold) and any error response. +fn shouldLogRequest(elapsed_ms: u64, status: u16, threshold_ms: u64) bool { + return elapsed_ms > threshold_ms or status >= 400; +} + +/// Pure predicate: is this request path part of the public surface that +/// never requires an API key? Public = the landing page, the help text, +/// and the LibreOffice returns endpoint (`/:symbol/returns`, including +/// its `?fmt=xml` and `?watch=true` variants - the query string is not +/// part of the path). Everything else (raw SRF cache files, quotes, the +/// symbol list, EDGAR maps, entity facts) is gated. +fn pathIsPublic(path: []const u8) bool { + if (std.mem.eql(u8, path, "/")) return true; + if (std.mem.eql(u8, path, "/help")) return true; + return isSymbolReturnsPath(path); +} + +/// True only for paths shaped exactly like `//returns`: a single +/// non-empty symbol segment followed by the literal `returns`. Guards +/// against `/returns` (no symbol) and multi-segment look-alikes such as +/// `/a/b/returns` or `/AAPL/returns/extra`. +fn isSymbolReturnsPath(path: []const u8) bool { + const suffix = "/returns"; + if (path.len <= suffix.len) return false; + if (!std.mem.endsWith(u8, path, suffix)) return false; + if (path[0] != '/') return false; + const symbol = path[1 .. path.len - suffix.len]; + if (symbol.len == 0) return false; + if (std.mem.indexOfScalar(u8, symbol, '/') != null) return false; + return true; +} + +/// Length-checked equality of a caller-supplied key against the +/// configured one. The threat model is casual/incidental traffic, not a +/// timing-attack adversary, so plain `mem.eql` is sufficient. A null +/// (absent header/param) never matches. +fn keyMatches(provided: ?[]const u8, expected: []const u8) bool { + const p = provided orelse return false; + return std.mem.eql(u8, p, expected); +} + +/// Read the caller-supplied API key from the `X-API-Key` header +/// (preferred) or an `api_key` query parameter (curl convenience) and +/// compare against the configured key. httpz wants the header name in +/// lowercase. A malformed query string is treated as "no key". +fn providedKeyMatches(req: *httpz.Request, expected: []const u8) bool { + if (keyMatches(req.header("x-api-key"), expected)) return true; + const q = req.query() catch return false; + return keyMatches(q.get("api_key"), expected); +} + +// ── Tests ──────────────────────────────────────────────────── + +test "shouldLogRequest: fast 2xx is silent" { + try std.testing.expect(!shouldLogRequest(10, 200, 500)); + try std.testing.expect(!shouldLogRequest(499, 200, 500)); + try std.testing.expect(!shouldLogRequest(0, 204, 500)); +} + +test "shouldLogRequest: slow 2xx logs" { + try std.testing.expect(shouldLogRequest(501, 200, 500)); + try std.testing.expect(shouldLogRequest(2000, 200, 500)); + // Boundary: == threshold is NOT logged (strict >). + try std.testing.expect(!shouldLogRequest(500, 200, 500)); +} + +test "shouldLogRequest: any error response logs regardless of timing" { + try std.testing.expect(shouldLogRequest(1, 400, 500)); + try std.testing.expect(shouldLogRequest(1, 404, 500)); + try std.testing.expect(shouldLogRequest(1, 500, 500)); + try std.testing.expect(shouldLogRequest(1, 503, 500)); + // 3xx is not flagged as error. + try std.testing.expect(!shouldLogRequest(1, 301, 500)); + try std.testing.expect(!shouldLogRequest(1, 304, 500)); +} + +test "shouldLogRequest: custom threshold respected" { + try std.testing.expect(!shouldLogRequest(50, 200, 100)); + try std.testing.expect(shouldLogRequest(150, 200, 100)); + // Higher threshold (e.g. user sets ZFIN_SERVER_SLOW_MS=2000). + try std.testing.expect(!shouldLogRequest(1500, 200, 2000)); + try std.testing.expect(shouldLogRequest(2500, 200, 2000)); +} + +test "pathIsPublic: public surface (no key required)" { + try std.testing.expect(pathIsPublic("/")); + try std.testing.expect(pathIsPublic("/help")); + try std.testing.expect(pathIsPublic("/AAPL/returns")); + // Symbols with dots (class shares) still match. + try std.testing.expect(pathIsPublic("/BRK.B/returns")); +} + +test "pathIsPublic: gated surface (key required)" { + try std.testing.expect(!pathIsPublic("/AAPL/candles")); + try std.testing.expect(!pathIsPublic("/AAPL/quote")); + try std.testing.expect(!pathIsPublic("/symbols")); + try std.testing.expect(!pathIsPublic("/_edgar/tickers_funds")); + try std.testing.expect(!pathIsPublic("/0000320193/entity_facts")); + // Diagnostics leaks the operator's tracked set and cache layout, so it must + // stay gated. It needs no entry in `pathIsPublic` to be gated - the default + // is closed - and this asserts the default rather than trusting it, because + // the cost of that assumption being wrong is silent disclosure. + try std.testing.expect(!pathIsPublic("/AAPL/diagnostics")); + try std.testing.expect(!pathIsPublic("/BRK.B/diagnostics")); +} + +test "pathIsPublic: returns look-alikes do not slip through" { + try std.testing.expect(!pathIsPublic("/returns")); // no symbol + try std.testing.expect(!pathIsPublic("/a/b/returns")); // extra segment + try std.testing.expect(!pathIsPublic("/AAPL/returns/extra")); // suffix, not exact + try std.testing.expect(!pathIsPublic("/help/secret")); // help prefix only + try std.testing.expect(!pathIsPublic("//returns")); // empty symbol +} + +test "keyMatches" { + try std.testing.expect(keyMatches("s3cret", "s3cret")); + try std.testing.expect(!keyMatches("s3cret", "other")); + try std.testing.expect(!keyMatches(null, "s3cret")); + try std.testing.expect(!keyMatches("", "s3cret")); + // Length-checked: neither a prefix nor an extension matches. + try std.testing.expect(!keyMatches("s3cre", "s3cret")); + try std.testing.expect(!keyMatches("s3cretX", "s3cret")); +} diff --git a/src/handlers.zig b/src/handlers.zig new file mode 100644 index 0000000..f1cd24b --- /dev/null +++ b/src/handlers.zig @@ -0,0 +1,940 @@ +//! Route handlers: every HTTP endpoint, the SRF cache passthrough, and the +//! portfolio writes behind `/:symbol/watch`. +//! +//! Imports `refresh.zig` for `collectRefreshSymbols`. That direction is correct +//! rather than incidental: `/:symbol/diagnostics` exists to report what the +//! refresh loop intends, so it must use refresh's own definition of its symbol +//! set - a second definition here would be a second answer to one question. + +const std = @import("std"); +const zfin = @import("zfin"); +const httpz = @import("httpz"); + +const App = @import("App.zig").App; +const refresh_cmd = @import("refresh.zig"); + +const version = @import("build_options").version; +const log = std.log.scoped(.@"zfin-server"); + +/// Case-insensitive User-Agent substrings permitted to add to the +/// watchlist via the public `/:symbol/returns?watch=true` path. This is +/// obscurity-grade (a UA is trivially spoofable) and matches the +/// casual-traffic threat model: it keeps crawlers and stray browsers +/// from growing the tracked set - and thus the recurring cron-refresh +/// load - while letting the non-technical user's LibreOffice WEBSERVICE +/// calls through. The authenticated `/:symbol/watch` route bypasses this +/// (a valid API key is a stronger signal than any UA). +/// +/// Confirmed empirically - LibreOffice's WEBSERVICE sends e.g. +/// "LibreOffice 24.2.7.2 denylistedbackend/8.5.0 OpenSSL/3.0.13" +/// (it also fires a WebDAV OPTIONS preflight that 404s harmlessly; the +/// real GET carries the same User-Agent). Matching the version-agnostic +/// "LibreOffice" token keeps this robust across releases. +const watch_user_agents = [_][]const u8{"LibreOffice"}; + +/// True if `ua` matches one of `watch_user_agents` (case-insensitive +/// substring). A null/absent User-Agent never matches. +fn userAgentMayWatch(ua: ?[]const u8) bool { + const agent = ua orelse return false; + for (watch_user_agents) |needle| { + if (std.ascii.indexOfIgnoreCase(agent, needle) != null) return true; + } + return false; +} + +/// Sanity gate for symbols entering the tracked set (and thus recurring +/// cron load): non-empty, <=16 chars, and only the characters real +/// tickers use - uppercase letters, digits, and `.`/`-` for class +/// shares. Not a real ticker validator; just enough to keep junk like an +/// over-long or path-shaped segment out of the portfolio file. Symbols +/// reach here already upper-cased by `upperDupe`. +fn isPlausibleSymbol(sym: []const u8) bool { + if (sym.len == 0 or sym.len > 16) return false; + for (sym) |c| { + const ok = (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '.' or c == '-'; + if (!ok) return false; + } + return true; +} + +// ── Route handlers ─────────────────────────────────────────── + +pub fn handleIndex(_: *App, _: *httpz.Request, res: *httpz.Response) !void { + res.content_type = httpz.ContentType.HTML; + res.body = + \\ + \\zfin-server + \\ + \\

zfin-server

+ \\

This is a financial data API server. Not intended for browser use.

+ \\

See /help for endpoint documentation.

+ \\ + ; +} + +pub fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void { + res.content_type = httpz.ContentType.TEXT; + res.body = "zfin-server " ++ version ++ " - financial data API" ++ + \\ + \\ + \\Endpoints: + \\ GET /{SYMBOL}/returns Trailing 1/3/5/10yr returns (JSON) + \\ GET /{SYMBOL}/returns?fmt=xml Trailing returns (XML, for LibreCalc) + \\ GET /{SYMBOL}/watch Add SYMBOL to the watchlist (authenticated) + \\ GET /{SYMBOL}/quote Latest quote (JSON) + \\ GET /{SYMBOL}/candles Raw SRF cache file + \\ GET /{SYMBOL}/candles_meta Candle freshness metadata (SRF) + \\ GET /{SYMBOL}/dividends Raw SRF cache file + \\ GET /{SYMBOL}/splits Raw SRF cache file + \\ GET /{SYMBOL}/earnings Raw SRF cache file + \\ GET /{SYMBOL}/options Raw SRF cache file + \\ GET /{SYMBOL}/classification Wikidata classification (SRF) + \\ GET /{SYMBOL}/etf_metrics EDGAR NPORT-P fund metrics (SRF; 404 for non-funds) + \\ GET /{CIK}/entity_facts EDGAR XBRL entity facts (SRF; CIK-keyed) + \\ GET /_edgar/tickers_funds EDGAR mutual-fund ticker map (SRF) + \\ GET /_edgar/tickers_companies EDGAR company ticker map (SRF) + \\ GET /symbols List of tracked symbols + \\ + \\Auth: + \\ All endpoints except /, /help, and /{SYMBOL}/returns require an + \\ API key when ZFIN_SERVER_API_KEY is set (X-API-Key header or + \\ ?api_key= query parameter). + \\ + \\Caching: + \\ SRF endpoints serve from the local cache; on a miss the server + \\ fetches once from the provider, fills the cache, then serves + \\ (404 only if that fetch also fails). + \\ + \\Watchlist (add a symbol to the cron refresh set): + \\ GET /{SYMBOL}/watch authenticated; for your own tooling + \\ GET /{SYMBOL}/returns?watch=true public, but only LibreOffice's + \\ WEBSERVICE User-Agent is honored + \\ + \\Returns fields: + \\ lastClose Last closing price + \\ trailing{1,3,5,10}YearReturn Total return with dividend reinvestment + \\ price{1,3,5,10}YearReturn Price-only return (from adjusted close) + \\ volatility Longest-term available annualized volatility + \\ volatilityTerm Period (years) of the volatility field + \\ volatility{1,3,5,10}Year Per-period annualized volatility + \\ + \\XML example (LibreCalc): + \\ =FILTERXML(WEBSERVICE("http://host/AAPL/returns?fmt=xml"),"//total10YearReturn") + \\ + ; +} + +pub fn handleSymbols(app: *App, _: *httpz.Request, res: *httpz.Response) !void { + const arena = res.arena; + const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; + + const file_data = std.Io.Dir.cwd().readFileAlloc(app.io, portfolio_path, arena, .limited(10 * 1024 * 1024)) catch { + res.content_type = httpz.ContentType.JSON; + res.body = "[]"; + return; + }; + + var portfolio = zfin.cache.deserializePortfolio(arena, file_data) catch { + res.content_type = httpz.ContentType.JSON; + res.body = "[]"; + return; + }; + defer portfolio.deinit(); + + // Collect unique symbols + var seen = std.StringHashMap(void).init(arena); + var symbols = std.ArrayList([]const u8).empty; + for (portfolio.lots) |lot| { + if (lot.symbol.len == 0) continue; + if (seen.contains(lot.symbol)) continue; + try seen.put(lot.symbol, {}); + try symbols.append(arena, lot.symbol); + } + + // Build JSON array + var aw: std.Io.Writer.Allocating = .init(arena); + try aw.writer.writeByte('['); + for (symbols.items, 0..) |sym, i| { + if (i > 0) try aw.writer.writeByte(','); + try aw.writer.print("\"{s}\"", .{sym}); + } + try aw.writer.writeByte(']'); + + res.content_type = httpz.ContentType.JSON; + res.body = try aw.toOwnedSlice(); +} + +pub fn handleReturns(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + const raw_symbol = req.param("symbol") orelse { + res.status = 404; + res.body = "Missing symbol"; + return; + }; + const arena = res.arena; + const symbol = try upperDupe(arena, raw_symbol); + + // Auto-add to watchlist if requested. UA-gated (obscurity) so only + // LibreOffice WEBSERVICE calls - not random browsers/crawlers - can + // grow the tracked set via this public endpoint. Best-effort: the + // returns response below is served regardless of whether the add ran. + const q = try req.query(); + if (q.get("watch")) |w| { + if (std.ascii.eqlIgnoreCase(w, "true")) { + if (userAgentMayWatch(req.header("user-agent"))) { + appendWatchSymbol(app, symbol) catch |err| { + log.warn("failed to append watch symbol {s}: {t}", .{ symbol, err }); + }; + } else { + log.debug("watch add for {s} skipped: User-Agent not allowlisted", .{symbol}); + } + } + } + + const result = app.svc.getTrailingReturns(symbol, .{}) catch { + res.status = 404; + res.body = "Symbol not found or fetch failed"; + return; + }; + defer app.allocator.free(result.candles); + if (result.dividends) |divs| { + defer zfin.Dividend.freeSlice(app.allocator, divs); + } + + const candles = result.candles; + if (candles.len == 0) { + res.status = 404; + res.body = "No candle data"; + return; + } + + const last_close = candles[candles.len - 1].close; + var date_buf: [10]u8 = undefined; + const date_str = try std.fmt.bufPrint(&date_buf, "{f}", .{candles[candles.len - 1].date}); + + // Price-only returns (split-adjusted, NOT dividend-adjusted — + // see analytics/performance.zig:trailingReturnsPriceOnly). + // Matches the "price return" numbers public sources publish + // (Yahoo chart-bar, FMP, Barchart, Fidelity stock pages). + const p1y = if (result.asof_price.one_year) |r| r.annualized_return else null; + const p3y = if (result.asof_price.three_year) |r| r.annualized_return else null; + const p5y = if (result.asof_price.five_year) |r| r.annualized_return else null; + const p10y = if (result.asof_price.ten_year) |r| r.annualized_return else null; + + // Total returns (dividend reinvestment when dividends are + // available; falls back to adj_close-based total return when + // dividend records are missing). Matches Morningstar + // "Trailing Returns" / Yahoo "Performance Overview" / Koyfin + // "Total Return". + const total = result.asof_total orelse result.asof_price; + const t1y = if (total.one_year) |r| r.annualized_return else null; + const t3y = if (total.three_year) |r| r.annualized_return else null; + const t5y = if (total.five_year) |r| r.annualized_return else null; + const t10y = if (total.ten_year) |r| r.annualized_return else null; + + // Per-period volatility + const risk = zfin.risk.trailingRisk(candles); + const v1y = if (risk.one_year) |r| r.volatility else null; + const v3y = if (risk.three_year) |r| r.volatility else null; + const v5y = if (risk.five_year) |r| r.volatility else null; + const v10y = if (risk.ten_year) |r| r.volatility else null; + + // Longest-term volatility convenience fields + const vol_best = v10y orelse v5y orelse v3y orelse v1y; + const vol_term: ?u8 = if (v10y != null) 10 else if (v5y != null) 5 else if (v3y != null) 3 else if (v1y != null) 1 else null; + + // Check if XML requested + if (q.get("fmt")) |fmt| { + if (std.ascii.eqlIgnoreCase(fmt, "xml")) { + res.content_type = httpz.ContentType.XML; + res.body = try std.fmt.allocPrint(arena, + \\ + \\ {s} + \\ {s} + \\ {d:.2} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ {s} + \\ + \\ + , .{ + symbol, + date_str, + last_close, + fmtPct(arena, t1y), + fmtPct(arena, t3y), + fmtPct(arena, t5y), + fmtPct(arena, t10y), + fmtPct(arena, p1y), + fmtPct(arena, p3y), + fmtPct(arena, p5y), + fmtPct(arena, p10y), + fmtPct(arena, vol_best), + fmtInt(arena, vol_term), + fmtPct(arena, v1y), + fmtPct(arena, v3y), + fmtPct(arena, v5y), + fmtPct(arena, v10y), + }); + return; + } + } + + res.content_type = httpz.ContentType.JSON; + res.body = try std.fmt.allocPrint(arena, + \\{{"ticker":"{s}","returnDate":"{s}","lastClose":{d:.2},"trailing1YearReturn":{s},"trailing3YearReturn":{s},"trailing5YearReturn":{s},"trailing10YearReturn":{s},"price1YearReturn":{s},"price3YearReturn":{s},"price5YearReturn":{s},"price10YearReturn":{s},"volatility":{s},"volatilityTerm":{s},"volatility1Year":{s},"volatility3Year":{s},"volatility5Year":{s},"volatility10Year":{s}}} + , .{ + symbol, + date_str, + last_close, + fmtPct(arena, t1y), + fmtPct(arena, t3y), + fmtPct(arena, t5y), + fmtPct(arena, t10y), + fmtPct(arena, p1y), + fmtPct(arena, p3y), + fmtPct(arena, p5y), + fmtPct(arena, p10y), + fmtPct(arena, vol_best), + fmtInt(arena, vol_term), + fmtPct(arena, v1y), + fmtPct(arena, v3y), + fmtPct(arena, v5y), + fmtPct(arena, v10y), + }); +} + +/// Authenticated explicit watchlist add. Not on the public allowlist, so +/// `dispatch` requires the API key - unlike the UA-gated `?watch=true` +/// path, this is for the operator's own tooling deliberately growing the +/// tracked set (and accepting the recurring cron-refresh cost). +pub fn handleWatch(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + const raw_symbol = req.param("symbol") orelse { + res.status = 400; + res.body = "Missing symbol"; + return; + }; + const arena = res.arena; + const symbol = try upperDupe(arena, raw_symbol); + + appendWatchSymbol(app, symbol) catch |err| switch (err) { + error.InvalidSymbol => { + res.status = 400; + res.body = "Invalid symbol"; + return; + }, + else => { + res.status = 500; + res.body = try std.fmt.allocPrint(arena, "Failed to add watch symbol: {t}", .{err}); + return; + }, + }; + + res.content_type = httpz.ContentType.JSON; + res.body = try std.fmt.allocPrint(arena, "{{\"symbol\":\"{s}\",\"watched\":true}}", .{symbol}); +} + +pub fn handleQuote(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + const raw_symbol = req.param("symbol") orelse { + res.status = 400; + res.body = "Missing symbol"; + return; + }; + const arena = res.arena; + const symbol = try upperDupe(arena, raw_symbol); + + const q = app.svc.getQuote(symbol, .{}) catch { + res.status = 404; + res.body = "Quote not available"; + return; + }; + + res.content_type = httpz.ContentType.JSON; + res.body = try std.fmt.allocPrint(arena, + \\{{"symbol":"{s}","close":{d:.2},"open":{d:.2},"high":{d:.2},"low":{d:.2},"volume":{d},"previous_close":{d:.2}}} + , .{ symbol, q.close, q.open, q.high, q.low, q.volume, q.previous_close }); +} + +/// Identifies which `DataService` fetch to run when a served SRF file is +/// absent (see `fetchOnMiss`). Kept separate from `zfin.cache.DataType` +/// because the mapping isn't 1:1 - both `candles_daily.srf` and +/// `candles_meta.srf` are populated by a single `getCandles` call. +const SrfKind = enum { + candles, + dividends, + splits, + earnings, + options, + classification, + etf_metrics, + entity_facts, + tickers_funds, + tickers_companies, +}; + +/// What the server INTENDS for a symbol, as opposed to the data it happens to +/// hold. Every other endpoint answers the second question; nothing answered the +/// first, which is how a symbol the refresh loop never touches sat six weeks +/// behind while being served to clients as though it were maintained. +const SymbolDiagnostics = struct { + /// Will the refresh loop fetch this symbol? See `collectRefreshSymbols`. + tracked: bool, + /// Newest cached bar, or null when there is no candle meta at all. + last_date: ?zfin.Date, + /// When the cached copy was written (Unix seconds), null when uncached. + created: ?i64, + /// Consecutive transient provider failures on the primary provider. + fail_count: u8, + /// Is the cached copy stamped fresh by its own `#!expires=`? Reported rather + /// than the raw expiry because `fresh` alongside a non-zero `days_behind` is + /// precisely the pathology that started this: a copy stamped good until + /// tomorrow while sitting days behind its peers. The raw directive is still + /// on the wire via `/:symbol/candles_meta` for anyone who wants it. + fresh: bool, + /// Newest bar held by any same-kind peer in this cache, or null when there is + /// no peer to compare against. + peer_date: ?zfin.Date, + /// Calendar days behind `peer_date`; 0 when not behind or incomparable. + days_behind: i64, +}; + +/// Calendar days `last` sits behind `peer`, or 0 when it is not behind. +/// +/// Computed here rather than read out of the sweep's findings. `scan` emits a +/// Finding only for a TRACKED symbol (untracked ones divert to `orphans`), so +/// reading `days_behind` from there returned 0 for every untracked symbol - +/// exactly the class this endpoint exists to expose. An untracked symbol sitting +/// 43 days behind, reported as `days_behind:0`, is the worst available answer: +/// it reads as "current" for the one case nobody is watching. +/// +/// Calendar days, truncated, to match `zfin.freshness.Finding.days_behind` - the +/// magnitude an operator weighs against `max_normal_lag_days`. +fn daysBehind(last: ?zfin.Date, peer: ?zfin.Date) i64 { + const l = last orelse return 0; + const p = peer orelse return 0; + if (!l.lessThan(p)) return 0; + return @divTrunc(p.toEpoch() - l.toEpoch(), std.time.s_per_day); +} + +/// The peer reference date for `kind`, or null when the group cannot yield a +/// comparison. `conclusive()` is the gate: with a single cached symbol of a kind +/// there are no peers, and reporting that symbol's own date as its `peer_date` +/// would manufacture agreement out of nothing. +fn groupPeerDate(report: zfin.freshness.Report, kind: zfin.market.InstrumentKind) ?zfin.Date { + for (report.groups) |g| { + if (g.kind != kind) continue; + if (!g.conclusive()) return null; + return g.peer_date; + } + return null; +} + +pub fn handleDiagnostics(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + const raw_symbol = req.param("symbol") orelse { + res.status = 400; + res.body = "Missing symbol"; + return; + }; + const arena = res.arena; + const symbol = try upperDupe(arena, raw_symbol); + + var store = zfin.cache.Store.init(app.io, arena, app.config.cache_dir); + + // An unreadable or unparseable portfolio leaves the set empty, which reports + // `tracked:false` - honest, because a refresh run reading the same file would + // fetch nothing either. + // + // NO `portfolio.deinit()` here, deliberately. The set's keys BORROW from the + // parsed lots, and every `contains` happens below this block, so freeing the + // portfolio dangles them - observed as all 25 tracked symbols reading as + // untracked. The request arena owns this memory and releases it with the + // response. This is the same trap `zfin`'s `trackedSymbols` documents: "the + // scoped-defer-frees version dangled its keys before the caller read them, + // which made every cached symbol look untracked." + var tracked = std.StringHashMap(void).init(arena); + const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; + if (std.Io.Dir.cwd().readFileAlloc(app.io, portfolio_path, arena, .limited(10 * 1024 * 1024))) |data| { + if (zfin.cache.deserializePortfolio(arena, data)) |parsed| { + try refresh_cmd.collectRefreshSymbols(&tracked, parsed.lots); + } else |_| {} + } else |_| {} + + // The peer sweep. `collect` + `scan` rather than a local "newest of this + // kind" loop: the definition of behind-its-peers lives in one place, and this + // endpoint exists to report that definition, not a second opinion on it. + const keys = store.cacheKeys(arena) catch &.{}; + const entries = try zfin.freshness.collect(arena, &store, keys, &tracked, &.{}); + // wall-clock required: peer freshness is judged against the market calendar. + const now_s = std.Io.Timestamp.now(app.io, .real).toSeconds(); + const report = try zfin.freshness.scan(arena, entries, now_s); + + const meta = store.readCandleMeta(symbol); + const last_date: ?zfin.Date = if (meta) |m| m.meta.last_date else null; + const peer_date = groupPeerDate(report, zfin.market.classify(symbol)); + const d = SymbolDiagnostics{ + .tracked = tracked.contains(symbol), + .last_date = last_date, + .created = if (meta) |m| m.created else null, + .fail_count = if (meta) |m| m.meta.fail_count else 0, + .fresh = store.isCandleMetaFresh(symbol), + .peer_date = peer_date, + .days_behind = daysBehind(last_date, peer_date), + }; + + var aw: std.Io.Writer.Allocating = .init(arena); + try aw.writer.print( + \\{{"symbol":"{s}","tracked":{},"fresh":{},"fail_count":{d},"days_behind":{d} + , .{ symbol, d.tracked, d.fresh, d.fail_count, d.days_behind }); + // Null rather than a sentinel date for the absent cases: a client must be + // able to tell "no cached bar" from "a bar dated the epoch". + if (d.last_date) |ld| { + try aw.writer.print(",\"last_date\":\"{f}\"", .{ld}); + } else { + try aw.writer.writeAll(",\"last_date\":null"); + } + if (d.peer_date) |pd| { + try aw.writer.print(",\"peer_date\":\"{f}\"", .{pd}); + } else { + try aw.writer.writeAll(",\"peer_date\":null"); + } + if (d.created) |c| { + try aw.writer.print(",\"created\":{d}", .{c}); + } else { + try aw.writer.writeAll(",\"created\":null"); + } + try aw.writer.writeByte('}'); + + res.content_type = httpz.ContentType.JSON; + res.body = aw.written(); +} + +pub fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8, kind: SrfKind) !void { + return handleSrfFileByKey(app, req, res, "symbol", filename, kind); +} + +/// Generalized SRF cache-file passthrough: reads +/// `//` where `` is whatever URL +/// parameter `key_param` resolves to. The default `handleSrfFile` +/// uses `"symbol"`; CIK-keyed routes (e.g. `/:cik/entity_facts`) +/// pass `"cik"` instead. The cache-key segment is uppercased +/// (safe for both symbols and zero-padded CIK digit strings). +pub fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8, kind: SrfKind) !void { + const raw_key = req.param(key_param) orelse { + res.status = 400; + res.body = "Missing key"; + return; + }; + const arena = res.arena; + const key = try upperDupe(arena, raw_key); + return serveSrfFile(app, res, key, filename, kind); +} + +/// Static-key SRF cache-file passthrough for routes that don't +/// take a path parameter (e.g. `/_edgar/tickers_funds` reads +/// `/_edgar/tickers_funds.srf` directly). The `key` +/// is a literal directory name; not uppercased because the +/// cache uses `_edgar` as-is. +pub fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void { + return serveSrfFile(app, res, key, filename, kind); +} + +/// Inner shared helper. Serves `//` as raw SRF +/// with a sha256 ETag. L2-cache contract: a *present* file is served +/// as-is even when stale - cron is the freshness authority, so reads +/// never trigger a refetch - while an *absent* file triggers a one-shot +/// provider fetch (`fetchOnMiss`) to populate it, after which we re-read +/// and serve. If the fetch still can't produce the file, we fall back to +/// the original 404. +fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void { + const arena = res.arena; + const path = try std.fs.path.join(arena, &.{ app.config.cache_dir, key, filename }); + + const content = readCacheFile(app, arena, path) orelse blk: { + // Cache miss -> fetch from the provider, fill the cache, re-read. + fetchOnMiss(app, key, kind); + break :blk readCacheFile(app, arena, path) orelse { + res.status = 404; + res.body = "Cache file not found"; + return; + }; + }; + + // Body integrity header: sha256 of the bytes we're about to send. + // Clients can use this to detect mid-stream truncation that Zig's + // std.http.Client.fetch silently accepts on the Content-Length path + // (a premature EOF from the transport bubbles up as EndOfStream and + // is swallowed as a normal end-of-body). Shaped as a standard + // `ETag` value so future conditional-request work gets it for free. + var hash: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; + std.crypto.hash.sha2.Sha256.hash(content, &hash, .{}); + var etag_buf: [std.crypto.hash.sha2.Sha256.digest_length * 2 + "\"sha256:\"".len]u8 = undefined; + const etag = try std.fmt.bufPrint(&etag_buf, "\"sha256:{x}\"", .{&hash}); + // httpz.Response.header borrows the value — duplicate into the + // per-request arena so the slice outlives `etag_buf`. + const etag_owned = try arena.dupe(u8, etag); + + res.content_type = httpz.ContentType.BINARY; + res.header("content-type", "application/x-srf"); + res.header("etag", etag_owned); + res.body = content; +} + +/// Read a cache file into the request arena, or null when it can't be +/// read. Absent is the common case; any read error is treated as a miss +/// (matches the pre-fetch-on-miss behavior of falling back to 404). +fn readCacheFile(app: *App, arena: std.mem.Allocator, path: []const u8) ?[]u8 { + return std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch null; +} + +/// Populate the cache for `key` by running the matching `DataService` +/// fetch, then discard the parsed result - the caller re-reads the +/// canonical bytes off disk. Best-effort and synchronous: any provider +/// error (NotFound, rate limit, auth, transient, parse) is logged and +/// swallowed so the caller falls back to a 404. +/// +/// Backpressure caveat: these fetches share zfin's rate limiter, so when +/// the token bucket is drained (e.g. mid-cron) this blocks the httpz +/// worker until a token frees. Accepted for now; a 202 + poll path is +/// the planned escape hatch if blocking becomes a problem. +fn fetchOnMiss(app: *App, key: []const u8, kind: SrfKind) void { + const svc = &app.svc; + switch (kind) { + // getCandles writes both candles_daily.srf and candles_meta.srf. + .candles => { + const r = svc.getCandles(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .dividends => { + const r = svc.getDividends(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .splits => { + const r = svc.getSplits(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .earnings => { + const r = svc.getEarnings(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .options => { + const r = svc.getOptions(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .classification => { + const r = svc.getClassification(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .etf_metrics => { + const r = svc.getEtfMetrics(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + // `key` is the CIK here (resolved from the :cik route param). + .entity_facts => { + const r = svc.getEntityFacts(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .tickers_funds => { + var m = svc.loadMutualFundTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err); + m.deinit(); + }, + .tickers_companies => { + var m = svc.loadCompanyTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err); + m.deinit(); + }, + } +} + +/// Log a failed populate. NotFound is the normal "no such symbol / no +/// data" outcome (debug); everything else is operator-relevant (warn). +fn logFetchMiss(key: []const u8, kind: SrfKind, err: anyerror) void { + if (err == error.NotFound) { + log.info("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) }); + } else { + log.warn("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) }); + } +} + +pub fn handleCandles(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + return handleSrfFile(app, req, res, "candles_daily.srf", .candles); +} + +pub fn handleCandlesMeta(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + return handleSrfFile(app, req, res, "candles_meta.srf", .candles); +} + +pub fn handleDividends(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + return handleSrfFile(app, req, res, "dividends.srf", .dividends); +} + +pub fn handleSplits(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + return handleSrfFile(app, req, res, "splits.srf", .splits); +} + +pub fn handleEarnings(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + return handleSrfFile(app, req, res, "earnings.srf", .earnings); +} + +pub fn handleOptions(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + return handleSrfFile(app, req, res, "options.srf", .options); +} + +pub fn handleClassification(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + return handleSrfFile(app, req, res, "classification.srf", .classification); +} + +pub fn handleEtfMetrics(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + return handleSrfFile(app, req, res, "etf_metrics.srf", .etf_metrics); +} + +pub fn handleEntityFacts(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + // CIK-keyed route: cache layout is + // `//entity_facts.srf` (the CIK is the + // zero-padded 10-digit string Wikidata's P5531 emits). + return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf", .entity_facts); +} + +pub fn handleTickersFunds(app: *App, _: *httpz.Request, res: *httpz.Response) !void { + // Static-key route: `/_edgar/tickers_funds.srf` + // is a single file shared across all symbol lookups, not a + // per-symbol cache. + return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf", .tickers_funds); +} + +pub fn handleTickersCompanies(app: *App, _: *httpz.Request, res: *httpz.Response) !void { + return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf", .tickers_companies); +} + +// ── Helpers ────────────────────────────────────────────────── + +fn upperDupe(allocator: std.mem.Allocator, s: []const u8) ![]u8 { + const d = try allocator.dupe(u8, s); + for (d) |*c| c.* = std.ascii.toUpper(c.*); + return d; +} + +/// Print an inline rate-limit estimate tag like "[~14s] " before the +/// next fetch of `data_type`, then flush so the tag is visible before +/// the (possibly blocking) fetch runs. An interactive caller sees the +/// estimate, then the pause, then the result land on a single line; a +fn fmtPct(arena: std.mem.Allocator, value: ?f64) []const u8 { + if (value) |v| return std.fmt.allocPrint(arena, "{d:.5}", .{v * 100.0}) catch "null"; + return "null"; +} + +/// Format an optional integer, or "null" if absent. +fn fmtInt(arena: std.mem.Allocator, value: ?u8) []const u8 { + if (value) |v| return std.fmt.allocPrint(arena, "{d}", .{v}) catch "null"; + return "null"; +} + +/// Append a watch lot for `symbol` to the portfolio SRF file, unless it +/// is already tracked. Serialized across requests via `app.watch_mutex` +/// and written atomically, so a concurrent add or a mid-write crash can't +/// clobber or truncate the portfolio file. Returns `error.InvalidSymbol` +/// for implausible symbols; callers decide how loud to be. +fn appendWatchSymbol(app: *App, symbol: []const u8) !void { + if (!isPlausibleSymbol(symbol)) return error.InvalidSymbol; + + const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; + const allocator = app.allocator; + const io = app.io; + + // Serialize the whole read-modify-write so concurrent adds don't lose + // updates (a last-writer-wins race would otherwise drop a symbol). + // Uncancelable so a canceled request can't abandon a half-done write. + app.watch_mutex.lockUncancelable(io); + defer app.watch_mutex.unlock(io); + + // Read and deserialize existing portfolio (or start empty) + const file_data = std.Io.Dir.cwd().readFileAlloc(io, portfolio_path, allocator, .limited(10 * 1024 * 1024)) catch |err| { + if (err == error.FileNotFound) return writeNewPortfolio(io, allocator, portfolio_path, symbol); + return err; + }; + defer allocator.free(file_data); + + var portfolio = zfin.cache.deserializePortfolio(allocator, file_data) catch return; + defer portfolio.deinit(); + + // Check if symbol already tracked + for (portfolio.lots) |lot| { + if (std.ascii.eqlIgnoreCase(lot.symbol, symbol)) return; + } + + // Build new lot list with the watch entry appended + const new_lots = try allocator.alloc(zfin.Lot, portfolio.lots.len + 1); + defer allocator.free(new_lots); + @memcpy(new_lots[0..portfolio.lots.len], portfolio.lots); + new_lots[portfolio.lots.len] = .{ + .symbol = symbol, + .shares = 0, + .open_date = zfin.Date.fromYmd(2026, 1, 1), + .open_price = 0, + .security_type = .watch, + }; + + // Serialize and write atomically. + const output = try zfin.cache.serializePortfolio(allocator, new_lots); + defer allocator.free(output); + try writeFileAtomic(io, allocator, portfolio_path, output); + + log.info("added watch symbol {s} to {s}", .{ symbol, portfolio_path }); +} + +fn writeNewPortfolio(io: std.Io, allocator: std.mem.Allocator, path: []const u8, symbol: []const u8) !void { + const lot = [_]zfin.Lot{.{ + .symbol = symbol, + .shares = 0, + .open_date = zfin.Date.fromYmd(2026, 1, 1), + .open_price = 0, + .security_type = .watch, + }}; + const output = try zfin.cache.serializePortfolio(allocator, &lot); + defer allocator.free(output); + try writeFileAtomic(io, allocator, path, output); + + log.info("created {s} with watch symbol {s}", .{ path, symbol }); +} + +/// Crash-safe file write: write to `.tmp`, fsync, then rename over +/// `path`. A mid-write crash leaves the prior file intact rather than a +/// truncated portfolio. (zfin's internal `atomic.writeFileAtomic` isn't +/// part of its public module, so we keep a small local copy.) +fn writeFileAtomic(io: std.Io, allocator: std.mem.Allocator, path: []const u8, bytes: []const u8) !void { + const tmp_path = try std.fmt.allocPrint(allocator, "{s}.tmp", .{path}); + defer allocator.free(tmp_path); + + { + var tmp_file = try std.Io.Dir.cwd().createFile(io, tmp_path, .{ .truncate = true, .exclusive = false }); + errdefer { + tmp_file.close(io); + std.Io.Dir.cwd().deleteFile(io, tmp_path) catch |err| { + log.debug("atomic write cleanup deleteFile({s}): {t}", .{ tmp_path, err }); + }; + } + try tmp_file.writeStreamingAll(io, bytes); + // fsync so the data is durable before the rename appears. + try tmp_file.sync(io); + tmp_file.close(io); + } + + std.Io.Dir.cwd().rename(tmp_path, std.Io.Dir.cwd(), path, io) catch |err| { + std.Io.Dir.cwd().deleteFile(io, tmp_path) catch |del_err| { + log.debug("atomic write cleanup deleteFile({s}): {t}", .{ tmp_path, del_err }); + }; + return err; + }; +} + +// ── Tests ──────────────────────────────────────────────────── + +test "fmtPct" { + var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + try std.testing.expectEqualStrings("null", fmtPct(arena, null)); + const result = fmtPct(arena, 0.1234); + try std.testing.expect(std.mem.startsWith(u8, result, "12.34")); +} + +test "upperDupe" { + const result = try upperDupe(std.testing.allocator, "aapl"); + defer std.testing.allocator.free(result); + try std.testing.expectEqualStrings("AAPL", result); +} + +test "userAgentMayWatch" { + try std.testing.expect(userAgentMayWatch("LibreOffice 24.8")); + try std.testing.expect(userAgentMayWatch("libreoffice")); // case-insensitive + try std.testing.expect(userAgentMayWatch("Mozilla/5.0 LibreOffice/7.6")); + try std.testing.expect(!userAgentMayWatch("Mozilla/5.0 (X11; Linux x86_64)")); + try std.testing.expect(!userAgentMayWatch("curl/8.14.1")); + try std.testing.expect(!userAgentMayWatch(null)); + try std.testing.expect(!userAgentMayWatch("")); +} + +test "isPlausibleSymbol" { + try std.testing.expect(isPlausibleSymbol("AAPL")); + try std.testing.expect(isPlausibleSymbol("BRK.B")); + try std.testing.expect(isPlausibleSymbol("BRK-B")); + try std.testing.expect(isPlausibleSymbol("X")); + try std.testing.expect(!isPlausibleSymbol("")); // empty + try std.testing.expect(!isPlausibleSymbol("aapl")); // lowercase (upper-cased before this) + try std.testing.expect(!isPlausibleSymbol("AB CD")); // space + try std.testing.expect(!isPlausibleSymbol("../etc/passwd")); // path-shaped junk + try std.testing.expect(!isPlausibleSymbol("ABCDEFGHIJKLMNOPQ")); // 17 chars, too long +} + +test "daysBehind: an UNTRACKED symbol behind its peers still reports the gap" { + // The regression this replaces: `days_behind` was read out of + // `zfin.freshness.Report.stale`/`far_behind`, and `scan` only emits findings + // for tracked symbols - untracked ones divert to `orphans`. So every + // untracked symbol reported 0, and an untracked symbol is precisely what this + // endpoint was built to expose. AGG's real numbers, observed against a live + // cache copy: five days behind, reported as current. + try std.testing.expectEqual(@as(i64, 5), daysBehind( + zfin.Date.fromYmd(2026, 8, 6), + zfin.Date.fromYmd(2026, 8, 11), + )); + // SPCX's real gap. + try std.testing.expectEqual(@as(i64, 43), daysBehind( + zfin.Date.fromYmd(2026, 6, 29), + zfin.Date.fromYmd(2026, 8, 11), + )); +} + +test "daysBehind: not behind, or incomparable, is zero rather than negative" { + const d = zfin.Date.fromYmd(2026, 8, 11); + // Level with peers. + try std.testing.expectEqual(@as(i64, 0), daysBehind(d, d)); + // AHEAD of peers - this symbol IS the peer maximum. Must not report a + // negative gap, which would sort as "most behind" in any worst-first list. + try std.testing.expectEqual(@as(i64, 0), daysBehind(d, zfin.Date.fromYmd(2026, 8, 1))); + // No cached bar, and no peer to compare against: unanswerable, not zero-ish. + // Callers distinguish these from "current" via the null `last_date`. + try std.testing.expectEqual(@as(i64, 0), daysBehind(null, d)); + try std.testing.expectEqual(@as(i64, 0), daysBehind(d, null)); +} + +test "groupPeerDate: an inconclusive group has no peer date" { + const d = zfin.Date.fromYmd(2026, 8, 11); + // `dated = 1` is the symbol itself and nothing else. Returning its own date + // as `peer_date` would read as "agrees with its peers" when there are none. + const lonely = [_]zfin.freshness.GroupState{.{ + .kind = .equity, + .peer_date = d, + .freshness = null, + .dated = 1, + }}; + var report = zfin.freshness.Report{ + .stale = &.{}, + .far_behind = &.{}, + .orphans = &.{}, + .missing = &.{}, + .groups = @constCast(lonely[0..]), + }; + try std.testing.expectEqual(@as(?zfin.Date, null), groupPeerDate(report, .equity)); + + // Two dated entries make a comparison possible. + const peers = [_]zfin.freshness.GroupState{.{ + .kind = .equity, + .peer_date = d, + .freshness = null, + .dated = 2, + }}; + report.groups = @constCast(peers[0..]); + try std.testing.expectEqual(@as(?zfin.Date, d), groupPeerDate(report, .equity)); + + // A kind with no group at all is not an error, just unanswerable. + try std.testing.expectEqual(@as(?zfin.Date, null), groupPeerDate(report, .mutual_fund)); +} diff --git a/src/main.zig b/src/main.zig index 600a143..90f6b18 100644 --- a/src/main.zig +++ b/src/main.zig @@ -7,1759 +7,15 @@ //! See GET /help for endpoint documentation. const std = @import("std"); -const zfin = @import("zfin"); const httpz = @import("httpz"); const build_options = @import("build_options"); const version = build_options.version; const log = std.log.scoped(.@"zfin-server"); -// ── App ────────────────────────────────────────────────────── - -const App = struct { - io: std.Io, - environ: *const std.process.Environ.Map, - allocator: std.mem.Allocator, - config: zfin.Config, - svc: zfin.DataService, - /// Threshold in milliseconds above which a request is logged - /// as slow. Tunable via `ZFIN_SERVER_SLOW_MS` env var; defaults - /// to 500ms. Captured once at App.init so the dispatch hot path - /// doesn't re-parse on every request. - slow_threshold_ms: u64, - /// Optional shared API key. When set (via `ZFIN_SERVER_API_KEY`), - /// every endpoint except the public surface (`/`, `/help`, and - /// `/:symbol/returns`) requires a matching key, supplied either as - /// an `X-API-Key` header or an `api_key` query parameter. When null - /// (env var unset or empty) the server is fully open - this - /// soft cutover lets the key roll out to clients before enforcement - /// is switched on. Captured once at init. - api_key: ?[]const u8, - /// Serializes the portfolio read-modify-write across concurrent - /// watchlist adds (httpz dispatches requests on multiple threads). - /// Without it two simultaneous adds could both read the old file and - /// the second writer would clobber the first's new symbol. - watch_mutex: std.Io.Mutex = .init, - - fn init(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) App { - const config = zfin.Config.fromEnv(io, allocator, environ); - const svc = zfin.DataService.init(io, allocator, config); - const slow_threshold_ms = if (environ.get("ZFIN_SERVER_SLOW_MS")) |s| - std.fmt.parseInt(u64, s, 10) catch 500 - else - 500; - // Treat an empty value as unset so `ZFIN_SERVER_API_KEY=` can't - // accidentally enable enforcement with an empty (never-matching) key. - const api_key: ?[]const u8 = if (environ.get("ZFIN_SERVER_API_KEY")) |v| - (if (v.len == 0) null else v) - else - null; - return .{ - .io = io, - .environ = environ, - .allocator = allocator, - .config = config, - .svc = svc, - .slow_threshold_ms = slow_threshold_ms, - .api_key = api_key, - }; - } - - fn deinit(self: *App) void { - self.svc.deinit(); - self.config.deinit(); - } - - /// httpz dispatch hook: every request flows through here so we - /// have a single place to wrap timing and error logging without - /// modifying every handler. Slow requests (above - /// `slow_threshold_ms`) and error responses (status >= 400) - /// emit a structured stderr line; everything else stays silent. - pub fn dispatch(self: *App, action: httpz.Action(*App), req: *httpz.Request, res: *httpz.Response) !void { - // wall-clock required: per-request elapsed for slow-request - // logging. `.awake` (monotonic) avoids spurious negatives - // on system clock skew. - const start_ns = std.Io.Timestamp.now(self.io, .awake).nanoseconds; - - // Registered before the gate and the action so every exit path - - // a 401 from the gate, an error from the action, or a normal - // response - emits the slow/error log line. - defer { - const elapsed_ns = std.Io.Timestamp.now(self.io, .awake).nanoseconds - start_ns; - const elapsed_ms: u64 = @intCast(@divTrunc(elapsed_ns, std.time.ns_per_ms)); - if (shouldLogRequest(elapsed_ms, res.status, self.slow_threshold_ms)) { - // wall-clock required: ts in stderr line lets the - // operator correlate slow requests with cron / system - // events using `date -d @`. - const ts = std.Io.Timestamp.now(self.io, .real).toSeconds(); - log.warn("ts={d} elapsed_ms={d} status={d} method={s} path={s}", .{ - ts, - elapsed_ms, - res.status, - @tagName(req.method), - req.url.path, - }); - } - } - - // API-key gate. Soft cutover: only enforced when a key is - // configured. The public surface (`/`, `/help`, and the - // LibreOffice `/:symbol/returns` endpoint) is never gated. - if (self.api_key) |expected| { - if (!pathIsPublic(req.url.path) and !providedKeyMatches(req, expected)) { - res.status = 401; - res.content_type = httpz.ContentType.TEXT; - res.body = "Unauthorized: missing or invalid API key\n"; - return; - } - } - - try action(self, req, res); - } -}; - -/// Pure predicate: should this request emit a stderr log line? -/// Logs slow successes (above threshold) and any error response. -fn shouldLogRequest(elapsed_ms: u64, status: u16, threshold_ms: u64) bool { - return elapsed_ms > threshold_ms or status >= 400; -} - -/// Pure predicate: is this request path part of the public surface that -/// never requires an API key? Public = the landing page, the help text, -/// and the LibreOffice returns endpoint (`/:symbol/returns`, including -/// its `?fmt=xml` and `?watch=true` variants - the query string is not -/// part of the path). Everything else (raw SRF cache files, quotes, the -/// symbol list, EDGAR maps, entity facts) is gated. -fn pathIsPublic(path: []const u8) bool { - if (std.mem.eql(u8, path, "/")) return true; - if (std.mem.eql(u8, path, "/help")) return true; - return isSymbolReturnsPath(path); -} - -/// True only for paths shaped exactly like `//returns`: a single -/// non-empty symbol segment followed by the literal `returns`. Guards -/// against `/returns` (no symbol) and multi-segment look-alikes such as -/// `/a/b/returns` or `/AAPL/returns/extra`. -fn isSymbolReturnsPath(path: []const u8) bool { - const suffix = "/returns"; - if (path.len <= suffix.len) return false; - if (!std.mem.endsWith(u8, path, suffix)) return false; - if (path[0] != '/') return false; - const symbol = path[1 .. path.len - suffix.len]; - if (symbol.len == 0) return false; - if (std.mem.indexOfScalar(u8, symbol, '/') != null) return false; - return true; -} - -/// Length-checked equality of a caller-supplied key against the -/// configured one. The threat model is casual/incidental traffic, not a -/// timing-attack adversary, so plain `mem.eql` is sufficient. A null -/// (absent header/param) never matches. -fn keyMatches(provided: ?[]const u8, expected: []const u8) bool { - const p = provided orelse return false; - return std.mem.eql(u8, p, expected); -} - -/// Read the caller-supplied API key from the `X-API-Key` header -/// (preferred) or an `api_key` query parameter (curl convenience) and -/// compare against the configured key. httpz wants the header name in -/// lowercase. A malformed query string is treated as "no key". -fn providedKeyMatches(req: *httpz.Request, expected: []const u8) bool { - if (keyMatches(req.header("x-api-key"), expected)) return true; - const q = req.query() catch return false; - return keyMatches(q.get("api_key"), expected); -} - -/// Case-insensitive User-Agent substrings permitted to add to the -/// watchlist via the public `/:symbol/returns?watch=true` path. This is -/// obscurity-grade (a UA is trivially spoofable) and matches the -/// casual-traffic threat model: it keeps crawlers and stray browsers -/// from growing the tracked set - and thus the recurring cron-refresh -/// load - while letting the non-technical user's LibreOffice WEBSERVICE -/// calls through. The authenticated `/:symbol/watch` route bypasses this -/// (a valid API key is a stronger signal than any UA). -/// -/// Confirmed empirically - LibreOffice's WEBSERVICE sends e.g. -/// "LibreOffice 24.2.7.2 denylistedbackend/8.5.0 OpenSSL/3.0.13" -/// (it also fires a WebDAV OPTIONS preflight that 404s harmlessly; the -/// real GET carries the same User-Agent). Matching the version-agnostic -/// "LibreOffice" token keeps this robust across releases. -const watch_user_agents = [_][]const u8{"LibreOffice"}; - -/// True if `ua` matches one of `watch_user_agents` (case-insensitive -/// substring). A null/absent User-Agent never matches. -fn userAgentMayWatch(ua: ?[]const u8) bool { - const agent = ua orelse return false; - for (watch_user_agents) |needle| { - if (std.ascii.indexOfIgnoreCase(agent, needle) != null) return true; - } - return false; -} - -/// Sanity gate for symbols entering the tracked set (and thus recurring -/// cron load): non-empty, <=16 chars, and only the characters real -/// tickers use - uppercase letters, digits, and `.`/`-` for class -/// shares. Not a real ticker validator; just enough to keep junk like an -/// over-long or path-shaped segment out of the portfolio file. Symbols -/// reach here already upper-cased by `upperDupe`. -fn isPlausibleSymbol(sym: []const u8) bool { - if (sym.len == 0 or sym.len > 16) return false; - for (sym) |c| { - const ok = (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '.' or c == '-'; - if (!ok) return false; - } - return true; -} - -// ── Route handlers ─────────────────────────────────────────── - -fn handleIndex(_: *App, _: *httpz.Request, res: *httpz.Response) !void { - res.content_type = httpz.ContentType.HTML; - res.body = - \\ - \\zfin-server - \\ - \\

zfin-server

- \\

This is a financial data API server. Not intended for browser use.

- \\

See /help for endpoint documentation.

- \\ - ; -} - -fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void { - res.content_type = httpz.ContentType.TEXT; - res.body = "zfin-server " ++ version ++ " - financial data API" ++ - \\ - \\ - \\Endpoints: - \\ GET /{SYMBOL}/returns Trailing 1/3/5/10yr returns (JSON) - \\ GET /{SYMBOL}/returns?fmt=xml Trailing returns (XML, for LibreCalc) - \\ GET /{SYMBOL}/watch Add SYMBOL to the watchlist (authenticated) - \\ GET /{SYMBOL}/quote Latest quote (JSON) - \\ GET /{SYMBOL}/candles Raw SRF cache file - \\ GET /{SYMBOL}/candles_meta Candle freshness metadata (SRF) - \\ GET /{SYMBOL}/dividends Raw SRF cache file - \\ GET /{SYMBOL}/splits Raw SRF cache file - \\ GET /{SYMBOL}/earnings Raw SRF cache file - \\ GET /{SYMBOL}/options Raw SRF cache file - \\ GET /{SYMBOL}/classification Wikidata classification (SRF) - \\ GET /{SYMBOL}/etf_metrics EDGAR NPORT-P fund metrics (SRF; 404 for non-funds) - \\ GET /{CIK}/entity_facts EDGAR XBRL entity facts (SRF; CIK-keyed) - \\ GET /_edgar/tickers_funds EDGAR mutual-fund ticker map (SRF) - \\ GET /_edgar/tickers_companies EDGAR company ticker map (SRF) - \\ GET /symbols List of tracked symbols - \\ - \\Auth: - \\ All endpoints except /, /help, and /{SYMBOL}/returns require an - \\ API key when ZFIN_SERVER_API_KEY is set (X-API-Key header or - \\ ?api_key= query parameter). - \\ - \\Caching: - \\ SRF endpoints serve from the local cache; on a miss the server - \\ fetches once from the provider, fills the cache, then serves - \\ (404 only if that fetch also fails). - \\ - \\Watchlist (add a symbol to the cron refresh set): - \\ GET /{SYMBOL}/watch authenticated; for your own tooling - \\ GET /{SYMBOL}/returns?watch=true public, but only LibreOffice's - \\ WEBSERVICE User-Agent is honored - \\ - \\Returns fields: - \\ lastClose Last closing price - \\ trailing{1,3,5,10}YearReturn Total return with dividend reinvestment - \\ price{1,3,5,10}YearReturn Price-only return (from adjusted close) - \\ volatility Longest-term available annualized volatility - \\ volatilityTerm Period (years) of the volatility field - \\ volatility{1,3,5,10}Year Per-period annualized volatility - \\ - \\XML example (LibreCalc): - \\ =FILTERXML(WEBSERVICE("http://host/AAPL/returns?fmt=xml"),"//total10YearReturn") - \\ - ; -} - -fn handleSymbols(app: *App, _: *httpz.Request, res: *httpz.Response) !void { - const arena = res.arena; - const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; - - const file_data = std.Io.Dir.cwd().readFileAlloc(app.io, portfolio_path, arena, .limited(10 * 1024 * 1024)) catch { - res.content_type = httpz.ContentType.JSON; - res.body = "[]"; - return; - }; - - var portfolio = zfin.cache.deserializePortfolio(arena, file_data) catch { - res.content_type = httpz.ContentType.JSON; - res.body = "[]"; - return; - }; - defer portfolio.deinit(); - - // Collect unique symbols - var seen = std.StringHashMap(void).init(arena); - var symbols = std.ArrayList([]const u8).empty; - for (portfolio.lots) |lot| { - if (lot.symbol.len == 0) continue; - if (seen.contains(lot.symbol)) continue; - try seen.put(lot.symbol, {}); - try symbols.append(arena, lot.symbol); - } - - // Build JSON array - var aw: std.Io.Writer.Allocating = .init(arena); - try aw.writer.writeByte('['); - for (symbols.items, 0..) |sym, i| { - if (i > 0) try aw.writer.writeByte(','); - try aw.writer.print("\"{s}\"", .{sym}); - } - try aw.writer.writeByte(']'); - - res.content_type = httpz.ContentType.JSON; - res.body = try aw.toOwnedSlice(); -} - -fn handleReturns(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - const raw_symbol = req.param("symbol") orelse { - res.status = 404; - res.body = "Missing symbol"; - return; - }; - const arena = res.arena; - const symbol = try upperDupe(arena, raw_symbol); - - // Auto-add to watchlist if requested. UA-gated (obscurity) so only - // LibreOffice WEBSERVICE calls - not random browsers/crawlers - can - // grow the tracked set via this public endpoint. Best-effort: the - // returns response below is served regardless of whether the add ran. - const q = try req.query(); - if (q.get("watch")) |w| { - if (std.ascii.eqlIgnoreCase(w, "true")) { - if (userAgentMayWatch(req.header("user-agent"))) { - appendWatchSymbol(app, symbol) catch |err| { - log.warn("failed to append watch symbol {s}: {t}", .{ symbol, err }); - }; - } else { - log.debug("watch add for {s} skipped: User-Agent not allowlisted", .{symbol}); - } - } - } - - const result = app.svc.getTrailingReturns(symbol, .{}) catch { - res.status = 404; - res.body = "Symbol not found or fetch failed"; - return; - }; - defer app.allocator.free(result.candles); - if (result.dividends) |divs| { - defer zfin.Dividend.freeSlice(app.allocator, divs); - } - - const candles = result.candles; - if (candles.len == 0) { - res.status = 404; - res.body = "No candle data"; - return; - } - - const last_close = candles[candles.len - 1].close; - var date_buf: [10]u8 = undefined; - const date_str = try std.fmt.bufPrint(&date_buf, "{f}", .{candles[candles.len - 1].date}); - - // Price-only returns (split-adjusted, NOT dividend-adjusted — - // see analytics/performance.zig:trailingReturnsPriceOnly). - // Matches the "price return" numbers public sources publish - // (Yahoo chart-bar, FMP, Barchart, Fidelity stock pages). - const p1y = if (result.asof_price.one_year) |r| r.annualized_return else null; - const p3y = if (result.asof_price.three_year) |r| r.annualized_return else null; - const p5y = if (result.asof_price.five_year) |r| r.annualized_return else null; - const p10y = if (result.asof_price.ten_year) |r| r.annualized_return else null; - - // Total returns (dividend reinvestment when dividends are - // available; falls back to adj_close-based total return when - // dividend records are missing). Matches Morningstar - // "Trailing Returns" / Yahoo "Performance Overview" / Koyfin - // "Total Return". - const total = result.asof_total orelse result.asof_price; - const t1y = if (total.one_year) |r| r.annualized_return else null; - const t3y = if (total.three_year) |r| r.annualized_return else null; - const t5y = if (total.five_year) |r| r.annualized_return else null; - const t10y = if (total.ten_year) |r| r.annualized_return else null; - - // Per-period volatility - const risk = zfin.risk.trailingRisk(candles); - const v1y = if (risk.one_year) |r| r.volatility else null; - const v3y = if (risk.three_year) |r| r.volatility else null; - const v5y = if (risk.five_year) |r| r.volatility else null; - const v10y = if (risk.ten_year) |r| r.volatility else null; - - // Longest-term volatility convenience fields - const vol_best = v10y orelse v5y orelse v3y orelse v1y; - const vol_term: ?u8 = if (v10y != null) 10 else if (v5y != null) 5 else if (v3y != null) 3 else if (v1y != null) 1 else null; - - // Check if XML requested - if (q.get("fmt")) |fmt| { - if (std.ascii.eqlIgnoreCase(fmt, "xml")) { - res.content_type = httpz.ContentType.XML; - res.body = try std.fmt.allocPrint(arena, - \\ - \\ {s} - \\ {s} - \\ {d:.2} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ {s} - \\ - \\ - , .{ - symbol, - date_str, - last_close, - fmtPct(arena, t1y), - fmtPct(arena, t3y), - fmtPct(arena, t5y), - fmtPct(arena, t10y), - fmtPct(arena, p1y), - fmtPct(arena, p3y), - fmtPct(arena, p5y), - fmtPct(arena, p10y), - fmtPct(arena, vol_best), - fmtInt(arena, vol_term), - fmtPct(arena, v1y), - fmtPct(arena, v3y), - fmtPct(arena, v5y), - fmtPct(arena, v10y), - }); - return; - } - } - - res.content_type = httpz.ContentType.JSON; - res.body = try std.fmt.allocPrint(arena, - \\{{"ticker":"{s}","returnDate":"{s}","lastClose":{d:.2},"trailing1YearReturn":{s},"trailing3YearReturn":{s},"trailing5YearReturn":{s},"trailing10YearReturn":{s},"price1YearReturn":{s},"price3YearReturn":{s},"price5YearReturn":{s},"price10YearReturn":{s},"volatility":{s},"volatilityTerm":{s},"volatility1Year":{s},"volatility3Year":{s},"volatility5Year":{s},"volatility10Year":{s}}} - , .{ - symbol, - date_str, - last_close, - fmtPct(arena, t1y), - fmtPct(arena, t3y), - fmtPct(arena, t5y), - fmtPct(arena, t10y), - fmtPct(arena, p1y), - fmtPct(arena, p3y), - fmtPct(arena, p5y), - fmtPct(arena, p10y), - fmtPct(arena, vol_best), - fmtInt(arena, vol_term), - fmtPct(arena, v1y), - fmtPct(arena, v3y), - fmtPct(arena, v5y), - fmtPct(arena, v10y), - }); -} - -/// Authenticated explicit watchlist add. Not on the public allowlist, so -/// `dispatch` requires the API key - unlike the UA-gated `?watch=true` -/// path, this is for the operator's own tooling deliberately growing the -/// tracked set (and accepting the recurring cron-refresh cost). -fn handleWatch(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - const raw_symbol = req.param("symbol") orelse { - res.status = 400; - res.body = "Missing symbol"; - return; - }; - const arena = res.arena; - const symbol = try upperDupe(arena, raw_symbol); - - appendWatchSymbol(app, symbol) catch |err| switch (err) { - error.InvalidSymbol => { - res.status = 400; - res.body = "Invalid symbol"; - return; - }, - else => { - res.status = 500; - res.body = try std.fmt.allocPrint(arena, "Failed to add watch symbol: {t}", .{err}); - return; - }, - }; - - res.content_type = httpz.ContentType.JSON; - res.body = try std.fmt.allocPrint(arena, "{{\"symbol\":\"{s}\",\"watched\":true}}", .{symbol}); -} - -fn handleQuote(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - const raw_symbol = req.param("symbol") orelse { - res.status = 400; - res.body = "Missing symbol"; - return; - }; - const arena = res.arena; - const symbol = try upperDupe(arena, raw_symbol); - - const q = app.svc.getQuote(symbol, .{}) catch { - res.status = 404; - res.body = "Quote not available"; - return; - }; - - res.content_type = httpz.ContentType.JSON; - res.body = try std.fmt.allocPrint(arena, - \\{{"symbol":"{s}","close":{d:.2},"open":{d:.2},"high":{d:.2},"low":{d:.2},"volume":{d},"previous_close":{d:.2}}} - , .{ symbol, q.close, q.open, q.high, q.low, q.volume, q.previous_close }); -} - -/// Identifies which `DataService` fetch to run when a served SRF file is -/// absent (see `fetchOnMiss`). Kept separate from `zfin.cache.DataType` -/// because the mapping isn't 1:1 - both `candles_daily.srf` and -/// `candles_meta.srf` are populated by a single `getCandles` call. -const SrfKind = enum { - candles, - dividends, - splits, - earnings, - options, - classification, - etf_metrics, - entity_facts, - tickers_funds, - tickers_companies, -}; - -/// What the server INTENDS for a symbol, as opposed to the data it happens to -/// hold. Every other endpoint answers the second question; nothing answered the -/// first, which is how a symbol the refresh loop never touches sat six weeks -/// behind while being served to clients as though it were maintained. -const SymbolDiagnostics = struct { - /// Will the refresh loop fetch this symbol? See `collectRefreshSymbols`. - tracked: bool, - /// Newest cached bar, or null when there is no candle meta at all. - last_date: ?zfin.Date, - /// When the cached copy was written (Unix seconds), null when uncached. - created: ?i64, - /// Consecutive transient provider failures on the primary provider. - fail_count: u8, - /// Is the cached copy stamped fresh by its own `#!expires=`? Reported rather - /// than the raw expiry because `fresh` alongside a non-zero `days_behind` is - /// precisely the pathology that started this: a copy stamped good until - /// tomorrow while sitting days behind its peers. The raw directive is still - /// on the wire via `/:symbol/candles_meta` for anyone who wants it. - fresh: bool, - /// Newest bar held by any same-kind peer in this cache, or null when there is - /// no peer to compare against. - peer_date: ?zfin.Date, - /// Calendar days behind `peer_date`; 0 when not behind or incomparable. - days_behind: i64, -}; - -/// Calendar days `last` sits behind `peer`, or 0 when it is not behind. -/// -/// Computed here rather than read out of the sweep's findings. `scan` emits a -/// Finding only for a TRACKED symbol (untracked ones divert to `orphans`), so -/// reading `days_behind` from there returned 0 for every untracked symbol - -/// exactly the class this endpoint exists to expose. An untracked symbol sitting -/// 43 days behind, reported as `days_behind:0`, is the worst available answer: -/// it reads as "current" for the one case nobody is watching. -/// -/// Calendar days, truncated, to match `zfin.freshness.Finding.days_behind` - the -/// magnitude an operator weighs against `max_normal_lag_days`. -fn daysBehind(last: ?zfin.Date, peer: ?zfin.Date) i64 { - const l = last orelse return 0; - const p = peer orelse return 0; - if (!l.lessThan(p)) return 0; - return @divTrunc(p.toEpoch() - l.toEpoch(), std.time.s_per_day); -} - -/// The peer reference date for `kind`, or null when the group cannot yield a -/// comparison. `conclusive()` is the gate: with a single cached symbol of a kind -/// there are no peers, and reporting that symbol's own date as its `peer_date` -/// would manufacture agreement out of nothing. -fn groupPeerDate(report: zfin.freshness.Report, kind: zfin.market.InstrumentKind) ?zfin.Date { - for (report.groups) |g| { - if (g.kind != kind) continue; - if (!g.conclusive()) return null; - return g.peer_date; - } - return null; -} - -fn handleDiagnostics(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - const raw_symbol = req.param("symbol") orelse { - res.status = 400; - res.body = "Missing symbol"; - return; - }; - const arena = res.arena; - const symbol = try upperDupe(arena, raw_symbol); - - var store = zfin.cache.Store.init(app.io, arena, app.config.cache_dir); - - // An unreadable or unparseable portfolio leaves the set empty, which reports - // `tracked:false` - honest, because a refresh run reading the same file would - // fetch nothing either. - // - // NO `portfolio.deinit()` here, deliberately. The set's keys BORROW from the - // parsed lots, and every `contains` happens below this block, so freeing the - // portfolio dangles them - observed as all 25 tracked symbols reading as - // untracked. The request arena owns this memory and releases it with the - // response. This is the same trap `zfin`'s `trackedSymbols` documents: "the - // scoped-defer-frees version dangled its keys before the caller read them, - // which made every cached symbol look untracked." - var tracked = std.StringHashMap(void).init(arena); - const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; - if (std.Io.Dir.cwd().readFileAlloc(app.io, portfolio_path, arena, .limited(10 * 1024 * 1024))) |data| { - if (zfin.cache.deserializePortfolio(arena, data)) |parsed| { - try collectRefreshSymbols(&tracked, parsed.lots); - } else |_| {} - } else |_| {} - - // The peer sweep. `collect` + `scan` rather than a local "newest of this - // kind" loop: the definition of behind-its-peers lives in one place, and this - // endpoint exists to report that definition, not a second opinion on it. - const keys = store.cacheKeys(arena) catch &.{}; - const entries = try zfin.freshness.collect(arena, &store, keys, &tracked, &.{}); - // wall-clock required: peer freshness is judged against the market calendar. - const now_s = std.Io.Timestamp.now(app.io, .real).toSeconds(); - const report = try zfin.freshness.scan(arena, entries, now_s); - - const meta = store.readCandleMeta(symbol); - const last_date: ?zfin.Date = if (meta) |m| m.meta.last_date else null; - const peer_date = groupPeerDate(report, zfin.market.classify(symbol)); - const d = SymbolDiagnostics{ - .tracked = tracked.contains(symbol), - .last_date = last_date, - .created = if (meta) |m| m.created else null, - .fail_count = if (meta) |m| m.meta.fail_count else 0, - .fresh = store.isCandleMetaFresh(symbol), - .peer_date = peer_date, - .days_behind = daysBehind(last_date, peer_date), - }; - - var aw: std.Io.Writer.Allocating = .init(arena); - try aw.writer.print( - \\{{"symbol":"{s}","tracked":{},"fresh":{},"fail_count":{d},"days_behind":{d} - , .{ symbol, d.tracked, d.fresh, d.fail_count, d.days_behind }); - // Null rather than a sentinel date for the absent cases: a client must be - // able to tell "no cached bar" from "a bar dated the epoch". - if (d.last_date) |ld| { - try aw.writer.print(",\"last_date\":\"{f}\"", .{ld}); - } else { - try aw.writer.writeAll(",\"last_date\":null"); - } - if (d.peer_date) |pd| { - try aw.writer.print(",\"peer_date\":\"{f}\"", .{pd}); - } else { - try aw.writer.writeAll(",\"peer_date\":null"); - } - if (d.created) |c| { - try aw.writer.print(",\"created\":{d}", .{c}); - } else { - try aw.writer.writeAll(",\"created\":null"); - } - try aw.writer.writeByte('}'); - - res.content_type = httpz.ContentType.JSON; - res.body = aw.written(); -} - -fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8, kind: SrfKind) !void { - return handleSrfFileByKey(app, req, res, "symbol", filename, kind); -} - -/// Generalized SRF cache-file passthrough: reads -/// `//` where `` is whatever URL -/// parameter `key_param` resolves to. The default `handleSrfFile` -/// uses `"symbol"`; CIK-keyed routes (e.g. `/:cik/entity_facts`) -/// pass `"cik"` instead. The cache-key segment is uppercased -/// (safe for both symbols and zero-padded CIK digit strings). -fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8, kind: SrfKind) !void { - const raw_key = req.param(key_param) orelse { - res.status = 400; - res.body = "Missing key"; - return; - }; - const arena = res.arena; - const key = try upperDupe(arena, raw_key); - return serveSrfFile(app, res, key, filename, kind); -} - -/// Static-key SRF cache-file passthrough for routes that don't -/// take a path parameter (e.g. `/_edgar/tickers_funds` reads -/// `/_edgar/tickers_funds.srf` directly). The `key` -/// is a literal directory name; not uppercased because the -/// cache uses `_edgar` as-is. -fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void { - return serveSrfFile(app, res, key, filename, kind); -} - -/// Inner shared helper. Serves `//` as raw SRF -/// with a sha256 ETag. L2-cache contract: a *present* file is served -/// as-is even when stale - cron is the freshness authority, so reads -/// never trigger a refetch - while an *absent* file triggers a one-shot -/// provider fetch (`fetchOnMiss`) to populate it, after which we re-read -/// and serve. If the fetch still can't produce the file, we fall back to -/// the original 404. -fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void { - const arena = res.arena; - const path = try std.fs.path.join(arena, &.{ app.config.cache_dir, key, filename }); - - const content = readCacheFile(app, arena, path) orelse blk: { - // Cache miss -> fetch from the provider, fill the cache, re-read. - fetchOnMiss(app, key, kind); - break :blk readCacheFile(app, arena, path) orelse { - res.status = 404; - res.body = "Cache file not found"; - return; - }; - }; - - // Body integrity header: sha256 of the bytes we're about to send. - // Clients can use this to detect mid-stream truncation that Zig's - // std.http.Client.fetch silently accepts on the Content-Length path - // (a premature EOF from the transport bubbles up as EndOfStream and - // is swallowed as a normal end-of-body). Shaped as a standard - // `ETag` value so future conditional-request work gets it for free. - var hash: [std.crypto.hash.sha2.Sha256.digest_length]u8 = undefined; - std.crypto.hash.sha2.Sha256.hash(content, &hash, .{}); - var etag_buf: [std.crypto.hash.sha2.Sha256.digest_length * 2 + "\"sha256:\"".len]u8 = undefined; - const etag = try std.fmt.bufPrint(&etag_buf, "\"sha256:{x}\"", .{&hash}); - // httpz.Response.header borrows the value — duplicate into the - // per-request arena so the slice outlives `etag_buf`. - const etag_owned = try arena.dupe(u8, etag); - - res.content_type = httpz.ContentType.BINARY; - res.header("content-type", "application/x-srf"); - res.header("etag", etag_owned); - res.body = content; -} - -/// Read a cache file into the request arena, or null when it can't be -/// read. Absent is the common case; any read error is treated as a miss -/// (matches the pre-fetch-on-miss behavior of falling back to 404). -fn readCacheFile(app: *App, arena: std.mem.Allocator, path: []const u8) ?[]u8 { - return std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch null; -} - -/// Populate the cache for `key` by running the matching `DataService` -/// fetch, then discard the parsed result - the caller re-reads the -/// canonical bytes off disk. Best-effort and synchronous: any provider -/// error (NotFound, rate limit, auth, transient, parse) is logged and -/// swallowed so the caller falls back to a 404. -/// -/// Backpressure caveat: these fetches share zfin's rate limiter, so when -/// the token bucket is drained (e.g. mid-cron) this blocks the httpz -/// worker until a token frees. Accepted for now; a 202 + poll path is -/// the planned escape hatch if blocking becomes a problem. -fn fetchOnMiss(app: *App, key: []const u8, kind: SrfKind) void { - const svc = &app.svc; - switch (kind) { - // getCandles writes both candles_daily.srf and candles_meta.srf. - .candles => { - const r = svc.getCandles(key, .{}) catch |err| return logFetchMiss(key, kind, err); - r.deinit(); - }, - .dividends => { - const r = svc.getDividends(key, .{}) catch |err| return logFetchMiss(key, kind, err); - r.deinit(); - }, - .splits => { - const r = svc.getSplits(key, .{}) catch |err| return logFetchMiss(key, kind, err); - r.deinit(); - }, - .earnings => { - const r = svc.getEarnings(key, .{}) catch |err| return logFetchMiss(key, kind, err); - r.deinit(); - }, - .options => { - const r = svc.getOptions(key, .{}) catch |err| return logFetchMiss(key, kind, err); - r.deinit(); - }, - .classification => { - const r = svc.getClassification(key, .{}) catch |err| return logFetchMiss(key, kind, err); - r.deinit(); - }, - .etf_metrics => { - const r = svc.getEtfMetrics(key, .{}) catch |err| return logFetchMiss(key, kind, err); - r.deinit(); - }, - // `key` is the CIK here (resolved from the :cik route param). - .entity_facts => { - const r = svc.getEntityFacts(key, .{}) catch |err| return logFetchMiss(key, kind, err); - r.deinit(); - }, - .tickers_funds => { - var m = svc.loadMutualFundTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err); - m.deinit(); - }, - .tickers_companies => { - var m = svc.loadCompanyTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err); - m.deinit(); - }, - } -} - -/// Log a failed populate. NotFound is the normal "no such symbol / no -/// data" outcome (debug); everything else is operator-relevant (warn). -fn logFetchMiss(key: []const u8, kind: SrfKind, err: anyerror) void { - if (err == error.NotFound) { - log.info("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) }); - } else { - log.warn("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) }); - } -} - -fn handleCandles(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "candles_daily.srf", .candles); -} - -fn handleCandlesMeta(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "candles_meta.srf", .candles); -} - -fn handleDividends(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "dividends.srf", .dividends); -} - -fn handleSplits(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "splits.srf", .splits); -} - -fn handleEarnings(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "earnings.srf", .earnings); -} - -fn handleOptions(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "options.srf", .options); -} - -fn handleClassification(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "classification.srf", .classification); -} - -fn handleEtfMetrics(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "etf_metrics.srf", .etf_metrics); -} - -fn handleEntityFacts(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - // CIK-keyed route: cache layout is - // `//entity_facts.srf` (the CIK is the - // zero-padded 10-digit string Wikidata's P5531 emits). - return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf", .entity_facts); -} - -fn handleTickersFunds(app: *App, _: *httpz.Request, res: *httpz.Response) !void { - // Static-key route: `/_edgar/tickers_funds.srf` - // is a single file shared across all symbol lookups, not a - // per-symbol cache. - return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf", .tickers_funds); -} - -fn handleTickersCompanies(app: *App, _: *httpz.Request, res: *httpz.Response) !void { - return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf", .tickers_companies); -} - -// ── Helpers ────────────────────────────────────────────────── - -fn upperDupe(allocator: std.mem.Allocator, s: []const u8) ![]u8 { - const d = try allocator.dupe(u8, s); - for (d) |*c| c.* = std.ascii.toUpper(c.*); - return d; -} - -/// Print an inline rate-limit estimate tag like "[~14s] " before the -/// next fetch of `data_type`, then flush so the tag is visible before -/// the (possibly blocking) fetch runs. An interactive caller sees the -/// estimate, then the pause, then the result land on a single line; a -/// cron log just gets the tag inline. No-op when a token is available -/// (estimate 0) or the provider isn't instantiated yet. -/// -/// The number is the pause a *live* fetch would incur right now -- not -/// a promise that we will wait. A cache hit consumes no token and skips -/// the wait entirely (see the one-time legend at the top of a refresh -/// run). The estimate is read before the fetch, so it reflects the -/// pre-fetch bucket state. -fn printRateLimitTag(svc: *zfin.DataService, data_type: zfin.cache.DataType, stdout: *std.Io.Writer) !void { - if (svc.estimateWaitSeconds(data_type)) |wait| { - if (wait > 0) { - try stdout.print("[~{d}s] ", .{wait}); - try stdout.flush(); - } - } -} - -/// Per-data-type outcome tally for a refresh run. `fetched` vs `cached` -/// tracks network-vs-cache (useful for spotting cache expiry / TTL -/// tuning and for gauging load against provider rate limits); `na` is a -/// legitimate "no data for this symbol" outcome (NotFound, or an -/// entity_facts skip); `failed` is a hard error. -const TypeStat = struct { - fetched: usize = 0, - cached: usize = 0, - na: usize = 0, - failed: usize = 0, - - /// Record a successful fetch: network hit vs served-from-cache. - fn hit(self: *TypeStat, was_fetched: bool) void { - if (was_fetched) self.fetched += 1 else self.cached += 1; - } -}; - -/// Per-type tallies for the seven per-symbol data types a refresh -/// touches. Reported as the summary table at the end of a run. -const RefreshStats = struct { - candles: TypeStat = .{}, - dividends: TypeStat = .{}, - splits: TypeStat = .{}, - earnings: TypeStat = .{}, - classification: TypeStat = .{}, - etf_metrics: TypeStat = .{}, - entity_facts: TypeStat = .{}, -}; - -/// Symbol-level status partition. Precedence, highest first: -/// failed > lagging > overdue > current -- the same precedence -/// `refreshExit` uses, so the printed counts always agree with the -/// process exit code (lagging is NOT folded into "current"). -const SymbolCounts = struct { - current: usize = 0, - lagging: usize = 0, - overdue: usize = 0, - failed: usize = 0, -}; - -/// Candle freshness for one symbol this run. Set during the candle -/// check; stays `.current` if candles were empty or the symbol failed -/// before the check (a failed symbol is bucketed as `failed` regardless). -const Freshness = enum { current, lagging, overdue }; - -/// Record a hard failure of `tag` for the current symbol: bump that -/// type's failed counter and remember the tag, in one call, so the two -/// can't drift apart. The remembered tags become the end-of-symbol -/// `failed:` entry, e.g. "SYM (candles, earnings)". Each data-type block -/// runs once per symbol and `fail_types` is cleared per symbol, so a tag -/// is recorded at most once per symbol (no duplicates). -fn recordFailure(stat: *TypeStat, fail_types: *std.ArrayList([]const u8), allocator: std.mem.Allocator, tag: []const u8) !void { - stat.failed += 1; - try fail_types.append(allocator, tag); -} - -/// Print one right-aligned data-type row of the summary table. Shares -/// its width specifiers with the header in `refresh` so columns line up. -fn printStatRow(stdout: *std.Io.Writer, name: []const u8, s: TypeStat) !void { - try stdout.print(" {s:<14}{d:>7}{d:>8}{d:>5}{d:>8}\n", .{ name, s.fetched, s.cached, s.na, s.failed }); -} - -/// Print a `label: a, b, c` line, or `label: (none)` when empty. Always -/// printed (even when empty) so the failed/lagging/overdue lines are -/// reliable grep targets. -fn printSymbolList(stdout: *std.Io.Writer, label: []const u8, items: []const []const u8) !void { - try stdout.print(" {s:<9}", .{label}); - if (items.len == 0) { - try stdout.print("(none)\n", .{}); - return; - } - for (items, 0..) |it, i| { - if (i > 0) try stdout.print(", ", .{}); - try stdout.print("{s}", .{it}); - } - try stdout.print("\n", .{}); -} - -/// Format as percentage (e.g., 0.1234 -> "12.34000"), or "null" if absent. -fn fmtPct(arena: std.mem.Allocator, value: ?f64) []const u8 { - if (value) |v| return std.fmt.allocPrint(arena, "{d:.5}", .{v * 100.0}) catch "null"; - return "null"; -} - -/// Format an optional integer, or "null" if absent. -fn fmtInt(arena: std.mem.Allocator, value: ?u8) []const u8 { - if (value) |v| return std.fmt.allocPrint(arena, "{d}", .{v}) catch "null"; - return "null"; -} - -/// Append a watch lot for `symbol` to the portfolio SRF file, unless it -/// is already tracked. Serialized across requests via `app.watch_mutex` -/// and written atomically, so a concurrent add or a mid-write crash can't -/// clobber or truncate the portfolio file. Returns `error.InvalidSymbol` -/// for implausible symbols; callers decide how loud to be. -fn appendWatchSymbol(app: *App, symbol: []const u8) !void { - if (!isPlausibleSymbol(symbol)) return error.InvalidSymbol; - - const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; - const allocator = app.allocator; - const io = app.io; - - // Serialize the whole read-modify-write so concurrent adds don't lose - // updates (a last-writer-wins race would otherwise drop a symbol). - // Uncancelable so a canceled request can't abandon a half-done write. - app.watch_mutex.lockUncancelable(io); - defer app.watch_mutex.unlock(io); - - // Read and deserialize existing portfolio (or start empty) - const file_data = std.Io.Dir.cwd().readFileAlloc(io, portfolio_path, allocator, .limited(10 * 1024 * 1024)) catch |err| { - if (err == error.FileNotFound) return writeNewPortfolio(io, allocator, portfolio_path, symbol); - return err; - }; - defer allocator.free(file_data); - - var portfolio = zfin.cache.deserializePortfolio(allocator, file_data) catch return; - defer portfolio.deinit(); - - // Check if symbol already tracked - for (portfolio.lots) |lot| { - if (std.ascii.eqlIgnoreCase(lot.symbol, symbol)) return; - } - - // Build new lot list with the watch entry appended - const new_lots = try allocator.alloc(zfin.Lot, portfolio.lots.len + 1); - defer allocator.free(new_lots); - @memcpy(new_lots[0..portfolio.lots.len], portfolio.lots); - new_lots[portfolio.lots.len] = .{ - .symbol = symbol, - .shares = 0, - .open_date = zfin.Date.fromYmd(2026, 1, 1), - .open_price = 0, - .security_type = .watch, - }; - - // Serialize and write atomically. - const output = try zfin.cache.serializePortfolio(allocator, new_lots); - defer allocator.free(output); - try writeFileAtomic(io, allocator, portfolio_path, output); - - log.info("added watch symbol {s} to {s}", .{ symbol, portfolio_path }); -} - -fn writeNewPortfolio(io: std.Io, allocator: std.mem.Allocator, path: []const u8, symbol: []const u8) !void { - const lot = [_]zfin.Lot{.{ - .symbol = symbol, - .shares = 0, - .open_date = zfin.Date.fromYmd(2026, 1, 1), - .open_price = 0, - .security_type = .watch, - }}; - const output = try zfin.cache.serializePortfolio(allocator, &lot); - defer allocator.free(output); - try writeFileAtomic(io, allocator, path, output); - - log.info("created {s} with watch symbol {s}", .{ path, symbol }); -} - -/// Crash-safe file write: write to `.tmp`, fsync, then rename over -/// `path`. A mid-write crash leaves the prior file intact rather than a -/// truncated portfolio. (zfin's internal `atomic.writeFileAtomic` isn't -/// part of its public module, so we keep a small local copy.) -fn writeFileAtomic(io: std.Io, allocator: std.mem.Allocator, path: []const u8, bytes: []const u8) !void { - const tmp_path = try std.fmt.allocPrint(allocator, "{s}.tmp", .{path}); - defer allocator.free(tmp_path); - - { - var tmp_file = try std.Io.Dir.cwd().createFile(io, tmp_path, .{ .truncate = true, .exclusive = false }); - errdefer { - tmp_file.close(io); - std.Io.Dir.cwd().deleteFile(io, tmp_path) catch |err| { - log.debug("atomic write cleanup deleteFile({s}): {t}", .{ tmp_path, err }); - }; - } - try tmp_file.writeStreamingAll(io, bytes); - // fsync so the data is durable before the rename appears. - try tmp_file.sync(io); - tmp_file.close(io); - } - - std.Io.Dir.cwd().rename(tmp_path, std.Io.Dir.cwd(), path, io) catch |err| { - std.Io.Dir.cwd().deleteFile(io, tmp_path) catch |del_err| { - log.debug("atomic write cleanup deleteFile({s}): {t}", .{ tmp_path, del_err }); - }; - return err; - }; -} - -// ── Refresh command ────────────────────────────────────────── - -/// The symbols the refresh loop will fetch: `.stock` and `.watch` lots, keyed by -/// `priceSymbol()` so a `ticker::` alias resolves the same way a fetch does. -/// -/// Extracted rather than inlined because `/:symbol/diagnostics` reports a -/// `tracked` flag, and a `tracked` that meant anything other than "refresh will -/// fetch this" would be worse than not reporting it at all - an operator would -/// read it as a promise the loop never made. One definition, two callers. -fn collectRefreshSymbols(out: *std.StringHashMap(void), lots: []const zfin.Lot) !void { - for (lots) |lot| { - if (lot.security_type != .stock and lot.security_type != .watch) continue; - if (lot.symbol.len == 0) continue; - const sym = lot.priceSymbol(); - if (!out.contains(sym)) try out.put(sym, {}); - } -} - -fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) !u8 { - var config = zfin.Config.fromEnv(io, allocator, environ); - defer config.deinit(); - var svc = zfin.DataService.init(io, allocator, config); - defer svc.deinit(); - - // wall-clock required: the provider-lag check compares each symbol's - // newest cached bar against the most recent session the market should - // already have data for (see zfin.market.candleFreshness). Captured - // once so the whole run shares a consistent "now". - const now_s = std.Io.Timestamp.now(io, .real).toSeconds(); - - // wall-clock required: end-to-end run duration for the summary line. - // .awake (monotonic) avoids skew-induced negatives like dispatch does. - const start_ns = std.Io.Timestamp.now(io, .awake).nanoseconds; - - const portfolio_path = environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; - - const data = std.Io.Dir.cwd().readFileAlloc(io, portfolio_path, allocator, .limited(10 * 1024 * 1024)) catch { - log.err("failed to read portfolio: {s}", .{portfolio_path}); - return error.ReadFailed; - }; - defer allocator.free(data); - - var portfolio = zfin.cache.deserializePortfolio(allocator, data) catch { - log.err("failed to parse portfolio", .{}); - return error.ParseFailed; - }; - defer portfolio.deinit(); - - var symbols = std.StringHashMap(void).init(allocator); - defer symbols.deinit(); - try collectRefreshSymbols(&symbols, portfolio.lots); - - const stdout_file = std.Io.File.stdout(); - var buf: [4096]u8 = undefined; - var writer = stdout_file.writer(io, &buf); - const stdout = &writer.interface; - - try stdout.print("zfin-server {s}\n", .{version}); - try stdout.print("Refreshing {d} symbols from {s}\n", .{ symbols.count(), portfolio_path }); - try stdout.print("note: [~Ns] = est. pause before the next live fetch (bucket empty); cache hits skip it\n", .{}); - try stdout.flush(); - - var counts: SymbolCounts = .{}; - var stats: RefreshStats = .{}; - // Symbols whose candles came back WITHOUT a provider call this run: either - // the TTL was still fresh, or a server sync satisfied the request - - // `service.zig` returns `.cached` for both (see the sync at its line 934). - // These are the only symbols a forced refresh in the post-pass sweep can - // help; for every other finding the provider was asked seconds ago and did - // not have the bar, so asking again in the same run is pure quota burn and - // bypasses the very TTL pacing `expiryAfterFetch` exists to get right. - // - // Keys borrow from `portfolio`, whose `deinit` is function-scoped and so - // outlives the sweep below. Block-scoping that deinit is what dangled the - // equivalent keys in `handleDiagnostics`. - var unfetched = std.StringHashMap(void).init(allocator); - defer unfetched.deinit(); - var failed_list = std.ArrayList([]const u8).empty; - var lagging_list = std.ArrayList([]const u8).empty; - var overdue_list = std.ArrayList([]const u8).empty; - // Reused per symbol (clearRetainingCapacity) to collect the data-type - // tags that failed, so the failed list can read "SYM (candles, earnings)". - var fail_types = std.ArrayList([]const u8).empty; - - // Warm the EDGAR ticker maps once per refresh run. They're - // ~3-5 MB each, cached for 30 days; warming guarantees the - // shared `/_edgar/tickers_funds.srf` and - // `tickers_companies.srf` files exist for the static-route - // handlers to serve. Per-symbol `getEtfMetrics` calls below - // also rely on these maps being loaded. - { - try printRateLimitTag(&svc, .tickers_funds, stdout); - if (svc.loadMutualFundTickerMap(.{})) |mut_map| { - var m = mut_map; - m.deinit(); - try stdout.print("EDGAR mutual-fund ticker map ok\n", .{}); - } else |err| { - try stdout.print("EDGAR mutual-fund ticker map FAILED ({t})\n", .{err}); - } - try printRateLimitTag(&svc, .tickers_companies, stdout); - if (svc.loadCompanyTickerMap(.{})) |co_map| { - var m = co_map; - m.deinit(); - try stdout.print("EDGAR company ticker map ok\n", .{}); - } else |err| { - try stdout.print("EDGAR company ticker map FAILED ({t})\n", .{err}); - } - try stdout.flush(); - } - - var it = symbols.iterator(); - while (it.next()) |entry| { - const sym = entry.key_ptr.*; - try stdout.print("{s}: ", .{sym}); - try stdout.flush(); - - var sym_freshness: Freshness = .current; - fail_types.clearRetainingCapacity(); - - // Candles - try printRateLimitTag(&svc, .candles_daily, stdout); - if (svc.getCandles(sym, .{})) |result| { - defer result.deinit(); - try stdout.print("candles ok ({s})", .{@tagName(result.source)}); - stats.candles.hit(result.source == .fetched); - // `== .cached` rather than `!= .fetched` on purpose: should a third - // `Source` ever appear, this defaults to NOT forcing. That failure - // mode leaves a symbol behind, which surfaces via `stuck` and mails - // the operator; the opposite default burns quota in silence. - if (result.source == .cached) try unfetched.put(sym, {}); - - // Provider-data-lag check: did we end up with the latest bar - // the market should have posted by now? A `.lagging` bar is - // merely unposted -> flag it so the run exits EX_TEMPFAIL and - // cron retries. An `.overdue` bar is almost certainly an - // un-modeled closure (e.g. Good Friday) -> note it, no retry. - if (result.data.len > 0) { - const last = result.data[result.data.len - 1].date; - var date_buf: [10]u8 = undefined; - const ds = std.fmt.bufPrint(&date_buf, "{f}", .{last}) catch "?"; - switch (zfin.market.candleFreshness(now_s, zfin.market.classify(sym), last)) { - .lagging => { - sym_freshness = .lagging; - try stdout.print(" LAGGING (latest {s})", .{ds}); - log.info("provider data lag: {s} latest bar {s}; newer session due but unposted", .{ sym, ds }); - }, - .overdue => { - sym_freshness = .overdue; - // States the observation, NOT an inference. This used to - // say "assuming market closure (no retry)", which the - // pass cannot know: `candleFreshness` judges this symbol - // against the trading calendar in isolation, and a - // closure moves EVERY symbol together. A single symbol - // nothing refreshes produces the identical reading. The - // corpus question is answered after the pass, by - // `sweepAfterPass`, which can see the peers. - log.info("{s}: latest bar {s} overdue past grace window; no retry this pass", .{ sym, ds }); - }, - .current => {}, - } - } - } else |err| { - try stdout.print("candles FAILED ({s})", .{@errorName(err)}); - try recordFailure(&stats.candles, &fail_types, allocator, "candles"); - if (err == zfin.DataError.TransientError or err == zfin.DataError.AuthError) { - const reason = if (err == zfin.DataError.AuthError) "auth failure" else "transient provider failure"; - try stdout.print("\n", .{}); - try stdout.print("\nStopping refresh: {s}\n", .{reason}); - try stdout.print("Refresh aborted after {d} current, {d} lagging, {d} overdue, {d} failed\n", .{ counts.current, counts.lagging, counts.overdue, counts.failed + 1 }); - try stdout.flush(); - return error.RefreshFailed; - } - } - - // Dividends - try stdout.print(", ", .{}); - try printRateLimitTag(&svc, .dividends, stdout); - if (svc.getDividends(sym, .{})) |result| { - defer result.deinit(); - try stdout.print("dividends ok ({s})", .{@tagName(result.source)}); - stats.dividends.hit(result.source == .fetched); - } else |err| { - try stdout.print("dividends FAILED ({s})", .{@errorName(err)}); - try recordFailure(&stats.dividends, &fail_types, allocator, "dividends"); - } - - // Splits - try stdout.print(", ", .{}); - try printRateLimitTag(&svc, .splits, stdout); - if (svc.getSplits(sym, .{})) |result| { - defer result.deinit(); - try stdout.print("splits ok ({s})", .{@tagName(result.source)}); - stats.splits.hit(result.source == .fetched); - } else |err| { - try stdout.print("splits FAILED ({s})", .{@errorName(err)}); - try recordFailure(&stats.splits, &fail_types, allocator, "splits"); - } - - // Earnings - try stdout.print(", ", .{}); - try printRateLimitTag(&svc, .earnings, stdout); - if (svc.getEarnings(sym, .{})) |result| { - defer result.deinit(); - try stdout.print("earnings ok ({s})", .{@tagName(result.source)}); - stats.earnings.hit(result.source == .fetched); - } else |err| { - try stdout.print("earnings FAILED ({s})", .{@errorName(err)}); - try recordFailure(&stats.earnings, &fail_types, allocator, "earnings"); - } - - // Classification (Wikidata + EDGAR fallback). Captures - // CIK and is_etf — used to chain into entity_facts below. - // NotFound is logged as `n/a` (symbol genuinely has no - // Wikidata or EDGAR entry) and isn't counted as a failure. - var cik_buf: ?[]u8 = null; - defer if (cik_buf) |b| allocator.free(b); - var is_etf = false; - try stdout.print(", ", .{}); - try printRateLimitTag(&svc, .classification, stdout); - if (svc.getClassification(sym, .{})) |result| { - defer result.deinit(); - if (result.data.len > 0) { - if (result.data[0].cik) |cik| { - cik_buf = allocator.dupe(u8, cik) catch null; - } - is_etf = result.data[0].is_etf; - } - try stdout.print("classification ok ({s})", .{@tagName(result.source)}); - stats.classification.hit(result.source == .fetched); - } else |err| switch (err) { - zfin.DataError.NotFound => { - try stdout.print("classification n/a", .{}); - stats.classification.na += 1; - }, - else => { - try stdout.print("classification FAILED ({t})", .{err}); - try recordFailure(&stats.classification, &fail_types, allocator, "classification"); - }, - } - - // ETF metrics. NotFound is the expected outcome for - // non-funds (NPORT-P only exists for funds + UITs); a - // negative-cache entry suppresses retries. Logged as - // `n/a` and isn't counted as a failure. - try stdout.print(", ", .{}); - try printRateLimitTag(&svc, .etf_metrics, stdout); - if (svc.getEtfMetrics(sym, .{})) |result| { - defer result.deinit(); - try stdout.print("etf_metrics ok ({s})", .{@tagName(result.source)}); - stats.etf_metrics.hit(result.source == .fetched); - } else |err| switch (err) { - zfin.DataError.NotFound => { - try stdout.print("etf_metrics n/a", .{}); - stats.etf_metrics.na += 1; - }, - else => { - try stdout.print("etf_metrics FAILED ({t})", .{err}); - try recordFailure(&stats.etf_metrics, &fail_types, allocator, "etf_metrics"); - }, - } - - // Entity facts (XBRL). Only attempted when the - // classification step yielded a CIK from a non-fund - // record. ETFs/funds CIKs (iShares Trust, Fidelity series - // CIKs, etc.) don't file the operating-company XBRL - // concepts entity_facts looks for; calling EDGAR for - // them is guaranteed-404 noise. Skip them up front. - if (cik_buf) |cik| { - try stdout.print(", ", .{}); - if (is_etf) { - try stdout.print("entity_facts n/a (ETF)", .{}); - stats.entity_facts.na += 1; - } else { - try printRateLimitTag(&svc, .entity_facts, stdout); - if (svc.getEntityFacts(cik, .{})) |result| { - defer result.deinit(); - try stdout.print("entity_facts ok ({s})", .{@tagName(result.source)}); - stats.entity_facts.hit(result.source == .fetched); - } else |err| switch (err) { - zfin.DataError.NotFound => { - try stdout.print("entity_facts n/a", .{}); - stats.entity_facts.na += 1; - }, - else => { - try stdout.print("entity_facts FAILED ({t})", .{err}); - try recordFailure(&stats.entity_facts, &fail_types, allocator, "entity_facts"); - }, - } - } - } else { - // No CIK resolved: entity_facts is not applicable for this - // symbol, so it counts as n/a (keeps the table row summing - // to the total symbol count). - stats.entity_facts.na += 1; - } - - try stdout.print("\n", .{}); - try stdout.flush(); - - // Bucket the symbol by precedence failed > lagging > overdue > - // current (same order refreshExit uses for the exit code). A - // symbol failed iff any data type recorded a hard error. - if (fail_types.items.len > 0) { - counts.failed += 1; - const types = try std.mem.join(allocator, ", ", fail_types.items); - try failed_list.append(allocator, try std.fmt.allocPrint(allocator, "{s} ({s})", .{ sym, types })); - } else switch (sym_freshness) { - .current => counts.current += 1, - .lagging => { - counts.lagging += 1; - try lagging_list.append(allocator, sym); - }, - .overdue => { - counts.overdue += 1; - try overdue_list.append(allocator, sym); - }, - } - } - - // Sweep AFTER every symbol has been through, because the question it answers - // - is anything behind its peers? - has no answer until the corpus is whole. - const sweep = sweepAfterPass(io, allocator, &svc, config.cache_dir, &symbols, &unfetched, now_s, stdout) catch |err| blk: { - log.warn("post-pass sweep failed: {t}", .{err}); - break :blk SweepOutcome{}; - }; - try stdout.flush(); - - const elapsed_ns = std.Io.Timestamp.now(io, .awake).nanoseconds - start_ns; - const elapsed_s: u64 = @intCast(@divTrunc(elapsed_ns, std.time.ns_per_s)); - const stuck = if (sweepIsActionable(sweep)) sweep.stuck else 0; - const code = refreshExit(counts.failed, counts.lagging, stuck); - // Distinguishes the two paths to `1`, since the code alone cannot. - const reason = if (counts.failed > 0) - "failures" - else if (stuck > 0) - "stuck behind peers" - else if (code == 75) - "lagging" - else - "clean"; - - try stdout.print("\nRefresh complete in {d}s (exit {d}: {s})\n", .{ elapsed_s, code, reason }); - try stdout.print(" symbols: {d} current, {d} lagging, {d} overdue, {d} failed ({d} total)\n", .{ counts.current, counts.lagging, counts.overdue, counts.failed, symbols.count() }); - - try stdout.print("\n {s:<14}{s:>7}{s:>8}{s:>5}{s:>8}\n", .{ "type", "fetched", "cached", "n/a", "failed" }); - try printStatRow(stdout, "candles", stats.candles); - try printStatRow(stdout, "dividends", stats.dividends); - try printStatRow(stdout, "splits", stats.splits); - try printStatRow(stdout, "earnings", stats.earnings); - try printStatRow(stdout, "classification", stats.classification); - try printStatRow(stdout, "etf_metrics", stats.etf_metrics); - try printStatRow(stdout, "entity_facts", stats.entity_facts); - - try stdout.print("\n", .{}); - try printSymbolList(stdout, "failed:", failed_list.items); - try printSymbolList(stdout, "lagging:", lagging_list.items); - try printSymbolList(stdout, "overdue:", overdue_list.items); - try stdout.flush(); - - return code; -} - -/// What the post-pass sweep found and what it managed to fix. -const SweepOutcome = struct { - /// Tracked symbols still further behind their peers than ordinary lag - /// explains, AFTER a forced refresh. These will not clear on the next cron - /// tick, which is why they get their own exit disposition. - stuck: usize = 0, - /// Findings the sweep saw before it acted on anything. - /// - /// Distinct from `attempted` because "nothing was behind" and "things were - /// behind but none were worth re-asking" are different states, and only the - /// first may print a market-closure conclusion. Conflating them would - /// reintroduce the false inference the pass-time log used to make. - found: usize = 0, - /// Symbols the sweep force-refreshed. - attempted: usize = 0, - /// Findings deliberately left alone: the pass already asked the provider for - /// these this run, so a forced re-ask cannot produce a different answer. - skipped: usize = 0, - /// Of those, how many the forced fetch actually brought level with peers. - recovered: usize = 0, - /// Symbols still behind their peers after the retry, INCLUDING those within - /// what ordinary lag explains. Tracked separately from `stuck` because - /// keying the all-clear message on `stuck` alone announced "all clear" while - /// a symbol was still behind - it had merely crossed back inside - /// `max_normal_lag_days`, which is progress, not resolution. - still_behind: usize = 0, - /// Was the sweep able to reach a conclusion at all? False when the cache - /// could not be enumerated, in which case nothing here is a finding. - ran: bool = false, - /// True when no individual symbol is behind its peers but the corpus as a - /// whole sits behind the calendar. That is the shape a market closure makes, - /// and the only shape from which one can honestly be inferred. - corpus_behind: bool = false, - /// The peer reference the conclusion was drawn against, for the operator to - /// check the reasoning rather than take it on faith. - peer_date: ?zfin.Date = null, -}; - -/// Should this run exit non-zero for a gap that will not self-heal? -/// -/// Split from `refreshExit` so the "is a multi-day gap actionable" question is -/// testable without constructing a whole run: `stuck` counts only symbols still -/// past `max_normal_lag_days` after a FORCED refresh, so it excludes both -/// ordinary provider lag (which `75` already covers) and anything a retry fixes. -fn sweepIsActionable(o: SweepOutcome) bool { - return o.ran and o.stuck > 0; -} - -/// Which findings a forced refresh can plausibly help, worst first. -/// -/// Only symbols the pass did NOT already ask the provider about. A symbol whose -/// candles were fetched seconds ago will get the identical answer from an -/// immediate re-ask, so forcing it wastes a request AND bypasses the TTL pacing -/// that exists to space retries out. The case that DOES benefit is a symbol the -/// pass never reached the provider for - a fresh TTL, or a server sync that -/// satisfied the request while serving a copy weeks old. That is precisely the -/// shape of the bug this whole sweep was built for. -/// -/// `far_behind` before `stale` so a rate limit, if one bites mid-sweep, bites the -/// least-behind symbols. Both input lists arrive already sorted worst-first. -/// -/// Pure given the report and the set, which is the point: the "don't waste calls" -/// rule is testable without a provider. -fn selectForForcing( - arena: std.mem.Allocator, - before: zfin.freshness.Report, - unfetched: *const std.StringHashMap(void), -) ![]const []const u8 { - var out = std.ArrayList([]const u8).empty; - for (before.far_behind) |f| { - if (unfetched.contains(f.symbol)) try out.append(arena, f.symbol); - } - for (before.stale) |f| { - if (unfetched.contains(f.symbol)) try out.append(arena, f.symbol); - } - return out.items; -} - -/// Is `symbol` still behind its peers in `report`? -fn isFinding(report: zfin.freshness.Report, symbol: []const u8) bool { - for (report.far_behind) |f| { - if (std.mem.eql(u8, f.symbol, symbol)) return true; - } - for (report.stale) |f| { - if (std.mem.eql(u8, f.symbol, symbol)) return true; - } - return false; -} - -/// Sweep the whole cache after the main pass, force-refresh whatever is behind -/// its peers, then re-check. -/// -/// Why this cannot be folded into the pass: the pass judges each symbol against -/// the trading calendar in isolation (`market.candleFreshness`), which cannot -/// tell a market closure from a single symbol nothing refreshes - both leave a -/// bar sitting past the grace window. A closure moves every symbol together, so -/// "behind its own peers" is disproof of one. That is a corpus question, and the -/// corpus is only complete once every symbol has been through the pass. -/// -/// The forced refresh is the point, not a nicety: a symbol can be behind purely -/// because nothing ever asked for it, and one forced fetch brings it fully -/// current (observed on a watchlist symbol 42 days behind). Reporting the gap -/// without attempting the fix would file a ticket for something the run could -/// have closed itself. -fn sweepAfterPass( - io: std.Io, - allocator: std.mem.Allocator, - svc: *zfin.DataService, - cache_dir: []const u8, - tracked: *const std.StringHashMap(void), - unfetched: *const std.StringHashMap(void), - now_s: i64, - stdout: *std.Io.Writer, -) !SweepOutcome { - // One arena for the whole sweep: it runs once, at the end of a process that - // is about to exit, and the alternative is five separate ownership dances - // across two scans. - var arena_state = std.heap.ArenaAllocator.init(allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - var store = zfin.cache.Store.init(io, arena, cache_dir); - const keys = store.cacheKeys(arena) catch |err| { - // Not a finding: an unreadable cache directory means the sweep has no - // opinion, and reporting `stuck = 0` as though it had checked would be - // the same lie the old log line told. - log.warn("post-pass sweep skipped: cannot enumerate cache: {t}", .{err}); - return .{}; - }; - - const before = try zfin.freshness.scan( - arena, - try zfin.freshness.collect(arena, &store, keys, tracked, &.{}), - now_s, - ); - - var out = SweepOutcome{ .ran = true }; - out.peer_date = widestPeerDate(before); - - out.found = before.stale.len + before.far_behind.len; - if (out.found == 0) { - // Nothing is behind its peers. The only remaining question is whether the - // corpus itself has moved, which is what distinguishes "quiet market" from - // "nothing ran" - and it is the ONLY state from which a market closure can - // honestly be inferred. - out.corpus_behind = corpusBehind(before); - try printSweepConclusion(stdout, out, &.{}); - return out; - } - - const forced = try selectForForcing(arena, before, unfetched); - out.attempted = forced.len; - out.skipped = out.found - forced.len; - for (forced) |sym| forceOne(svc, sym); - - // Nothing was forced, so the cache is byte-identical to the pre-sweep scan - - // re-reading it would cost ~40 file reads to learn what we already know, and - // could only differ by disagreeing with itself. - const after = if (forced.len == 0) before else blk: { - // Re-read from disk rather than trusting the fetch's return value: the - // question is what a CLIENT will now be served, and that is whatever - // landed in the cache. - var after_store = zfin.cache.Store.init(io, arena, cache_dir); - const after_keys = after_store.cacheKeys(arena) catch keys; - break :blk try zfin.freshness.scan( - arena, - try zfin.freshness.collect(arena, &after_store, after_keys, tracked, &.{}), - now_s, - ); - }; - - out.still_behind = after.stale.len + after.far_behind.len; - out.stuck = after.far_behind.len; - // Counted by membership, not by subtracting totals. The arithmetic form - // (`attempted - still_behind`) already produced a wrong number once, and - // skipped symbols make it wrong in a second way: they inflate - // `still_behind` without ever having been attempted. - for (forced) |sym| { - if (!isFinding(after, sym)) out.recovered += 1; - } - out.peer_date = widestPeerDate(after) orelse out.peer_date; - // A symbol behind its peers is disproof of a closure, so this stays false - // whenever there are findings - regardless of how the corpus looks. - out.corpus_behind = false; - - try printSweepConclusion(stdout, out, after.far_behind); - return out; -} - -/// Force one symbol's candles, swallowing failures by design: the sweep's job is -/// to report the post-attempt state, and a fetch error here is already visible in -/// the re-scan as "still behind". Logged by name so the cause is not lost. -fn forceOne(svc: *zfin.DataService, symbol: []const u8) void { - if (svc.getCandles(symbol, .{ .force_refresh = true })) |result| { - result.deinit(); - } else |err| { - log.warn("post-pass sweep: forced refresh of {s} failed: {t}", .{ symbol, err }); - } -} - -/// The newest bar any conclusive peer group holds. The reference the sweep's -/// conclusions are measured against. -fn widestPeerDate(report: zfin.freshness.Report) ?zfin.Date { - var newest: ?zfin.Date = null; - for (report.groups) |g| { - if (!g.conclusive()) continue; - const p = g.peer_date orelse continue; - if (newest == null or newest.?.lessThan(p)) newest = p; - } - return newest; -} - -/// Is every conclusive group's own newest bar behind the calendar? -/// -/// Only meaningful when nothing is behind its peers - which the caller enforces. -/// `GroupState.freshness` is the same verdict the fetch gate uses, so this does -/// not re-derive the market calendar. -fn corpusBehind(report: zfin.freshness.Report) bool { - var conclusive: usize = 0; - for (report.groups) |g| { - if (!g.conclusive()) continue; - conclusive += 1; - const f = g.freshness orelse return false; - if (f == .current) return false; - } - return conclusive > 0; -} - -fn printSweepConclusion( - stdout: *std.Io.Writer, - o: SweepOutcome, - stuck: []const zfin.freshness.Finding, -) !void { - if (!o.ran) return; - try stdout.print("\nPost-pass sweep:\n", .{}); - // Keyed on `found`, NOT `attempted`. Those diverged the moment the sweep - // learned to skip symbols the pass had already asked about, and keying the - // closure conclusion on `attempted` would have claimed a market closure while - // symbols were demonstrably behind their peers - the exact false inference - // that was removed from the pass-time log. - if (o.found == 0) { - if (o.corpus_behind) { - // The one case where a closure CAN be inferred, and it is inferred - // from the corpus moving together rather than from one symbol. - try stdout.print(" no symbol is behind its peers, but the whole cache sits behind the\n", .{}); - try stdout.print(" calendar - consistent with a market closure, not a refresh problem\n", .{}); - } else { - try stdout.print(" nothing behind its peers\n", .{}); - } - return; - } - - if (o.attempted == 0) { - // The 5pm case: equities posted, a couple of symbols have not, and the - // pass already asked the provider about them this run. Re-asking cannot - // change the answer, so the sweep does nothing and says why. - try stdout.print(" {d} behind peers; none re-asked - the provider was already queried this pass\n", .{o.found}); - } else if (o.skipped > 0) { - try stdout.print(" {d} behind peers; {d} re-asked, {d} skipped (already queried this pass); {d} now level\n", .{ o.found, o.attempted, o.skipped, o.recovered }); - } else { - try stdout.print(" {d} symbol(s) behind peers -> forced refresh; {d} now level with peers\n", .{ o.attempted, o.recovered }); - } - - // Ordered by severity, NOT by field convenience. An earlier arrangement - // tested `still_behind == 0` first, which would print "all clear" for any - // outcome whose counters disagreed - the one direction this must never fail - // in, since the all-clear is what suppresses the non-zero exit. - if (o.stuck > 0) { - // Named explicitly as non-self-healing, because the whole reason this run - // exits non-zero is to stop cron from looping on it silently. - try stdout.print(" {d} still further behind than lag explains - a retry will not clear these:\n", .{o.stuck}); - for (stuck) |f| { - try stdout.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date }); - } - // Deliberately not naming a cause. The candidates are many and none - // visible from here (see `zfin.freshness.max_normal_lag_days`); - // `zfin diagnose SYMBOL` is the tool that narrows it. - try stdout.print(" run `zfin diagnose SYMBOL` against one of these to narrow it\n", .{}); - return; - } - if (o.still_behind > 0) { - // Partial progress, and saying so matters: these crossed back inside - // `max_normal_lag_days`, so the next pass should close them and this run - // must NOT exit as though a human were needed. - try stdout.print(" {d} still behind, but within what ordinary lag explains - the next pass should close it\n", .{o.still_behind}); - return; - } - try stdout.print(" all clear after the retry\n", .{}); -} - -/// Map a refresh run's failure/lag counts to a process exit code: -/// 0 - every symbol current and fetched cleanly -/// 75 - EX_TEMPFAIL: no hard failures, but at least one symbol's -/// just-closed bar hadn't posted yet (provider lag); cron should -/// retry shortly -/// 1 - at least one hard failure (fetch error), OR a tracked symbol -/// still further behind its peers than lag explains after the -/// post-pass sweep forced a refresh -/// Hard failure dominates lag - if anything failed outright that's the -/// code the operator needs to act on. -/// -/// The stuck-symbol case is deliberately `1` rather than `75`, and this is the -/// distinction that matters: `75` tells cron "retry soon", which is right for an -/// unposted bar and wrong for a multi-day gap. A gap the sweep could not close -/// with a forced fetch will not close on the next tick either, so the run has to -/// mail rather than loop. It shares `1` with a fetch failure because both mean -/// "a human should look"; the summary line names which one it was. -fn refreshExit(fail_count: usize, lag_count: usize, stuck_count: usize) u8 { - if (fail_count > 0) return 1; - if (stuck_count > 0) return 1; - if (lag_count > 0) return 75; - return 0; -} +const App = @import("App.zig").App; +const handlers = @import("handlers.zig"); +const refresh_cmd = @import("refresh.zig"); // ── Main ───────────────────────────────────────────────────── @@ -1800,41 +56,41 @@ pub fn main(init: std.process.Init) !u8 { var router = try server.router(.{}); // Static routes - router.get("/", handleIndex, .{}); - router.get("/help", handleHelp, .{}); - router.get("/symbols", handleSymbols, .{}); + router.get("/", handlers.handleIndex, .{}); + router.get("/help", handlers.handleHelp, .{}); + router.get("/symbols", handlers.handleSymbols, .{}); // Symbol routes - router.get("/:symbol/returns", handleReturns, .{}); + router.get("/:symbol/returns", handlers.handleReturns, .{}); // Authenticated explicit watchlist add (API key required - not on // the public allowlist). Distinct from the UA-gated // /:symbol/returns?watch=true path used by LibreOffice. - router.get("/:symbol/watch", handleWatch, .{}); - router.get("/:symbol/quote", handleQuote, .{}); - router.get("/:symbol/candles", handleCandles, .{}); - router.get("/:symbol/candles_meta", handleCandlesMeta, .{}); - router.get("/:symbol/diagnostics", handleDiagnostics, .{}); - router.get("/:symbol/dividends", handleDividends, .{}); - router.get("/:symbol/splits", handleSplits, .{}); - router.get("/:symbol/earnings", handleEarnings, .{}); - router.get("/:symbol/options", handleOptions, .{}); + router.get("/:symbol/watch", handlers.handleWatch, .{}); + router.get("/:symbol/quote", handlers.handleQuote, .{}); + router.get("/:symbol/candles", handlers.handleCandles, .{}); + router.get("/:symbol/candles_meta", handlers.handleCandlesMeta, .{}); + router.get("/:symbol/diagnostics", handlers.handleDiagnostics, .{}); + router.get("/:symbol/dividends", handlers.handleDividends, .{}); + router.get("/:symbol/splits", handlers.handleSplits, .{}); + router.get("/:symbol/earnings", handlers.handleEarnings, .{}); + router.get("/:symbol/options", handlers.handleOptions, .{}); // Wikidata + EDGAR derived data — populated by `refresh`. - router.get("/:symbol/classification", handleClassification, .{}); - router.get("/:symbol/etf_metrics", handleEtfMetrics, .{}); - router.get("/:cik/entity_facts", handleEntityFacts, .{}); + router.get("/:symbol/classification", handlers.handleClassification, .{}); + router.get("/:symbol/etf_metrics", handlers.handleEtfMetrics, .{}); + router.get("/:cik/entity_facts", handlers.handleEntityFacts, .{}); // EDGAR shared ticker maps (~3-5 MB each, refreshed // every 30 days). Static-key routes — single file // shared across every symbol lookup. - router.get("/_edgar/tickers_funds", handleTickersFunds, .{}); - router.get("/_edgar/tickers_companies", handleTickersCompanies, .{}); + router.get("/_edgar/tickers_funds", handlers.handleTickersFunds, .{}); + router.get("/_edgar/tickers_companies", handlers.handleTickersCompanies, .{}); log.info("zfin-server {s}", .{version}); log.info("listening on port {d}", .{port}); try server.listen(); } else if (std.mem.eql(u8, command, "refresh")) { - return try refresh(io, init.arena.allocator(), environ); + return try refresh_cmd.refresh(io, init.arena.allocator(), environ); } else { try printUsage(io); } @@ -1874,611 +130,13 @@ fn printUsage(io: std.Io) !void { // ── Tests ──────────────────────────────────────────────────── -test "fmtPct" { - var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena_state.deinit(); - const arena = arena_state.allocator(); - - try std.testing.expectEqualStrings("null", fmtPct(arena, null)); - const result = fmtPct(arena, 0.1234); - try std.testing.expect(std.mem.startsWith(u8, result, "12.34")); -} - -test "upperDupe" { - const result = try upperDupe(std.testing.allocator, "aapl"); - defer std.testing.allocator.free(result); - try std.testing.expectEqualStrings("AAPL", result); -} - -test "shouldLogRequest: fast 2xx is silent" { - try std.testing.expect(!shouldLogRequest(10, 200, 500)); - try std.testing.expect(!shouldLogRequest(499, 200, 500)); - try std.testing.expect(!shouldLogRequest(0, 204, 500)); -} - -test "shouldLogRequest: slow 2xx logs" { - try std.testing.expect(shouldLogRequest(501, 200, 500)); - try std.testing.expect(shouldLogRequest(2000, 200, 500)); - // Boundary: == threshold is NOT logged (strict >). - try std.testing.expect(!shouldLogRequest(500, 200, 500)); -} - -test "shouldLogRequest: any error response logs regardless of timing" { - try std.testing.expect(shouldLogRequest(1, 400, 500)); - try std.testing.expect(shouldLogRequest(1, 404, 500)); - try std.testing.expect(shouldLogRequest(1, 500, 500)); - try std.testing.expect(shouldLogRequest(1, 503, 500)); - // 3xx is not flagged as error. - try std.testing.expect(!shouldLogRequest(1, 301, 500)); - try std.testing.expect(!shouldLogRequest(1, 304, 500)); -} - -test "shouldLogRequest: custom threshold respected" { - try std.testing.expect(!shouldLogRequest(50, 200, 100)); - try std.testing.expect(shouldLogRequest(150, 200, 100)); - // Higher threshold (e.g. user sets ZFIN_SERVER_SLOW_MS=2000). - try std.testing.expect(!shouldLogRequest(1500, 200, 2000)); - try std.testing.expect(shouldLogRequest(2500, 200, 2000)); -} - -test "pathIsPublic: public surface (no key required)" { - try std.testing.expect(pathIsPublic("/")); - try std.testing.expect(pathIsPublic("/help")); - try std.testing.expect(pathIsPublic("/AAPL/returns")); - // Symbols with dots (class shares) still match. - try std.testing.expect(pathIsPublic("/BRK.B/returns")); -} - -test "pathIsPublic: gated surface (key required)" { - try std.testing.expect(!pathIsPublic("/AAPL/candles")); - try std.testing.expect(!pathIsPublic("/AAPL/quote")); - try std.testing.expect(!pathIsPublic("/symbols")); - try std.testing.expect(!pathIsPublic("/_edgar/tickers_funds")); - try std.testing.expect(!pathIsPublic("/0000320193/entity_facts")); - // Diagnostics leaks the operator's tracked set and cache layout, so it must - // stay gated. It needs no entry in `pathIsPublic` to be gated - the default - // is closed - and this asserts the default rather than trusting it, because - // the cost of that assumption being wrong is silent disclosure. - try std.testing.expect(!pathIsPublic("/AAPL/diagnostics")); - try std.testing.expect(!pathIsPublic("/BRK.B/diagnostics")); -} - -test "pathIsPublic: returns look-alikes do not slip through" { - try std.testing.expect(!pathIsPublic("/returns")); // no symbol - try std.testing.expect(!pathIsPublic("/a/b/returns")); // extra segment - try std.testing.expect(!pathIsPublic("/AAPL/returns/extra")); // suffix, not exact - try std.testing.expect(!pathIsPublic("/help/secret")); // help prefix only - try std.testing.expect(!pathIsPublic("//returns")); // empty symbol -} - -test "keyMatches" { - try std.testing.expect(keyMatches("s3cret", "s3cret")); - try std.testing.expect(!keyMatches("s3cret", "other")); - try std.testing.expect(!keyMatches(null, "s3cret")); - try std.testing.expect(!keyMatches("", "s3cret")); - // Length-checked: neither a prefix nor an extension matches. - try std.testing.expect(!keyMatches("s3cre", "s3cret")); - try std.testing.expect(!keyMatches("s3cretX", "s3cret")); -} - -test "userAgentMayWatch" { - try std.testing.expect(userAgentMayWatch("LibreOffice 24.8")); - try std.testing.expect(userAgentMayWatch("libreoffice")); // case-insensitive - try std.testing.expect(userAgentMayWatch("Mozilla/5.0 LibreOffice/7.6")); - try std.testing.expect(!userAgentMayWatch("Mozilla/5.0 (X11; Linux x86_64)")); - try std.testing.expect(!userAgentMayWatch("curl/8.14.1")); - try std.testing.expect(!userAgentMayWatch(null)); - try std.testing.expect(!userAgentMayWatch("")); -} - -test "isPlausibleSymbol" { - try std.testing.expect(isPlausibleSymbol("AAPL")); - try std.testing.expect(isPlausibleSymbol("BRK.B")); - try std.testing.expect(isPlausibleSymbol("BRK-B")); - try std.testing.expect(isPlausibleSymbol("X")); - try std.testing.expect(!isPlausibleSymbol("")); // empty - try std.testing.expect(!isPlausibleSymbol("aapl")); // lowercase (upper-cased before this) - try std.testing.expect(!isPlausibleSymbol("AB CD")); // space - try std.testing.expect(!isPlausibleSymbol("../etc/passwd")); // path-shaped junk - try std.testing.expect(!isPlausibleSymbol("ABCDEFGHIJKLMNOPQ")); // 17 chars, too long -} - -test "refreshExit: hard failure dominates, then stuck, then lag, else clean" { - try std.testing.expectEqual(@as(u8, 0), refreshExit(0, 0, 0)); - try std.testing.expectEqual(@as(u8, 75), refreshExit(0, 3, 0)); - try std.testing.expectEqual(@as(u8, 1), refreshExit(2, 0, 0)); - // A hard failure outranks lag. - try std.testing.expectEqual(@as(u8, 1), refreshExit(1, 5, 0)); - - // A symbol still behind its peers after a FORCED refresh exits 1, not 75. - // This is the whole point of the third argument: 75 means EX_TEMPFAIL, which - // tells cron to retry soon - correct for an unposted bar, wrong for a - // multi-day gap that a forced fetch already failed to close. Retrying that on - // a schedule loops silently forever, which is how SPCX went 43 days unnoticed. - try std.testing.expectEqual(@as(u8, 1), refreshExit(0, 0, 1)); - // Stuck outranks lag: the actionable finding wins over the retryable one. - try std.testing.expectEqual(@as(u8, 1), refreshExit(0, 9, 1)); - // But a hard failure still outranks stuck - it is the more proximate problem - // and may well be the CAUSE of the gap. - try std.testing.expectEqual(@as(u8, 1), refreshExit(4, 0, 2)); -} - -test "sweepIsActionable: only a sweep that actually ran can be a finding" { - // A sweep that could not enumerate the cache reports `stuck = 0`, and reading - // that as "nothing is behind" would repeat the exact mistake the old - // market-closure log line made: asserting a conclusion from missing evidence. - // `ran` is what separates "checked and found nothing" from "did not check". - try std.testing.expect(!sweepIsActionable(.{ .ran = false, .stuck = 3 })); - try std.testing.expect(!sweepIsActionable(.{ .ran = true, .stuck = 0 })); - try std.testing.expect(sweepIsActionable(.{ .ran = true, .stuck = 1 })); - // The zero value must never be actionable - it is what the error paths return. - try std.testing.expect(!sweepIsActionable(.{})); -} - -test "corpusBehind: a closure is only inferable when every group moved together" { - const d = zfin.Date.fromYmd(2026, 8, 12); - const empty: []zfin.freshness.Finding = &.{}; - - // Two conclusive groups, both overdue, nothing behind its peers. This is the - // one shape from which a market closure can honestly be read. - var groups = [_]zfin.freshness.GroupState{ - .{ .kind = .equity, .peer_date = d, .freshness = .overdue, .dated = 9 }, - .{ .kind = .mutual_fund, .peer_date = d, .freshness = .overdue, .dated = 4 }, - }; - var r = zfin.freshness.Report{ - .stale = empty, - .far_behind = empty, - .orphans = &.{}, - .missing = &.{}, - .groups = &groups, - }; - try std.testing.expect(corpusBehind(r)); - - // One group current: the market plainly was not closed. - groups[1].freshness = .current; - try std.testing.expect(!corpusBehind(r)); - - // Unknown freshness is not evidence of a closure. Absence of information must - // not become a conclusion. - groups[1].freshness = null; - try std.testing.expect(!corpusBehind(r)); - - // No conclusive group at all - a single cached symbol per kind - concludes - // nothing rather than vacuously true. - var lonely = [_]zfin.freshness.GroupState{ - .{ .kind = .equity, .peer_date = d, .freshness = .overdue, .dated = 1 }, - }; - r.groups = &lonely; - try std.testing.expect(!corpusBehind(r)); - r.groups = &.{}; - try std.testing.expect(!corpusBehind(r)); -} - -test "widestPeerDate: the newest conclusive group wins, inconclusive ignored" { - const aug12 = zfin.Date.fromYmd(2026, 8, 12); - const aug11 = zfin.Date.fromYmd(2026, 8, 11); - const jun01 = zfin.Date.fromYmd(2026, 6, 1); - var groups = [_]zfin.freshness.GroupState{ - .{ .kind = .equity, .peer_date = aug11, .freshness = null, .dated = 5 }, - .{ .kind = .mutual_fund, .peer_date = aug12, .freshness = null, .dated = 3 }, - }; - var r = zfin.freshness.Report{ - .stale = &.{}, - .far_behind = &.{}, - .orphans = &.{}, - .missing = &.{}, - .groups = &groups, - }; - try std.testing.expectEqual(@as(?zfin.Date, aug12), widestPeerDate(r)); - - // An inconclusive group's date must not become the reference - with one cached - // symbol its "peer date" is just its own bar. - groups[1] = .{ .kind = .mutual_fund, .peer_date = jun01, .freshness = null, .dated = 1 }; - try std.testing.expectEqual(@as(?zfin.Date, aug11), widestPeerDate(r)); - - r.groups = &.{}; - try std.testing.expectEqual(@as(?zfin.Date, null), widestPeerDate(r)); -} - -test "printStatRow aligns with the summary-table header" { - var hdr: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer hdr.deinit(); - try hdr.writer.print(" {s:<14}{s:>7}{s:>8}{s:>5}{s:>8}\n", .{ "type", "fetched", "cached", "n/a", "failed" }); - const hdr_out = try hdr.toOwnedSlice(); - defer std.testing.allocator.free(hdr_out); - - var row: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer row.deinit(); - // Multi-digit values and the widest type name still fit the columns. - try printStatRow(&row.writer, "classification", .{ .fetched = 0, .cached = 31, .na = 14, .failed = 0 }); - const row_out = try row.toOwnedSlice(); - defer std.testing.allocator.free(row_out); - - // Identical width specifiers => identical rendered length => columns line up. - try std.testing.expectEqual(hdr_out.len, row_out.len); -} - -test "printSymbolList: empty shows (none), non-empty joins with commas" { - var a: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer a.deinit(); - try printSymbolList(&a.writer, "failed:", &.{}); - const a_out = try a.toOwnedSlice(); - defer std.testing.allocator.free(a_out); - try std.testing.expectEqualStrings(" failed: (none)\n", a_out); - - var b: std.Io.Writer.Allocating = .init(std.testing.allocator); - defer b.deinit(); - try printSymbolList(&b.writer, "lagging:", &.{ "NKE", "AMZN" }); - const b_out = try b.toOwnedSlice(); - defer std.testing.allocator.free(b_out); - try std.testing.expectEqualStrings(" lagging: NKE, AMZN\n", b_out); -} - -test "collectRefreshSymbols: only .stock and .watch, keyed by priceSymbol" { - const a = std.testing.allocator; - var set = std.StringHashMap(void).init(a); - defer set.deinit(); - - const lots = [_]zfin.Lot{ - .{ .symbol = "AAPL", .security_type = .stock, .shares = 1, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, - .{ .symbol = "SPCX", .security_type = .watch, .shares = 0, .open_price = 0, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, - // Cash is not fetched, so it must not appear as tracked - reporting it - // would promise a refresh that never runs. - .{ .symbol = "USD", .security_type = .cash, .shares = 100, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, - // A duplicate holding is one symbol, not two. - .{ .symbol = "AAPL", .security_type = .stock, .shares = 5, .open_price = 5, .open_date = zfin.Date.fromYmd(2026, 2, 1) }, - // A CUSIP priced through a `ticker::` alias must register under the - // symbol a FETCH uses, not the one the statement shows - otherwise - // `tracked` says false for a symbol the loop refreshes every night. - .{ .symbol = "922908736", .ticker = "VTTHX", .security_type = .stock, .shares = 3, .open_price = 27, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, - }; - try collectRefreshSymbols(&set, &lots); - - try std.testing.expectEqual(@as(u32, 3), set.count()); - try std.testing.expect(set.contains("AAPL")); - try std.testing.expect(set.contains("SPCX")); - try std.testing.expect(set.contains("VTTHX")); - try std.testing.expect(!set.contains("922908736")); - try std.testing.expect(!set.contains("USD")); -} - -test "daysBehind: an UNTRACKED symbol behind its peers still reports the gap" { - // The regression this replaces: `days_behind` was read out of - // `zfin.freshness.Report.stale`/`far_behind`, and `scan` only emits findings - // for tracked symbols - untracked ones divert to `orphans`. So every - // untracked symbol reported 0, and an untracked symbol is precisely what this - // endpoint was built to expose. AGG's real numbers, observed against a live - // cache copy: five days behind, reported as current. - try std.testing.expectEqual(@as(i64, 5), daysBehind( - zfin.Date.fromYmd(2026, 8, 6), - zfin.Date.fromYmd(2026, 8, 11), - )); - // SPCX's real gap. - try std.testing.expectEqual(@as(i64, 43), daysBehind( - zfin.Date.fromYmd(2026, 6, 29), - zfin.Date.fromYmd(2026, 8, 11), - )); -} - -test "daysBehind: not behind, or incomparable, is zero rather than negative" { - const d = zfin.Date.fromYmd(2026, 8, 11); - // Level with peers. - try std.testing.expectEqual(@as(i64, 0), daysBehind(d, d)); - // AHEAD of peers - this symbol IS the peer maximum. Must not report a - // negative gap, which would sort as "most behind" in any worst-first list. - try std.testing.expectEqual(@as(i64, 0), daysBehind(d, zfin.Date.fromYmd(2026, 8, 1))); - // No cached bar, and no peer to compare against: unanswerable, not zero-ish. - // Callers distinguish these from "current" via the null `last_date`. - try std.testing.expectEqual(@as(i64, 0), daysBehind(null, d)); - try std.testing.expectEqual(@as(i64, 0), daysBehind(d, null)); -} - -test "groupPeerDate: an inconclusive group has no peer date" { - const d = zfin.Date.fromYmd(2026, 8, 11); - // `dated = 1` is the symbol itself and nothing else. Returning its own date - // as `peer_date` would read as "agrees with its peers" when there are none. - const lonely = [_]zfin.freshness.GroupState{.{ - .kind = .equity, - .peer_date = d, - .freshness = null, - .dated = 1, - }}; - var report = zfin.freshness.Report{ - .stale = &.{}, - .far_behind = &.{}, - .orphans = &.{}, - .missing = &.{}, - .groups = @constCast(lonely[0..]), - }; - try std.testing.expectEqual(@as(?zfin.Date, null), groupPeerDate(report, .equity)); - - // Two dated entries make a comparison possible. - const peers = [_]zfin.freshness.GroupState{.{ - .kind = .equity, - .peer_date = d, - .freshness = null, - .dated = 2, - }}; - report.groups = @constCast(peers[0..]); - try std.testing.expectEqual(@as(?zfin.Date, d), groupPeerDate(report, .equity)); - - // A kind with no group at all is not an error, just unanswerable. - try std.testing.expectEqual(@as(?zfin.Date, null), groupPeerDate(report, .mutual_fund)); -} - -test "collectRefreshSymbols: keys stay readable while the source lots live" { - // Guards the bug this shipped with: the handler parsed the portfolio in an - // inner scope with a `defer portfolio.deinit()`, so by the time it asked - // `tracked.contains(symbol)` the keys pointed at released memory and all 25 - // tracked symbols reported false. `zfin`'s own `trackedSymbols` carries a - // comment about the identical failure - "made every cached symbol look - // untracked" - which is what makes it worth a test rather than a comment. - // - // The invariant is ownership, not content: the set BORROWS from `lots`, so a - // lookup is only valid while `lots` is alive. Asserting a hit after the - // insert-scope has closed is the cheapest way to pin that. - const a = std.testing.allocator; - var set = std.StringHashMap(void).init(a); - defer set.deinit(); - - const lots = [_]zfin.Lot{ - .{ .symbol = "AMZN", .security_type = .stock, .shares = 1, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, - }; - { - // A nested scope that ends before the lookup, mirroring the handler's - // shape. `lots` outlives it, so the keys remain valid. - try collectRefreshSymbols(&set, &lots); - } - try std.testing.expect(set.contains("AMZN")); - try std.testing.expectEqual(@as(u32, 1), set.count()); -} - -test "printSweepConclusion: a closure is claimed only when the corpus moved together" { - const a = std.testing.allocator; - - // Nothing behind peers, corpus itself behind: the ONE case where a closure is - // a legitimate inference, and it must say so from the corpus, not one symbol. - { - var w: std.Io.Writer.Allocating = .init(a); - defer w.deinit(); - try printSweepConclusion(&w.writer, .{ .ran = true, .corpus_behind = true }, &.{}); - const s = w.written(); - try std.testing.expect(std.mem.indexOf(u8, s, "market closure") != null); - try std.testing.expect(std.mem.indexOf(u8, s, "not a refresh problem") != null); - } - - // Nothing behind peers and the corpus is current: no closure claim at all. - // The old pass-time log asserted closure from a single symbol's calendar - // position; nothing may reintroduce that from an empty finding list. - { - var w: std.Io.Writer.Allocating = .init(a); - defer w.deinit(); - try printSweepConclusion(&w.writer, .{ .ran = true, .corpus_behind = false }, &.{}); - const s = w.written(); - try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); - try std.testing.expect(std.mem.indexOf(u8, s, "nothing behind its peers") != null); - } - - // A sweep that never ran prints nothing - it has no opinion to report. - { - var w: std.Io.Writer.Allocating = .init(a); - defer w.deinit(); - try printSweepConclusion(&w.writer, .{}, &.{}); - try std.testing.expectEqual(@as(usize, 0), w.written().len); - } -} - -test "printSweepConclusion: a stuck symbol is named and marked non-self-healing" { - const a = std.testing.allocator; - const stuck = [_]zfin.freshness.Finding{.{ - .symbol = "SPCX", - .kind = .equity, - .last_date = zfin.Date.fromYmd(2026, 6, 29), - .peer_date = zfin.Date.fromYmd(2026, 8, 12), - .days_behind = 44, - }}; - var w: std.Io.Writer.Allocating = .init(a); - defer w.deinit(); - try printSweepConclusion(&w.writer, .{ - .ran = true, - .found = 3, - .attempted = 3, - .recovered = 2, - .still_behind = 1, - .stuck = 1, - }, &stuck); - const s = w.written(); - - // The symbol and both dates, so the operator can check the reasoning rather - // than trust the verdict. - try std.testing.expect(std.mem.indexOf(u8, s, "SPCX") != null); - try std.testing.expect(std.mem.indexOf(u8, s, "2026-06-29") != null); - try std.testing.expect(std.mem.indexOf(u8, s, "2026-08-12") != null); - try std.testing.expect(std.mem.indexOf(u8, s, "44d behind") != null); - // The recovery tally, so a partial success is not read as total failure. - try std.testing.expect(std.mem.indexOf(u8, s, "2 now level with peers") != null); - // Says a retry will NOT help - the justification for exiting 1 over 75. - try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") != null); - // Points at the tool that narrows a cause instead of guessing one. - try std.testing.expect(std.mem.indexOf(u8, s, "zfin diagnose") != null); - // And must NOT guess. `freshness.max_normal_lag_days` documents that the - // candidates are many and none visible from here. - try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); -} - -test "printSweepConclusion: partial progress is not an all-clear, and not a page" { - // The bug this pins, found by running it: `all clear after the retry` printed - // whenever `stuck == 0`, ignoring symbols still behind by a day or two. A - // forced refresh that drags a symbol from 53 days behind to 1 day behind has - // made progress and resolved nothing, and the operator must be able to tell - // those apart from the output alone. - const a = std.testing.allocator; - var w: std.Io.Writer.Allocating = .init(a); - defer w.deinit(); - try printSweepConclusion(&w.writer, .{ - .ran = true, - .found = 1, - .attempted = 1, - .recovered = 0, - .still_behind = 1, - .stuck = 0, - }, &.{}); - const s = w.written(); - try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); - try std.testing.expect(std.mem.indexOf(u8, s, "within what ordinary lag explains") != null); - // Must not claim non-recovery is permanent: that language belongs to `stuck`, - // which is what exits 1. - try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") == null); - - // Genuinely resolved: every attempted symbol is level again. - var w2: std.Io.Writer.Allocating = .init(a); - defer w2.deinit(); - try printSweepConclusion(&w2.writer, .{ - .ran = true, - .found = 2, - .attempted = 2, - .recovered = 2, - .still_behind = 0, - .stuck = 0, - }, &.{}); - try std.testing.expect(std.mem.indexOf(u8, w2.written(), "all clear") != null); -} - -test "printSweepConclusion: disagreeing counters never produce a false all-clear" { - // Defence in the one direction that matters. `stuck > 0` with a `still_behind` - // that failed to keep up is a bug in the caller, but the output must degrade - // to the LOUD reading, never the quiet one - the all-clear is what suppresses - // the non-zero exit, so a false all-clear is silent data rot. - const a = std.testing.allocator; - const stuck = [_]zfin.freshness.Finding{.{ - .symbol = "SPCX", - .kind = .equity, - .last_date = zfin.Date.fromYmd(2026, 6, 29), - .peer_date = zfin.Date.fromYmd(2026, 8, 12), - .days_behind = 44, - }}; - var w: std.Io.Writer.Allocating = .init(a); - defer w.deinit(); - try printSweepConclusion(&w.writer, .{ - .ran = true, - .found = 1, - .attempted = 1, - .recovered = 0, - .still_behind = 0, // deliberately inconsistent with `stuck` - .stuck = 1, - }, &stuck); - const s = w.written(); - try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); - try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") != null); - try std.testing.expect(std.mem.indexOf(u8, s, "SPCX") != null); -} - -/// Build a Report from findings, for the pure decision tests. -fn testReport( - stale: []zfin.freshness.Finding, - far: []zfin.freshness.Finding, -) zfin.freshness.Report { - return .{ .stale = stale, .far_behind = far, .orphans = &.{}, .missing = &.{}, .groups = &.{} }; -} - -fn testFinding(symbol: []const u8, days: i64) zfin.freshness.Finding { - return .{ - .symbol = symbol, - .kind = .equity, - .last_date = zfin.Date.fromYmd(2026, 8, 12).addDays(@intCast(-days)), - .peer_date = zfin.Date.fromYmd(2026, 8, 12), - .days_behind = days, - }; -} - -test "selectForForcing: a symbol the pass already fetched is not re-asked" { - // THE 5pm CASE. 13 equities post, 2 do not. Those 2 had their TTL lapse, so - // the pass made a real provider call and got nothing newer. Re-asking seconds - // later cannot change the answer: it burns quota (50 req/hr on the free Tiingo - // plan) and bypasses the TTL pacing `expiryAfterFetch` exists to enforce. - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const a = arena.allocator(); - - var stale = [_]zfin.freshness.Finding{ testFinding("MSFT", 1), testFinding("NKE", 1) }; - const report = testReport(&stale, &.{}); - - // Empty set: the pass fetched everything, so nothing is worth re-asking. - var none = std.StringHashMap(void).init(a); - try std.testing.expectEqual(@as(usize, 0), (try selectForForcing(a, report, &none)).len); -} - -test "selectForForcing: a symbol the pass never fetched IS re-asked" { - // The SPCX case, and the reason the sweep exists. A server sync stamped a - // future `#!expires=`, so the pass returned `.cached` without ever reaching a - // provider - while serving a copy six weeks old. Forcing is the only thing - // that moves it. - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const a = arena.allocator(); - - var far = [_]zfin.freshness.Finding{testFinding("SPCX", 44)}; - var stale = [_]zfin.freshness.Finding{testFinding("NKE", 1)}; - const report = testReport(&stale, &far); - - var cached = std.StringHashMap(void).init(a); - try cached.put("SPCX", {}); - try cached.put("NKE", {}); - const sel = try selectForForcing(a, report, &cached); - try std.testing.expectEqual(@as(usize, 2), sel.len); - // far_behind first, so a mid-sweep rate limit bites the least-behind symbol. - try std.testing.expectEqualStrings("SPCX", sel[0]); - try std.testing.expectEqualStrings("NKE", sel[1]); -} - -test "selectForForcing: mixed - only the unfetched half is re-asked" { - var arena = std.heap.ArenaAllocator.init(std.testing.allocator); - defer arena.deinit(); - const a = arena.allocator(); - - var far = [_]zfin.freshness.Finding{ testFinding("SPCX", 44), testFinding("ORC42", 30) }; - var stale = [_]zfin.freshness.Finding{ testFinding("MSFT", 1), testFinding("NKE", 2) }; - const report = testReport(&stale, &far); - - var cached = std.StringHashMap(void).init(a); - try cached.put("SPCX", {}); // TTL-held, worth forcing - try cached.put("NKE", {}); // TTL-held, worth forcing - const sel = try selectForForcing(a, report, &cached); - try std.testing.expectEqual(@as(usize, 2), sel.len); - try std.testing.expectEqualStrings("SPCX", sel[0]); - try std.testing.expectEqualStrings("NKE", sel[1]); -} - -test "isFinding: consulted across BOTH lists" { - // An earlier shape of the recovered-count checked only one list, which - // credited recovery to symbols that had merely crossed from far_behind into - // stale - still behind, reported as fixed. - var far = [_]zfin.freshness.Finding{testFinding("SPCX", 44)}; - var stale = [_]zfin.freshness.Finding{testFinding("NKE", 1)}; - const report = testReport(&stale, &far); - try std.testing.expect(isFinding(report, "SPCX")); - try std.testing.expect(isFinding(report, "NKE")); - try std.testing.expect(!isFinding(report, "AMZN")); -} - -test "printSweepConclusion: findings with nothing re-asked is not a closure" { - // The 5pm output. Symbols ARE behind, so no closure may be claimed, and the - // run must explain why it did nothing rather than looking like it missed them. - const a = std.testing.allocator; - var w: std.Io.Writer.Allocating = .init(a); - defer w.deinit(); - try printSweepConclusion(&w.writer, .{ - .ran = true, - .found = 2, - .attempted = 0, - .skipped = 2, - .still_behind = 2, - .stuck = 0, - }, &.{}); - const s = w.written(); - try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); - try std.testing.expect(std.mem.indexOf(u8, s, "already queried this pass") != null); - try std.testing.expect(std.mem.indexOf(u8, s, "within what ordinary lag explains") != null); - // Must not read as resolved - the symbols are still behind. - try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); +test { + // Pulls each module's tests into the build. NOT optional bookkeeping: a test + // build never analyses `main`, so nothing else references these files, and + // without this the runner reports "test success" having executed ZERO tests. + // That is the worst possible failure - a green tick read as coverage - and it + // is exactly what the first attempt at this split produced. + _ = @import("App.zig"); + _ = @import("handlers.zig"); + _ = @import("refresh.zig"); } diff --git a/src/refresh.zig b/src/refresh.zig new file mode 100644 index 0000000..3f88e68 --- /dev/null +++ b/src/refresh.zig @@ -0,0 +1,1221 @@ +//! The `refresh` subcommand: the cron-driven pass over every tracked symbol, +//! and the post-pass sweep that judges the result against peers. +//! +//! Deliberately free of any `App` dependency - this is a CLI path, not a request +//! path - which is what lets `handlers.zig` import it for the one definition they +//! genuinely share (`collectRefreshSymbols`, the answer to "what will refresh +//! fetch?", which `/:symbol/diagnostics` reports). + +const std = @import("std"); +const zfin = @import("zfin"); + +const version = @import("build_options").version; +const log = std.log.scoped(.@"zfin-server"); + +/// (estimate 0) or the provider isn't instantiated yet. +/// +/// The number is the pause a *live* fetch would incur right now -- not +/// a promise that we will wait. A cache hit consumes no token and skips +/// the wait entirely (see the one-time legend at the top of a refresh +/// run). The estimate is read before the fetch, so it reflects the +/// pre-fetch bucket state. +fn printRateLimitTag(svc: *zfin.DataService, data_type: zfin.cache.DataType, stdout: *std.Io.Writer) !void { + if (svc.estimateWaitSeconds(data_type)) |wait| { + if (wait > 0) { + try stdout.print("[~{d}s] ", .{wait}); + try stdout.flush(); + } + } +} + +/// Per-data-type outcome tally for a refresh run. `fetched` vs `cached` +/// tracks network-vs-cache (useful for spotting cache expiry / TTL +/// tuning and for gauging load against provider rate limits); `na` is a +/// legitimate "no data for this symbol" outcome (NotFound, or an +/// entity_facts skip); `failed` is a hard error. +const TypeStat = struct { + fetched: usize = 0, + cached: usize = 0, + na: usize = 0, + failed: usize = 0, + + /// Record a successful fetch: network hit vs served-from-cache. + fn hit(self: *TypeStat, was_fetched: bool) void { + if (was_fetched) self.fetched += 1 else self.cached += 1; + } +}; + +/// Per-type tallies for the seven per-symbol data types a refresh +/// touches. Reported as the summary table at the end of a run. +const RefreshStats = struct { + candles: TypeStat = .{}, + dividends: TypeStat = .{}, + splits: TypeStat = .{}, + earnings: TypeStat = .{}, + classification: TypeStat = .{}, + etf_metrics: TypeStat = .{}, + entity_facts: TypeStat = .{}, +}; + +/// Symbol-level status partition. Precedence, highest first: +/// failed > lagging > overdue > current -- the same precedence +/// `refreshExit` uses, so the printed counts always agree with the +/// process exit code (lagging is NOT folded into "current"). +const SymbolCounts = struct { + current: usize = 0, + lagging: usize = 0, + overdue: usize = 0, + failed: usize = 0, +}; + +/// Candle freshness for one symbol this run. Set during the candle +/// check; stays `.current` if candles were empty or the symbol failed +/// before the check (a failed symbol is bucketed as `failed` regardless). +const Freshness = enum { current, lagging, overdue }; + +/// Record a hard failure of `tag` for the current symbol: bump that +/// type's failed counter and remember the tag, in one call, so the two +/// can't drift apart. The remembered tags become the end-of-symbol +/// `failed:` entry, e.g. "SYM (candles, earnings)". Each data-type block +/// runs once per symbol and `fail_types` is cleared per symbol, so a tag +/// is recorded at most once per symbol (no duplicates). +fn recordFailure(stat: *TypeStat, fail_types: *std.ArrayList([]const u8), allocator: std.mem.Allocator, tag: []const u8) !void { + stat.failed += 1; + try fail_types.append(allocator, tag); +} + +/// Print one right-aligned data-type row of the summary table. Shares +/// its width specifiers with the header in `refresh` so columns line up. +fn printStatRow(stdout: *std.Io.Writer, name: []const u8, s: TypeStat) !void { + try stdout.print(" {s:<14}{d:>7}{d:>8}{d:>5}{d:>8}\n", .{ name, s.fetched, s.cached, s.na, s.failed }); +} + +/// Print a `label: a, b, c` line, or `label: (none)` when empty. Always +/// printed (even when empty) so the failed/lagging/overdue lines are +/// reliable grep targets. +fn printSymbolList(stdout: *std.Io.Writer, label: []const u8, items: []const []const u8) !void { + try stdout.print(" {s:<9}", .{label}); + if (items.len == 0) { + try stdout.print("(none)\n", .{}); + return; + } + for (items, 0..) |it, i| { + if (i > 0) try stdout.print(", ", .{}); + try stdout.print("{s}", .{it}); + } + try stdout.print("\n", .{}); +} + +// ── Refresh command ────────────────────────────────────────── + +/// The symbols the refresh loop will fetch: `.stock` and `.watch` lots, keyed by +/// `priceSymbol()` so a `ticker::` alias resolves the same way a fetch does. +/// +/// Extracted rather than inlined because `/:symbol/diagnostics` reports a +/// `tracked` flag, and a `tracked` that meant anything other than "refresh will +/// fetch this" would be worse than not reporting it at all - an operator would +/// read it as a promise the loop never made. One definition, two callers. +pub fn collectRefreshSymbols(out: *std.StringHashMap(void), lots: []const zfin.Lot) !void { + for (lots) |lot| { + if (lot.security_type != .stock and lot.security_type != .watch) continue; + if (lot.symbol.len == 0) continue; + const sym = lot.priceSymbol(); + if (!out.contains(sym)) try out.put(sym, {}); + } +} + +pub fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) !u8 { + var config = zfin.Config.fromEnv(io, allocator, environ); + defer config.deinit(); + var svc = zfin.DataService.init(io, allocator, config); + defer svc.deinit(); + + // wall-clock required: the provider-lag check compares each symbol's + // newest cached bar against the most recent session the market should + // already have data for (see zfin.market.candleFreshness). Captured + // once so the whole run shares a consistent "now". + const now_s = std.Io.Timestamp.now(io, .real).toSeconds(); + + // wall-clock required: end-to-end run duration for the summary line. + // .awake (monotonic) avoids skew-induced negatives like dispatch does. + const start_ns = std.Io.Timestamp.now(io, .awake).nanoseconds; + + const portfolio_path = environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; + + const data = std.Io.Dir.cwd().readFileAlloc(io, portfolio_path, allocator, .limited(10 * 1024 * 1024)) catch { + log.err("failed to read portfolio: {s}", .{portfolio_path}); + return error.ReadFailed; + }; + defer allocator.free(data); + + var portfolio = zfin.cache.deserializePortfolio(allocator, data) catch { + log.err("failed to parse portfolio", .{}); + return error.ParseFailed; + }; + defer portfolio.deinit(); + + var symbols = std.StringHashMap(void).init(allocator); + defer symbols.deinit(); + try collectRefreshSymbols(&symbols, portfolio.lots); + + const stdout_file = std.Io.File.stdout(); + var buf: [4096]u8 = undefined; + var writer = stdout_file.writer(io, &buf); + const stdout = &writer.interface; + + try stdout.print("zfin-server {s}\n", .{version}); + try stdout.print("Refreshing {d} symbols from {s}\n", .{ symbols.count(), portfolio_path }); + try stdout.print("note: [~Ns] = est. pause before the next live fetch (bucket empty); cache hits skip it\n", .{}); + try stdout.flush(); + + var counts: SymbolCounts = .{}; + var stats: RefreshStats = .{}; + // Symbols whose candles came back WITHOUT a provider call this run: either + // the TTL was still fresh, or a server sync satisfied the request - + // `service.zig` returns `.cached` for both (see the sync at its line 934). + // These are the only symbols a forced refresh in the post-pass sweep can + // help; for every other finding the provider was asked seconds ago and did + // not have the bar, so asking again in the same run is pure quota burn and + // bypasses the very TTL pacing `expiryAfterFetch` exists to get right. + // + // Keys borrow from `portfolio`, whose `deinit` is function-scoped and so + // outlives the sweep below. Block-scoping that deinit is what dangled the + // equivalent keys in `handleDiagnostics`. + var unfetched = std.StringHashMap(void).init(allocator); + defer unfetched.deinit(); + var failed_list = std.ArrayList([]const u8).empty; + var lagging_list = std.ArrayList([]const u8).empty; + var overdue_list = std.ArrayList([]const u8).empty; + // Reused per symbol (clearRetainingCapacity) to collect the data-type + // tags that failed, so the failed list can read "SYM (candles, earnings)". + var fail_types = std.ArrayList([]const u8).empty; + + // Warm the EDGAR ticker maps once per refresh run. They're + // ~3-5 MB each, cached for 30 days; warming guarantees the + // shared `/_edgar/tickers_funds.srf` and + // `tickers_companies.srf` files exist for the static-route + // handlers to serve. Per-symbol `getEtfMetrics` calls below + // also rely on these maps being loaded. + { + try printRateLimitTag(&svc, .tickers_funds, stdout); + if (svc.loadMutualFundTickerMap(.{})) |mut_map| { + var m = mut_map; + m.deinit(); + try stdout.print("EDGAR mutual-fund ticker map ok\n", .{}); + } else |err| { + try stdout.print("EDGAR mutual-fund ticker map FAILED ({t})\n", .{err}); + } + try printRateLimitTag(&svc, .tickers_companies, stdout); + if (svc.loadCompanyTickerMap(.{})) |co_map| { + var m = co_map; + m.deinit(); + try stdout.print("EDGAR company ticker map ok\n", .{}); + } else |err| { + try stdout.print("EDGAR company ticker map FAILED ({t})\n", .{err}); + } + try stdout.flush(); + } + + var it = symbols.iterator(); + while (it.next()) |entry| { + const sym = entry.key_ptr.*; + try stdout.print("{s}: ", .{sym}); + try stdout.flush(); + + var sym_freshness: Freshness = .current; + fail_types.clearRetainingCapacity(); + + // Candles + try printRateLimitTag(&svc, .candles_daily, stdout); + if (svc.getCandles(sym, .{})) |result| { + defer result.deinit(); + try stdout.print("candles ok ({s})", .{@tagName(result.source)}); + stats.candles.hit(result.source == .fetched); + // `== .cached` rather than `!= .fetched` on purpose: should a third + // `Source` ever appear, this defaults to NOT forcing. That failure + // mode leaves a symbol behind, which surfaces via `stuck` and mails + // the operator; the opposite default burns quota in silence. + if (result.source == .cached) try unfetched.put(sym, {}); + + // Provider-data-lag check: did we end up with the latest bar + // the market should have posted by now? A `.lagging` bar is + // merely unposted -> flag it so the run exits EX_TEMPFAIL and + // cron retries. An `.overdue` bar is almost certainly an + // un-modeled closure (e.g. Good Friday) -> note it, no retry. + if (result.data.len > 0) { + const last = result.data[result.data.len - 1].date; + var date_buf: [10]u8 = undefined; + const ds = std.fmt.bufPrint(&date_buf, "{f}", .{last}) catch "?"; + switch (zfin.market.candleFreshness(now_s, zfin.market.classify(sym), last)) { + .lagging => { + sym_freshness = .lagging; + try stdout.print(" LAGGING (latest {s})", .{ds}); + log.info("provider data lag: {s} latest bar {s}; newer session due but unposted", .{ sym, ds }); + }, + .overdue => { + sym_freshness = .overdue; + // States the observation, NOT an inference. This used to + // say "assuming market closure (no retry)", which the + // pass cannot know: `candleFreshness` judges this symbol + // against the trading calendar in isolation, and a + // closure moves EVERY symbol together. A single symbol + // nothing refreshes produces the identical reading. The + // corpus question is answered after the pass, by + // `sweepAfterPass`, which can see the peers. + log.info("{s}: latest bar {s} overdue past grace window; no retry this pass", .{ sym, ds }); + }, + .current => {}, + } + } + } else |err| { + try stdout.print("candles FAILED ({s})", .{@errorName(err)}); + try recordFailure(&stats.candles, &fail_types, allocator, "candles"); + if (err == zfin.DataError.TransientError or err == zfin.DataError.AuthError) { + const reason = if (err == zfin.DataError.AuthError) "auth failure" else "transient provider failure"; + try stdout.print("\n", .{}); + try stdout.print("\nStopping refresh: {s}\n", .{reason}); + try stdout.print("Refresh aborted after {d} current, {d} lagging, {d} overdue, {d} failed\n", .{ counts.current, counts.lagging, counts.overdue, counts.failed + 1 }); + try stdout.flush(); + return error.RefreshFailed; + } + } + + // Dividends + try stdout.print(", ", .{}); + try printRateLimitTag(&svc, .dividends, stdout); + if (svc.getDividends(sym, .{})) |result| { + defer result.deinit(); + try stdout.print("dividends ok ({s})", .{@tagName(result.source)}); + stats.dividends.hit(result.source == .fetched); + } else |err| { + try stdout.print("dividends FAILED ({s})", .{@errorName(err)}); + try recordFailure(&stats.dividends, &fail_types, allocator, "dividends"); + } + + // Splits + try stdout.print(", ", .{}); + try printRateLimitTag(&svc, .splits, stdout); + if (svc.getSplits(sym, .{})) |result| { + defer result.deinit(); + try stdout.print("splits ok ({s})", .{@tagName(result.source)}); + stats.splits.hit(result.source == .fetched); + } else |err| { + try stdout.print("splits FAILED ({s})", .{@errorName(err)}); + try recordFailure(&stats.splits, &fail_types, allocator, "splits"); + } + + // Earnings + try stdout.print(", ", .{}); + try printRateLimitTag(&svc, .earnings, stdout); + if (svc.getEarnings(sym, .{})) |result| { + defer result.deinit(); + try stdout.print("earnings ok ({s})", .{@tagName(result.source)}); + stats.earnings.hit(result.source == .fetched); + } else |err| { + try stdout.print("earnings FAILED ({s})", .{@errorName(err)}); + try recordFailure(&stats.earnings, &fail_types, allocator, "earnings"); + } + + // Classification (Wikidata + EDGAR fallback). Captures + // CIK and is_etf — used to chain into entity_facts below. + // NotFound is logged as `n/a` (symbol genuinely has no + // Wikidata or EDGAR entry) and isn't counted as a failure. + var cik_buf: ?[]u8 = null; + defer if (cik_buf) |b| allocator.free(b); + var is_etf = false; + try stdout.print(", ", .{}); + try printRateLimitTag(&svc, .classification, stdout); + if (svc.getClassification(sym, .{})) |result| { + defer result.deinit(); + if (result.data.len > 0) { + if (result.data[0].cik) |cik| { + cik_buf = allocator.dupe(u8, cik) catch null; + } + is_etf = result.data[0].is_etf; + } + try stdout.print("classification ok ({s})", .{@tagName(result.source)}); + stats.classification.hit(result.source == .fetched); + } else |err| switch (err) { + zfin.DataError.NotFound => { + try stdout.print("classification n/a", .{}); + stats.classification.na += 1; + }, + else => { + try stdout.print("classification FAILED ({t})", .{err}); + try recordFailure(&stats.classification, &fail_types, allocator, "classification"); + }, + } + + // ETF metrics. NotFound is the expected outcome for + // non-funds (NPORT-P only exists for funds + UITs); a + // negative-cache entry suppresses retries. Logged as + // `n/a` and isn't counted as a failure. + try stdout.print(", ", .{}); + try printRateLimitTag(&svc, .etf_metrics, stdout); + if (svc.getEtfMetrics(sym, .{})) |result| { + defer result.deinit(); + try stdout.print("etf_metrics ok ({s})", .{@tagName(result.source)}); + stats.etf_metrics.hit(result.source == .fetched); + } else |err| switch (err) { + zfin.DataError.NotFound => { + try stdout.print("etf_metrics n/a", .{}); + stats.etf_metrics.na += 1; + }, + else => { + try stdout.print("etf_metrics FAILED ({t})", .{err}); + try recordFailure(&stats.etf_metrics, &fail_types, allocator, "etf_metrics"); + }, + } + + // Entity facts (XBRL). Only attempted when the + // classification step yielded a CIK from a non-fund + // record. ETFs/funds CIKs (iShares Trust, Fidelity series + // CIKs, etc.) don't file the operating-company XBRL + // concepts entity_facts looks for; calling EDGAR for + // them is guaranteed-404 noise. Skip them up front. + if (cik_buf) |cik| { + try stdout.print(", ", .{}); + if (is_etf) { + try stdout.print("entity_facts n/a (ETF)", .{}); + stats.entity_facts.na += 1; + } else { + try printRateLimitTag(&svc, .entity_facts, stdout); + if (svc.getEntityFacts(cik, .{})) |result| { + defer result.deinit(); + try stdout.print("entity_facts ok ({s})", .{@tagName(result.source)}); + stats.entity_facts.hit(result.source == .fetched); + } else |err| switch (err) { + zfin.DataError.NotFound => { + try stdout.print("entity_facts n/a", .{}); + stats.entity_facts.na += 1; + }, + else => { + try stdout.print("entity_facts FAILED ({t})", .{err}); + try recordFailure(&stats.entity_facts, &fail_types, allocator, "entity_facts"); + }, + } + } + } else { + // No CIK resolved: entity_facts is not applicable for this + // symbol, so it counts as n/a (keeps the table row summing + // to the total symbol count). + stats.entity_facts.na += 1; + } + + try stdout.print("\n", .{}); + try stdout.flush(); + + // Bucket the symbol by precedence failed > lagging > overdue > + // current (same order refreshExit uses for the exit code). A + // symbol failed iff any data type recorded a hard error. + if (fail_types.items.len > 0) { + counts.failed += 1; + const types = try std.mem.join(allocator, ", ", fail_types.items); + try failed_list.append(allocator, try std.fmt.allocPrint(allocator, "{s} ({s})", .{ sym, types })); + } else switch (sym_freshness) { + .current => counts.current += 1, + .lagging => { + counts.lagging += 1; + try lagging_list.append(allocator, sym); + }, + .overdue => { + counts.overdue += 1; + try overdue_list.append(allocator, sym); + }, + } + } + + // Sweep AFTER every symbol has been through, because the question it answers + // - is anything behind its peers? - has no answer until the corpus is whole. + const sweep = sweepAfterPass(io, allocator, &svc, config.cache_dir, &symbols, &unfetched, now_s, stdout) catch |err| blk: { + log.warn("post-pass sweep failed: {t}", .{err}); + break :blk SweepOutcome{}; + }; + try stdout.flush(); + + const elapsed_ns = std.Io.Timestamp.now(io, .awake).nanoseconds - start_ns; + const elapsed_s: u64 = @intCast(@divTrunc(elapsed_ns, std.time.ns_per_s)); + const stuck = if (sweepIsActionable(sweep)) sweep.stuck else 0; + const code = refreshExit(counts.failed, counts.lagging, stuck); + // Distinguishes the two paths to `1`, since the code alone cannot. + const reason = if (counts.failed > 0) + "failures" + else if (stuck > 0) + "stuck behind peers" + else if (code == 75) + "lagging" + else + "clean"; + + try stdout.print("\nRefresh complete in {d}s (exit {d}: {s})\n", .{ elapsed_s, code, reason }); + try stdout.print(" symbols: {d} current, {d} lagging, {d} overdue, {d} failed ({d} total)\n", .{ counts.current, counts.lagging, counts.overdue, counts.failed, symbols.count() }); + + try stdout.print("\n {s:<14}{s:>7}{s:>8}{s:>5}{s:>8}\n", .{ "type", "fetched", "cached", "n/a", "failed" }); + try printStatRow(stdout, "candles", stats.candles); + try printStatRow(stdout, "dividends", stats.dividends); + try printStatRow(stdout, "splits", stats.splits); + try printStatRow(stdout, "earnings", stats.earnings); + try printStatRow(stdout, "classification", stats.classification); + try printStatRow(stdout, "etf_metrics", stats.etf_metrics); + try printStatRow(stdout, "entity_facts", stats.entity_facts); + + try stdout.print("\n", .{}); + try printSymbolList(stdout, "failed:", failed_list.items); + try printSymbolList(stdout, "lagging:", lagging_list.items); + try printSymbolList(stdout, "overdue:", overdue_list.items); + try stdout.flush(); + + return code; +} + +/// What the post-pass sweep found and what it managed to fix. +const SweepOutcome = struct { + /// Tracked symbols still further behind their peers than ordinary lag + /// explains, AFTER a forced refresh. These will not clear on the next cron + /// tick, which is why they get their own exit disposition. + stuck: usize = 0, + /// Findings the sweep saw before it acted on anything. + /// + /// Distinct from `attempted` because "nothing was behind" and "things were + /// behind but none were worth re-asking" are different states, and only the + /// first may print a market-closure conclusion. Conflating them would + /// reintroduce the false inference the pass-time log used to make. + found: usize = 0, + /// Symbols the sweep force-refreshed. + attempted: usize = 0, + /// Findings deliberately left alone: the pass already asked the provider for + /// these this run, so a forced re-ask cannot produce a different answer. + skipped: usize = 0, + /// Of those, how many the forced fetch actually brought level with peers. + recovered: usize = 0, + /// Symbols still behind their peers after the retry, INCLUDING those within + /// what ordinary lag explains. Tracked separately from `stuck` because + /// keying the all-clear message on `stuck` alone announced "all clear" while + /// a symbol was still behind - it had merely crossed back inside + /// `max_normal_lag_days`, which is progress, not resolution. + still_behind: usize = 0, + /// Was the sweep able to reach a conclusion at all? False when the cache + /// could not be enumerated, in which case nothing here is a finding. + ran: bool = false, + /// True when no individual symbol is behind its peers but the corpus as a + /// whole sits behind the calendar. That is the shape a market closure makes, + /// and the only shape from which one can honestly be inferred. + corpus_behind: bool = false, + /// The peer reference the conclusion was drawn against, for the operator to + /// check the reasoning rather than take it on faith. + peer_date: ?zfin.Date = null, +}; + +/// Should this run exit non-zero for a gap that will not self-heal? +/// +/// Split from `refreshExit` so the "is a multi-day gap actionable" question is +/// testable without constructing a whole run: `stuck` counts only symbols still +/// past `max_normal_lag_days` after a FORCED refresh, so it excludes both +/// ordinary provider lag (which `75` already covers) and anything a retry fixes. +fn sweepIsActionable(o: SweepOutcome) bool { + return o.ran and o.stuck > 0; +} + +/// Which findings a forced refresh can plausibly help, worst first. +/// +/// Only symbols the pass did NOT already ask the provider about. A symbol whose +/// candles were fetched seconds ago will get the identical answer from an +/// immediate re-ask, so forcing it wastes a request AND bypasses the TTL pacing +/// that exists to space retries out. The case that DOES benefit is a symbol the +/// pass never reached the provider for - a fresh TTL, or a server sync that +/// satisfied the request while serving a copy weeks old. That is precisely the +/// shape of the bug this whole sweep was built for. +/// +/// `far_behind` before `stale` so a rate limit, if one bites mid-sweep, bites the +/// least-behind symbols. Both input lists arrive already sorted worst-first. +/// +/// Pure given the report and the set, which is the point: the "don't waste calls" +/// rule is testable without a provider. +fn selectForForcing( + arena: std.mem.Allocator, + before: zfin.freshness.Report, + unfetched: *const std.StringHashMap(void), +) ![]const []const u8 { + var out = std.ArrayList([]const u8).empty; + for (before.far_behind) |f| { + if (unfetched.contains(f.symbol)) try out.append(arena, f.symbol); + } + for (before.stale) |f| { + if (unfetched.contains(f.symbol)) try out.append(arena, f.symbol); + } + return out.items; +} + +/// Is `symbol` still behind its peers in `report`? +fn isFinding(report: zfin.freshness.Report, symbol: []const u8) bool { + for (report.far_behind) |f| { + if (std.mem.eql(u8, f.symbol, symbol)) return true; + } + for (report.stale) |f| { + if (std.mem.eql(u8, f.symbol, symbol)) return true; + } + return false; +} + +/// Sweep the whole cache after the main pass, force-refresh whatever is behind +/// its peers, then re-check. +/// +/// Why this cannot be folded into the pass: the pass judges each symbol against +/// the trading calendar in isolation (`market.candleFreshness`), which cannot +/// tell a market closure from a single symbol nothing refreshes - both leave a +/// bar sitting past the grace window. A closure moves every symbol together, so +/// "behind its own peers" is disproof of one. That is a corpus question, and the +/// corpus is only complete once every symbol has been through the pass. +/// +/// The forced refresh is the point, not a nicety: a symbol can be behind purely +/// because nothing ever asked for it, and one forced fetch brings it fully +/// current (observed on a watchlist symbol 42 days behind). Reporting the gap +/// without attempting the fix would file a ticket for something the run could +/// have closed itself. +fn sweepAfterPass( + io: std.Io, + allocator: std.mem.Allocator, + svc: *zfin.DataService, + cache_dir: []const u8, + tracked: *const std.StringHashMap(void), + unfetched: *const std.StringHashMap(void), + now_s: i64, + stdout: *std.Io.Writer, +) !SweepOutcome { + // One arena for the whole sweep: it runs once, at the end of a process that + // is about to exit, and the alternative is five separate ownership dances + // across two scans. + var arena_state = std.heap.ArenaAllocator.init(allocator); + defer arena_state.deinit(); + const arena = arena_state.allocator(); + + var store = zfin.cache.Store.init(io, arena, cache_dir); + const keys = store.cacheKeys(arena) catch |err| { + // Not a finding: an unreadable cache directory means the sweep has no + // opinion, and reporting `stuck = 0` as though it had checked would be + // the same lie the old log line told. + log.warn("post-pass sweep skipped: cannot enumerate cache: {t}", .{err}); + return .{}; + }; + + const before = try zfin.freshness.scan( + arena, + try zfin.freshness.collect(arena, &store, keys, tracked, &.{}), + now_s, + ); + + var out = SweepOutcome{ .ran = true }; + out.peer_date = widestPeerDate(before); + + out.found = before.stale.len + before.far_behind.len; + if (out.found == 0) { + // Nothing is behind its peers. The only remaining question is whether the + // corpus itself has moved, which is what distinguishes "quiet market" from + // "nothing ran" - and it is the ONLY state from which a market closure can + // honestly be inferred. + out.corpus_behind = corpusBehind(before); + try printSweepConclusion(stdout, out, &.{}); + return out; + } + + const forced = try selectForForcing(arena, before, unfetched); + out.attempted = forced.len; + out.skipped = out.found - forced.len; + for (forced) |sym| forceOne(svc, sym); + + // Nothing was forced, so the cache is byte-identical to the pre-sweep scan - + // re-reading it would cost ~40 file reads to learn what we already know, and + // could only differ by disagreeing with itself. + const after = if (forced.len == 0) before else blk: { + // Re-read from disk rather than trusting the fetch's return value: the + // question is what a CLIENT will now be served, and that is whatever + // landed in the cache. + var after_store = zfin.cache.Store.init(io, arena, cache_dir); + const after_keys = after_store.cacheKeys(arena) catch keys; + break :blk try zfin.freshness.scan( + arena, + try zfin.freshness.collect(arena, &after_store, after_keys, tracked, &.{}), + now_s, + ); + }; + + out.still_behind = after.stale.len + after.far_behind.len; + out.stuck = after.far_behind.len; + // Counted by membership, not by subtracting totals. The arithmetic form + // (`attempted - still_behind`) already produced a wrong number once, and + // skipped symbols make it wrong in a second way: they inflate + // `still_behind` without ever having been attempted. + for (forced) |sym| { + if (!isFinding(after, sym)) out.recovered += 1; + } + out.peer_date = widestPeerDate(after) orelse out.peer_date; + // A symbol behind its peers is disproof of a closure, so this stays false + // whenever there are findings - regardless of how the corpus looks. + out.corpus_behind = false; + + try printSweepConclusion(stdout, out, after.far_behind); + return out; +} + +/// Force one symbol's candles, swallowing failures by design: the sweep's job is +/// to report the post-attempt state, and a fetch error here is already visible in +/// the re-scan as "still behind". Logged by name so the cause is not lost. +fn forceOne(svc: *zfin.DataService, symbol: []const u8) void { + if (svc.getCandles(symbol, .{ .force_refresh = true })) |result| { + result.deinit(); + } else |err| { + log.warn("post-pass sweep: forced refresh of {s} failed: {t}", .{ symbol, err }); + } +} + +/// The newest bar any conclusive peer group holds. The reference the sweep's +/// conclusions are measured against. +fn widestPeerDate(report: zfin.freshness.Report) ?zfin.Date { + var newest: ?zfin.Date = null; + for (report.groups) |g| { + if (!g.conclusive()) continue; + const p = g.peer_date orelse continue; + if (newest == null or newest.?.lessThan(p)) newest = p; + } + return newest; +} + +/// Is every conclusive group's own newest bar behind the calendar? +/// +/// Only meaningful when nothing is behind its peers - which the caller enforces. +/// `GroupState.freshness` is the same verdict the fetch gate uses, so this does +/// not re-derive the market calendar. +fn corpusBehind(report: zfin.freshness.Report) bool { + var conclusive: usize = 0; + for (report.groups) |g| { + if (!g.conclusive()) continue; + conclusive += 1; + const f = g.freshness orelse return false; + if (f == .current) return false; + } + return conclusive > 0; +} + +fn printSweepConclusion( + stdout: *std.Io.Writer, + o: SweepOutcome, + stuck: []const zfin.freshness.Finding, +) !void { + if (!o.ran) return; + try stdout.print("\nPost-pass sweep:\n", .{}); + // Keyed on `found`, NOT `attempted`. Those diverged the moment the sweep + // learned to skip symbols the pass had already asked about, and keying the + // closure conclusion on `attempted` would have claimed a market closure while + // symbols were demonstrably behind their peers - the exact false inference + // that was removed from the pass-time log. + if (o.found == 0) { + if (o.corpus_behind) { + // The one case where a closure CAN be inferred, and it is inferred + // from the corpus moving together rather than from one symbol. + try stdout.print(" no symbol is behind its peers, but the whole cache sits behind the\n", .{}); + try stdout.print(" calendar - consistent with a market closure, not a refresh problem\n", .{}); + } else { + try stdout.print(" nothing behind its peers\n", .{}); + } + return; + } + + if (o.attempted == 0) { + // The 5pm case: equities posted, a couple of symbols have not, and the + // pass already asked the provider about them this run. Re-asking cannot + // change the answer, so the sweep does nothing and says why. + try stdout.print(" {d} behind peers; none re-asked - the provider was already queried this pass\n", .{o.found}); + } else if (o.skipped > 0) { + try stdout.print(" {d} behind peers; {d} re-asked, {d} skipped (already queried this pass); {d} now level\n", .{ o.found, o.attempted, o.skipped, o.recovered }); + } else { + try stdout.print(" {d} symbol(s) behind peers -> forced refresh; {d} now level with peers\n", .{ o.attempted, o.recovered }); + } + + // Ordered by severity, NOT by field convenience. An earlier arrangement + // tested `still_behind == 0` first, which would print "all clear" for any + // outcome whose counters disagreed - the one direction this must never fail + // in, since the all-clear is what suppresses the non-zero exit. + if (o.stuck > 0) { + // Named explicitly as non-self-healing, because the whole reason this run + // exits non-zero is to stop cron from looping on it silently. + try stdout.print(" {d} still further behind than lag explains - a retry will not clear these:\n", .{o.stuck}); + for (stuck) |f| { + try stdout.print(" {s:<10} {f} {d}d behind {f}\n", .{ f.symbol, f.last_date, f.days_behind, f.peer_date }); + } + // Deliberately not naming a cause. The candidates are many and none + // visible from here (see `zfin.freshness.max_normal_lag_days`); + // `zfin diagnose SYMBOL` is the tool that narrows it. + try stdout.print(" run `zfin diagnose SYMBOL` against one of these to narrow it\n", .{}); + return; + } + if (o.still_behind > 0) { + // Partial progress, and saying so matters: these crossed back inside + // `max_normal_lag_days`, so the next pass should close them and this run + // must NOT exit as though a human were needed. + try stdout.print(" {d} still behind, but within what ordinary lag explains - the next pass should close it\n", .{o.still_behind}); + return; + } + try stdout.print(" all clear after the retry\n", .{}); +} + +/// Map a refresh run's failure/lag counts to a process exit code: +/// 0 - every symbol current and fetched cleanly +/// 75 - EX_TEMPFAIL: no hard failures, but at least one symbol's +/// just-closed bar hadn't posted yet (provider lag); cron should +/// retry shortly +/// 1 - at least one hard failure (fetch error), OR a tracked symbol +/// still further behind its peers than lag explains after the +/// post-pass sweep forced a refresh +/// Hard failure dominates lag - if anything failed outright that's the +/// code the operator needs to act on. +/// +/// The stuck-symbol case is deliberately `1` rather than `75`, and this is the +/// distinction that matters: `75` tells cron "retry soon", which is right for an +/// unposted bar and wrong for a multi-day gap. A gap the sweep could not close +/// with a forced fetch will not close on the next tick either, so the run has to +/// mail rather than loop. It shares `1` with a fetch failure because both mean +/// "a human should look"; the summary line names which one it was. +fn refreshExit(fail_count: usize, lag_count: usize, stuck_count: usize) u8 { + if (fail_count > 0) return 1; + if (stuck_count > 0) return 1; + if (lag_count > 0) return 75; + return 0; +} + +// ── Tests ──────────────────────────────────────────────────── + +test "refreshExit: hard failure dominates, then stuck, then lag, else clean" { + try std.testing.expectEqual(@as(u8, 0), refreshExit(0, 0, 0)); + try std.testing.expectEqual(@as(u8, 75), refreshExit(0, 3, 0)); + try std.testing.expectEqual(@as(u8, 1), refreshExit(2, 0, 0)); + // A hard failure outranks lag. + try std.testing.expectEqual(@as(u8, 1), refreshExit(1, 5, 0)); + + // A symbol still behind its peers after a FORCED refresh exits 1, not 75. + // This is the whole point of the third argument: 75 means EX_TEMPFAIL, which + // tells cron to retry soon - correct for an unposted bar, wrong for a + // multi-day gap that a forced fetch already failed to close. Retrying that on + // a schedule loops silently forever, which is how SPCX went 43 days unnoticed. + try std.testing.expectEqual(@as(u8, 1), refreshExit(0, 0, 1)); + // Stuck outranks lag: the actionable finding wins over the retryable one. + try std.testing.expectEqual(@as(u8, 1), refreshExit(0, 9, 1)); + // But a hard failure still outranks stuck - it is the more proximate problem + // and may well be the CAUSE of the gap. + try std.testing.expectEqual(@as(u8, 1), refreshExit(4, 0, 2)); +} + +test "sweepIsActionable: only a sweep that actually ran can be a finding" { + // A sweep that could not enumerate the cache reports `stuck = 0`, and reading + // that as "nothing is behind" would repeat the exact mistake the old + // market-closure log line made: asserting a conclusion from missing evidence. + // `ran` is what separates "checked and found nothing" from "did not check". + try std.testing.expect(!sweepIsActionable(.{ .ran = false, .stuck = 3 })); + try std.testing.expect(!sweepIsActionable(.{ .ran = true, .stuck = 0 })); + try std.testing.expect(sweepIsActionable(.{ .ran = true, .stuck = 1 })); + // The zero value must never be actionable - it is what the error paths return. + try std.testing.expect(!sweepIsActionable(.{})); +} + +test "corpusBehind: a closure is only inferable when every group moved together" { + const d = zfin.Date.fromYmd(2026, 8, 12); + const empty: []zfin.freshness.Finding = &.{}; + + // Two conclusive groups, both overdue, nothing behind its peers. This is the + // one shape from which a market closure can honestly be read. + var groups = [_]zfin.freshness.GroupState{ + .{ .kind = .equity, .peer_date = d, .freshness = .overdue, .dated = 9 }, + .{ .kind = .mutual_fund, .peer_date = d, .freshness = .overdue, .dated = 4 }, + }; + var r = zfin.freshness.Report{ + .stale = empty, + .far_behind = empty, + .orphans = &.{}, + .missing = &.{}, + .groups = &groups, + }; + try std.testing.expect(corpusBehind(r)); + + // One group current: the market plainly was not closed. + groups[1].freshness = .current; + try std.testing.expect(!corpusBehind(r)); + + // Unknown freshness is not evidence of a closure. Absence of information must + // not become a conclusion. + groups[1].freshness = null; + try std.testing.expect(!corpusBehind(r)); + + // No conclusive group at all - a single cached symbol per kind - concludes + // nothing rather than vacuously true. + var lonely = [_]zfin.freshness.GroupState{ + .{ .kind = .equity, .peer_date = d, .freshness = .overdue, .dated = 1 }, + }; + r.groups = &lonely; + try std.testing.expect(!corpusBehind(r)); + r.groups = &.{}; + try std.testing.expect(!corpusBehind(r)); +} + +test "widestPeerDate: the newest conclusive group wins, inconclusive ignored" { + const aug12 = zfin.Date.fromYmd(2026, 8, 12); + const aug11 = zfin.Date.fromYmd(2026, 8, 11); + const jun01 = zfin.Date.fromYmd(2026, 6, 1); + var groups = [_]zfin.freshness.GroupState{ + .{ .kind = .equity, .peer_date = aug11, .freshness = null, .dated = 5 }, + .{ .kind = .mutual_fund, .peer_date = aug12, .freshness = null, .dated = 3 }, + }; + var r = zfin.freshness.Report{ + .stale = &.{}, + .far_behind = &.{}, + .orphans = &.{}, + .missing = &.{}, + .groups = &groups, + }; + try std.testing.expectEqual(@as(?zfin.Date, aug12), widestPeerDate(r)); + + // An inconclusive group's date must not become the reference - with one cached + // symbol its "peer date" is just its own bar. + groups[1] = .{ .kind = .mutual_fund, .peer_date = jun01, .freshness = null, .dated = 1 }; + try std.testing.expectEqual(@as(?zfin.Date, aug11), widestPeerDate(r)); + + r.groups = &.{}; + try std.testing.expectEqual(@as(?zfin.Date, null), widestPeerDate(r)); +} + +test "printStatRow aligns with the summary-table header" { + var hdr: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer hdr.deinit(); + try hdr.writer.print(" {s:<14}{s:>7}{s:>8}{s:>5}{s:>8}\n", .{ "type", "fetched", "cached", "n/a", "failed" }); + const hdr_out = try hdr.toOwnedSlice(); + defer std.testing.allocator.free(hdr_out); + + var row: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer row.deinit(); + // Multi-digit values and the widest type name still fit the columns. + try printStatRow(&row.writer, "classification", .{ .fetched = 0, .cached = 31, .na = 14, .failed = 0 }); + const row_out = try row.toOwnedSlice(); + defer std.testing.allocator.free(row_out); + + // Identical width specifiers => identical rendered length => columns line up. + try std.testing.expectEqual(hdr_out.len, row_out.len); +} + +test "printSymbolList: empty shows (none), non-empty joins with commas" { + var a: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer a.deinit(); + try printSymbolList(&a.writer, "failed:", &.{}); + const a_out = try a.toOwnedSlice(); + defer std.testing.allocator.free(a_out); + try std.testing.expectEqualStrings(" failed: (none)\n", a_out); + + var b: std.Io.Writer.Allocating = .init(std.testing.allocator); + defer b.deinit(); + try printSymbolList(&b.writer, "lagging:", &.{ "NKE", "AMZN" }); + const b_out = try b.toOwnedSlice(); + defer std.testing.allocator.free(b_out); + try std.testing.expectEqualStrings(" lagging: NKE, AMZN\n", b_out); +} + +test "collectRefreshSymbols: only .stock and .watch, keyed by priceSymbol" { + const a = std.testing.allocator; + var set = std.StringHashMap(void).init(a); + defer set.deinit(); + + const lots = [_]zfin.Lot{ + .{ .symbol = "AAPL", .security_type = .stock, .shares = 1, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + .{ .symbol = "SPCX", .security_type = .watch, .shares = 0, .open_price = 0, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + // Cash is not fetched, so it must not appear as tracked - reporting it + // would promise a refresh that never runs. + .{ .symbol = "USD", .security_type = .cash, .shares = 100, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + // A duplicate holding is one symbol, not two. + .{ .symbol = "AAPL", .security_type = .stock, .shares = 5, .open_price = 5, .open_date = zfin.Date.fromYmd(2026, 2, 1) }, + // A CUSIP priced through a `ticker::` alias must register under the + // symbol a FETCH uses, not the one the statement shows - otherwise + // `tracked` says false for a symbol the loop refreshes every night. + .{ .symbol = "922908736", .ticker = "VTTHX", .security_type = .stock, .shares = 3, .open_price = 27, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + }; + try collectRefreshSymbols(&set, &lots); + + try std.testing.expectEqual(@as(u32, 3), set.count()); + try std.testing.expect(set.contains("AAPL")); + try std.testing.expect(set.contains("SPCX")); + try std.testing.expect(set.contains("VTTHX")); + try std.testing.expect(!set.contains("922908736")); + try std.testing.expect(!set.contains("USD")); +} + +test "collectRefreshSymbols: keys stay readable while the source lots live" { + // Guards the bug this shipped with: the handler parsed the portfolio in an + // inner scope with a `defer portfolio.deinit()`, so by the time it asked + // `tracked.contains(symbol)` the keys pointed at released memory and all 25 + // tracked symbols reported false. `zfin`'s own `trackedSymbols` carries a + // comment about the identical failure - "made every cached symbol look + // untracked" - which is what makes it worth a test rather than a comment. + // + // The invariant is ownership, not content: the set BORROWS from `lots`, so a + // lookup is only valid while `lots` is alive. Asserting a hit after the + // insert-scope has closed is the cheapest way to pin that. + const a = std.testing.allocator; + var set = std.StringHashMap(void).init(a); + defer set.deinit(); + + const lots = [_]zfin.Lot{ + .{ .symbol = "AMZN", .security_type = .stock, .shares = 1, .open_price = 1, .open_date = zfin.Date.fromYmd(2026, 1, 1) }, + }; + { + // A nested scope that ends before the lookup, mirroring the handler's + // shape. `lots` outlives it, so the keys remain valid. + try collectRefreshSymbols(&set, &lots); + } + try std.testing.expect(set.contains("AMZN")); + try std.testing.expectEqual(@as(u32, 1), set.count()); +} + +test "printSweepConclusion: a closure is claimed only when the corpus moved together" { + const a = std.testing.allocator; + + // Nothing behind peers, corpus itself behind: the ONE case where a closure is + // a legitimate inference, and it must say so from the corpus, not one symbol. + { + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ .ran = true, .corpus_behind = true }, &.{}); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "market closure") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "not a refresh problem") != null); + } + + // Nothing behind peers and the corpus is current: no closure claim at all. + // The old pass-time log asserted closure from a single symbol's calendar + // position; nothing may reintroduce that from an empty finding list. + { + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ .ran = true, .corpus_behind = false }, &.{}); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); + try std.testing.expect(std.mem.indexOf(u8, s, "nothing behind its peers") != null); + } + + // A sweep that never ran prints nothing - it has no opinion to report. + { + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{}, &.{}); + try std.testing.expectEqual(@as(usize, 0), w.written().len); + } +} + +test "printSweepConclusion: a stuck symbol is named and marked non-self-healing" { + const a = std.testing.allocator; + const stuck = [_]zfin.freshness.Finding{.{ + .symbol = "SPCX", + .kind = .equity, + .last_date = zfin.Date.fromYmd(2026, 6, 29), + .peer_date = zfin.Date.fromYmd(2026, 8, 12), + .days_behind = 44, + }}; + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ + .ran = true, + .found = 3, + .attempted = 3, + .recovered = 2, + .still_behind = 1, + .stuck = 1, + }, &stuck); + const s = w.written(); + + // The symbol and both dates, so the operator can check the reasoning rather + // than trust the verdict. + try std.testing.expect(std.mem.indexOf(u8, s, "SPCX") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "2026-06-29") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "2026-08-12") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "44d behind") != null); + // The recovery tally, so a partial success is not read as total failure. + try std.testing.expect(std.mem.indexOf(u8, s, "2 now level with peers") != null); + // Says a retry will NOT help - the justification for exiting 1 over 75. + try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") != null); + // Points at the tool that narrows a cause instead of guessing one. + try std.testing.expect(std.mem.indexOf(u8, s, "zfin diagnose") != null); + // And must NOT guess. `freshness.max_normal_lag_days` documents that the + // candidates are many and none visible from here. + try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); +} + +test "printSweepConclusion: partial progress is not an all-clear, and not a page" { + // The bug this pins, found by running it: `all clear after the retry` printed + // whenever `stuck == 0`, ignoring symbols still behind by a day or two. A + // forced refresh that drags a symbol from 53 days behind to 1 day behind has + // made progress and resolved nothing, and the operator must be able to tell + // those apart from the output alone. + const a = std.testing.allocator; + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ + .ran = true, + .found = 1, + .attempted = 1, + .recovered = 0, + .still_behind = 1, + .stuck = 0, + }, &.{}); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); + try std.testing.expect(std.mem.indexOf(u8, s, "within what ordinary lag explains") != null); + // Must not claim non-recovery is permanent: that language belongs to `stuck`, + // which is what exits 1. + try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") == null); + + // Genuinely resolved: every attempted symbol is level again. + var w2: std.Io.Writer.Allocating = .init(a); + defer w2.deinit(); + try printSweepConclusion(&w2.writer, .{ + .ran = true, + .found = 2, + .attempted = 2, + .recovered = 2, + .still_behind = 0, + .stuck = 0, + }, &.{}); + try std.testing.expect(std.mem.indexOf(u8, w2.written(), "all clear") != null); +} + +test "printSweepConclusion: disagreeing counters never produce a false all-clear" { + // Defence in the one direction that matters. `stuck > 0` with a `still_behind` + // that failed to keep up is a bug in the caller, but the output must degrade + // to the LOUD reading, never the quiet one - the all-clear is what suppresses + // the non-zero exit, so a false all-clear is silent data rot. + const a = std.testing.allocator; + const stuck = [_]zfin.freshness.Finding{.{ + .symbol = "SPCX", + .kind = .equity, + .last_date = zfin.Date.fromYmd(2026, 6, 29), + .peer_date = zfin.Date.fromYmd(2026, 8, 12), + .days_behind = 44, + }}; + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ + .ran = true, + .found = 1, + .attempted = 1, + .recovered = 0, + .still_behind = 0, // deliberately inconsistent with `stuck` + .stuck = 1, + }, &stuck); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); + try std.testing.expect(std.mem.indexOf(u8, s, "will not clear") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "SPCX") != null); +} + +/// Build a Report from findings, for the pure decision tests. +fn testReport( + stale: []zfin.freshness.Finding, + far: []zfin.freshness.Finding, +) zfin.freshness.Report { + return .{ .stale = stale, .far_behind = far, .orphans = &.{}, .missing = &.{}, .groups = &.{} }; +} + +fn testFinding(symbol: []const u8, days: i64) zfin.freshness.Finding { + return .{ + .symbol = symbol, + .kind = .equity, + .last_date = zfin.Date.fromYmd(2026, 8, 12).addDays(@intCast(-days)), + .peer_date = zfin.Date.fromYmd(2026, 8, 12), + .days_behind = days, + }; +} + +test "selectForForcing: a symbol the pass already fetched is not re-asked" { + // THE 5pm CASE. 13 equities post, 2 do not. Those 2 had their TTL lapse, so + // the pass made a real provider call and got nothing newer. Re-asking seconds + // later cannot change the answer: it burns quota (50 req/hr on the free Tiingo + // plan) and bypasses the TTL pacing `expiryAfterFetch` exists to enforce. + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + var stale = [_]zfin.freshness.Finding{ testFinding("MSFT", 1), testFinding("NKE", 1) }; + const report = testReport(&stale, &.{}); + + // Empty set: the pass fetched everything, so nothing is worth re-asking. + var none = std.StringHashMap(void).init(a); + try std.testing.expectEqual(@as(usize, 0), (try selectForForcing(a, report, &none)).len); +} + +test "selectForForcing: a symbol the pass never fetched IS re-asked" { + // The SPCX case, and the reason the sweep exists. A server sync stamped a + // future `#!expires=`, so the pass returned `.cached` without ever reaching a + // provider - while serving a copy six weeks old. Forcing is the only thing + // that moves it. + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + var far = [_]zfin.freshness.Finding{testFinding("SPCX", 44)}; + var stale = [_]zfin.freshness.Finding{testFinding("NKE", 1)}; + const report = testReport(&stale, &far); + + var cached = std.StringHashMap(void).init(a); + try cached.put("SPCX", {}); + try cached.put("NKE", {}); + const sel = try selectForForcing(a, report, &cached); + try std.testing.expectEqual(@as(usize, 2), sel.len); + // far_behind first, so a mid-sweep rate limit bites the least-behind symbol. + try std.testing.expectEqualStrings("SPCX", sel[0]); + try std.testing.expectEqualStrings("NKE", sel[1]); +} + +test "selectForForcing: mixed - only the unfetched half is re-asked" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const a = arena.allocator(); + + var far = [_]zfin.freshness.Finding{ testFinding("SPCX", 44), testFinding("ORC42", 30) }; + var stale = [_]zfin.freshness.Finding{ testFinding("MSFT", 1), testFinding("NKE", 2) }; + const report = testReport(&stale, &far); + + var cached = std.StringHashMap(void).init(a); + try cached.put("SPCX", {}); // TTL-held, worth forcing + try cached.put("NKE", {}); // TTL-held, worth forcing + const sel = try selectForForcing(a, report, &cached); + try std.testing.expectEqual(@as(usize, 2), sel.len); + try std.testing.expectEqualStrings("SPCX", sel[0]); + try std.testing.expectEqualStrings("NKE", sel[1]); +} + +test "isFinding: consulted across BOTH lists" { + // An earlier shape of the recovered-count checked only one list, which + // credited recovery to symbols that had merely crossed from far_behind into + // stale - still behind, reported as fixed. + var far = [_]zfin.freshness.Finding{testFinding("SPCX", 44)}; + var stale = [_]zfin.freshness.Finding{testFinding("NKE", 1)}; + const report = testReport(&stale, &far); + try std.testing.expect(isFinding(report, "SPCX")); + try std.testing.expect(isFinding(report, "NKE")); + try std.testing.expect(!isFinding(report, "AMZN")); +} + +test "printSweepConclusion: findings with nothing re-asked is not a closure" { + // The 5pm output. Symbols ARE behind, so no closure may be claimed, and the + // run must explain why it did nothing rather than looking like it missed them. + const a = std.testing.allocator; + var w: std.Io.Writer.Allocating = .init(a); + defer w.deinit(); + try printSweepConclusion(&w.writer, .{ + .ran = true, + .found = 2, + .attempted = 0, + .skipped = 2, + .still_behind = 2, + .stuck = 0, + }, &.{}); + const s = w.written(); + try std.testing.expect(std.mem.indexOf(u8, s, "market closure") == null); + try std.testing.expect(std.mem.indexOf(u8, s, "already queried this pass") != null); + try std.testing.expect(std.mem.indexOf(u8, s, "within what ordinary lag explains") != null); + // Must not read as resolved - the symbols are still behind. + try std.testing.expect(std.mem.indexOf(u8, s, "all clear") == null); +}