From 3b014c2609d1bb203238cdc7bc93cda7798d269f Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Mon, 29 Jun 2026 14:05:09 -0700 Subject: [PATCH] fetch on-demand if file not available --- README.md | 7 +++ src/main.zig | 161 +++++++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 143 insertions(+), 25 deletions(-) diff --git a/README.md b/README.md index 60ada17..bbf4dbc 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,13 @@ curl http://localhost:8080/AAPL/returns?fmt=xml | `GET /_edgar/tickers_funds` | `application/x-srf` | EDGAR mutual-fund ticker map (~3 MB) | | `GET /_edgar/tickers_companies` | `application/x-srf` | EDGAR company ticker map (~5 MB) | +The SRF endpoints behave as an L2 cache. A present cache file is served +as-is (even if stale -- the `refresh` cron is the freshness authority). +On a miss the server fetches once from the upstream provider, fills the +cache, and serves the result; it returns 404 only if that fetch also +fails. A miss never adds the symbol to the tracked set, so an ad-hoc +lookup doesn't create recurring `refresh` load. + ## Authentication `/`, `/help`, and `/:symbol/returns` (the LibreCalc endpoint) are always diff --git a/src/main.zig b/src/main.zig index 9981aa6..3426ec2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -202,6 +202,16 @@ fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void { \\ GET /_edgar/tickers_companies EDGAR company ticker map (SRF) \\ GET /symbols List of tracked symbols \\ + \\Auth: + \\ All endpoints except /, /help, and /{SYMBOL}/returns require an + \\ API key when ZFIN_SERVER_API_KEY is set (X-API-Key header or + \\ ?api_key= query parameter). + \\ + \\Caching: + \\ SRF endpoints serve from the local cache; on a miss the server + \\ fetches once from the provider, fills the cache, then serves + \\ (404 only if that fetch also fails). + \\ \\Returns fields: \\ lastClose Last closing price \\ trailing{1,3,5,10}YearReturn Total return with dividend reinvestment @@ -420,8 +430,25 @@ fn handleQuote(app: *App, req: *httpz.Request, res: *httpz.Response) !void { , .{ symbol, q.close, q.open, q.high, q.low, q.volume, q.previous_close }); } -fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8) !void { - return handleSrfFileByKey(app, req, res, "symbol", filename); +/// Identifies which `DataService` fetch to run when a served SRF file is +/// absent (see `fetchOnMiss`). Kept separate from `zfin.cache.DataType` +/// because the mapping isn't 1:1 - both `candles_daily.srf` and +/// `candles_meta.srf` are populated by a single `getCandles` call. +const SrfKind = enum { + candles, + dividends, + splits, + earnings, + options, + classification, + etf_metrics, + entity_facts, + tickers_funds, + tickers_companies, +}; + +fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8, kind: SrfKind) !void { + return handleSrfFileByKey(app, req, res, "symbol", filename, kind); } /// Generalized SRF cache-file passthrough: reads @@ -430,7 +457,7 @@ fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: /// uses `"symbol"`; CIK-keyed routes (e.g. `/:cik/entity_facts`) /// pass `"cik"` instead. The cache-key segment is uppercased /// (safe for both symbols and zero-padded CIK digit strings). -fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8) !void { +fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8, kind: SrfKind) !void { const raw_key = req.param(key_param) orelse { res.status = 400; res.body = "Missing key"; @@ -438,7 +465,7 @@ fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_ }; const arena = res.arena; const key = try upperDupe(arena, raw_key); - return serveSrfFile(app, res, key, filename); + return serveSrfFile(app, res, key, filename, kind); } /// Static-key SRF cache-file passthrough for routes that don't @@ -446,20 +473,29 @@ fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_ /// `/_edgar/tickers_funds.srf` directly). The `key` /// is a literal directory name; not uppercased because the /// cache uses `_edgar` as-is. -fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8) !void { - return serveSrfFile(app, res, key, filename); +fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void { + return serveSrfFile(app, res, key, filename, kind); } -/// Inner shared helper. Reads the file, computes etag, sets -/// headers, sends. Caller has already resolved the cache-key -/// segment (per-request param or static literal). -fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8) !void { +/// Inner shared helper. Serves `//` as raw SRF +/// with a sha256 ETag. L2-cache contract: a *present* file is served +/// as-is even when stale - cron is the freshness authority, so reads +/// never trigger a refetch - while an *absent* file triggers a one-shot +/// provider fetch (`fetchOnMiss`) to populate it, after which we re-read +/// and serve. If the fetch still can't produce the file, we fall back to +/// the original 404. +fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void { const arena = res.arena; const path = try std.fs.path.join(arena, &.{ app.config.cache_dir, key, filename }); - const content = std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch { - res.status = 404; - res.body = "Cache file not found"; - return; + + const content = readCacheFile(app, arena, path) orelse blk: { + // Cache miss -> fetch from the provider, fill the cache, re-read. + fetchOnMiss(app, key, kind); + break :blk readCacheFile(app, arena, path) orelse { + res.status = 404; + res.body = "Cache file not found"; + return; + }; }; // Body integrity header: sha256 of the bytes we're about to send. @@ -482,54 +518,129 @@ fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []co res.body = content; } +/// Read a cache file into the request arena, or null when it can't be +/// read. Absent is the common case; any read error is treated as a miss +/// (matches the pre-fetch-on-miss behavior of falling back to 404). +fn readCacheFile(app: *App, arena: std.mem.Allocator, path: []const u8) ?[]u8 { + return std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch null; +} + +/// Populate the cache for `key` by running the matching `DataService` +/// fetch, then discard the parsed result - the caller re-reads the +/// canonical bytes off disk. Best-effort and synchronous: any provider +/// error (NotFound, rate limit, auth, transient, parse) is logged and +/// swallowed so the caller falls back to a 404. +/// +/// Backpressure caveat: these fetches share zfin's rate limiter, so when +/// the token bucket is drained (e.g. mid-cron) this blocks the httpz +/// worker until a token frees. Accepted for now; a 202 + poll path is +/// the planned escape hatch if blocking becomes a problem. +fn fetchOnMiss(app: *App, key: []const u8, kind: SrfKind) void { + const svc = &app.svc; + switch (kind) { + // getCandles writes both candles_daily.srf and candles_meta.srf. + .candles => { + const r = svc.getCandles(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .dividends => { + const r = svc.getDividends(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .splits => { + const r = svc.getSplits(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .earnings => { + const r = svc.getEarnings(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .options => { + const r = svc.getOptions(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .classification => { + const r = svc.getClassification(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .etf_metrics => { + const r = svc.getEtfMetrics(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + // `key` is the CIK here (resolved from the :cik route param). + .entity_facts => { + const r = svc.getEntityFacts(key, .{}) catch |err| return logFetchMiss(key, kind, err); + r.deinit(); + }, + .tickers_funds => { + var m = svc.loadMutualFundTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err); + m.deinit(); + }, + .tickers_companies => { + var m = svc.loadCompanyTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err); + m.deinit(); + }, + } +} + +/// Log a failed populate. NotFound is the normal "no such symbol / no +/// data" outcome (debug); everything else is operator-relevant (warn). +fn logFetchMiss(key: []const u8, kind: SrfKind, err: anyerror) void { + if (err == error.NotFound) { + log.info("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) }); + } else { + log.warn("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) }); + } +} + fn handleCandles(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "candles_daily.srf"); + return handleSrfFile(app, req, res, "candles_daily.srf", .candles); } fn handleCandlesMeta(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "candles_meta.srf"); + return handleSrfFile(app, req, res, "candles_meta.srf", .candles); } fn handleDividends(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "dividends.srf"); + return handleSrfFile(app, req, res, "dividends.srf", .dividends); } fn handleSplits(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "splits.srf"); + return handleSrfFile(app, req, res, "splits.srf", .splits); } fn handleEarnings(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "earnings.srf"); + return handleSrfFile(app, req, res, "earnings.srf", .earnings); } fn handleOptions(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "options.srf"); + return handleSrfFile(app, req, res, "options.srf", .options); } fn handleClassification(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "classification.srf"); + return handleSrfFile(app, req, res, "classification.srf", .classification); } fn handleEtfMetrics(app: *App, req: *httpz.Request, res: *httpz.Response) !void { - return handleSrfFile(app, req, res, "etf_metrics.srf"); + return handleSrfFile(app, req, res, "etf_metrics.srf", .etf_metrics); } fn handleEntityFacts(app: *App, req: *httpz.Request, res: *httpz.Response) !void { // CIK-keyed route: cache layout is // `//entity_facts.srf` (the CIK is the // zero-padded 10-digit string Wikidata's P5531 emits). - return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf"); + return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf", .entity_facts); } fn handleTickersFunds(app: *App, _: *httpz.Request, res: *httpz.Response) !void { // Static-key route: `/_edgar/tickers_funds.srf` // is a single file shared across all symbol lookups, not a // per-symbol cache. - return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf"); + return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf", .tickers_funds); } fn handleTickersCompanies(app: *App, _: *httpz.Request, res: *httpz.Response) !void { - return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf"); + return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf", .tickers_companies); } // ── Helpers ──────────────────────────────────────────────────