fetch on-demand if file not available

README.md

@@ -41,6 +41,13 @@ curl http://localhost:8080/AAPL/returns?fmt=xml
 | `GET /_edgar/tickers_funds` | `application/x-srf` | EDGAR mutual-fund ticker map (~3 MB) |
 | `GET /_edgar/tickers_companies` | `application/x-srf` | EDGAR company ticker map (~5 MB) |
 
+The SRF endpoints behave as an L2 cache. A present cache file is served
+as-is (even if stale -- the `refresh` cron is the freshness authority).
+On a miss the server fetches once from the upstream provider, fills the
+cache, and serves the result; it returns 404 only if that fetch also
+fails. A miss never adds the symbol to the tracked set, so an ad-hoc
+lookup doesn't create recurring `refresh` load.
+
 ## Authentication
 
 `/`, `/help`, and `/:symbol/returns` (the LibreCalc endpoint) are always

src/main.zig

@@ -202,6 +202,16 @@ fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void {
         \\  GET /_edgar/tickers_companies   EDGAR company ticker map (SRF)
         \\  GET /symbols                    List of tracked symbols
         \\
+        \\Auth:
+        \\  All endpoints except /, /help, and /{SYMBOL}/returns require an
+        \\  API key when ZFIN_SERVER_API_KEY is set (X-API-Key header or
+        \\  ?api_key= query parameter).
+        \\
+        \\Caching:
+        \\  SRF endpoints serve from the local cache; on a miss the server
+        \\  fetches once from the provider, fills the cache, then serves
+        \\  (404 only if that fetch also fails).
+        \\
         \\Returns fields:
         \\  lastClose                       Last closing price
         \\  trailing{1,3,5,10}YearReturn    Total return with dividend reinvestment
@@ -420,8 +430,25 @@ fn handleQuote(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
     , .{ symbol, q.close, q.open, q.high, q.low, q.volume, q.previous_close });
 }
 
-fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8) !void {
+/// Identifies which `DataService` fetch to run when a served SRF file is
-    return handleSrfFileByKey(app, req, res, "symbol", filename);
+/// absent (see `fetchOnMiss`). Kept separate from `zfin.cache.DataType`
+/// because the mapping isn't 1:1 - both `candles_daily.srf` and
+/// `candles_meta.srf` are populated by a single `getCandles` call.
+const SrfKind = enum {
+    candles,
+    dividends,
+    splits,
+    earnings,
+    options,
+    classification,
+    etf_metrics,
+    entity_facts,
+    tickers_funds,
+    tickers_companies,
+};
+
+fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8, kind: SrfKind) !void {
+    return handleSrfFileByKey(app, req, res, "symbol", filename, kind);
 }
 
 /// Generalized SRF cache-file passthrough: reads
@@ -430,7 +457,7 @@ fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename:
 /// uses `"symbol"`; CIK-keyed routes (e.g. `/:cik/entity_facts`)
 /// pass `"cik"` instead. The cache-key segment is uppercased
 /// (safe for both symbols and zero-padded CIK digit strings).
-fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8) !void {
+fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8, kind: SrfKind) !void {
     const raw_key = req.param(key_param) orelse {
         res.status = 400;
         res.body = "Missing key";
@@ -438,7 +465,7 @@ fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_
     };
     const arena = res.arena;
     const key = try upperDupe(arena, raw_key);
-    return serveSrfFile(app, res, key, filename);
+    return serveSrfFile(app, res, key, filename, kind);
 }
 
 /// Static-key SRF cache-file passthrough for routes that don't
@@ -446,20 +473,29 @@ fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_
 /// `<cache_dir>/_edgar/tickers_funds.srf` directly). The `key`
 /// is a literal directory name; not uppercased because the
 /// cache uses `_edgar` as-is.
-fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8) !void {
+fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void {
-    return serveSrfFile(app, res, key, filename);
+    return serveSrfFile(app, res, key, filename, kind);
 }
 
-/// Inner shared helper. Reads the file, computes etag, sets
+/// Inner shared helper. Serves `<cache_dir>/<key>/<filename>` as raw SRF
-/// headers, sends. Caller has already resolved the cache-key
+/// with a sha256 ETag. L2-cache contract: a *present* file is served
-/// segment (per-request param or static literal).
+/// as-is even when stale - cron is the freshness authority, so reads
-fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8) !void {
+/// never trigger a refetch - while an *absent* file triggers a one-shot
+/// provider fetch (`fetchOnMiss`) to populate it, after which we re-read
+/// and serve. If the fetch still can't produce the file, we fall back to
+/// the original 404.
+fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void {
     const arena = res.arena;
     const path = try std.fs.path.join(arena, &.{ app.config.cache_dir, key, filename });
-    const content = std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch {
+
-        res.status = 404;
+    const content = readCacheFile(app, arena, path) orelse blk: {
-        res.body = "Cache file not found";
+        // Cache miss -> fetch from the provider, fill the cache, re-read.
-        return;
+        fetchOnMiss(app, key, kind);
+        break :blk readCacheFile(app, arena, path) orelse {
+            res.status = 404;
+            res.body = "Cache file not found";
+            return;
+        };
     };
 
     // Body integrity header: sha256 of the bytes we're about to send.
@@ -482,54 +518,129 @@ fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []co
     res.body = content;
 }
 
+/// Read a cache file into the request arena, or null when it can't be
+/// read. Absent is the common case; any read error is treated as a miss
+/// (matches the pre-fetch-on-miss behavior of falling back to 404).
+fn readCacheFile(app: *App, arena: std.mem.Allocator, path: []const u8) ?[]u8 {
+    return std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch null;
+}
+
+/// Populate the cache for `key` by running the matching `DataService`
+/// fetch, then discard the parsed result - the caller re-reads the
+/// canonical bytes off disk. Best-effort and synchronous: any provider
+/// error (NotFound, rate limit, auth, transient, parse) is logged and
+/// swallowed so the caller falls back to a 404.
+///
+/// Backpressure caveat: these fetches share zfin's rate limiter, so when
+/// the token bucket is drained (e.g. mid-cron) this blocks the httpz
+/// worker until a token frees. Accepted for now; a 202 + poll path is
+/// the planned escape hatch if blocking becomes a problem.
+fn fetchOnMiss(app: *App, key: []const u8, kind: SrfKind) void {
+    const svc = &app.svc;
+    switch (kind) {
+        // getCandles writes both candles_daily.srf and candles_meta.srf.
+        .candles => {
+            const r = svc.getCandles(key, .{}) catch |err| return logFetchMiss(key, kind, err);
+            r.deinit();
+        },
+        .dividends => {
+            const r = svc.getDividends(key, .{}) catch |err| return logFetchMiss(key, kind, err);
+            r.deinit();
+        },
+        .splits => {
+            const r = svc.getSplits(key, .{}) catch |err| return logFetchMiss(key, kind, err);
+            r.deinit();
+        },
+        .earnings => {
+            const r = svc.getEarnings(key, .{}) catch |err| return logFetchMiss(key, kind, err);
+            r.deinit();
+        },
+        .options => {
+            const r = svc.getOptions(key, .{}) catch |err| return logFetchMiss(key, kind, err);
+            r.deinit();
+        },
+        .classification => {
+            const r = svc.getClassification(key, .{}) catch |err| return logFetchMiss(key, kind, err);
+            r.deinit();
+        },
+        .etf_metrics => {
+            const r = svc.getEtfMetrics(key, .{}) catch |err| return logFetchMiss(key, kind, err);
+            r.deinit();
+        },
+        // `key` is the CIK here (resolved from the :cik route param).
+        .entity_facts => {
+            const r = svc.getEntityFacts(key, .{}) catch |err| return logFetchMiss(key, kind, err);
+            r.deinit();
+        },
+        .tickers_funds => {
+            var m = svc.loadMutualFundTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err);
+            m.deinit();
+        },
+        .tickers_companies => {
+            var m = svc.loadCompanyTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err);
+            m.deinit();
+        },
+    }
+}
+
+/// Log a failed populate. NotFound is the normal "no such symbol / no
+/// data" outcome (debug); everything else is operator-relevant (warn).
+fn logFetchMiss(key: []const u8, kind: SrfKind, err: anyerror) void {
+    if (err == error.NotFound) {
+        log.info("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) });
+    } else {
+        log.warn("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) });
+    }
+}
+
 fn handleCandles(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
-    return handleSrfFile(app, req, res, "candles_daily.srf");
+    return handleSrfFile(app, req, res, "candles_daily.srf", .candles);
 }
 
 fn handleCandlesMeta(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
-    return handleSrfFile(app, req, res, "candles_meta.srf");
+    return handleSrfFile(app, req, res, "candles_meta.srf", .candles);
 }
 
 fn handleDividends(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
-    return handleSrfFile(app, req, res, "dividends.srf");
+    return handleSrfFile(app, req, res, "dividends.srf", .dividends);
 }
 
 fn handleSplits(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
-    return handleSrfFile(app, req, res, "splits.srf");
+    return handleSrfFile(app, req, res, "splits.srf", .splits);
 }
 
 fn handleEarnings(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
-    return handleSrfFile(app, req, res, "earnings.srf");
+    return handleSrfFile(app, req, res, "earnings.srf", .earnings);
 }
 
 fn handleOptions(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
-    return handleSrfFile(app, req, res, "options.srf");
+    return handleSrfFile(app, req, res, "options.srf", .options);
 }
 
 fn handleClassification(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
-    return handleSrfFile(app, req, res, "classification.srf");
+    return handleSrfFile(app, req, res, "classification.srf", .classification);
 }
 
 fn handleEtfMetrics(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
-    return handleSrfFile(app, req, res, "etf_metrics.srf");
+    return handleSrfFile(app, req, res, "etf_metrics.srf", .etf_metrics);
 }
 
 fn handleEntityFacts(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
     // CIK-keyed route: cache layout is
     // `<cache_dir>/<CIK>/entity_facts.srf` (the CIK is the
     // zero-padded 10-digit string Wikidata's P5531 emits).
-    return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf");
+    return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf", .entity_facts);
 }
 
 fn handleTickersFunds(app: *App, _: *httpz.Request, res: *httpz.Response) !void {
     // Static-key route: `<cache_dir>/_edgar/tickers_funds.srf`
     // is a single file shared across all symbol lookups, not a
     // per-symbol cache.
-    return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf");
+    return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf", .tickers_funds);
 }
 
 fn handleTickersCompanies(app: *App, _: *httpz.Request, res: *httpz.Response) !void {
-    return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf");
+    return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf", .tickers_companies);
 }
 
 // ── Helpers ──────────────────────────────────────────────────