lobo/zfin-server
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

fetch on-demand if file not available

Browse source
This commit is contained in:
Emil Lerch 2026-06-29 14:05:09 -07:00
parent ae664da40d
commit 3b014c2609
Signed by: lobo
GPG key ID: A7B62D657EF764F8
2 changed files with 143 additions and 25 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
README.md
View file

@ -41,6 +41,13 @@ curl http://localhost:8080/AAPL/returns?fmt=xml
| `GET /_edgar/tickers_funds` | `application/x-srf` | EDGAR mutual-fund ticker map (~3 MB) |
| `GET /_edgar/tickers_companies` | `application/x-srf` | EDGAR company ticker map (~5 MB) |
The SRF endpoints behave as an L2 cache. A present cache file is served
as-is (even if stale -- the `refresh` cron is the freshness authority).
On a miss the server fetches once from the upstream provider, fills the
cache, and serves the result; it returns 404 only if that fetch also
fails. A miss never adds the symbol to the tracked set, so an ad-hoc
lookup doesn't create recurring `refresh` load.
## Authentication
`/`, `/help`, and `/:symbol/returns` (the LibreCalc endpoint) are always

161
src/main.zig
View file

@ -202,6 +202,16 @@ fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void {
\\ GET /_edgar/tickers_companies EDGAR company ticker map (SRF)
\\ GET /symbols List of tracked symbols
\\
\\Auth:
\\ All endpoints except /, /help, and /{SYMBOL}/returns require an
\\ API key when ZFIN_SERVER_API_KEY is set (X-API-Key header or
\\ ?api_key= query parameter).
\\
\\Caching:
\\ SRF endpoints serve from the local cache; on a miss the server
\\ fetches once from the provider, fills the cache, then serves
\\ (404 only if that fetch also fails).
\\
\\Returns fields:
\\ lastClose Last closing price
\\ trailing{1,3,5,10}YearReturn Total return with dividend reinvestment
@ -420,8 +430,25 @@ fn handleQuote(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
, .{ symbol, q.close, q.open, q.high, q.low, q.volume, q.previous_close });
}
fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8) !void {
return handleSrfFileByKey(app, req, res, "symbol", filename);
/// Identifies which `DataService` fetch to run when a served SRF file is
/// absent (see `fetchOnMiss`). Kept separate from `zfin.cache.DataType`
/// because the mapping isn't 1:1 - both `candles_daily.srf` and
/// `candles_meta.srf` are populated by a single `getCandles` call.
const SrfKind = enum {
candles,
dividends,
splits,
earnings,
options,
classification,
etf_metrics,
entity_facts,
tickers_funds,
tickers_companies,
};
fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8, kind: SrfKind) !void {
return handleSrfFileByKey(app, req, res, "symbol", filename, kind);
}
/// Generalized SRF cache-file passthrough: reads
@ -430,7 +457,7 @@ fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename:
/// uses `"symbol"`; CIK-keyed routes (e.g. `/:cik/entity_facts`)
/// pass `"cik"` instead. The cache-key segment is uppercased
/// (safe for both symbols and zero-padded CIK digit strings).
fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8) !void {
fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_param: []const u8, filename: []const u8, kind: SrfKind) !void {
const raw_key = req.param(key_param) orelse {
res.status = 400;
res.body = "Missing key";
@ -438,7 +465,7 @@ fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_
};
const arena = res.arena;
const key = try upperDupe(arena, raw_key);
return serveSrfFile(app, res, key, filename);
return serveSrfFile(app, res, key, filename, kind);
}
/// Static-key SRF cache-file passthrough for routes that don't
@ -446,20 +473,29 @@ fn handleSrfFileByKey(app: *App, req: *httpz.Request, res: *httpz.Response, key_
/// `<cache_dir>/_edgar/tickers_funds.srf` directly). The `key`
/// is a literal directory name; not uppercased because the
/// cache uses `_edgar` as-is.
fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8) !void {
return serveSrfFile(app, res, key, filename);
fn handleStaticSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void {
return serveSrfFile(app, res, key, filename, kind);
}
/// Inner shared helper. Reads the file, computes etag, sets
/// headers, sends. Caller has already resolved the cache-key
/// segment (per-request param or static literal).
fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8) !void {
/// Inner shared helper. Serves `<cache_dir>/<key>/<filename>` as raw SRF
/// with a sha256 ETag. L2-cache contract: a *present* file is served
/// as-is even when stale - cron is the freshness authority, so reads
/// never trigger a refetch - while an *absent* file triggers a one-shot
/// provider fetch (`fetchOnMiss`) to populate it, after which we re-read
/// and serve. If the fetch still can't produce the file, we fall back to
/// the original 404.
fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []const u8, kind: SrfKind) !void {
const arena = res.arena;
const path = try std.fs.path.join(arena, &.{ app.config.cache_dir, key, filename });
const content = std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch {
res.status = 404;
res.body = "Cache file not found";
return;
const content = readCacheFile(app, arena, path) orelse blk: {
// Cache miss -> fetch from the provider, fill the cache, re-read.
fetchOnMiss(app, key, kind);
break :blk readCacheFile(app, arena, path) orelse {
res.status = 404;
res.body = "Cache file not found";
return;
};
};
// Body integrity header: sha256 of the bytes we're about to send.
@ -482,54 +518,129 @@ fn serveSrfFile(app: *App, res: *httpz.Response, key: []const u8, filename: []co
res.body = content;
}
/// Read a cache file into the request arena, or null when it can't be
/// read. Absent is the common case; any read error is treated as a miss
/// (matches the pre-fetch-on-miss behavior of falling back to 404).
fn readCacheFile(app: *App, arena: std.mem.Allocator, path: []const u8) ?[]u8 {
return std.Io.Dir.cwd().readFileAlloc(app.io, path, arena, .limited(10 * 1024 * 1024)) catch null;
}
/// Populate the cache for `key` by running the matching `DataService`
/// fetch, then discard the parsed result - the caller re-reads the
/// canonical bytes off disk. Best-effort and synchronous: any provider
/// error (NotFound, rate limit, auth, transient, parse) is logged and
/// swallowed so the caller falls back to a 404.
///
/// Backpressure caveat: these fetches share zfin's rate limiter, so when
/// the token bucket is drained (e.g. mid-cron) this blocks the httpz
/// worker until a token frees. Accepted for now; a 202 + poll path is
/// the planned escape hatch if blocking becomes a problem.
fn fetchOnMiss(app: *App, key: []const u8, kind: SrfKind) void {
const svc = &app.svc;
switch (kind) {
// getCandles writes both candles_daily.srf and candles_meta.srf.
.candles => {
const r = svc.getCandles(key, .{}) catch |err| return logFetchMiss(key, kind, err);
r.deinit();
},
.dividends => {
const r = svc.getDividends(key, .{}) catch |err| return logFetchMiss(key, kind, err);
r.deinit();
},
.splits => {
const r = svc.getSplits(key, .{}) catch |err| return logFetchMiss(key, kind, err);
r.deinit();
},
.earnings => {
const r = svc.getEarnings(key, .{}) catch |err| return logFetchMiss(key, kind, err);
r.deinit();
},
.options => {
const r = svc.getOptions(key, .{}) catch |err| return logFetchMiss(key, kind, err);
r.deinit();
},
.classification => {
const r = svc.getClassification(key, .{}) catch |err| return logFetchMiss(key, kind, err);
r.deinit();
},
.etf_metrics => {
const r = svc.getEtfMetrics(key, .{}) catch |err| return logFetchMiss(key, kind, err);
r.deinit();
},
// `key` is the CIK here (resolved from the :cik route param).
.entity_facts => {
const r = svc.getEntityFacts(key, .{}) catch |err| return logFetchMiss(key, kind, err);
r.deinit();
},
.tickers_funds => {
var m = svc.loadMutualFundTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err);
m.deinit();
},
.tickers_companies => {
var m = svc.loadCompanyTickerMap(.{}) catch |err| return logFetchMiss(key, kind, err);
m.deinit();
},
}
}
/// Log a failed populate. NotFound is the normal "no such symbol / no
/// data" outcome (debug); everything else is operator-relevant (warn).
fn logFetchMiss(key: []const u8, kind: SrfKind, err: anyerror) void {
if (err == error.NotFound) {
log.info("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) });
} else {
log.warn("fetch-on-miss {s} {s}: {s}", .{ key, @tagName(kind), @errorName(err) });
}
}
fn handleCandles(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "candles_daily.srf");
return handleSrfFile(app, req, res, "candles_daily.srf", .candles);
}
fn handleCandlesMeta(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "candles_meta.srf");
return handleSrfFile(app, req, res, "candles_meta.srf", .candles);
}
fn handleDividends(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "dividends.srf");
return handleSrfFile(app, req, res, "dividends.srf", .dividends);
}
fn handleSplits(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "splits.srf");
return handleSrfFile(app, req, res, "splits.srf", .splits);
}
fn handleEarnings(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "earnings.srf");
return handleSrfFile(app, req, res, "earnings.srf", .earnings);
}
fn handleOptions(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "options.srf");
return handleSrfFile(app, req, res, "options.srf", .options);
}
fn handleClassification(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "classification.srf");
return handleSrfFile(app, req, res, "classification.srf", .classification);
}
fn handleEtfMetrics(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "etf_metrics.srf");
return handleSrfFile(app, req, res, "etf_metrics.srf", .etf_metrics);
}
fn handleEntityFacts(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
// CIK-keyed route: cache layout is
// `<cache_dir>/<CIK>/entity_facts.srf` (the CIK is the
// zero-padded 10-digit string Wikidata's P5531 emits).
return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf");
return handleSrfFileByKey(app, req, res, "cik", "entity_facts.srf", .entity_facts);
}
fn handleTickersFunds(app: *App, _: *httpz.Request, res: *httpz.Response) !void {
// Static-key route: `<cache_dir>/_edgar/tickers_funds.srf`
// is a single file shared across all symbol lookups, not a
// per-symbol cache.
return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf");
return handleStaticSrfFile(app, res, "_edgar", "tickers_funds.srf", .tickers_funds);
}
fn handleTickersCompanies(app: *App, _: *httpz.Request, res: *httpz.Response) !void {
return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf");
return handleStaticSrfFile(app, res, "_edgar", "tickers_companies.srf", .tickers_companies);
}
// ── Helpers ──────────────────────────────────────────────────