ai all the things

This commit is contained in:
Emil Lerch 2026-03-10 17:29:33 -07:00
parent 238160ba90
commit 1c3bb524c5
Signed by: lobo
GPG key ID: A7B62D657EF764F8
5 changed files with 696 additions and 0 deletions

97
README.md Normal file
View file

@ -0,0 +1,97 @@
# zfin-server
HTTP data service backed by [zfin](https://git.lerch.org/lobo/zfin)'s financial
data providers. Serves cached market data over HTTP for two consumers:
1. **LibreCalc spreadsheets** — trailing returns via `WEBSERVICE`/`FILTERXML`
2. **zfin CLI/TUI** — raw SRF cache files as an L1 data cache
## Quick start
```sh
# Set API keys (same as zfin)
export TWELVEDATA_API_KEY=...
export POLYGON_API_KEY=...
# Start the server
zig build run -- serve --port=8080
# In another terminal
curl http://localhost:8080/AAPL/returns
curl http://localhost:8080/AAPL/returns?fmt=xml
```
## Endpoints
| Route | Content-Type | Description |
|-------|-------------|-------------|
| `GET /` | `text/html` | Landing page |
| `GET /help` | `text/plain` | Endpoint documentation |
| `GET /symbols` | `application/json` | List of tracked symbols |
| `GET /:symbol/returns` | `application/json` | Trailing 1/3/5/10yr returns + volatility |
| `GET /:symbol/returns?fmt=xml` | `application/xml` | Same, XML for LibreCalc |
| `GET /:symbol/quote` | `application/json` | Latest quote |
| `GET /:symbol/candles` | `application/x-srf` | Raw SRF cache file |
| `GET /:symbol/dividends` | `application/x-srf` | Raw SRF cache file |
| `GET /:symbol/earnings` | `application/x-srf` | Raw SRF cache file |
| `GET /:symbol/options` | `application/x-srf` | Raw SRF cache file |
## LibreCalc usage
```
=FILTERXML(WEBSERVICE("http://localhost:8080/AAPL/returns?fmt=xml"), "//trailing1YearReturn")
```
XML response format:
```xml
<returns>
<ticker>AAPL</ticker>
<returnDate>2026-03-07</returnDate>
<trailing1YearReturn>12.34000</trailing1YearReturn>
<trailing3YearReturn>8.56000</trailing3YearReturn>
<trailing5YearReturn>15.78000</trailing5YearReturn>
<trailing10YearReturn>22.10000</trailing10YearReturn>
<volatility>18.50000</volatility>
</returns>
```
## Cache refresh
Use cron to keep the cache warm for all symbols in your portfolio:
```sh
# Refresh all portfolio symbols nightly at 5pm ET (after market close)
0 17 * * 1-5 cd /path/to/workdir && ZFIN_PORTFOLIO=portfolio.srf zfin-server refresh
```
The `refresh` command reads the portfolio SRF file (defaults to `portfolio.srf`
in the current directory if `ZFIN_PORTFOLIO` is not set), extracts all stock and
watch symbols, and fetches candles, dividends, and earnings for each.
## Building
Requires [Zig 0.15.2](https://ziglang.org/download/).
```sh
zig build # build
zig build test # run tests
zig build run -- serve --port=8080
```
## Configuration
All configuration is via environment variables:
| Variable | Required | Description |
|----------|----------|-------------|
| `TWELVEDATA_API_KEY` | Yes | TwelveData API key |
| `POLYGON_API_KEY` | No | Polygon API key |
| `FINNHUB_API_KEY` | No | Finnhub API key |
| `ALPHAVANTAGE_API_KEY` | No | Alpha Vantage API key |
| `ZFIN_PORTFOLIO` | No | Path to portfolio SRF (default: `portfolio.srf`) |
| `ZFIN_CACHE_DIR` | No | Cache directory (default: `~/.cache/zfin`) |
## License
MIT

51
build.zig Normal file
View file

@ -0,0 +1,51 @@
const std = @import("std");
pub fn build(b: *std.Build) void {
const target = b.standardTargetOptions(.{});
const optimize = b.standardOptimizeOption(.{});
const zfin_dep = b.dependency("zfin", .{
.target = target,
.optimize = optimize,
});
const httpz_dep = b.dependency("httpz", .{
.target = target,
.optimize = optimize,
});
const imports: []const std.Build.Module.Import = &.{
.{ .name = "zfin", .module = zfin_dep.module("zfin") },
.{ .name = "httpz", .module = httpz_dep.module("httpz") },
};
const exe = b.addExecutable(.{
.name = "zfin-server",
.root_module = b.createModule(.{
.root_source_file = b.path("src/main.zig"),
.target = target,
.optimize = optimize,
.imports = imports,
}),
});
b.installArtifact(exe);
// Run step: `zig build run -- <args>`
const run_step = b.step("run", "Run the server");
const run_cmd = b.addRunArtifact(exe);
run_step.dependOn(&run_cmd.step);
run_cmd.step.dependOn(b.getInstallStep());
if (b.args) |args| {
run_cmd.addArgs(args);
}
// Tests
const test_step = b.step("test", "Run all tests");
const tests = b.addTest(.{ .root_module = b.createModule(.{
.root_source_file = b.path("src/main.zig"),
.target = target,
.optimize = optimize,
.imports = imports,
}) });
test_step.dependOn(&b.addRunArtifact(tests).step);
}

20
build.zig.zon Normal file
View file

@ -0,0 +1,20 @@
.{
.name = .zfin_server,
.version = "0.1.0",
.fingerprint = 0xa060a7d1947dfe8f,
.minimum_zig_version = "0.15.0",
.paths = .{
"build.zig",
"build.zig.zon",
"src",
},
.dependencies = .{
.zfin = .{
.path = "../zfin",
},
.httpz = .{
.url = "git+https://github.com/karlseguin/http.zig#844f8016e6616f00b05d4cc3c713307b0fe586c7",
.hash = "httpz-0.0.0-PNVzrBtMBwAPcQx3mNEgat3Xbsynw-eIC9SmOX5M9XtP",
},
},
}

2
portfolio.srf Normal file
View file

@ -0,0 +1,2 @@
#!srfv1
symbol::AMZN,shares:num:0,open_date::2026-01-01,open_price:num:0,security_type::watch

526
src/main.zig Normal file
View file

@ -0,0 +1,526 @@
//! zfin-server HTTP data service backed by zfin's provider infrastructure.
//!
//! Two modes:
//! zfin-server serve [--port=8080] Start the HTTP server
//! zfin-server refresh Refresh cache for all tracked symbols (for cron)
//!
//! See GET /help for endpoint documentation.
const std = @import("std");
const zfin = @import("zfin");
const httpz = @import("httpz");
const log = std.log.scoped(.@"zfin-server");
// App
const App = struct {
allocator: std.mem.Allocator,
config: zfin.Config,
svc: zfin.DataService,
fn init(allocator: std.mem.Allocator) App {
const config = zfin.Config.fromEnv(allocator);
const svc = zfin.DataService.init(allocator, config);
return .{ .allocator = allocator, .config = config, .svc = svc };
}
fn deinit(self: *App) void {
self.svc.deinit();
self.config.deinit();
}
};
// Route handlers
fn handleIndex(_: *App, _: *httpz.Request, res: *httpz.Response) !void {
res.content_type = httpz.ContentType.HTML;
res.body =
\\<!DOCTYPE html>
\\<html><head><title>zfin-server</title></head>
\\<body>
\\<h1>zfin-server</h1>
\\<p>This is a financial data API server. Not intended for browser use.</p>
\\<p>See <a href="/help">/help</a> for endpoint documentation.</p>
\\</body></html>
;
}
fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void {
res.content_type = httpz.ContentType.TEXT;
res.body =
\\zfin-server — financial data API
\\
\\Endpoints:
\\ GET /{SYMBOL}/returns Trailing 1/3/5/10yr returns (JSON)
\\ GET /{SYMBOL}/returns?fmt=xml Trailing returns (XML, for LibreCalc)
\\ GET /{SYMBOL}/quote Latest quote (JSON)
\\ GET /{SYMBOL}/candles Raw SRF cache file
\\ GET /{SYMBOL}/dividends Raw SRF cache file
\\ GET /{SYMBOL}/earnings Raw SRF cache file
\\ GET /{SYMBOL}/options Raw SRF cache file
\\ GET /symbols List of tracked symbols
\\
\\XML example (LibreCalc):
\\ =FILTERXML(WEBSERVICE("http://host/AAPL/returns?fmt=xml"),"//trailing1YearReturn")
\\
;
}
fn handleSymbols(_: *App, _: *httpz.Request, res: *httpz.Response) !void {
const arena = res.arena;
const portfolio_path = std.posix.getenv("ZFIN_PORTFOLIO") orelse "portfolio.srf";
const file_data = std.fs.cwd().readFileAlloc(arena, portfolio_path, 10 * 1024 * 1024) catch {
res.content_type = httpz.ContentType.JSON;
res.body = "[]";
return;
};
var portfolio = zfin.cache.deserializePortfolio(arena, file_data) catch {
res.content_type = httpz.ContentType.JSON;
res.body = "[]";
return;
};
defer portfolio.deinit();
// Collect unique symbols
var seen = std.StringHashMap(void).init(arena);
var symbols = std.ArrayList([]const u8).empty;
for (portfolio.lots) |lot| {
if (lot.symbol.len == 0) continue;
if (seen.contains(lot.symbol)) continue;
try seen.put(lot.symbol, {});
try symbols.append(arena, lot.symbol);
}
// Build JSON array
var aw: std.Io.Writer.Allocating = .init(arena);
try aw.writer.writeByte('[');
for (symbols.items, 0..) |sym, i| {
if (i > 0) try aw.writer.writeByte(',');
try aw.writer.print("\"{s}\"", .{sym});
}
try aw.writer.writeByte(']');
res.content_type = httpz.ContentType.JSON;
res.body = try aw.toOwnedSlice();
}
fn handleReturns(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
const raw_symbol = req.param("symbol") orelse {
res.status = 400;
res.body = "Missing symbol";
return;
};
const arena = res.arena;
const symbol = try upperDupe(arena, raw_symbol);
// Auto-add to watchlist if requested
const q = try req.query();
if (q.get("watch")) |w| {
if (std.ascii.eqlIgnoreCase(w, "true")) {
appendWatchSymbol(symbol) catch |err| {
log.warn("failed to append watch symbol {s}: {}", .{ symbol, err });
};
}
}
const candle_result = app.svc.getCandles(symbol) catch {
res.status = 404;
res.body = "Symbol not found or fetch failed";
return;
};
defer app.allocator.free(candle_result.data);
const candles = candle_result.data;
if (candles.len == 0) {
res.status = 404;
res.body = "No candle data";
return;
}
const ret = zfin.performance.trailingReturns(candles);
var date_buf: [10]u8 = undefined;
const date_str = candles[candles.len - 1].date.format(&date_buf);
const risk = zfin.risk.computeRisk(candles, zfin.risk.default_risk_free_rate);
const r1y = annualizedOrTotal(ret.one_year);
const r3y = annualizedOrTotal(ret.three_year);
const r5y = annualizedOrTotal(ret.five_year);
const r10y = annualizedOrTotal(ret.ten_year);
const vol = if (risk) |r| r.volatility else null;
// Check if XML requested
if (q.get("fmt")) |fmt| {
if (std.ascii.eqlIgnoreCase(fmt, "xml")) {
res.content_type = httpz.ContentType.XML;
res.body = try std.fmt.allocPrint(arena,
\\<returns>
\\ <ticker>{s}</ticker>
\\ <returnDate>{s}</returnDate>
\\ <trailing1YearReturn>{s}</trailing1YearReturn>
\\ <trailing3YearReturn>{s}</trailing3YearReturn>
\\ <trailing5YearReturn>{s}</trailing5YearReturn>
\\ <trailing10YearReturn>{s}</trailing10YearReturn>
\\ <volatility>{s}</volatility>
\\</returns>
\\
, .{
symbol,
date_str,
fmtPct(arena, r1y),
fmtPct(arena, r3y),
fmtPct(arena, r5y),
fmtPct(arena, r10y),
fmtPct(arena, vol),
});
return;
}
}
res.content_type = httpz.ContentType.JSON;
res.body = try std.fmt.allocPrint(arena,
\\{{"ticker":"{s}","returnDate":"{s}","trailing1YearReturn":{s},"trailing3YearReturn":{s},"trailing5YearReturn":{s},"trailing10YearReturn":{s},"volatility":{s}}}
, .{
symbol,
date_str,
fmtJsonNum(arena, r1y),
fmtJsonNum(arena, r3y),
fmtJsonNum(arena, r5y),
fmtJsonNum(arena, r10y),
fmtJsonNum(arena, vol),
});
}
fn handleQuote(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
const raw_symbol = req.param("symbol") orelse {
res.status = 400;
res.body = "Missing symbol";
return;
};
const arena = res.arena;
const symbol = try upperDupe(arena, raw_symbol);
const q = app.svc.getQuote(symbol) catch {
res.status = 404;
res.body = "Quote not available";
return;
};
res.content_type = httpz.ContentType.JSON;
res.body = try std.fmt.allocPrint(arena,
\\{{"symbol":"{s}","close":{d:.2},"open":{d:.2},"high":{d:.2},"low":{d:.2},"volume":{d},"previous_close":{d:.2}}}
, .{ symbol, q.close, q.open, q.high, q.low, q.volume, q.previous_close });
}
fn handleSrfFile(app: *App, req: *httpz.Request, res: *httpz.Response, filename: []const u8) !void {
const raw_symbol = req.param("symbol") orelse {
res.status = 400;
res.body = "Missing symbol";
return;
};
const arena = res.arena;
const symbol = try upperDupe(arena, raw_symbol);
const path = try std.fmt.allocPrint(arena, "{s}/{s}/{s}", .{ app.config.cache_dir, symbol, filename });
const content = std.fs.cwd().readFileAlloc(arena, path, 10 * 1024 * 1024) catch {
res.status = 404;
res.body = "Cache file not found";
return;
};
res.content_type = httpz.ContentType.BINARY;
res.header("content-type", "application/x-srf");
res.body = content;
}
fn handleCandles(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "candles_daily.srf");
}
fn handleDividends(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "dividends.srf");
}
fn handleEarnings(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "earnings.srf");
}
fn handleOptions(app: *App, req: *httpz.Request, res: *httpz.Response) !void {
return handleSrfFile(app, req, res, "options.srf");
}
// Helpers
fn upperDupe(allocator: std.mem.Allocator, s: []const u8) ![]u8 {
const d = try allocator.dupe(u8, s);
for (d) |*c| c.* = std.ascii.toUpper(c.*);
return d;
}
fn annualizedOrTotal(result: ?zfin.performance.PerformanceResult) ?f64 {
if (result) |r| return r.annualized_return orelse r.total_return;
return null;
}
/// Format as percentage string for XML (e.g., 0.1234 -> "12.34000")
fn fmtPct(arena: std.mem.Allocator, value: ?f64) []const u8 {
if (value) |v| return std.fmt.allocPrint(arena, "{d:.5}", .{v * 100.0}) catch "null";
return "null";
}
/// Format for JSON: number or null literal
fn fmtJsonNum(arena: std.mem.Allocator, value: ?f64) []const u8 {
if (value) |v| return std.fmt.allocPrint(arena, "{d:.5}", .{v * 100.0}) catch "null";
return "null";
}
/// Append a watch lot for the given symbol to the portfolio SRF file,
/// unless it already exists. Best-effort errors are logged, not fatal.
fn appendWatchSymbol(symbol: []const u8) !void {
const portfolio_path = std.posix.getenv("ZFIN_PORTFOLIO") orelse "portfolio.srf";
const allocator = std.heap.page_allocator;
// Read and deserialize existing portfolio (or start empty)
const file_data = std.fs.cwd().readFileAlloc(allocator, portfolio_path, 10 * 1024 * 1024) catch |err| {
if (err == error.FileNotFound) return writeNewPortfolio(allocator, portfolio_path, symbol);
return err;
};
defer allocator.free(file_data);
var portfolio = zfin.cache.deserializePortfolio(allocator, file_data) catch return;
defer portfolio.deinit();
// Check if symbol already tracked
for (portfolio.lots) |lot| {
if (std.ascii.eqlIgnoreCase(lot.symbol, symbol)) return;
}
// Build new lot list with the watch entry appended
var new_lots = try allocator.alloc(zfin.Lot, portfolio.lots.len + 1);
defer allocator.free(new_lots);
@memcpy(new_lots[0..portfolio.lots.len], portfolio.lots);
new_lots[portfolio.lots.len] = .{
.symbol = symbol,
.shares = 0,
.open_date = zfin.Date.fromYmd(2026, 1, 1),
.open_price = 0,
.security_type = .watch,
};
// Serialize and write
const output = try zfin.cache.serializePortfolio(allocator, new_lots);
defer allocator.free(output);
const file = try std.fs.cwd().createFile(portfolio_path, .{});
defer file.close();
try file.writeAll(output);
log.info("added watch symbol {s} to {s}", .{ symbol, portfolio_path });
}
fn writeNewPortfolio(allocator: std.mem.Allocator, path: []const u8, symbol: []const u8) !void {
const lot = [_]zfin.Lot{.{
.symbol = symbol,
.shares = 0,
.open_date = zfin.Date.fromYmd(2026, 1, 1),
.open_price = 0,
.security_type = .watch,
}};
const output = try zfin.cache.serializePortfolio(allocator, &lot);
defer allocator.free(output);
const file = try std.fs.cwd().createFile(path, .{});
defer file.close();
try file.writeAll(output);
log.info("created {s} with watch symbol {s}", .{ path, symbol });
}
// Refresh command
fn refresh(allocator: std.mem.Allocator) !void {
var config = zfin.Config.fromEnv(allocator);
defer config.deinit();
var svc = zfin.DataService.init(allocator, config);
defer svc.deinit();
const portfolio_path = std.posix.getenv("ZFIN_PORTFOLIO") orelse "portfolio.srf";
const data = std.fs.cwd().readFileAlloc(allocator, portfolio_path, 10 * 1024 * 1024) catch {
log.err("failed to read portfolio: {s}", .{portfolio_path});
return error.ReadFailed;
};
defer allocator.free(data);
const portfolio = zfin.cache.deserializePortfolio(allocator, data) catch {
log.err("failed to parse portfolio", .{});
return error.ParseFailed;
};
defer {
for (portfolio.lots) |*lot| {
if (lot.note) |n| allocator.free(n);
}
allocator.free(portfolio.lots);
}
var symbols = std.StringHashMap(void).init(allocator);
defer symbols.deinit();
for (portfolio.lots) |lot| {
if (lot.security_type != .stock and lot.security_type != .watch) continue;
if (lot.symbol.len == 0) continue;
const sym = lot.priceSymbol();
if (!symbols.contains(sym)) {
try symbols.put(sym, {});
}
}
log.info("refreshing {d} symbols...", .{symbols.count()});
var it = symbols.iterator();
while (it.next()) |entry| {
const sym = entry.key_ptr.*;
log.info(" {s}", .{sym});
_ = svc.getCandles(sym) catch |err| {
log.warn(" {s}: candles: {}", .{ sym, err });
};
_ = svc.getDividends(sym) catch |err| {
log.warn(" {s}: dividends: {}", .{ sym, err });
};
_ = svc.getEarnings(sym) catch |err| {
log.warn(" {s}: earnings: {}", .{ sym, err });
};
}
log.info("refresh complete", .{});
}
// Main
pub fn main() !void {
var gpa = std.heap.GeneralPurposeAllocator(.{}){};
defer _ = gpa.deinit();
const allocator = gpa.allocator();
const args = try std.process.argsAlloc(allocator);
defer std.process.argsFree(allocator, args);
if (args.len < 2) {
printUsage();
return;
}
const command = args[1];
if (std.mem.eql(u8, command, "serve")) {
var port: u16 = 8080;
for (args[2..]) |arg| {
if (std.mem.startsWith(u8, arg, "--port=")) {
port = std.fmt.parseInt(u16, arg["--port=".len..], 10) catch 8080;
}
}
var app = App.init(allocator);
defer app.deinit();
var server = try httpz.Server(*App).init(allocator, .{
.address = .all(port),
}, &app);
defer {
server.stop();
server.deinit();
}
var router = try server.router(.{});
// Static routes
router.get("/", handleIndex, .{});
router.get("/help", handleHelp, .{});
router.get("/symbols", handleSymbols, .{});
// Symbol routes
router.get("/:symbol/returns", handleReturns, .{});
router.get("/:symbol/quote", handleQuote, .{});
router.get("/:symbol/candles", handleCandles, .{});
router.get("/:symbol/dividends", handleDividends, .{});
router.get("/:symbol/earnings", handleEarnings, .{});
router.get("/:symbol/options", handleOptions, .{});
log.info("listening on port {d}", .{port});
try server.listen();
} else if (std.mem.eql(u8, command, "refresh")) {
try refresh(allocator);
} else {
printUsage();
}
}
fn printUsage() void {
std.debug.print(
\\Usage: zfin-server <command>
\\
\\Commands:
\\ serve [--port=8080] Start the HTTP server
\\ refresh Refresh cache for all tracked symbols
\\
\\Environment:
\\ ZFIN_PORTFOLIO Path to portfolio SRF file (default: portfolio.srf)
\\ TWELVEDATA_API_KEY TwelveData API key
\\ POLYGON_API_KEY Polygon API key
\\ FINNHUB_API_KEY Finnhub API key
\\ ALPHAVANTAGE_API_KEY Alpha Vantage API key
\\
, .{});
}
// Tests
test "fmtPct" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
try std.testing.expectEqualStrings("null", fmtPct(arena, null));
const result = fmtPct(arena, 0.1234);
try std.testing.expect(std.mem.startsWith(u8, result, "12.34"));
}
test "fmtJsonNum" {
var arena_state = std.heap.ArenaAllocator.init(std.testing.allocator);
defer arena_state.deinit();
const arena = arena_state.allocator();
try std.testing.expectEqualStrings("null", fmtJsonNum(arena, null));
const result = fmtJsonNum(arena, 0.1234);
try std.testing.expect(std.mem.startsWith(u8, result, "12.34"));
}
test "wantsXml" {
// Tested through handleReturns query parsing
}
test "upperDupe" {
const result = try upperDupe(std.testing.allocator, "aapl");
defer std.testing.allocator.free(result);
try std.testing.expectEqualStrings("AAPL", result);
}
test "annualizedOrTotal" {
try std.testing.expect(annualizedOrTotal(null) == null);
const with_ann = zfin.performance.PerformanceResult{
.total_return = 0.5,
.annualized_return = 0.15,
.from = .{ .days = 0 },
.to = .{ .days = 365 },
};
try std.testing.expectApproxEqAbs(@as(f64, 0.15), annualizedOrTotal(with_ann).?, 0.001);
const without_ann = zfin.performance.PerformanceResult{
.total_return = 0.08,
.annualized_return = null,
.from = .{ .days = 0 },
.to = .{ .days = 180 },
};
try std.testing.expectApproxEqAbs(@as(f64, 0.08), annualizedOrTotal(without_ann).?, 0.001);
}