diff --git a/README.md b/README.md index bbf4dbc..10915d3 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,7 @@ curl http://localhost:8080/AAPL/returns?fmt=xml | `GET /symbols` | `application/json` | List of tracked symbols | | `GET /:symbol/returns` | `application/json` | Trailing 1/3/5/10yr returns + volatility | | `GET /:symbol/returns?fmt=xml` | `application/xml` | Same, XML for LibreCalc | +| `GET /:symbol/watch` | `application/json` | Add the symbol to the watchlist (authenticated) | | `GET /:symbol/quote` | `application/json` | Latest quote | | `GET /:symbol/candles` | `application/x-srf` | Raw SRF cache file | | `GET /:symbol/dividends` | `application/x-srf` | Raw SRF cache file | @@ -68,6 +69,25 @@ If `ZFIN_SERVER_API_KEY` is unset or empty, the server is fully open -- a soft cutover so the key can roll out to clients before enforcement is switched on. +## Watchlist + +Adding a symbol to the watchlist enrolls it in the `refresh` cron, so it +gets fetched on every run from then on -- the one operation that creates +recurring provider load. There are two ways in, gated to fit each +consumer: + +- **`GET /:symbol/watch`** -- authenticated (API key), for your own + tooling deliberately growing the tracked set. +- **`GET /:symbol/returns?watch=true`** -- public, but the add only + happens when the request carries LibreOffice's `WEBSERVICE` + User-Agent. This lets the non-technical user add symbols from a + spreadsheet while keeping random callers from growing the cron set. + (Obscurity-grade: a User-Agent is trivially spoofable, which is + acceptable under the casual-traffic threat model.) + +Adds are serialized and written atomically, and implausible symbols are +rejected, so a stray request can't corrupt or balloon `portfolio.srf`. + ## LibreCalc usage ``` diff --git a/src/main.zig b/src/main.zig index 3426ec2..4294c22 100644 --- a/src/main.zig +++ b/src/main.zig @@ -35,6 +35,11 @@ const App = struct { /// soft cutover lets the key roll out to clients before enforcement /// is switched on. Captured once at init. api_key: ?[]const u8, + /// Serializes the portfolio read-modify-write across concurrent + /// watchlist adds (httpz dispatches requests on multiple threads). + /// Without it two simultaneous adds could both read the old file and + /// the second writer would clobber the first's new symbol. + watch_mutex: std.Io.Mutex = .init, fn init(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) App { const config = zfin.Config.fromEnv(io, allocator, environ); @@ -165,6 +170,47 @@ fn providedKeyMatches(req: *httpz.Request, expected: []const u8) bool { return keyMatches(q.get("api_key"), expected); } +/// Case-insensitive User-Agent substrings permitted to add to the +/// watchlist via the public `/:symbol/returns?watch=true` path. This is +/// obscurity-grade (a UA is trivially spoofable) and matches the +/// casual-traffic threat model: it keeps crawlers and stray browsers +/// from growing the tracked set - and thus the recurring cron-refresh +/// load - while letting the non-technical user's LibreOffice WEBSERVICE +/// calls through. The authenticated `/:symbol/watch` route bypasses this +/// (a valid API key is a stronger signal than any UA). +/// +/// Confirmed empirically - LibreOffice's WEBSERVICE sends e.g. +/// "LibreOffice 24.2.7.2 denylistedbackend/8.5.0 OpenSSL/3.0.13" +/// (it also fires a WebDAV OPTIONS preflight that 404s harmlessly; the +/// real GET carries the same User-Agent). Matching the version-agnostic +/// "LibreOffice" token keeps this robust across releases. +const watch_user_agents = [_][]const u8{"LibreOffice"}; + +/// True if `ua` matches one of `watch_user_agents` (case-insensitive +/// substring). A null/absent User-Agent never matches. +fn userAgentMayWatch(ua: ?[]const u8) bool { + const agent = ua orelse return false; + for (watch_user_agents) |needle| { + if (std.ascii.indexOfIgnoreCase(agent, needle) != null) return true; + } + return false; +} + +/// Sanity gate for symbols entering the tracked set (and thus recurring +/// cron load): non-empty, <=16 chars, and only the characters real +/// tickers use - uppercase letters, digits, and `.`/`-` for class +/// shares. Not a real ticker validator; just enough to keep junk like an +/// over-long or path-shaped segment out of the portfolio file. Symbols +/// reach here already upper-cased by `upperDupe`. +fn isPlausibleSymbol(sym: []const u8) bool { + if (sym.len == 0 or sym.len > 16) return false; + for (sym) |c| { + const ok = (c >= 'A' and c <= 'Z') or (c >= '0' and c <= '9') or c == '.' or c == '-'; + if (!ok) return false; + } + return true; +} + // ── Route handlers ─────────────────────────────────────────── fn handleIndex(_: *App, _: *httpz.Request, res: *httpz.Response) !void { @@ -188,6 +234,7 @@ fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void { \\Endpoints: \\ GET /{SYMBOL}/returns Trailing 1/3/5/10yr returns (JSON) \\ GET /{SYMBOL}/returns?fmt=xml Trailing returns (XML, for LibreCalc) + \\ GET /{SYMBOL}/watch Add SYMBOL to the watchlist (authenticated) \\ GET /{SYMBOL}/quote Latest quote (JSON) \\ GET /{SYMBOL}/candles Raw SRF cache file \\ GET /{SYMBOL}/candles_meta Candle freshness metadata (SRF) @@ -212,6 +259,11 @@ fn handleHelp(_: *App, _: *httpz.Request, res: *httpz.Response) !void { \\ fetches once from the provider, fills the cache, then serves \\ (404 only if that fetch also fails). \\ + \\Watchlist (add a symbol to the cron refresh set): + \\ GET /{SYMBOL}/watch authenticated; for your own tooling + \\ GET /{SYMBOL}/returns?watch=true public, but only LibreOffice's + \\ WEBSERVICE User-Agent is honored + \\ \\Returns fields: \\ lastClose Last closing price \\ trailing{1,3,5,10}YearReturn Total return with dividend reinvestment @@ -275,13 +327,20 @@ fn handleReturns(app: *App, req: *httpz.Request, res: *httpz.Response) !void { const arena = res.arena; const symbol = try upperDupe(arena, raw_symbol); - // Auto-add to watchlist if requested + // Auto-add to watchlist if requested. UA-gated (obscurity) so only + // LibreOffice WEBSERVICE calls - not random browsers/crawlers - can + // grow the tracked set via this public endpoint. Best-effort: the + // returns response below is served regardless of whether the add ran. const q = try req.query(); if (q.get("watch")) |w| { if (std.ascii.eqlIgnoreCase(w, "true")) { - appendWatchSymbol(app, symbol) catch |err| { - log.warn("failed to append watch symbol {s}: {}", .{ symbol, err }); - }; + if (userAgentMayWatch(req.header("user-agent"))) { + appendWatchSymbol(app, symbol) catch |err| { + log.warn("failed to append watch symbol {s}: {t}", .{ symbol, err }); + }; + } else { + log.debug("watch add for {s} skipped: User-Agent not allowlisted", .{symbol}); + } } } @@ -409,6 +468,36 @@ fn handleReturns(app: *App, req: *httpz.Request, res: *httpz.Response) !void { }); } +/// Authenticated explicit watchlist add. Not on the public allowlist, so +/// `dispatch` requires the API key - unlike the UA-gated `?watch=true` +/// path, this is for the operator's own tooling deliberately growing the +/// tracked set (and accepting the recurring cron-refresh cost). +fn handleWatch(app: *App, req: *httpz.Request, res: *httpz.Response) !void { + const raw_symbol = req.param("symbol") orelse { + res.status = 400; + res.body = "Missing symbol"; + return; + }; + const arena = res.arena; + const symbol = try upperDupe(arena, raw_symbol); + + appendWatchSymbol(app, symbol) catch |err| switch (err) { + error.InvalidSymbol => { + res.status = 400; + res.body = "Invalid symbol"; + return; + }, + else => { + res.status = 500; + res.body = try std.fmt.allocPrint(arena, "Failed to add watch symbol: {t}", .{err}); + return; + }, + }; + + res.content_type = httpz.ContentType.JSON; + res.body = try std.fmt.allocPrint(arena, "{{\"symbol\":\"{s}\",\"watched\":true}}", .{symbol}); +} + fn handleQuote(app: *App, req: *httpz.Request, res: *httpz.Response) !void { const raw_symbol = req.param("symbol") orelse { res.status = 400; @@ -762,13 +851,24 @@ fn fmtInt(arena: std.mem.Allocator, value: ?u8) []const u8 { return "null"; } -/// Append a watch lot for the given symbol to the portfolio SRF file, -/// unless it already exists. Best-effort — errors are logged, not fatal. +/// Append a watch lot for `symbol` to the portfolio SRF file, unless it +/// is already tracked. Serialized across requests via `app.watch_mutex` +/// and written atomically, so a concurrent add or a mid-write crash can't +/// clobber or truncate the portfolio file. Returns `error.InvalidSymbol` +/// for implausible symbols; callers decide how loud to be. fn appendWatchSymbol(app: *App, symbol: []const u8) !void { + if (!isPlausibleSymbol(symbol)) return error.InvalidSymbol; + const portfolio_path = app.environ.get("ZFIN_PORTFOLIO") orelse "portfolio.srf"; const allocator = app.allocator; const io = app.io; + // Serialize the whole read-modify-write so concurrent adds don't lose + // updates (a last-writer-wins race would otherwise drop a symbol). + // Uncancelable so a canceled request can't abandon a half-done write. + app.watch_mutex.lockUncancelable(io); + defer app.watch_mutex.unlock(io); + // Read and deserialize existing portfolio (or start empty) const file_data = std.Io.Dir.cwd().readFileAlloc(io, portfolio_path, allocator, .limited(10 * 1024 * 1024)) catch |err| { if (err == error.FileNotFound) return writeNewPortfolio(io, allocator, portfolio_path, symbol); @@ -785,7 +885,7 @@ fn appendWatchSymbol(app: *App, symbol: []const u8) !void { } // Build new lot list with the watch entry appended - var new_lots = try allocator.alloc(zfin.Lot, portfolio.lots.len + 1); + const new_lots = try allocator.alloc(zfin.Lot, portfolio.lots.len + 1); defer allocator.free(new_lots); @memcpy(new_lots[0..portfolio.lots.len], portfolio.lots); new_lots[portfolio.lots.len] = .{ @@ -796,16 +896,10 @@ fn appendWatchSymbol(app: *App, symbol: []const u8) !void { .security_type = .watch, }; - // Serialize and write + // Serialize and write atomically. const output = try zfin.cache.serializePortfolio(allocator, new_lots); defer allocator.free(output); - - const file = try std.Io.Dir.cwd().createFile(io, portfolio_path, .{}); - defer file.close(io); - var write_buf: [4096]u8 = undefined; - var fw = file.writer(io, &write_buf); - try fw.interface.writeAll(output); - try fw.interface.flush(); + try writeFileAtomic(io, allocator, portfolio_path, output); log.info("added watch symbol {s} to {s}", .{ symbol, portfolio_path }); } @@ -820,17 +914,41 @@ fn writeNewPortfolio(io: std.Io, allocator: std.mem.Allocator, path: []const u8, }}; const output = try zfin.cache.serializePortfolio(allocator, &lot); defer allocator.free(output); - - const file = try std.Io.Dir.cwd().createFile(io, path, .{}); - defer file.close(io); - var write_buf: [4096]u8 = undefined; - var fw = file.writer(io, &write_buf); - try fw.interface.writeAll(output); - try fw.interface.flush(); + try writeFileAtomic(io, allocator, path, output); log.info("created {s} with watch symbol {s}", .{ path, symbol }); } +/// Crash-safe file write: write to `.tmp`, fsync, then rename over +/// `path`. A mid-write crash leaves the prior file intact rather than a +/// truncated portfolio. (zfin's internal `atomic.writeFileAtomic` isn't +/// part of its public module, so we keep a small local copy.) +fn writeFileAtomic(io: std.Io, allocator: std.mem.Allocator, path: []const u8, bytes: []const u8) !void { + const tmp_path = try std.fmt.allocPrint(allocator, "{s}.tmp", .{path}); + defer allocator.free(tmp_path); + + { + var tmp_file = try std.Io.Dir.cwd().createFile(io, tmp_path, .{ .truncate = true, .exclusive = false }); + errdefer { + tmp_file.close(io); + std.Io.Dir.cwd().deleteFile(io, tmp_path) catch |err| { + log.debug("atomic write cleanup deleteFile({s}): {t}", .{ tmp_path, err }); + }; + } + try tmp_file.writeStreamingAll(io, bytes); + // fsync so the data is durable before the rename appears. + try tmp_file.sync(io); + tmp_file.close(io); + } + + std.Io.Dir.cwd().rename(tmp_path, std.Io.Dir.cwd(), path, io) catch |err| { + std.Io.Dir.cwd().deleteFile(io, tmp_path) catch |del_err| { + log.debug("atomic write cleanup deleteFile({s}): {t}", .{ tmp_path, del_err }); + }; + return err; + }; +} + // ── Refresh command ────────────────────────────────────────── fn refresh(io: std.Io, allocator: std.mem.Allocator, environ: *const std.process.Environ.Map) !u8 { @@ -1204,6 +1322,10 @@ pub fn main(init: std.process.Init) !u8 { // Symbol routes router.get("/:symbol/returns", handleReturns, .{}); + // Authenticated explicit watchlist add (API key required - not on + // the public allowlist). Distinct from the UA-gated + // /:symbol/returns?watch=true path used by LibreOffice. + router.get("/:symbol/watch", handleWatch, .{}); router.get("/:symbol/quote", handleQuote, .{}); router.get("/:symbol/candles", handleCandles, .{}); router.get("/:symbol/candles_meta", handleCandlesMeta, .{}); @@ -1348,6 +1470,28 @@ test "keyMatches" { try std.testing.expect(!keyMatches("s3cretX", "s3cret")); } +test "userAgentMayWatch" { + try std.testing.expect(userAgentMayWatch("LibreOffice 24.8")); + try std.testing.expect(userAgentMayWatch("libreoffice")); // case-insensitive + try std.testing.expect(userAgentMayWatch("Mozilla/5.0 LibreOffice/7.6")); + try std.testing.expect(!userAgentMayWatch("Mozilla/5.0 (X11; Linux x86_64)")); + try std.testing.expect(!userAgentMayWatch("curl/8.14.1")); + try std.testing.expect(!userAgentMayWatch(null)); + try std.testing.expect(!userAgentMayWatch("")); +} + +test "isPlausibleSymbol" { + try std.testing.expect(isPlausibleSymbol("AAPL")); + try std.testing.expect(isPlausibleSymbol("BRK.B")); + try std.testing.expect(isPlausibleSymbol("BRK-B")); + try std.testing.expect(isPlausibleSymbol("X")); + try std.testing.expect(!isPlausibleSymbol("")); // empty + try std.testing.expect(!isPlausibleSymbol("aapl")); // lowercase (upper-cased before this) + try std.testing.expect(!isPlausibleSymbol("AB CD")); // space + try std.testing.expect(!isPlausibleSymbol("../etc/passwd")); // path-shaped junk + try std.testing.expect(!isPlausibleSymbol("ABCDEFGHIJKLMNOPQ")); // 17 chars, too long +} + test "refreshExit: hard failure dominates, then lag, else clean" { try std.testing.expectEqual(@as(u8, 0), refreshExit(0, 0)); try std.testing.expectEqual(@as(u8, 75), refreshExit(0, 3));