From 0a2dd47f3eca4f504655a8c61a937c0b79330327 Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Fri, 6 Mar 2026 18:04:45 -0800 Subject: [PATCH] use a candle metadata file for current price, append only to candle data --- build.zig.zon | 4 +- src/cache/store.zig | 155 ++++++++++++++++++++++++++++-------------- src/service.zig | 162 ++++++++++++++++++++++++++++++++++++-------- 3 files changed, 242 insertions(+), 79 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index ad96cf1..88dfbc9 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -13,8 +13,8 @@ .hash = "z2d-0.10.0-j5P_Hu-6FgBsZNgwphIqh17jDnj8_yPtD8yzjO6PpHRQ", }, .srf = .{ - .url = "git+https://git.lerch.org/lobo/srf.git#7aa7ec112af736490ce7b5a887886a1da1212c6a", - .hash = "srf-0.0.0-qZj57-ZRAQARvv95mwaqA_39lAEKbuhQdUdmSQ5qhei5", + .url = "git+https://git.lerch.org/lobo/srf.git#95036e83e26bb885641c62aaf1e26dbfbb147ea9", + .hash = "srf-0.0.0-qZj575xZAQB4wzO6J8wf0hBFTZMDjCfFFCtHx6BCQifK", }, }, .paths = .{ diff --git a/src/cache/store.zig b/src/cache/store.zig index a3879d5..c78c4b2 100644 --- a/src/cache/store.zig +++ b/src/cache/store.zig @@ -37,6 +37,7 @@ pub const Ttl = struct { pub const DataType = enum { candles_daily, + candles_meta, dividends, splits, options, @@ -47,6 +48,7 @@ pub const DataType = enum { pub fn fileName(self: DataType) []const u8 { return switch (self) { .candles_daily => "candles_daily.srf", + .candles_meta => "candles_meta.srf", .dividends => "dividends.srf", .splits => "splits.srf", .options => "options.srf", @@ -108,6 +110,21 @@ pub const Store = struct { try file.writeAll(data); } + /// Append raw SRF record data to an existing file for a symbol and data type. + /// The data must be serialized with `emit_directives = false` (no header). + pub fn appendRaw(self: *Store, symbol: []const u8, data_type: DataType, data: []const u8) !void { + const path = try self.symbolPath(symbol, data_type.fileName()); + defer self.allocator.free(path); + + const file = std.fs.cwd().openFile(path, .{ .mode = .write_only }) catch { + // File doesn't exist — fall back to full write (caller should handle this) + return error.FileNotFound; + }; + defer file.close(); + try file.seekFromEnd(0); + try file.writeAll(data); + } + /// Check if raw SRF data is fresh using the embedded `#!expires=` directive. /// - Negative cache entries (# fetch_failed) are always fresh. /// - Data with `#!expires=` is fresh if the SRF library says so. @@ -126,19 +143,6 @@ pub const Store = struct { return parsed.isFresh(); } - /// Get the modification time (unix seconds) of a cached data file. - /// Returns null if the file does not exist. - pub fn getMtime(self: *Store, symbol: []const u8, data_type: DataType) ?i64 { - const path = self.symbolPath(symbol, data_type.fileName()) catch return null; - defer self.allocator.free(path); - - const file = std.fs.cwd().openFile(path, .{}) catch return null; - defer file.close(); - - const stat = file.stat() catch return null; - return @intCast(@divFloor(stat.mtime, std.time.ns_per_s)); - } - /// Clear all cached data for a symbol. pub fn clearSymbol(self: *Store, symbol: []const u8) !void { const path = try self.symbolPath(symbol, ""); @@ -179,40 +183,31 @@ pub const Store = struct { std.fs.cwd().deleteFile(path) catch {}; } - /// Read the close price from the last candle record without parsing the entire file. - /// Seeks to the end, reads the last ~256 bytes, and extracts `close:num:X`. - /// Returns null if the file doesn't exist or has no candle data. + /// Read the close price from the candle metadata file. + /// Returns null if no metadata exists. pub fn readLastClose(self: *Store, symbol: []const u8) ?f64 { - const path = self.symbolPath(symbol, DataType.candles_daily.fileName()) catch return null; - defer self.allocator.free(path); + const raw = self.readRaw(symbol, .candles_meta) catch return null; + const data = raw orelse return null; + defer self.allocator.free(data); + const meta = deserializeCandleMeta(self.allocator, data) catch return null; + return meta.last_close; + } - const file = std.fs.cwd().openFile(path, .{}) catch return null; - defer file.close(); + /// Read the full candle metadata (last_close, last_date, fetched_at). + /// Returns null if no metadata exists. + pub fn readCandleMeta(self: *Store, symbol: []const u8) ?CandleMeta { + const raw = self.readRaw(symbol, .candles_meta) catch return null; + const data = raw orelse return null; + defer self.allocator.free(data); + return deserializeCandleMeta(self.allocator, data) catch null; + } - const stat = file.stat() catch return null; - const file_size = stat.size; - if (file_size < 20) return null; // too small to have candle data - - // Read the last 256 bytes (one candle line is ~100 bytes, gives margin) - const read_size: u64 = @min(256, file_size); - file.seekTo(file_size - read_size) catch return null; - - var buf: [256]u8 = undefined; - const n = file.readAll(buf[0..@intCast(read_size)]) catch return null; - const chunk = buf[0..n]; - - // Find the last complete line (skip trailing newline, then find the previous newline) - const trimmed = std.mem.trimRight(u8, chunk, "\n"); - if (trimmed.len == 0) return null; - const last_nl = std.mem.lastIndexOfScalar(u8, trimmed, '\n'); - const last_line = if (last_nl) |pos| trimmed[pos + 1 ..] else trimmed; - - // Extract close:num:VALUE from the line - const marker = "close:num:"; - const close_start = std.mem.indexOf(u8, last_line, marker) orelse return null; - const val_start = close_start + marker.len; - const val_end = std.mem.indexOfScalar(u8, last_line[val_start..], ',') orelse (last_line.len - val_start); - return std.fmt.parseFloat(f64, last_line[val_start .. val_start + val_end]) catch null; + /// Check if candle metadata is fresh using the embedded `#!expires=` directive. + pub fn isCandleMetaFresh(self: *Store, symbol: []const u8) bool { + const raw = self.readRaw(symbol, .candles_meta) catch return false; + const data = raw orelse return false; + defer self.allocator.free(data); + return isFreshData(data, self.allocator); } /// Clear all cached data. @@ -248,21 +243,74 @@ pub const Store = struct { return candles.toOwnedSlice(allocator); } + /// Metadata stored in the separate candles_meta.srf file. + /// Allows fast price lookups and freshness checks without parsing the full candle file. + pub const CandleMeta = struct { + last_close: f64, + last_date: Date, + fetched_at: i64, + }; + + /// Serialize candle metadata to SRF format with an expiry directive. + pub fn serializeCandleMeta(allocator: std.mem.Allocator, meta: CandleMeta, options: srf.FormatOptions) ![]const u8 { + var buf: std.ArrayList(u8) = .empty; + errdefer buf.deinit(allocator); + const writer = buf.writer(allocator); + const items = [_]CandleMeta{meta}; + try writer.print("{f}", .{srf.fmtFrom(CandleMeta, allocator, &items, options)}); + return buf.toOwnedSlice(allocator); + } + + /// Deserialize candle metadata from SRF data. + pub fn deserializeCandleMeta(allocator: std.mem.Allocator, data: []const u8) !CandleMeta { + var reader = std.Io.Reader.fixed(data); + const parsed = srf.parse(&reader, allocator, .{ .alloc_strings = false }) catch return error.InvalidData; + defer parsed.deinit(); + + if (parsed.records.items.len == 0) return error.InvalidData; + return parsed.records.items[0].to(CandleMeta) catch error.InvalidData; + } + + /// Inline fetch metadata embedded as the first record in non-candle SRF files. + /// Uses a tag field to distinguish from data records. + pub const FetchMeta = struct { + fetched_at: i64, + }; + + /// Read the `fetched_at` timestamp from the first record of an SRF file. + /// Returns null if the file has no FetchMeta record or cannot be parsed. + pub fn readFetchedAt(allocator: std.mem.Allocator, data: []const u8) ?i64 { + var reader = std.Io.Reader.fixed(data); + const parsed = srf.parse(&reader, allocator, .{ .alloc_strings = false }) catch return null; + defer parsed.deinit(); + + if (parsed.records.items.len == 0) return null; + const meta = parsed.records.items[0].to(FetchMeta) catch return null; + return meta.fetched_at; + } + /// Serialize dividends to SRF compact format. + /// Prepends a FetchMeta record with the current timestamp. pub fn serializeDividends(allocator: std.mem.Allocator, dividends: []const Dividend, options: srf.FormatOptions) ![]const u8 { var buf: std.ArrayList(u8) = .empty; errdefer buf.deinit(allocator); const writer = buf.writer(allocator); - try writer.print("{f}", .{srf.fmtFrom(Dividend, allocator, dividends, options)}); + const meta = [_]FetchMeta{.{ .fetched_at = std.time.timestamp() }}; + try writer.print("{f}", .{srf.fmtFrom(FetchMeta, allocator, &meta, options)}); + // Append data records (no header -- already written by meta) + try writer.print("{f}", .{srf.fmtFrom(Dividend, allocator, dividends, .{ .emit_directives = false })}); return buf.toOwnedSlice(allocator); } /// Serialize splits to SRF compact format. + /// Prepends a FetchMeta record with the current timestamp. pub fn serializeSplits(allocator: std.mem.Allocator, splits: []const Split, options: srf.FormatOptions) ![]const u8 { var buf: std.ArrayList(u8) = .empty; errdefer buf.deinit(allocator); const writer = buf.writer(allocator); - try writer.print("{f}", .{srf.fmtFrom(Split, allocator, splits, options)}); + const meta = [_]FetchMeta{.{ .fetched_at = std.time.timestamp() }}; + try writer.print("{f}", .{srf.fmtFrom(FetchMeta, allocator, &meta, options)}); + try writer.print("{f}", .{srf.fmtFrom(Split, allocator, splits, .{ .emit_directives = false })}); return buf.toOwnedSlice(allocator); } @@ -308,11 +356,14 @@ pub const Store = struct { } /// Serialize earnings events to SRF compact format. + /// Prepends a FetchMeta record with the current timestamp. pub fn serializeEarnings(allocator: std.mem.Allocator, events: []const EarningsEvent, options: srf.FormatOptions) ![]const u8 { var buf: std.ArrayList(u8) = .empty; errdefer buf.deinit(allocator); const writer = buf.writer(allocator); - try writer.print("{f}", .{srf.fmtFrom(EarningsEvent, allocator, events, options)}); + const meta = [_]FetchMeta{.{ .fetched_at = std.time.timestamp() }}; + try writer.print("{f}", .{srf.fmtFrom(FetchMeta, allocator, &meta, options)}); + try writer.print("{f}", .{srf.fmtFrom(EarningsEvent, allocator, events, .{ .emit_directives = false })}); return buf.toOwnedSlice(allocator); } @@ -359,6 +410,7 @@ pub const Store = struct { }; /// Serialize ETF profile to SRF compact format. + /// Prepends a FetchMeta record with the current timestamp. /// Uses multiple record types: meta fields, then sector and holding records. pub fn serializeEtfProfile(allocator: std.mem.Allocator, profile: EtfProfile, options: srf.FormatOptions) ![]const u8 { var records: std.ArrayList(EtfRecord) = .empty; @@ -383,7 +435,9 @@ pub const Store = struct { var buf: std.ArrayList(u8) = .empty; errdefer buf.deinit(allocator); const writer = buf.writer(allocator); - try writer.print("{f}", .{srf.fmtFrom(EtfRecord, allocator, records.items, options)}); + const fetch_meta = [_]FetchMeta{.{ .fetched_at = std.time.timestamp() }}; + try writer.print("{f}", .{srf.fmtFrom(FetchMeta, allocator, &fetch_meta, options)}); + try writer.print("{f}", .{srf.fmtFrom(EtfRecord, allocator, records.items, .{ .emit_directives = false })}); return buf.toOwnedSlice(allocator); } @@ -502,6 +556,7 @@ pub const Store = struct { } /// Serialize options chains to SRF compact format. + /// Prepends a FetchMeta record with the current timestamp. pub fn serializeOptions(allocator: std.mem.Allocator, chains: []const OptionsChain, options: srf.FormatOptions) ![]const u8 { var records: std.ArrayList(OptionsRecord) = .empty; defer records.deinit(allocator); @@ -521,7 +576,9 @@ pub const Store = struct { var buf: std.ArrayList(u8) = .empty; errdefer buf.deinit(allocator); const writer = buf.writer(allocator); - try writer.print("{f}", .{srf.fmtFrom(OptionsRecord, allocator, records.items, options)}); + const meta = [_]FetchMeta{.{ .fetched_at = std.time.timestamp() }}; + try writer.print("{f}", .{srf.fmtFrom(FetchMeta, allocator, &meta, options)}); + try writer.print("{f}", .{srf.fmtFrom(OptionsRecord, allocator, records.items, .{ .emit_directives = false })}); return buf.toOwnedSlice(allocator); } diff --git a/src/service.zig b/src/service.zig index 4372c6b..774a089 100644 --- a/src/service.zig +++ b/src/service.zig @@ -119,46 +119,157 @@ pub const DataService = struct { pub fn invalidate(self: *DataService, symbol: []const u8, data_type: cache.DataType) void { var s = self.store(); s.clearData(symbol, data_type); + // Also clear candle metadata when invalidating candle data + if (data_type == .candles_daily) { + s.clearData(symbol, .candles_meta); + } } // ── Public data methods ────────────────────────────────────── /// Fetch daily candles for a symbol (10+ years for trailing returns). /// Checks cache first; fetches from TwelveData if stale/missing. + /// Uses incremental updates: when the cache is stale, only fetches + /// candles newer than the last cached date rather than re-fetching + /// the entire history. pub fn getCandles(self: *DataService, symbol: []const u8) DataError!struct { data: []Candle, source: Source, timestamp: i64 } { var s = self.store(); + const today = todayDate(); - // Try cache - const cached_raw = s.readRaw(symbol, .candles_daily) catch return DataError.CacheError; - if (cached_raw) |data| { - defer self.allocator.free(data); - if (cache.Store.isFreshData(data, self.allocator)) { - const candles = cache.Store.deserializeCandles(self.allocator, data) catch null; - if (candles) |c| return .{ .data = c, .source = .cached, .timestamp = s.getMtime(symbol, .candles_daily) orelse std.time.timestamp() }; + // Check candle metadata for freshness (tiny file, no candle deserialization) + const meta = s.readCandleMeta(symbol); + if (meta) |m| { + if (s.isCandleMetaFresh(symbol)) { + // Fresh — deserialize candles and return + const candles = self.loadCandleFile(&s, symbol); + if (candles) |c| return .{ .data = c, .source = .cached, .timestamp = m.fetched_at }; + } + + // Stale — try incremental update using last_date from meta + const fetch_from = m.last_date.addDays(1); + + // If last cached date is today or later, just refresh the TTL (meta only) + if (!fetch_from.lessThan(today)) { + self.updateCandleMeta(&s, symbol, m.last_close, m.last_date); + const candles = self.loadCandleFile(&s, symbol); + if (candles) |c| return .{ .data = c, .source = .cached, .timestamp = std.time.timestamp() }; + } else { + // Incremental fetch from day after last cached candle + var td = self.getTwelveData() catch { + // No API key — return stale data + const candles = self.loadCandleFile(&s, symbol); + if (candles) |c| return .{ .data = c, .source = .cached, .timestamp = m.fetched_at }; + return DataError.NoApiKey; + }; + const new_candles = td.fetchCandles(self.allocator, symbol, fetch_from, today) catch { + // Fetch failed — return stale data rather than erroring + const candles = self.loadCandleFile(&s, symbol); + if (candles) |c| return .{ .data = c, .source = .cached, .timestamp = m.fetched_at }; + return DataError.FetchFailed; + }; + + if (new_candles.len == 0) { + // No new candles (weekend/holiday) — refresh TTL only (meta rewrite) + self.allocator.free(new_candles); + self.updateCandleMeta(&s, symbol, m.last_close, m.last_date); + const candles = self.loadCandleFile(&s, symbol); + if (candles) |c| return .{ .data = c, .source = .cached, .timestamp = std.time.timestamp() }; + } else { + // Append new candles to existing file + update meta + self.appendCandles(&s, symbol, new_candles); + // Load the full (now-updated) file for the caller + const candles = self.loadCandleFile(&s, symbol); + if (candles) |c| { + self.allocator.free(new_candles); + return .{ .data = c, .source = .fetched, .timestamp = std.time.timestamp() }; + } + // Append failed or file unreadable — just return new candles + return .{ .data = new_candles, .source = .fetched, .timestamp = std.time.timestamp() }; + } } } - // Fetch from provider + // No usable cache — full fetch (~10 years, plus buffer for leap years) var td = try self.getTwelveData(); - const today = todayDate(); - const from = today.addDays(-365 * 10 - 60); + const from = today.addDays(-3700); const fetched = td.fetchCandles(self.allocator, symbol, from, today) catch { return DataError.FetchFailed; }; - // Cache the result if (fetched.len > 0) { - const expires = std.time.timestamp() + cache.Ttl.candles_latest; - if (cache.Store.serializeCandles(self.allocator, fetched, .{ .expires = expires })) |srf_data| { - defer self.allocator.free(srf_data); - s.writeRaw(symbol, .candles_daily, srf_data) catch {}; - } else |_| {} + self.cacheCandles(&s, symbol, fetched); } return .{ .data = fetched, .source = .fetched, .timestamp = std.time.timestamp() }; } + /// Load candle data from the cache file. Returns null if unavailable. + fn loadCandleFile(self: *DataService, s: *cache.Store, symbol: []const u8) ?[]Candle { + const raw = s.readRaw(symbol, .candles_daily) catch return null; + const data = raw orelse return null; + defer self.allocator.free(data); + return cache.Store.deserializeCandles(self.allocator, data) catch null; + } + + fn cacheCandles(self: *DataService, s: *cache.Store, symbol: []const u8, candles: []const Candle) void { + // Write candle data (no expiry -- historical facts don't expire) + if (cache.Store.serializeCandles(self.allocator, candles, .{})) |srf_data| { + defer self.allocator.free(srf_data); + s.writeRaw(symbol, .candles_daily, srf_data) catch {}; + } else |_| {} + + // Write candle metadata (with expiry for freshness checks) + if (candles.len > 0) { + const last = candles[candles.len - 1]; + self.updateCandleMeta(s, symbol, last.close, last.date); + } + } + + /// Append new candle records to the existing candle file and update metadata. + /// Falls back to a full rewrite if append fails (e.g. file doesn't exist). + fn appendCandles(self: *DataService, s: *cache.Store, symbol: []const u8, new_candles: []const Candle) void { + if (new_candles.len == 0) return; + + // Serialize just the new records with no SRF header + if (cache.Store.serializeCandles(self.allocator, new_candles, .{ .emit_directives = false })) |srf_data| { + defer self.allocator.free(srf_data); + s.appendRaw(symbol, .candles_daily, srf_data) catch { + // Append failed (file missing?) — fall back to full load + rewrite + if (self.loadCandleFile(s, symbol)) |existing| { + defer self.allocator.free(existing); + // Merge and do full write + const merged = self.allocator.alloc(Candle, existing.len + new_candles.len) catch return; + defer self.allocator.free(merged); + @memcpy(merged[0..existing.len], existing); + @memcpy(merged[existing.len..], new_candles); + if (cache.Store.serializeCandles(self.allocator, merged, .{})) |full_data| { + defer self.allocator.free(full_data); + s.writeRaw(symbol, .candles_daily, full_data) catch {}; + } else |_| {} + } + }; + } else |_| {} + + // Update metadata to reflect the new last candle + const last = new_candles[new_candles.len - 1]; + self.updateCandleMeta(s, symbol, last.close, last.date); + } + + /// Write (or refresh) candle metadata without touching the candle data file. + fn updateCandleMeta(self: *DataService, s: *cache.Store, symbol: []const u8, last_close: f64, last_date: Date) void { + const expires = std.time.timestamp() + cache.Ttl.candles_latest; + const meta = cache.Store.CandleMeta{ + .last_close = last_close, + .last_date = last_date, + .fetched_at = std.time.timestamp(), + }; + if (cache.Store.serializeCandleMeta(self.allocator, meta, .{ .expires = expires })) |meta_data| { + defer self.allocator.free(meta_data); + s.writeRaw(symbol, .candles_meta, meta_data) catch {}; + } else |_| {} + } + /// Fetch dividend history for a symbol. /// Checks cache first; fetches from Polygon if stale/missing. pub fn getDividends(self: *DataService, symbol: []const u8) DataError!struct { data: []Dividend, source: Source, timestamp: i64 } { @@ -169,7 +280,7 @@ pub const DataService = struct { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const divs = cache.Store.deserializeDividends(self.allocator, data) catch null; - if (divs) |d| return .{ .data = d, .source = .cached, .timestamp = s.getMtime(symbol, .dividends) orelse std.time.timestamp() }; + if (divs) |d| return .{ .data = d, .source = .cached, .timestamp = cache.Store.readFetchedAt(self.allocator, data) orelse std.time.timestamp() }; } } @@ -199,7 +310,7 @@ pub const DataService = struct { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const splits = cache.Store.deserializeSplits(self.allocator, data) catch null; - if (splits) |sp| return .{ .data = sp, .source = .cached, .timestamp = s.getMtime(symbol, .splits) orelse std.time.timestamp() }; + if (splits) |sp| return .{ .data = sp, .source = .cached, .timestamp = cache.Store.readFetchedAt(self.allocator, data) orelse std.time.timestamp() }; } } @@ -226,7 +337,7 @@ pub const DataService = struct { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const chains = cache.Store.deserializeOptions(self.allocator, data) catch null; - if (chains) |c| return .{ .data = c, .source = .cached, .timestamp = s.getMtime(symbol, .options) orelse std.time.timestamp() }; + if (chains) |c| return .{ .data = c, .source = .cached, .timestamp = cache.Store.readFetchedAt(self.allocator, data) orelse std.time.timestamp() }; } } @@ -267,7 +378,7 @@ pub const DataService = struct { } else false; if (!needs_refresh) { - return .{ .data = e, .source = .cached, .timestamp = s.getMtime(symbol, .earnings) orelse std.time.timestamp() }; + return .{ .data = e, .source = .cached, .timestamp = cache.Store.readFetchedAt(self.allocator, data) orelse std.time.timestamp() }; } // Stale: free cached events and re-fetch below self.allocator.free(e); @@ -304,7 +415,7 @@ pub const DataService = struct { defer self.allocator.free(data); if (cache.Store.isFreshData(data, self.allocator)) { const profile = cache.Store.deserializeEtfProfile(self.allocator, data) catch null; - if (profile) |p| return .{ .data = p, .source = .cached, .timestamp = s.getMtime(symbol, .etf_profile) orelse std.time.timestamp() }; + if (profile) |p| return .{ .data = p, .source = .cached, .timestamp = cache.Store.readFetchedAt(self.allocator, data) orelse std.time.timestamp() }; } } @@ -388,12 +499,7 @@ pub const DataService = struct { /// Check if candle data is fresh in cache without full deserialization. pub fn isCandleCacheFresh(self: *DataService, symbol: []const u8) bool { var s = self.store(); - const data = s.readRaw(symbol, .candles_daily) catch return false; - if (data) |d| { - defer self.allocator.free(d); - return cache.Store.isFreshData(d, self.allocator); - } - return false; + return s.isCandleMetaFresh(symbol); } /// Read only the latest close price from cached candles (no full deserialization). @@ -670,7 +776,7 @@ pub const DataService = struct { fn todayDate() Date { const ts = std.time.timestamp(); - const days: i32 = @intCast(@divFloor(ts, 86400)); + const days: i32 = @intCast(@divFloor(ts, std.time.s_per_day)); return .{ .days = days }; } };