lobo/zfin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

ai: store sentinal on failed fetch so we do not eat api limits

Browse source
This commit is contained in:
Emil Lerch 2026-02-27 13:40:29 -08:00
parent 19a1403d40
commit d6007e4305
Signed by: lobo
GPG key ID: A7B62D657EF764F8
3 changed files with 141 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

140
src/cache/store.zig vendored
View file

@ -107,7 +107,11 @@ pub const Store = struct {
}
/// Check if a cached data file exists and is within its TTL.
/// Negative cache entries (fetch_failed) are always considered fresh.
pub fn isFresh(self: *Store, symbol: []const u8, data_type: DataType, ttl_seconds: i64) !bool {
// Negative cache entries never expire (cleared only by --refresh / invalidate)
if (self.isNegative(symbol, data_type)) return true;
if (ttl_seconds < 0) {
// Infinite TTL: just check existence
const path = try self.symbolPath(symbol, data_type.fileName());
@ -148,6 +152,32 @@ pub const Store = struct {
std.fs.cwd().deleteTree(path) catch {};
}
/// Content of a negative cache entry (fetch failed, don't retry until --refresh).
pub const negative_cache_content = "#!srfv1\n# fetch_failed\n";
/// Write a negative cache entry for a symbol + data type.
/// This records that a fetch was attempted and failed, preventing repeated
/// network requests for symbols that will never resolve.
/// Cleared by --refresh (which calls clearData/invalidate).
pub fn writeNegative(self: *Store, symbol: []const u8, data_type: DataType) void {
self.writeRaw(symbol, data_type, negative_cache_content) catch {};
}
/// Check if a cached data file is a negative entry (fetch_failed marker).
/// Negative entries are always considered "fresh" -- they never expire.
pub fn isNegative(self: *Store, symbol: []const u8, data_type: DataType) bool {
const path = self.symbolPath(symbol, data_type.fileName()) catch return false;
defer self.allocator.free(path);
const file = std.fs.cwd().openFile(path, .{}) catch return false;
defer file.close();
var buf: [negative_cache_content.len]u8 = undefined;
const n = file.readAll(&buf) catch return false;
return n == negative_cache_content.len and
std.mem.eql(u8, buf[0..n], negative_cache_content);
}
/// Clear a specific data type for a symbol.
pub fn clearData(self: *Store, symbol: []const u8, data_type: DataType) void {
const path = self.symbolPath(symbol, data_type.fileName()) catch return;
@ -401,7 +431,10 @@ pub const Store = struct {
for (record.fields) |field| {
if (std.mem.eql(u8, field.key, "date")) {
if (field.value) |v| {
const str = switch (v) { .string => |s| s, else => continue };
const str = switch (v) {
.string => |s| s,
else => continue,
};
ev.date = Date.parse(str) catch continue;
}
} else if (std.mem.eql(u8, field.key, "estimate")) {
@ -424,7 +457,10 @@ pub const Store = struct {
if (field.value) |v| ev.revenue_estimate = numVal(v);
} else if (std.mem.eql(u8, field.key, "report_time")) {
if (field.value) |v| {
const str = switch (v) { .string => |s| s, else => continue };
const str = switch (v) {
.string => |s| s,
else => continue,
};
ev.report_time = parseReportTimeTag(str);
}
}
@ -501,7 +537,10 @@ pub const Store = struct {
for (record.fields) |field| {
if (std.mem.eql(u8, field.key, "type")) {
if (field.value) |v| {
record_type = switch (v) { .string => |s| s, else => "" };
record_type = switch (v) {
.string => |s| s,
else => "",
};
}
}
}
@ -523,12 +562,18 @@ pub const Store = struct {
}
} else if (std.mem.eql(u8, field.key, "inception_date")) {
if (field.value) |v| {
const str = switch (v) { .string => |s| s, else => continue };
const str = switch (v) {
.string => |s| s,
else => continue,
};
profile.inception_date = Date.parse(str) catch null;
}
} else if (std.mem.eql(u8, field.key, "leveraged")) {
if (field.value) |v| {
const str = switch (v) { .string => |s| s, else => continue };
const str = switch (v) {
.string => |s| s,
else => continue,
};
profile.leveraged = std.mem.eql(u8, str, "yes");
}
}
@ -538,7 +583,10 @@ pub const Store = struct {
var weight: f64 = 0;
for (record.fields) |field| {
if (std.mem.eql(u8, field.key, "name")) {
if (field.value) |v| name = switch (v) { .string => |s| s, else => null };
if (field.value) |v| name = switch (v) {
.string => |s| s,
else => null,
};
} else if (std.mem.eql(u8, field.key, "weight")) {
if (field.value) |v| weight = numVal(v);
}
@ -553,9 +601,15 @@ pub const Store = struct {
var weight: f64 = 0;
for (record.fields) |field| {
if (std.mem.eql(u8, field.key, "symbol")) {
if (field.value) |v| sym = switch (v) { .string => |s| s, else => null };
if (field.value) |v| sym = switch (v) {
.string => |s| s,
else => null,
};
} else if (std.mem.eql(u8, field.key, "name")) {
if (field.value) |v| hname = switch (v) { .string => |s| s, else => null };
if (field.value) |v| hname = switch (v) {
.string => |s| s,
else => null,
};
} else if (std.mem.eql(u8, field.key, "weight")) {
if (field.value) |v| weight = numVal(v);
}
@ -663,14 +717,23 @@ pub const Store = struct {
for (record.fields) |field| {
if (std.mem.eql(u8, field.key, "type")) {
if (field.value) |v| rec_type = switch (v) { .string => |s| s, else => "" };
if (field.value) |v| rec_type = switch (v) {
.string => |s| s,
else => "",
};
} else if (std.mem.eql(u8, field.key, "expiration")) {
if (field.value) |v| {
exp_str = switch (v) { .string => |s| s, else => continue };
exp_str = switch (v) {
.string => |s| s,
else => continue,
};
expiration = Date.parse(exp_str) catch null;
}
} else if (std.mem.eql(u8, field.key, "symbol")) {
if (field.value) |v| symbol = switch (v) { .string => |s| s, else => "" };
if (field.value) |v| symbol = switch (v) {
.string => |s| s,
else => "",
};
} else if (std.mem.eql(u8, field.key, "price")) {
if (field.value) |v| price = numVal(v);
}
@ -716,10 +779,16 @@ pub const Store = struct {
for (record.fields) |field| {
if (std.mem.eql(u8, field.key, "type")) {
if (field.value) |v| rec_type = switch (v) { .string => |s| s, else => "" };
if (field.value) |v| rec_type = switch (v) {
.string => |s| s,
else => "",
};
} else if (std.mem.eql(u8, field.key, "expiration")) {
if (field.value) |v| {
exp_str = switch (v) { .string => |s| s, else => continue };
exp_str = switch (v) {
.string => |s| s,
else => continue,
};
contract.expiration = Date.parse(exp_str) catch Date.epoch;
}
} else if (std.mem.eql(u8, field.key, "strike")) {
@ -911,32 +980,53 @@ pub fn deserializePortfolio(allocator: std.mem.Allocator, data: []const u8) !Por
for (record.fields) |field| {
if (std.mem.eql(u8, field.key, "symbol")) {
if (field.value) |v| sym_raw = switch (v) { .string => |s| s, else => null };
if (field.value) |v| sym_raw = switch (v) {
.string => |s| s,
else => null,
};
} else if (std.mem.eql(u8, field.key, "shares")) {
if (field.value) |v| lot.shares = Store.numVal(v);
} else if (std.mem.eql(u8, field.key, "open_date")) {
if (field.value) |v| {
const str = switch (v) { .string => |s| s, else => continue };
const str = switch (v) {
.string => |s| s,
else => continue,
};
lot.open_date = Date.parse(str) catch continue;
}
} else if (std.mem.eql(u8, field.key, "open_price")) {
if (field.value) |v| lot.open_price = Store.numVal(v);
} else if (std.mem.eql(u8, field.key, "close_date")) {
if (field.value) |v| {
const str = switch (v) { .string => |s| s, else => continue };
const str = switch (v) {
.string => |s| s,
else => continue,
};
lot.close_date = Date.parse(str) catch null;
}
} else if (std.mem.eql(u8, field.key, "close_price")) {
if (field.value) |v| lot.close_price = Store.numVal(v);
} else if (std.mem.eql(u8, field.key, "note")) {
if (field.value) |v| note_raw = switch (v) { .string => |s| s, else => null };
if (field.value) |v| note_raw = switch (v) {
.string => |s| s,
else => null,
};
} else if (std.mem.eql(u8, field.key, "account")) {
if (field.value) |v| account_raw = switch (v) { .string => |s| s, else => null };
if (field.value) |v| account_raw = switch (v) {
.string => |s| s,
else => null,
};
} else if (std.mem.eql(u8, field.key, "security_type")) {
if (field.value) |v| sec_type_raw = switch (v) { .string => |s| s, else => null };
if (field.value) |v| sec_type_raw = switch (v) {
.string => |s| s,
else => null,
};
} else if (std.mem.eql(u8, field.key, "maturity_date")) {
if (field.value) |v| {
const str = switch (v) { .string => |s| s, else => continue };
const str = switch (v) {
.string => |s| s,
else => continue,
};
lot.maturity_date = Date.parse(str) catch null;
}
} else if (std.mem.eql(u8, field.key, "rate")) {
@ -953,7 +1043,10 @@ pub fn deserializePortfolio(allocator: std.mem.Allocator, data: []const u8) !Por
}
}
} else if (std.mem.eql(u8, field.key, "ticker")) {
if (field.value) |v| ticker_raw = switch (v) { .string => |s| s, else => null };
if (field.value) |v| ticker_raw = switch (v) {
.string => |s| s,
else => null,
};
} else if (std.mem.eql(u8, field.key, "price")) {
if (field.value) |v| {
const p = Store.numVal(v);
@ -961,7 +1054,10 @@ pub fn deserializePortfolio(allocator: std.mem.Allocator, data: []const u8) !Por
}
} else if (std.mem.eql(u8, field.key, "price_date")) {
if (field.value) |v| {
const str = switch (v) { .string => |s| s, else => continue };
const str = switch (v) {
.string => |s| s,
else => continue,
};
lot.price_date = Date.parse(str) catch null;
}
}

3
src/cli/commands/portfolio.zig
View file

@ -81,12 +81,13 @@ pub fn run(allocator: std.mem.Allocator, config: zfin.Config, svc: *zfin.DataSer
defer allocator.free(cs);
if (cs.len > 0) {
try prices.put(sym, cs[cs.len - 1].close);
}
// Cached (including negative cache entries with 0 candles)
cached_count += 1;
try cli.stderrProgress(sym, " (cached)", loaded_count, all_syms_count, color);
continue;
}
}
}
// Need to fetch from API
const wait_s = svc.estimateWaitSeconds();

25
src/service.zig
View file

@ -137,8 +137,11 @@ pub const DataService = struct {
const today = todayDate();
const from = today.addDays(-365 * 10 - 60);
const fetched = td.fetchCandles(self.allocator, symbol, from, today) catch
const fetched = td.fetchCandles(self.allocator, symbol, from, today) catch {
// Write negative cache entry so we don't retry on next run
s.writeNegative(symbol, .candles_daily);
return DataError.FetchFailed;
};
// Cache the result
if (fetched.len > 0) {
@ -167,8 +170,10 @@ pub const DataService = struct {
}
var pg = try self.getPolygon();
const fetched = pg.fetchDividends(self.allocator, symbol, null, null) catch
const fetched = pg.fetchDividends(self.allocator, symbol, null, null) catch {
s.writeNegative(symbol, .dividends);
return DataError.FetchFailed;
};
if (fetched.len > 0) {
if (cache.Store.serializeDividends(self.allocator, fetched)) |srf_data| {
@ -196,8 +201,10 @@ pub const DataService = struct {
}
var pg = try self.getPolygon();
const fetched = pg.fetchSplits(self.allocator, symbol) catch
const fetched = pg.fetchSplits(self.allocator, symbol) catch {
s.writeNegative(symbol, .splits);
return DataError.FetchFailed;
};
if (cache.Store.serializeSplits(self.allocator, fetched)) |srf_data| {
defer self.allocator.free(srf_data);
@ -223,8 +230,10 @@ pub const DataService = struct {
}
var cboe = self.getCboe();
const fetched = cboe.fetchOptionsChain(self.allocator, symbol) catch
const fetched = cboe.fetchOptionsChain(self.allocator, symbol) catch {
s.writeNegative(symbol, .options);
return DataError.FetchFailed;
};
if (fetched.len > 0) {
if (cache.Store.serializeOptions(self.allocator, fetched)) |srf_data| {
@ -256,8 +265,10 @@ pub const DataService = struct {
const from = today.subtractYears(5);
const to = today.addDays(365);
const fetched = fh.fetchEarnings(self.allocator, symbol, from, to) catch
const fetched = fh.fetchEarnings(self.allocator, symbol, from, to) catch {
s.writeNegative(symbol, .earnings);
return DataError.FetchFailed;
};
if (fetched.len > 0) {
if (cache.Store.serializeEarnings(self.allocator, fetched)) |srf_data| {
@ -285,8 +296,10 @@ pub const DataService = struct {
}
var av = try self.getAlphaVantage();
const fetched = av.fetchEtfProfile(self.allocator, symbol) catch
const fetched = av.fetchEtfProfile(self.allocator, symbol) catch {
s.writeNegative(symbol, .etf_profile);
return DataError.FetchFailed;
};
if (cache.Store.serializeEtfProfile(self.allocator, fetched)) |srf_data| {
defer self.allocator.free(srf_data);