lobo/zfin
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

rely on cache clear to delete data - force refresh now simply tops off the candle data

Browse source
This commit is contained in:
Emil Lerch 2026-06-19 09:56:56 -07:00
parent d29cafb298
commit e246d1e9fe
Signed by: lobo
GPG key ID: A7B62D657EF764F8
3 changed files with 65 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/commands/common.zig
View file

@ -126,7 +126,7 @@ pub const stderrPrint = stderr.print;
pub const stderrProgress = stderr.progress;
pub const stderrRateLimitWait = stderr.rateLimitWait;
/// Progress callback for loadPrices that prints to stderr.
/// Progress callback for loadAllPrices that prints to stderr.
/// Shared between the CLI portfolio command and TUI pre-fetch.
pub const LoadProgress = struct {
io: std.Io,
@ -134,7 +134,7 @@ pub const LoadProgress = struct {
color: bool,
/// Offset added to index for display (e.g. stock count when loading watch symbols).
index_offset: usize,
/// Grand total across all loadPrices calls (stocks + watch).
/// Grand total across all loadAllPrices calls (stocks + watch).
grand_total: usize,
fn onProgress(ctx: *anyopaque, index: usize, _: usize, symbol: []const u8, status: zfin.DataService.SymbolStatus) void {
@ -273,7 +273,7 @@ pub fn loadPortfolioPrices(
};
// Map RefreshPolicy → LoadAllConfig:
// .force → invalidate cache, fetch fresh.
// .force → ignore TTL; incremental candle top-up (no wipe).
// .auto → respect TTL, fetch on stale.
// .never → offline mode: never touch the network. Stale cache
// entries are returned; cache misses fail the symbol.

9
src/commands/framework.zig
View file

@ -180,10 +180,11 @@ pub fn isUserError(comptime UserErrors: type, err: anyerror) bool {
/// stale entries trigger a provider refetch (or server sync if
/// `ZFIN_SERVER` is configured). The right behavior for almost
/// every invocation; the default.
/// - `force`: invalidate cache before reading. Re-fetches every
/// symbol's data from providers regardless of TTL freshness.
/// Useful when you suspect cached data is wrong or after rotating
/// providers.
/// - `force`: ignore cache TTL and re-validate against providers.
/// Candle history is topped up incrementally (new bars appended,
/// existing history kept), not wiped. To force a full re-download
/// from scratch (e.g. suspected-bad data or a retroactive split
/// adjustment), use `cache clear`.
/// - `never`: serve whatever's in cache regardless of TTL. No
/// provider calls. Useful for offline operation, debugging, and
/// reproducible historical analysis.

161
src/service.zig
View file

@ -1934,20 +1934,6 @@ pub const DataService = struct {
// ── Portfolio price loading ──────────────────────────────────
/// Result of loading prices for a set of symbols.
pub const PriceLoadResult = struct {
/// Number of symbols whose price came from fresh cache.
cached_count: usize,
/// Number of symbols successfully fetched from API.
fetched_count: usize,
/// Number of symbols where API fetch failed.
fail_count: usize,
/// Number of failed symbols that fell back to stale cache.
stale_count: usize,
/// Latest candle date seen across all symbols.
latest_date: ?Date,
};
/// Status emitted for each symbol during price loading.
pub const SymbolStatus = enum {
/// Price resolved from fresh cache.
@ -1973,78 +1959,6 @@ pub const DataService = struct {
}
};
/// Load current prices for a list of symbols into `prices`.
///
/// For each symbol the resolution order is:
/// 1. Fresh cache -> use cached last close (fast, no deserialization)
/// 2. API fetch -> use latest candle close
/// 3. Stale cache -> use last close from expired cache entry
///
/// If `force_refresh` is true, cache is invalidated before checking freshness.
/// If `progress` is provided, it is called for each symbol with the outcome.
pub fn loadPrices(
self: *DataService,
symbols: []const []const u8,
prices: *std.StringHashMap(f64),
force_refresh: bool,
progress: ?ProgressCallback,
) PriceLoadResult {
var result = PriceLoadResult{
.cached_count = 0,
.fetched_count = 0,
.fail_count = 0,
.stale_count = 0,
.latest_date = null,
};
const total = symbols.len;
for (symbols, 0..) |sym, i| {
if (force_refresh) {
self.invalidate(sym, .candles_daily);
}
// 1. Fresh cache — fast path (no full deserialization)
if (!force_refresh and self.isCandleCacheFresh(sym)) {
if (self.getCachedLastClose(sym)) |close| {
prices.put(sym, close) catch |err| log.warn("loadPrices cache-hit put({s}): {t}", .{ sym, err });
}
result.cached_count += 1;
if (progress) |p| p.emit(i, total, sym, .cached);
continue;
}
// About to fetch — notify caller (so it can show rate-limit waits etc.)
if (progress) |p| p.emit(i, total, sym, .fetching);
// 2. Try API fetch
if (self.getCandles(sym, .{ .force_refresh = force_refresh })) |candle_result| {
defer self.allocator.free(candle_result.data);
if (candle_result.data.len > 0) {
const last = candle_result.data[candle_result.data.len - 1];
prices.put(sym, last.close) catch |err| log.warn("loadPrices candle-close put({s}): {t}", .{ sym, err });
if (result.latest_date == null or last.date.days > result.latest_date.?.days) {
result.latest_date = last.date;
}
}
result.fetched_count += 1;
if (progress) |p| p.emit(i, total, sym, .fetched);
continue;
} else |_| {}
// 3. Fetch failed — fall back to stale cache
result.fail_count += 1;
if (self.getCachedLastClose(sym)) |close| {
prices.put(sym, close) catch |err| log.warn("loadPrices stale-fallback put({s}): {t}", .{ sym, err });
result.stale_count += 1;
if (progress) |p| p.emit(i, total, sym, .failed_used_stale);
} else {
if (progress) |p| p.emit(i, total, sym, .failed);
}
}
return result;
}
// ── Consolidated Price Loading (Parallel Server + Sequential Provider) ──
/// Configuration for loadAllPrices.
@ -2173,12 +2087,13 @@ pub const DataService = struct {
}
for (watch_syms) |sym| all_symbols.append(self.allocator, sym) catch |err| log.warn("loadAllPrices append watch sym({s}): {t}", .{ sym, err });
// Invalidate cache if force refresh
if (config.force_refresh) {
for (all_symbols.items) |sym| {
self.invalidate(sym, .candles_daily);
}
}
// force_refresh does NOT wipe the candle cache. It flows
// through to getCandles (via config.fetchOptions()), which
// ignores the TTL and does an incremental top-up — see the
// `--refresh-data=force` contract. The Phase-1 fast path below
// is skipped on force_refresh so every symbol is re-validated
// against the provider. A full wipe + re-download from scratch
// is reserved for `cache clear`.
// Phase 1: Check local cache (fast path)
var needs_fetch: std.ArrayList([]const u8) = .empty;
@ -3122,18 +3037,6 @@ test "DataService getProvider initializes provider with key" {
try std.testing.expect(tg1 == tg2);
}
test "DataService PriceLoadResult default values" {
const result = DataService.PriceLoadResult{
.cached_count = 0,
.fetched_count = 0,
.fail_count = 0,
.stale_count = 0,
.latest_date = null,
};
try std.testing.expectEqual(@as(usize, 0), result.cached_count);
try std.testing.expect(result.latest_date == null);
}
test "DataService LoadAllResult default values" {
const allocator = std.testing.allocator;
var result = DataService.LoadAllResult{
@ -3342,6 +3245,56 @@ test "loadAllPrices offline mode skips network and returns cached" {
try std.testing.expectEqual(@as(usize, 1), result.failed_count);
}
test "loadAllPrices force_refresh tops up without wiping the candle cache" {
// Regression: force_refresh must mean "ignore TTL + incremental
// top-up", NOT "delete the cache and re-download from scratch".
// The old behavior invalidated (deleted) candles_daily before the
// fetch, which forced a full network re-download. With the cache
// already covering through today, force_refresh must serve from
// the surviving cache and touch no network.
const allocator = std.testing.allocator;
const io = std.testing.io;
var tmp = std.testing.tmpDir(.{});
defer tmp.cleanup();
const dir_path = try tmp.dir.realPathFileAlloc(io, ".", allocator);
defer allocator.free(dir_path);
const config = Config{ .cache_dir = dir_path };
var svc = DataService.init(io, allocator, config);
defer svc.deinit();
var store = svc.store();
// Dated far in the future so getCandles' "last cached date is
// today-or-later" branch fires deterministically regardless of the
// test clock — an incremental fetch would have nothing to pull and
// never reaches the network.
var candles = [_]Candle{
.{ .date = Date.fromYmd(2099, 12, 31), .open = 100, .high = 105, .low = 99, .close = 104, .adj_close = 104, .volume = 1000 },
};
store.cacheCandles("HELD", candles[0..], .tiingo, 0);
// Any provider/network attempt now panics. If force_refresh wiped
// the cache (old behavior), getCandles would fall through to a full
// re-fetch and trip this.
svc.panic_on_network_attempt = true;
const symbols = [_][]const u8{"HELD"};
var result = svc.loadAllPrices(
symbols[0..],
&.{},
.{ .force_refresh = true },
null,
null,
);
defer result.prices.deinit();
// Served from the (un-wiped) cache.
try std.testing.expect(result.prices.contains("HELD"));
try std.testing.expectEqual(@as(f64, 104), result.prices.get("HELD").?);
// The candle cache survived the force-refresh.
try std.testing.expect(svc.getCachedLastClose("HELD") != null);
}
test "getClassification: skip_network with no cache returns FetchFailed" {
const allocator = std.testing.allocator;
const io = std.testing.io;