ai: cli caching and portfolio fetch

src/cli/main.zig

@@ -36,6 +36,7 @@ const usage =
     \\
     \\Portfolio command options:
     \\  -w, --watchlist <FILE>   Watchlist file
+    \\  --refresh                Force refresh (ignore cache, re-fetch all prices)
     \\
     \\Environment Variables:
     \\  TWELVEDATA_API_KEY    Twelve Data API key (primary: prices)
@@ -127,16 +128,19 @@ pub fn main() !void {
         try cmdEtf(allocator, &svc, args[2], color);
     } else if (std.mem.eql(u8, command, "portfolio")) {
         if (args.len < 3) return try stderr_print("Error: 'portfolio' requires a file path argument\n");
-        // Parse -w/--watchlist flag
+        // Parse -w/--watchlist and --refresh flags
         var watchlist_path: ?[]const u8 = null;
+        var force_refresh = false;
         var pi: usize = 3;
         while (pi < args.len) : (pi += 1) {
             if ((std.mem.eql(u8, args[pi], "--watchlist") or std.mem.eql(u8, args[pi], "-w")) and pi + 1 < args.len) {
                 pi += 1;
                 watchlist_path = args[pi];
+            } else if (std.mem.eql(u8, args[pi], "--refresh")) {
+                force_refresh = true;
             }
         }
-        try cmdPortfolio(allocator, config, args[2], watchlist_path, color);
+        try cmdPortfolio(allocator, config, &svc, args[2], watchlist_path, force_refresh, color);
     } else if (std.mem.eql(u8, command, "cache")) {
         if (args.len < 3) return try stderr_print("Error: 'cache' requires a subcommand (stats, clear)\n");
         try cmdCache(allocator, config, args[2]);
@@ -970,7 +974,7 @@ fn printEtfProfile(profile: zfin.EtfProfile, symbol: []const u8, color: bool) !v
     try out.flush();
 }
 
-fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, file_path: []const u8, watchlist_path: ?[]const u8, color: bool) !void {
+fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, svc: *zfin.DataService, file_path: []const u8, watchlist_path: ?[]const u8, force_refresh: bool, color: bool) !void {
     // Load portfolio from SRF file
     const data = std.fs.cwd().readFileAlloc(allocator, file_path, 10 * 1024 * 1024) catch |err| {
         try stderr_print("Error reading portfolio file: ");
@@ -1004,37 +1008,119 @@ fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, file_path: []
 
     var fail_count: usize = 0;
 
-    if (config.twelvedata_key) |td_key| {
+    // Also collect watch symbols that need fetching
-        var td = zfin.TwelveData.init(allocator, td_key);
+    var watch_syms: std.ArrayList([]const u8) = .empty;
-        defer td.deinit();
+    defer watch_syms.deinit(allocator);
-
+    {
-        for (syms) |sym| {
+        var seen = std.StringHashMap(void).init(allocator);
-            try stderr_print("Fetching quote: ");
+        defer seen.deinit();
-            try stderr_print(sym);
+        for (syms) |s| try seen.put(s, {});
-            try stderr_print("...\n");
+        for (portfolio.lots) |lot| {
-            if (td.fetchQuote(allocator, sym)) |qr_val| {
+            if (lot.lot_type == .watch and !seen.contains(lot.symbol)) {
-                var qr = qr_val;
+                try seen.put(lot.symbol, {});
-                defer qr.deinit();
+                try watch_syms.append(allocator, lot.symbol);
-                if (qr.parse(allocator)) |q_val| {
-                    var q = q_val;
-                    defer q.deinit();
-                    const price = q.close();
-                    if (price > 0) try prices.put(sym, price);
-                } else |_| {
-                    fail_count += 1;
-                }
-            } else |_| {
-                fail_count += 1;
             }
         }
-    } else {
-        try stderr_print("Warning: TWELVEDATA_API_KEY not set. Cannot fetch current prices.\n");
     }
 
-    if (fail_count > 0) {
+    // All symbols to fetch (stock positions + watch)
-        var warn_msg_buf: [128]u8 = undefined;
+    const all_syms_count = syms.len + watch_syms.items.len;
-        const warn_msg = std.fmt.bufPrint(&warn_msg_buf, "Warning: {d} securities failed to load prices\n", .{fail_count}) catch "Warning: some securities failed\n";
+
-        try stderr_print(warn_msg);
+    if (all_syms_count > 0) {
+        if (config.twelvedata_key == null) {
+            try stderr_print("Warning: TWELVEDATA_API_KEY not set. Cannot fetch current prices.\n");
+        }
+
+        var loaded_count: usize = 0;
+        var cached_count: usize = 0;
+
+        // Fetch stock/ETF prices via DataService (respects cache TTL)
+        for (syms) |sym| {
+            loaded_count += 1;
+
+            // If --refresh, invalidate cache for this symbol
+            if (force_refresh) {
+                svc.invalidate(sym, .candles_daily);
+            }
+
+            // Check if cached and fresh (will be a fast no-op)
+            const is_fresh = svc.isCandleCacheFresh(sym);
+
+            if (is_fresh and !force_refresh) {
+                // Load from cache (no network)
+                if (svc.getCachedCandles(sym)) |cs| {
+                    defer allocator.free(cs);
+                    if (cs.len > 0) {
+                        try prices.put(sym, cs[cs.len - 1].close);
+                        cached_count += 1;
+                        try stderrProgress(sym, " (cached)", loaded_count, all_syms_count, color);
+                        continue;
+                    }
+                }
+            }
+
+            // Need to fetch from API
+            const wait_s = svc.estimateWaitSeconds();
+            if (wait_s) |w| {
+                if (w > 0) {
+                    try stderrRateLimitWait(w, color);
+                }
+            }
+            try stderrProgress(sym, " (fetching)", loaded_count, all_syms_count, color);
+
+            const result = svc.getCandles(sym) catch {
+                fail_count += 1;
+                try stderrProgress(sym, " FAILED", loaded_count, all_syms_count, color);
+                continue;
+            };
+            defer allocator.free(result.data);
+            if (result.data.len > 0) {
+                try prices.put(sym, result.data[result.data.len - 1].close);
+            }
+        }
+
+        // Fetch watch symbol candles (for watchlist display)
+        for (watch_syms.items) |sym| {
+            loaded_count += 1;
+
+            if (force_refresh) {
+                svc.invalidate(sym, .candles_daily);
+            }
+
+            const is_fresh = svc.isCandleCacheFresh(sym);
+            if (is_fresh and !force_refresh) {
+                cached_count += 1;
+                try stderrProgress(sym, " (cached)", loaded_count, all_syms_count, color);
+                continue;
+            }
+
+            const wait_s = svc.estimateWaitSeconds();
+            if (wait_s) |w| {
+                if (w > 0) {
+                    try stderrRateLimitWait(w, color);
+                }
+            }
+            try stderrProgress(sym, " (fetching)", loaded_count, all_syms_count, color);
+
+            const result = svc.getCandles(sym) catch {
+                try stderrProgress(sym, " FAILED", loaded_count, all_syms_count, color);
+                continue;
+            };
+            allocator.free(result.data);
+        }
+
+        // Summary line
+        {
+            var msg_buf: [256]u8 = undefined;
+            if (cached_count == all_syms_count) {
+                const msg = std.fmt.bufPrint(&msg_buf, "All {d} symbols loaded from cache\n", .{all_syms_count}) catch "Loaded from cache\n";
+                try stderr_print(msg);
+            } else {
+                const fetched_count = all_syms_count - cached_count - fail_count;
+                const msg = std.fmt.bufPrint(&msg_buf, "Loaded {d} symbols ({d} cached, {d} fetched, {d} failed)\n", .{ all_syms_count, cached_count, fetched_count, fail_count }) catch "Done loading\n";
+                try stderr_print(msg);
+            }
+        }
     }
 
     // Compute summary
@@ -1422,7 +1508,6 @@ fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, file_path: []
 
     // Watchlist (from watch lots in portfolio + separate watchlist file)
     {
-        var store = zfin.cache.Store.init(allocator, config.cache_dir);
         var any_watch = false;
         var watch_seen = std.StringHashMap(void).init(allocator);
         defer watch_seen.deinit();
@@ -1434,7 +1519,7 @@ fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, file_path: []
 
         // Helper to render a watch symbol
         const renderWatch = struct {
-            fn f(o: anytype, c: bool, s: *zfin.cache.Store, a2: std.mem.Allocator, sym: []const u8, any: *bool) !void {
+            fn f(o: anytype, c: bool, s: *zfin.DataService, a2: std.mem.Allocator, sym: []const u8, any: *bool) !void {
                 if (!any.*) {
                     try o.print("\n", .{});
                     try setBold(o, c);
@@ -1444,15 +1529,11 @@ fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, file_path: []
                 }
                 var price_str2: [16]u8 = undefined;
                 var ps2: []const u8 = "--";
-                const cached2 = s.readRaw(sym, .candles_daily) catch null;
+                if (s.getCachedCandles(sym)) |candles2| {
-                if (cached2) |cdata2| {
+                    defer a2.free(candles2);
-                    defer a2.free(cdata2);
+                    if (candles2.len > 0) {
-                    if (zfin.cache.Store.deserializeCandles(a2, cdata2)) |candles2| {
+                        ps2 = fmt.fmtMoney2(&price_str2, candles2[candles2.len - 1].close);
-                        defer a2.free(candles2);
+                    }
-                        if (candles2.len > 0) {
-                            ps2 = fmt.fmtMoney2(&price_str2, candles2[candles2.len - 1].close);
-                        }
-                    } else |_| {}
                 }
                 try o.print("  {s:<6} {s:>10}\n", .{ sym, ps2 });
             }
@@ -1463,7 +1544,7 @@ fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, file_path: []
             if (lot.lot_type == .watch) {
                 if (watch_seen.contains(lot.symbol)) continue;
                 try watch_seen.put(lot.symbol, {});
-                try renderWatch(out, color, &store, allocator, lot.symbol, &any_watch);
+                try renderWatch(out, color, svc, allocator, lot.symbol, &any_watch);
             }
         }
 
@@ -1483,7 +1564,7 @@ fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, file_path: []
                         if (sym.len > 0 and sym.len <= 10) {
                             if (watch_seen.contains(sym)) continue;
                             try watch_seen.put(sym, {});
-                            try renderWatch(out, color, &store, allocator, sym, &any_watch);
+                            try renderWatch(out, color, svc, allocator, sym, &any_watch);
                         }
                     }
                 }
@@ -1493,46 +1574,41 @@ fn cmdPortfolio(allocator: std.mem.Allocator, config: zfin.Config, file_path: []
 
     // Risk metrics
     {
-        var store = zfin.cache.Store.init(allocator, config.cache_dir);
         var any_risk = false;
 
         for (summary.allocations) |a| {
-            const cached = store.readRaw(a.symbol, .candles_daily) catch null;
+            if (svc.getCachedCandles(a.symbol)) |candles| {
-            if (cached) |cdata| {
+                defer allocator.free(candles);
-                defer allocator.free(cdata);
+                if (zfin.risk.computeRisk(candles)) |metrics| {
-                if (zfin.cache.Store.deserializeCandles(allocator, cdata)) |candles| {
+                    if (!any_risk) {
-                    defer allocator.free(candles);
-                    if (zfin.risk.computeRisk(candles)) |metrics| {
-                        if (!any_risk) {
-                            try out.print("\n", .{});
-                            try setBold(out, color);
-                            try out.print("  Risk Metrics (from cached price data):\n", .{});
-                            try reset(out, color);
-                            try setFg(out, color, CLR_MUTED);
-                            try out.print("  {s:>6} {s:>10} {s:>8} {s:>10}\n", .{
-                                "Symbol", "Volatility", "Sharpe", "Max DD",
-                            });
-                            try out.print("  {s:->6} {s:->10} {s:->8} {s:->10}\n", .{
-                                "", "", "", "",
-                            });
-                            try reset(out, color);
-                            any_risk = true;
-                        }
-                        try out.print("  {s:>6} {d:>9.1}% {d:>8.2} ", .{
-                            a.symbol, metrics.volatility * 100.0, metrics.sharpe,
-                        });
-                        try setFg(out, color, CLR_RED);
-                        try out.print("{d:>9.1}%", .{metrics.max_drawdown * 100.0});
-                        try reset(out, color);
-                        if (metrics.drawdown_trough) |dt| {
-                            var db: [10]u8 = undefined;
-                            try setFg(out, color, CLR_MUTED);
-                            try out.print("  (trough {s})", .{dt.format(&db)});
-                            try reset(out, color);
-                        }
                         try out.print("\n", .{});
+                        try setBold(out, color);
+                        try out.print("  Risk Metrics (from cached price data):\n", .{});
+                        try reset(out, color);
+                        try setFg(out, color, CLR_MUTED);
+                        try out.print("  {s:>6} {s:>10} {s:>8} {s:>10}\n", .{
+                            "Symbol", "Volatility", "Sharpe", "Max DD",
+                        });
+                        try out.print("  {s:->6} {s:->10} {s:->8} {s:->10}\n", .{
+                            "", "", "", "",
+                        });
+                        try reset(out, color);
+                        any_risk = true;
                     }
-                } else |_| {}
+                    try out.print("  {s:>6} {d:>9.1}% {d:>8.2} ", .{
+                        a.symbol, metrics.volatility * 100.0, metrics.sharpe,
+                    });
+                    try setFg(out, color, CLR_RED);
+                    try out.print("{d:>9.1}%", .{metrics.max_drawdown * 100.0});
+                    try reset(out, color);
+                    if (metrics.drawdown_trough) |dt| {
+                        var db: [10]u8 = undefined;
+                        try setFg(out, color, CLR_MUTED);
+                        try out.print("  (trough {s})", .{dt.format(&db)});
+                        try reset(out, color);
+                    }
+                    try out.print("\n", .{});
+                }
             }
         }
     }
@@ -1619,6 +1695,42 @@ fn stdout_print(msg: []const u8) !void {
     try out.flush();
 }
 
+/// Print progress line to stderr: "  [N/M] SYMBOL (status)"
+fn stderrProgress(symbol: []const u8, status: []const u8, current: usize, total: usize, color: bool) !void {
+    var buf: [256]u8 = undefined;
+    var writer = std.fs.File.stderr().writer(&buf);
+    const out = &writer.interface;
+    if (color) try fmt.ansiSetFg(out, CLR_MUTED[0], CLR_MUTED[1], CLR_MUTED[2]);
+    try out.print("  [{d}/{d}] ", .{ current, total });
+    if (color) try fmt.ansiReset(out);
+    try out.print("{s}", .{symbol});
+    if (color) try fmt.ansiSetFg(out, CLR_MUTED[0], CLR_MUTED[1], CLR_MUTED[2]);
+    try out.print("{s}\n", .{status});
+    if (color) try fmt.ansiReset(out);
+    try out.flush();
+}
+
+/// Print rate-limit wait message to stderr
+fn stderrRateLimitWait(wait_seconds: u64, color: bool) !void {
+    var buf: [256]u8 = undefined;
+    var writer = std.fs.File.stderr().writer(&buf);
+    const out = &writer.interface;
+    if (color) try fmt.ansiSetFg(out, CLR_RED[0], CLR_RED[1], CLR_RED[2]);
+    if (wait_seconds >= 60) {
+        const mins = wait_seconds / 60;
+        const secs = wait_seconds % 60;
+        if (secs > 0) {
+            try out.print("  (rate limit -- waiting {d}m {d}s)\n", .{ mins, secs });
+        } else {
+            try out.print("  (rate limit -- waiting {d}m)\n", .{mins});
+        }
+    } else {
+        try out.print("  (rate limit -- waiting {d}s)\n", .{wait_seconds});
+    }
+    if (color) try fmt.ansiReset(out);
+    try out.flush();
+}
+
 fn stderr_print(msg: []const u8) !void {
     var buf: [1024]u8 = undefined;
     var writer = std.fs.File.stderr().writer(&buf);

src/service.zig

@@ -374,6 +374,22 @@ pub const DataService = struct {
         };
     }
 
+    /// Check if candle data is fresh in cache (within TTL) without reading/deserializing.
+    pub fn isCandleCacheFresh(self: *DataService, symbol: []const u8) bool {
+        var s = self.store();
+        return s.isFresh(symbol, .candles_daily, cache.Ttl.candles_latest) catch false;
+    }
+
+    /// Estimate wait time (in seconds) before the next TwelveData API call can proceed.
+    /// Returns 0 if a request can be made immediately. Returns null if no API key.
+    pub fn estimateWaitSeconds(self: *DataService) ?u64 {
+        if (self.td) |*td| {
+            const ns = td.rate_limiter.estimateWaitNs();
+            return if (ns == 0) 0 else @max(1, ns / std.time.ns_per_s);
+        }
+        return null;
+    }
+
     /// Read candles from cache only (no network fetch). Used by TUI for display.
     /// Returns null if no cached data exists.
     pub fn getCachedCandles(self: *DataService, symbol: []const u8) ?[]Candle {

src/tui/main.zig

@@ -854,9 +854,11 @@ const App = struct {
 
         var latest_date: ?zfin.Date = null;
         var fail_count: usize = 0;
+        var fetch_count: usize = 0;
         for (syms) |sym| {
             // Try cache first; if miss, fetch (handles new securities / stale cache)
             const candles_slice = self.svc.getCachedCandles(sym) orelse blk: {
+                fetch_count += 1;
                 const result = self.svc.getCandles(sym) catch {
                     fail_count += 1;
                     break :blk null;
@@ -918,6 +920,10 @@ const App = struct {
             var warn_buf: [128]u8 = undefined;
             const warn_msg = std.fmt.bufPrint(&warn_buf, "Warning: {d} securities failed to load prices", .{fail_count}) catch "Warning: some securities failed";
             self.setStatus(warn_msg);
+        } else if (fetch_count > 0) {
+            var info_buf: [128]u8 = undefined;
+            const info_msg = std.fmt.bufPrint(&info_buf, "Loaded {d} symbols ({d} fetched) | r/F5 to refresh", .{ syms.len, fetch_count }) catch "Loaded | r/F5 to refresh";
+            self.setStatus(info_msg);
         } else {
             self.setStatus("j/k navigate | Enter expand | s select symbol | / search | ? help");
         }