Compare commits
4 commits
f0b382375e
...
ea88a88e31
| Author | SHA1 | Date | |
|---|---|---|---|
| ea88a88e31 | |||
| 6a6101c495 | |||
| 50e493aad5 | |||
| f936422b07 |
4 changed files with 87 additions and 33 deletions
|
|
@ -119,28 +119,26 @@ pub const RateLimitConfig = struct {
|
|||
|
||||
### Cache
|
||||
|
||||
**Single-layer cache with two storage backends:**
|
||||
**Single layer, two-tier cache system (L1 memory + L2 file):**
|
||||
|
||||
**Interface:**
|
||||
```zig
|
||||
pub const Cache = struct {
|
||||
lru: LRU,
|
||||
file_store: FileStore,
|
||||
lru: Lru, // L1: In-memory cache
|
||||
cache_dir: ?[]const u8, // L2: Optional file-backed storage
|
||||
|
||||
pub fn get(self: *Cache, key: []const u8) ?[]const u8;
|
||||
pub fn put(self: *Cache, key: []const u8, value: []const u8, ttl: u64) !void;
|
||||
};
|
||||
```
|
||||
|
||||
**Cache Key:**
|
||||
```
|
||||
{user_agent}:{path}:{query}:{client_ip}
|
||||
```
|
||||
|
||||
**Storage Strategy:**
|
||||
- Small responses (<1KB): In-memory LRU only
|
||||
- Large responses (≥1KB): LRU stores file path, data on disk
|
||||
- TTL: 1000-2000s (randomized to prevent thundering herd)
|
||||
- **L1 (Memory)**: LRU cache with configurable max entries
|
||||
- **L2 (Disk)**: Optional file-backed storage for persistence
|
||||
- Files stored as JSON with key, value, and expiration timestamp
|
||||
- On eviction from L1, data remains in L2
|
||||
- On cache miss in L1, checks L2 and promotes to L1 if found
|
||||
- **TTL**: 1000-2000s (randomized to prevent thundering herd)
|
||||
|
||||
**Cache Locations:**
|
||||
|
||||
|
|
@ -148,23 +146,32 @@ All caches default to `$XDG_CACHE_HOME/wttr` (typically `~/.cache/wttr`).
|
|||
|
||||
1. **Weather Response Cache**
|
||||
- Location: `$WTTR_CACHE_DIR` (default: `~/.cache/wttr/`)
|
||||
- Format: JSON (one file for each entry/location)
|
||||
- Size: 10,000 entries (configurable via `WTTR_CACHE_SIZE`)
|
||||
- Expiration: 1000-2000 seconds (randomized)
|
||||
- Eviction: LRU
|
||||
- Implementation: `src/cache/Cache.zig`, `src/cache/Lru.zig`
|
||||
|
||||
2. **Geocoding Cache**
|
||||
- Purpose: Caches results from nominatim API to minimize API calls
|
||||
- Location: `$WTTR_GEOCACHE_FILE` (default: `~/.cache/wttr/geocache.json`)
|
||||
- Format: JSON
|
||||
- Format: JSON (single file with all entries)
|
||||
- Expiration: None (persists indefinitely)
|
||||
- Implementation: `src/location/GeoCache.zig`
|
||||
|
||||
3. **IP2Location Cache**
|
||||
- Purpose: Caches results from IP2Location.io API to minimize API calls
|
||||
- Location: `$IP2LOCATION_CACHE_FILE` (default: `~/.cache/wttr/ip2location.cache`)
|
||||
- Format: Binary (32-byte records)
|
||||
- Format: Text file with header `#Ip2location:v2` followed by CSV lines: `ip,lat,lon,name`
|
||||
Note that name stored at the end so any bytes are included, including commas
|
||||
- Expiration: None (persists indefinitely)
|
||||
- Storage: In-memory hash map (u128 IP → Location) + append-only file
|
||||
- Implementation: `src/location/Ip2location.zig` (internal Cache struct)
|
||||
|
||||
4. **GeoIP Database**
|
||||
- Location: `$WTTR_GEOLITE_PATH` (default: `~/.cache/wttr/GeoLite2-City.mmdb`)
|
||||
- Auto-downloaded if missing
|
||||
- Implementation: `src/location/GeoLite2.zig`
|
||||
|
||||
### Location Resolver (`src/location/resolver.zig`)
|
||||
|
||||
|
|
|
|||
48
src/cache/Cache.zig
vendored
48
src/cache/Cache.zig
vendored
|
|
@ -7,17 +7,19 @@ const log = std.log.scoped(.cache);
|
|||
|
||||
allocator: std.mem.Allocator,
|
||||
lru: Lru,
|
||||
cache_dir: []const u8,
|
||||
/// Cache directory for L2 persistent cache
|
||||
cache_dir: ?[]const u8,
|
||||
|
||||
pub const Config = struct {
|
||||
max_entries: usize = 1_000,
|
||||
cache_dir: []const u8,
|
||||
cache_dir: ?[]const u8,
|
||||
};
|
||||
|
||||
pub fn init(allocator: std.mem.Allocator, config: Config) !*Cache {
|
||||
std.fs.makeDirAbsolute(config.cache_dir) catch |err| {
|
||||
if (err != error.PathAlreadyExists) return err;
|
||||
};
|
||||
if (config.cache_dir) |d|
|
||||
std.fs.makeDirAbsolute(d) catch |err| {
|
||||
if (err != error.PathAlreadyExists) return err;
|
||||
};
|
||||
|
||||
const cache = try allocator.create(Cache);
|
||||
errdefer allocator.destroy(cache);
|
||||
|
|
@ -25,15 +27,17 @@ pub fn init(allocator: std.mem.Allocator, config: Config) !*Cache {
|
|||
cache.* = Cache{
|
||||
.allocator = allocator,
|
||||
.lru = try Lru.init(allocator, config.max_entries),
|
||||
.cache_dir = try allocator.dupe(u8, config.cache_dir),
|
||||
.cache_dir = if (config.cache_dir) |d| try allocator.dupe(u8, d) else null,
|
||||
};
|
||||
|
||||
cache.lru.setEvictionCallback(cache, evictCallback);
|
||||
|
||||
// Clean up expired files and populate L1 cache from L2
|
||||
cache.loadFromDir() catch |err| {
|
||||
std.log.warn("Failed to load cache from {s}: {}", .{ config.cache_dir, err });
|
||||
};
|
||||
if (config.cache_dir) |d| {
|
||||
cache.loadFromDir() catch |err| {
|
||||
std.log.warn("Failed to load cache from {s}: {}", .{ d, err });
|
||||
};
|
||||
}
|
||||
|
||||
return cache;
|
||||
}
|
||||
|
|
@ -61,8 +65,10 @@ pub fn put(self: *Cache, key: []const u8, value: []const u8, ttl_seconds: u64) !
|
|||
const now = std.time.milliTimestamp();
|
||||
const expires = now + @as(i64, @intCast(ttl_seconds * 1000));
|
||||
|
||||
// Write to L2 (disk) first
|
||||
try self.saveToFile(key, value, expires);
|
||||
// Write to L2 (disk) first if cache_dir is set
|
||||
if (self.cache_dir != null) {
|
||||
try self.saveToFile(key, value, expires);
|
||||
}
|
||||
|
||||
// Add to L1 (memory)
|
||||
try self.lru.put(key, value, expires);
|
||||
|
|
@ -70,7 +76,7 @@ pub fn put(self: *Cache, key: []const u8, value: []const u8, ttl_seconds: u64) !
|
|||
|
||||
pub fn deinit(self: *Cache) void {
|
||||
self.lru.deinit();
|
||||
self.allocator.free(self.cache_dir);
|
||||
if (self.cache_dir) |d| self.allocator.free(d);
|
||||
self.allocator.destroy(self);
|
||||
}
|
||||
|
||||
|
|
@ -96,10 +102,12 @@ const CacheEntry = struct {
|
|||
/// if the file access fails OR if the data has expired.
|
||||
/// If the data has expired, the file will be deleted
|
||||
fn loadFromFile(self: *Cache, key: []const u8) !CacheEntry {
|
||||
if (self.cache_dir == null) return error.NoCacheDir;
|
||||
|
||||
const filename = try self.getCacheFilename(key);
|
||||
defer self.allocator.free(filename);
|
||||
|
||||
const file_path = try std.fs.path.join(self.allocator, &.{ self.cache_dir, filename });
|
||||
const file_path = try std.fs.path.join(self.allocator, &.{ self.cache_dir.?, filename });
|
||||
defer self.allocator.free(file_path);
|
||||
|
||||
return self.loadFromFilePath(file_path);
|
||||
|
|
@ -155,10 +163,12 @@ fn deserialize(allocator: std.mem.Allocator, reader: *std.Io.Reader) !CacheEntry
|
|||
}
|
||||
|
||||
fn saveToFile(self: *Cache, key: []const u8, value: []const u8, expires: i64) !void {
|
||||
if (self.cache_dir == null) return error.NoCacheDir;
|
||||
|
||||
const filename = try self.getCacheFilename(key);
|
||||
defer self.allocator.free(filename);
|
||||
|
||||
const file_path = try std.fs.path.join(self.allocator, &.{ self.cache_dir, filename });
|
||||
const file_path = try std.fs.path.join(self.allocator, &.{ self.cache_dir.?, filename });
|
||||
defer self.allocator.free(file_path);
|
||||
|
||||
const file = try std.fs.cwd().createFile(file_path, .{});
|
||||
|
|
@ -172,14 +182,16 @@ fn saveToFile(self: *Cache, key: []const u8, value: []const u8, expires: i64) !v
|
|||
}
|
||||
|
||||
fn loadFromDir(self: *Cache) !void {
|
||||
var dir = try std.fs.cwd().openDir(self.cache_dir, .{ .iterate = true });
|
||||
if (self.cache_dir == null) return error.NoCacheDir;
|
||||
|
||||
var dir = try std.fs.cwd().openDir(self.cache_dir.?, .{ .iterate = true });
|
||||
defer dir.close();
|
||||
|
||||
var it = dir.iterate();
|
||||
while (try it.next()) |entry| {
|
||||
if (entry.kind != .file) continue;
|
||||
|
||||
const file_path = try std.fs.path.join(self.allocator, &.{ self.cache_dir, entry.name });
|
||||
const file_path = try std.fs.path.join(self.allocator, &.{ self.cache_dir.?, entry.name });
|
||||
defer self.allocator.free(file_path);
|
||||
|
||||
const cached = self.loadFromFilePath(file_path) catch continue;
|
||||
|
|
@ -191,10 +203,12 @@ fn loadFromDir(self: *Cache) !void {
|
|||
}
|
||||
|
||||
fn deleteFile(self: *Cache, key: []const u8) void {
|
||||
if (self.cache_dir == null) return;
|
||||
|
||||
const filename = self.getCacheFilename(key) catch @panic("OOM");
|
||||
defer self.allocator.free(filename);
|
||||
|
||||
const file_path = std.fs.path.join(self.allocator, &.{ self.cache_dir, filename }) catch @panic("OOM");
|
||||
const file_path = std.fs.path.join(self.allocator, &.{ self.cache_dir.?, filename }) catch @panic("OOM");
|
||||
defer self.allocator.free(file_path);
|
||||
|
||||
std.fs.cwd().deleteFile(file_path) catch |e| {
|
||||
|
|
|
|||
|
|
@ -100,7 +100,10 @@ pub fn render(writer: *std.Io.Writer, weather: types.WeatherData, format: []cons
|
|||
}
|
||||
|
||||
fn nowAt(coords: Coordinates) !i64 {
|
||||
const now = try zeit.instant(.{});
|
||||
const now = if (@import("builtin").is_test)
|
||||
(try zeit.Time.fromISO8601("2026-01-09")).instant()
|
||||
else
|
||||
try zeit.instant(.{});
|
||||
const offset = TimeZoneOffsets.getTimezoneOffset(coords);
|
||||
const new = if (offset >= 0)
|
||||
try now.add(.{ .minutes = @abs(offset) })
|
||||
|
|
|
|||
|
|
@ -23,8 +23,8 @@ pub const VTable = struct {
|
|||
|
||||
pub fn fetch(self: WeatherProvider, allocator: std.mem.Allocator, coords: Coordinates) !types.WeatherData {
|
||||
// Generate cache key from coordinates
|
||||
const cache_key = try std.fmt.allocPrint(allocator, "{d:.4},{d:.4}", .{ coords.latitude, coords.longitude });
|
||||
defer allocator.free(cache_key);
|
||||
var buf: [256]u8 = undefined;
|
||||
const cache_key = try std.fmt.bufPrint(&buf, "{d:.4},{d:.4}", .{ coords.latitude, coords.longitude });
|
||||
|
||||
// Check cache first
|
||||
if (self.cache.get(cache_key)) |cached_raw|
|
||||
|
|
@ -45,3 +45,33 @@ pub fn fetch(self: WeatherProvider, allocator: std.mem.Allocator, coords: Coordi
|
|||
pub fn deinit(self: WeatherProvider) void {
|
||||
self.vtable.deinit(self.ptr);
|
||||
}
|
||||
|
||||
test "provider fetch" {
|
||||
const Mock = @import("Mock.zig");
|
||||
const MetNo = @import("MetNo.zig");
|
||||
|
||||
const cache = try Cache.init(std.testing.allocator, .{ .max_entries = 10, .cache_dir = null });
|
||||
defer cache.deinit();
|
||||
|
||||
var mock = try Mock.init(std.testing.allocator);
|
||||
defer mock.deinit();
|
||||
|
||||
mock.parse_fn = MetNo.parse;
|
||||
|
||||
const test_data = @embedFile("../tests/metno_test_data.json");
|
||||
try mock.addResponse(.{ .latitude = 33.4484, .longitude = -112.0740 }, test_data);
|
||||
|
||||
const provider = mock.provider(cache);
|
||||
const coords = Coordinates{ .latitude = 33.4484, .longitude = -112.0740 };
|
||||
|
||||
// First fetch - should allocate and cache
|
||||
const weather1 = try provider.fetch(std.testing.allocator, coords);
|
||||
defer weather1.deinit();
|
||||
|
||||
// Second fetch - should use cache
|
||||
const weather2 = try provider.fetch(std.testing.allocator, coords);
|
||||
defer weather2.deinit();
|
||||
|
||||
try std.testing.expect(weather1.forecast.len > 0);
|
||||
try std.testing.expect(weather2.forecast.len > 0);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue