From 16319a9f47e7da6a8ef87e50d480a8e3fecb34fb Mon Sep 17 00:00:00 2001 From: Emil Lerch Date: Wed, 5 Aug 2026 13:28:39 -0700 Subject: [PATCH] move to srf for all cache file formats --- src/Config.zig | 6 +- src/cache/Cache.zig | 116 ++++++++++--- src/location/GeoCache.zig | 158 ++++++++++++------ src/location/Ip2location.zig | 310 ++++++++++++++++++++--------------- 4 files changed, 382 insertions(+), 208 deletions(-) diff --git a/src/Config.zig b/src/Config.zig index dc693cc..1a2e264 100644 --- a/src/Config.zig +++ b/src/Config.zig @@ -110,7 +110,7 @@ pub fn load(allocator: std.mem.Allocator, env: *const std.process.Environ.Map) ! if (env.get("WTTR_GEOCACHE_FILE")) |v| { break :blk try allocator.dupe(u8, v); } - break :blk try std.fmt.allocPrint(allocator, "{s}/geocache.json", .{ + break :blk try std.fmt.allocPrint(allocator, "{s}/geocache.srf", .{ env.get("WTTR_CACHE_DIR") orelse default_cache_dir, }); }, @@ -125,7 +125,7 @@ pub fn load(allocator: std.mem.Allocator, env: *const std.process.Environ.Map) ! if (env.get("IP2LOCATION_CACHE_FILE")) |v| { break :blk try allocator.dupe(u8, v); } - break :blk try std.fmt.allocPrint(allocator, "{s}/ip2location.cache", .{env.get("WTTR_CACHE_DIR") orelse default_cache_dir}); + break :blk try std.fmt.allocPrint(allocator, "{s}/ip2location.srf", .{env.get("WTTR_CACHE_DIR") orelse default_cache_dir}); }, .pins_file = blk: { if (env.get("WTTR_PINS_FILE")) |v| { @@ -137,7 +137,7 @@ pub fn load(allocator: std.mem.Allocator, env: *const std.process.Environ.Map) ! if (env.get("IPWHOIS_CACHE_FILE")) |v| { break :blk try allocator.dupe(u8, v); } - break :blk try std.fmt.allocPrint(allocator, "{s}/ipwhois.cache", .{env.get("WTTR_CACHE_DIR") orelse default_cache_dir}); + break :blk try std.fmt.allocPrint(allocator, "{s}/ipwhois.srf", .{env.get("WTTR_CACHE_DIR") orelse default_cache_dir}); }, }; } diff --git a/src/cache/Cache.zig b/src/cache/Cache.zig index 70f66b7..176b758 100644 --- a/src/cache/Cache.zig +++ b/src/cache/Cache.zig @@ -1,5 +1,6 @@ const std = @import("std"); const Lru = @import("Lru.zig"); +const srf = @import("srf"); const Cache = @This(); @@ -83,11 +84,15 @@ pub fn deinit(self: *Cache) void { self.allocator.destroy(self); } +/// Extension for L2 entries. Named so `loadFromDir` can tell weather entries +/// apart from the other state living in the same directory. +const cache_file_extension = ".srf"; + fn getCacheFilename(self: *Cache, key: []const u8) ![]const u8 { var hasher = std.hash.Wyhash.init(0); hasher.update(key); const hash = hasher.final(); - return std.fmt.allocPrint(self.allocator, "{x}.json", .{hash}); + return std.fmt.allocPrint(self.allocator, "{x}" ++ cache_file_extension, .{hash}); } const CacheEntry = struct { @@ -143,25 +148,23 @@ fn loadFromFilePath(self: *Cache, file_path: []const u8) !CacheEntry { fn serialize(writer: *std.Io.Writer, key: []const u8, value: []const u8, expires: i64) !void { const entry = CacheEntry{ .key = key, .value = value, .expires = expires }; - try writer.print("{f}", .{std.json.fmt(entry, .{})}); + // Long format: the cached payload is a multi-kilobyte provider response, so + // one field per line keeps the file skimmable. SRF length-prefixes the value + // because it contains newlines, so the payload needs no escaping. + try writer.print("{f}", .{srf.fmt(CacheEntry, &.{entry}, .{ .long_format = true })}); } fn deserialize(allocator: std.mem.Allocator, reader: *std.Io.Reader) !CacheEntry { - var json_reader = std.json.Reader.init(allocator, reader); - defer json_reader.deinit(); + var records = try srf.iterator(reader, allocator, .{}); + defer records.deinit(); - const parsed = try std.json.parseFromTokenSource( - CacheEntry, - allocator, - &json_reader, - .{}, - ); - defer parsed.deinit(); + const fields = try records.next() orelse return error.EmptyCacheEntry; + const entry = try fields.to(CacheEntry, .{}); return .{ - .key = try allocator.dupe(u8, parsed.value.key), - .value = try allocator.dupe(u8, parsed.value.value), - .expires = parsed.value.expires, + .key = try allocator.dupe(u8, entry.key), + .value = try allocator.dupe(u8, entry.value), + .expires = entry.expires, }; } @@ -184,6 +187,22 @@ fn saveToFile(self: *Cache, key: []const u8, value: []const u8, expires: i64) !v try writer.flush(); } +/// Whether a directory entry is one of our L2 entries. +/// +/// Entry names are a hash rendered as hex plus the extension. Matching the shape +/// rather than just the extension keeps the descriptively-named state files in +/// the same directory from being mistaken for cache entries now that they share +/// a format. +fn isCacheFileName(name: []const u8) bool { + if (!std.mem.endsWith(u8, name, cache_file_extension)) return false; + const stem = name[0 .. name.len - cache_file_extension.len]; + if (stem.len == 0) return false; + for (stem) |ch| { + if (!std.ascii.isHex(ch)) return false; + } + return true; +} + fn loadFromDir(self: *Cache) !void { if (self.cache_dir == null) return error.NoCacheDir; @@ -193,6 +212,11 @@ fn loadFromDir(self: *Cache) !void { var it = dir.iterate(); while (try it.next(self.io)) |entry| { if (entry.kind != .file) continue; + // The cache directory also holds the GeoLite2 database and the named + // state files (pins, geocache, provider caches). Without this filter every + // startup opened and tried to parse all of them, including reading a + // megabyte of the 60+ MB database. + if (!isCacheFileName(entry.name)) continue; const file_path = try std.fs.path.join(self.allocator, &.{ self.cache_dir.?, entry.name }); defer self.allocator.free(file_path); @@ -233,7 +257,12 @@ test "serialize and deserialize" { try fixed_writer.flush(); const serialized = buffer[0..fixed_writer.end]; - try std.testing.expectEqualStrings("{\"key\":\"test_key\",\"value\":\"test_value\",\"expires\":1234567890}", serialized); + // Assert the shape as well as the round trip, so a change of format is a + // deliberate edit here rather than a silent difference on disk. + try std.testing.expect(std.mem.indexOf(u8, serialized, "#!srfv1") != null); + try std.testing.expect(std.mem.indexOf(u8, serialized, "#!long") != null); + try std.testing.expect(std.mem.indexOf(u8, serialized, "key::test_key") != null); + try std.testing.expect(std.mem.indexOf(u8, serialized, "value::test_value") != null); var fixed_reader = std.Io.Reader.fixed(serialized); @@ -245,19 +274,68 @@ test "serialize and deserialize" { try std.testing.expectEqual(expires, cached.expires); } -test "deserialize handles integer expires" { +test "deserialize preserves a millisecond timestamp exactly" { const allocator = std.testing.allocator; - const json = "{\"key\":\"k\",\"value\":\"v\",\"expires\":9999999999999}"; + // SRF carries numbers as f64. Millisecond timestamps are around 1.7e12, well + // inside the 2^53 range f64 represents exactly, so no precision is lost -- + // but that is worth pinning down rather than assuming. + const expires: i64 = 9999999999999; - var fixed_reader = std.Io.Reader.fixed(json); + var buffer: [1024]u8 = undefined; + var fixed_writer = std.Io.Writer.fixed(&buffer); + try serialize(&fixed_writer, "k", "v", expires); + try fixed_writer.flush(); + var fixed_reader = std.Io.Reader.fixed(buffer[0..fixed_writer.end]); const cached = try deserialize(allocator, &fixed_reader); defer cached.deinit(allocator); try std.testing.expectEqualStrings("k", cached.key); try std.testing.expectEqualStrings("v", cached.value); - try std.testing.expectEqual(9999999999999, cached.expires); + try std.testing.expectEqual(expires, cached.expires); +} + +test "serialize round-trips a payload containing newlines and commas" { + const allocator = std.testing.allocator; + + // The real payload is a provider JSON response: multi-line, comma-heavy, and + // quoted. It survives only because SRF length-prefixes such values. + const value = + \\{"properties":{"timeseries":[ + \\ {"time":"2026-08-05T00:00:00Z","data":{"instant":1.5}}, + \\ {"time":"2026-08-05T01:00:00Z","data":{"instant":2.5}} + \\]}} + ; + + var buffer: [4096]u8 = undefined; + var fixed_writer = std.Io.Writer.fixed(&buffer); + try serialize(&fixed_writer, "59.9,10.7", value, 9999999999999); + try fixed_writer.flush(); + + var fixed_reader = std.Io.Reader.fixed(buffer[0..fixed_writer.end]); + const cached = try deserialize(allocator, &fixed_reader); + defer cached.deinit(allocator); + + try std.testing.expectEqualStrings("59.9,10.7", cached.key); + try std.testing.expectEqualStrings(value, cached.value); +} + +test "isCacheFileName distinguishes entries from the other state files" { + // Entries are a hex hash plus the extension. + try std.testing.expect(isCacheFileName("1f44d18884c40d0a.srf")); + try std.testing.expect(isCacheFileName("abcdef.srf")); + + // Everything else sharing the directory must be left alone, including the + // state files that now use the same format. + try std.testing.expect(!isCacheFileName("pins.srf")); + try std.testing.expect(!isCacheFileName("geocache.srf")); + try std.testing.expect(!isCacheFileName("ipwhois.srf")); + try std.testing.expect(!isCacheFileName("ip2location.srf")); + try std.testing.expect(!isCacheFileName("GeoLite2-City.mmdb")); + try std.testing.expect(!isCacheFileName("GeoLite2-City.mmdb.download")); + try std.testing.expect(!isCacheFileName(".srf")); + try std.testing.expect(!isCacheFileName("1f44d18884c40d0a.json")); } test "L1/L2 cache flow" { diff --git a/src/location/GeoCache.zig b/src/location/GeoCache.zig index c53e6af..69c1272 100644 --- a/src/location/GeoCache.zig +++ b/src/location/GeoCache.zig @@ -1,5 +1,6 @@ const std = @import("std"); const Coordinates = @import("../Coordinates.zig"); +const srf = @import("srf"); const GeoCache = @This(); @@ -91,31 +92,48 @@ pub fn saveIfNeeded(self: *GeoCache) void { self.last_save = now; } +/// On-disk shape of one geocoded place. +/// +/// The query is stored alongside the result rather than used as an object key, +/// because SRF records are flat field lists rather than a nested map. +const Record = struct { + query: []const u8, + name: []const u8, + lat: f64, + lon: f64, +}; + fn load(allocator: std.mem.Allocator, cache: *std.StringHashMap(CachedLocation), content: []const u8) !void { - const CacheData = struct { - name: []const u8, - latitude: f64, - longitude: f64, - }; + if (std.mem.trim(u8, content, " \r\n\t").len == 0) return; - const parsed = try std.json.parseFromSlice( - std.json.ArrayHashMap(CacheData), - allocator, - content, - .{}, - ); - defer parsed.deinit(); + var reader = std.Io.Reader.fixed(content); + var records = try srf.iterator(&reader, allocator, .{}); + defer records.deinit(); - for (parsed.value.map.keys(), parsed.value.map.values()) |key, value| { - const cache_key = try allocator.dupe(u8, key); - const cache_value = CachedLocation{ - .name = try allocator.dupe(u8, value.name), - .coords = .{ - .latitude = value.latitude, - .longitude = value.longitude, - }, - }; - try cache.put(cache_key, cache_value); + var index: usize = 0; + while (records.next() catch |err| { + log.warn("stopped reading geocache after {d} entr(ies): {t}", .{ index, err }); + return; + }) |fields| { + index += 1; + const record = fields.to(Record, .{}) catch continue; + if (record.query.len == 0) continue; + + const cache_key = try allocator.dupe(u8, record.query); + errdefer allocator.free(cache_key); + const name_copy = try allocator.dupe(u8, record.name); + errdefer allocator.free(name_copy); + + // A duplicate query in the file would otherwise leak the key and name + // already stored under it. + const existing = try cache.fetchPut(cache_key, .{ + .name = name_copy, + .coords = .{ .latitude = record.lat, .longitude = record.lon }, + }); + if (existing) |old| { + allocator.free(old.key); + allocator.free(old.value.name); + } } } @@ -127,25 +145,24 @@ fn loadFromFile(allocator: std.mem.Allocator, io: std.Io, cache: *std.StringHash } fn save(self: *GeoCache, writer: *std.Io.Writer) !void { - try writer.writeAll("{\n"); + // Rewritten in full rather than appended to, because entries are replaced in + // place; SRF long format keeps the result legible for a file an operator may + // want to inspect or prune. + var records = try self.allocator.alloc(Record, self.cache.count()); + defer self.allocator.free(records); var it = self.cache.iterator(); - var first = true; - while (it.next()) |entry| { - if (!first) try writer.writeAll(",\n"); - first = false; - - try writer.print(" {f}: {f}", .{ - std.json.fmt(entry.key_ptr.*, .{}), - std.json.fmt(.{ - .name = entry.value_ptr.name, - .latitude = entry.value_ptr.coords.latitude, - .longitude = entry.value_ptr.coords.longitude, - }, .{}), - }); + var i: usize = 0; + while (it.next()) |entry| : (i += 1) { + records[i] = .{ + .query = entry.key_ptr.*, + .name = entry.value_ptr.name, + .lat = entry.value_ptr.coords.latitude, + .lon = entry.value_ptr.coords.longitude, + }; } - try writer.writeAll("\n}\n"); + try writer.print("{f}", .{srf.fmt(Record, records, .{ .long_format = true })}); } fn saveToFile(self: *GeoCache, file_path: []const u8) !void { @@ -189,7 +206,7 @@ test "GeoCache miss returns null" { try std.testing.expect(result == null); } -test "save produces valid JSON" { +test "save produces SRF an operator can read" { const allocator = std.testing.allocator; var cache = try GeoCache.init(allocator, std.testing.io, null); defer cache.deinit(); @@ -208,12 +225,16 @@ test "save produces valid JSON" { try cache.save(&writer); const output = buffer[0..writer.end]; - try std.testing.expect(std.mem.indexOf(u8, output, "London") != null); - try std.testing.expect(std.mem.indexOf(u8, output, "Paris") != null); + // Assert the shape, not just that the values appear: a silent switch to + // compact output would still round-trip while making the file harder to read. + try std.testing.expect(std.mem.indexOf(u8, output, "#!srfv1") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "#!long") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "query::London") != null); + try std.testing.expect(std.mem.indexOf(u8, output, "query::Paris") != null); try std.testing.expect(std.mem.indexOf(u8, output, "51.5074") != null); } -test "load parses valid JSON" { +test "load parses SRF records" { const allocator = std.testing.allocator; var cache_map = std.StringHashMap(CachedLocation).init(allocator); defer { @@ -225,22 +246,53 @@ test "load parses valid JSON" { cache_map.deinit(); } - const json = + const content = + \\#!srfv1 + \\#!long + \\query::London + \\name::London, UK + \\lat:num:51.5074 + \\lon:num:-0.1278 + \\ + \\query::Paris + \\name::Paris, France + \\lat:num:48.8566 + \\lon:num:2.3522 + \\ + ; + + try load(allocator, &cache_map, content); + + try std.testing.expectEqual(@as(usize, 2), cache_map.count()); + const london = cache_map.get("London") orelse return error.TestUnexpectedResult; + try std.testing.expectEqualStrings("London, UK", london.name); + try std.testing.expectApproxEqAbs(@as(f64, 51.5074), london.coords.latitude, 0.0001); + const paris = cache_map.get("Paris") orelse return error.TestUnexpectedResult; + try std.testing.expectApproxEqAbs(@as(f64, 2.3522), paris.coords.longitude, 0.0001); +} + +test "load ignores a legacy JSON file rather than failing" { + const allocator = std.testing.allocator; + var cache_map = std.StringHashMap(CachedLocation).init(allocator); + defer { + var it = cache_map.iterator(); + while (it.next()) |entry| { + allocator.free(entry.key_ptr.*); + allocator.free(entry.value_ptr.name); + } + cache_map.deinit(); + } + + // The previous format. Entries are re-geocodable, so the file is dropped + // rather than migrated; what matters is that this is not fatal. + const legacy = \\{ - \\ "London": {"name": "London, UK", "latitude": 51.5074, "longitude": -0.1278}, - \\ "Paris": {"name": "Paris, France", "latitude": 48.8566, "longitude": 2.3522} + \\ "London": {"name": "London, UK", "latitude": 51.5074, "longitude": -0.1278} \\} ; - try load(allocator, &cache_map, json); - - const london = cache_map.get("London"); - try std.testing.expect(london != null); - try std.testing.expectApproxEqAbs(@as(f64, 51.5074), london.?.coords.latitude, 0.0001); - - const paris = cache_map.get("Paris"); - try std.testing.expect(paris != null); - try std.testing.expectApproxEqAbs(@as(f64, 48.8566), paris.?.coords.latitude, 0.0001); + load(allocator, &cache_map, legacy) catch {}; + try std.testing.expectEqual(@as(usize, 0), cache_map.count()); } test "save and load round-trip" { diff --git a/src/location/Ip2location.zig b/src/location/Ip2location.zig index 639e8a9..95f5f94 100644 --- a/src/location/Ip2location.zig +++ b/src/location/Ip2location.zig @@ -1,6 +1,7 @@ const std = @import("std"); const Allocator = std.mem.Allocator; const Location = @import("resolver.zig").Location; +const srf = @import("srf"); const Self = @This(); @@ -174,6 +175,15 @@ inline fn getString(obj: std.json.ObjectMap, key: []const u8) []const u8 { return maybe_val.?.string; } +/// Permanent IP-to-location cache, shared by the online providers. +/// +/// Append-only: an online lookup for an address is answered once and the result +/// is kept forever, because an address's city does not meaningfully change and +/// every avoided request is one that does not count against a rate limit. +/// +/// Stored as SRF in compact form, one record per line. Compact is safe for place +/// names containing commas because SRF length-prefixes any string containing the +/// field delimiter, so no escaping rules are needed here. pub const Cache = struct { allocator: Allocator, io: std.Io, @@ -181,6 +191,17 @@ pub const Cache = struct { entries: std.AutoHashMap(u128, Location), file: ?std.Io.File, + /// On-disk shape of one entry. + /// + /// The address is stored as text rather than the `u128` used for the in-memory + /// key, so the file stays legible to whoever has to look at it. + const Record = struct { + ip: []const u8, + lat: f64, + lon: f64, + name: []const u8, + }; + pub fn init(allocator: Allocator, io: std.Io, path: []const u8) !Cache { var cache = Cache{ .allocator = allocator, @@ -190,19 +211,22 @@ pub const Cache = struct { .file = null, }; errdefer allocator.free(cache.path); + errdefer cache.entries.deinit(); // Try to open existing cache file if (std.Io.Dir.openFileAbsolute(io, path, .{ .mode = .read_write })) |file| { cache.file = file; - try cache.load(); + cache.load() catch |err| { + // A cache that cannot be read is worth strictly less than the + // service staying up: start empty and let it refill. + log.warn("could not read cache {s} ({t}); starting empty", .{ path, err }); + }; } else |err| switch (err) { error.FileNotFound => { - // Create new cache file const dir = std.fs.path.dirname(path) orelse return error.InvalidPath; try std.Io.Dir.cwd().createDirPath(io, dir); cache.file = try std.Io.Dir.createFileAbsolute(io, path, .{ .read = true }); - // Write header - try cache.file.?.writePositionalAll(io, "#Ip2location:v2\n", 0); + try cache.writeDirectives(); }, else => return err, } @@ -210,6 +234,17 @@ pub const Cache = struct { return cache; } + /// Writes the SRF front matter that opens a new cache file. + /// + /// Emitted once at creation; `put` appends bare records afterwards. + fn writeDirectives(self: *Cache) !void { + const file = self.file orelse return; + var buf: [64]u8 = undefined; + var fw = file.writer(self.io, &buf); + try fw.interface.print("{f}", .{srf.fmt(Record, &.{}, .{})}); + try fw.interface.flush(); + } + pub fn deinit(self: *Cache) void { if (self.file) |f| f.close(self.io); var it = self.entries.valueIterator(); @@ -236,61 +271,30 @@ pub const Cache = struct { ); defer self.allocator.free(content); - var lines = std.mem.splitScalar(u8, content, '\n'); + var reader = std.Io.Reader.fixed(content); + var records = try srf.iterator(&reader, self.allocator, .{}); + defer records.deinit(); - // Check for header magic string - if (lines.next()) |first_line| { - if (!std.mem.eql(u8, first_line, "#Ip2location:v2")) { - log.warn("Cache file missing or invalid header, discarding", .{}); - file.close(self.io); - self.file = null; - std.Io.Dir.deleteFileAbsolute(self.io, self.path) catch |e| { - log.err("error deleting {s}: {}", .{ self.path, e }); - }; - return; - } - } else { - return; // Empty file - } + var index: usize = 0; + while (records.next() catch |err| { + log.warn("stopped reading {s} after {d} entr(ies): {t}", .{ self.path, index, err }); + return; + }) |fields| { + index += 1; + const record = fields.to(Record, .{}) catch continue; + const parsed = packIp(record.ip) orelse continue; - while (lines.next()) |line| { - if (line.len == 0) continue; - const entry = parseCacheLine(self.allocator, line) catch continue; - try self.entries.put(entry.ip, entry.location); - } - } + const name_copy = try self.allocator.dupe(u8, record.name); + errdefer self.allocator.free(name_copy); - const CacheEntry = struct { - ip: u128, - location: Location, - }; - - fn parseCacheLine(allocator: Allocator, line: []const u8) !CacheEntry { - // Parse: ip,lat,lon,name - var parts = std.mem.splitScalar(u8, line, ','); - const ip_str = parts.next() orelse return error.InvalidFormat; - const lat_str = parts.next() orelse return error.InvalidFormat; - const lon_str = parts.next() orelse return error.InvalidFormat; - const name = parts.rest(); - - const lat = try std.fmt.parseFloat(f64, lat_str); - const lon = try std.fmt.parseFloat(f64, lon_str); - - // Try parsing as IP address first, fall back to u128 - const ip_u128 = if (packIp(ip_str)) |parsed| - parsed.key - else - try std.fmt.parseInt(u128, ip_str, 10); - - const name_copy = try allocator.dupe(u8, name); - return .{ - .ip = ip_u128, - .location = .{ - .allocator = allocator, + // Replacing an existing key would leak the name it already owns. + const existing = try self.entries.fetchPut(parsed.key, .{ + .allocator = self.allocator, .name = name_copy, - .coords = .{ .latitude = lat, .longitude = lon }, - }, - }; + .coords = .{ .latitude = record.lat, .longitude = record.lon }, + }); + if (existing) |old| self.allocator.free(old.value.name); + } } pub fn get(self: *Cache, ip: u128) ?Location { @@ -304,101 +308,141 @@ pub const Cache = struct { pub fn put(self: *Cache, ip: u128, family: u8, loc: Location) !void { const name_copy = try self.allocator.dupe(u8, loc.name); - try self.entries.put(ip, .{ + errdefer self.allocator.free(name_copy); + + const existing = try self.entries.fetchPut(ip, .{ .allocator = self.allocator, .name = name_copy, .coords = loc.coords, }); + if (existing) |old| self.allocator.free(old.value.name); - // Append to file: ip,lat,lon,name - if (self.file) |file| { - // 0.16 has no seek+write; append at the current end offset. - const end = try file.length(self.io); - // Format IP as string for file - var buf: [64]u8 = undefined; - const ip_str = if (family == 4) - try std.fmt.bufPrint(&buf, "{}.{}.{}.{}", .{ - @as(u8, @truncate(ip >> 24)), - @as(u8, @truncate(ip >> 16)), - @as(u8, @truncate(ip >> 8)), - @as(u8, @truncate(ip)), - }) - else - try std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}", .{ - @as(u16, @truncate(ip >> 112)), - @as(u16, @truncate(ip >> 96)), - @as(u16, @truncate(ip >> 80)), - @as(u16, @truncate(ip >> 64)), - @as(u16, @truncate(ip >> 48)), - @as(u16, @truncate(ip >> 32)), - @as(u16, @truncate(ip >> 16)), - @as(u16, @truncate(ip)), - }); - const line = try std.fmt.allocPrint(self.allocator, "{s},{d},{d},{s}\n", .{ - ip_str, - loc.coords.latitude, - loc.coords.longitude, - loc.name, + const file = self.file orelse return; + + var ip_buf: [64]u8 = undefined; + const ip_str = try formatKey(ip, family, &ip_buf); + + // Append one record with no front matter; the directives were written + // when the file was created. + var line_buf: [512]u8 = undefined; + const line = try std.fmt.bufPrint(&line_buf, "{f}", .{srf.fmt(Record, &.{.{ + .ip = ip_str, + .lat = loc.coords.latitude, + .lon = loc.coords.longitude, + .name = loc.name, + }}, .{ .emit_directives = false })}); + + // 0.16 has no seek+write; append at the current end offset. + const end = try file.length(self.io); + try file.writePositionalAll(self.io, line, end); + } + + /// Renders a packed key back to text for storage. + fn formatKey(ip: u128, family: u8, buf: []u8) ![]const u8 { + if (family == 4) { + return std.fmt.bufPrint(buf, "{d}.{d}.{d}.{d}", .{ + @as(u8, @truncate(ip >> 24)), + @as(u8, @truncate(ip >> 16)), + @as(u8, @truncate(ip >> 8)), + @as(u8, @truncate(ip)), }); - defer self.allocator.free(line); - try file.writePositionalAll(self.io, line, end); } + var bytes: [16]u8 = undefined; + std.mem.writeInt(u128, &bytes, ip, .big); + // `Ip6Address.format` would append ":port" and bracket the address; + // `Unresolved` is the bare-address formatter. + const bare: std.Io.net.Ip6Address.Unresolved = .{ .bytes = bytes, .interface_name = null }; + return std.fmt.bufPrint(buf, "{f}", .{&bare}); } }; -test "parseCacheLine: valid IPv4 line" { +test "Cache: round-trips IPv4 and IPv6 entries through the file" { const allocator = std.testing.allocator; - const line = "192.168.1.1,37.5,-122.5,San Francisco, California, United States"; - const entry = try Cache.parseCacheLine(allocator, line); - defer allocator.free(entry.location.name); + const io = std.testing.io; - try std.testing.expectEqual(@as(u128, 3232235777), entry.ip); // 192.168.1.1 as u128 - try std.testing.expectEqual(@as(f64, 37.5), entry.location.coords.latitude); - try std.testing.expectEqual(@as(f64, -122.5), entry.location.coords.longitude); - try std.testing.expectEqualStrings("San Francisco, California, United States", entry.location.name); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_len = try tmp.dir.realPath(io, &path_buf); + const path = try std.fmt.allocPrint(allocator, "{s}/ip.cache", .{path_buf[0..dir_len]}); + defer allocator.free(path); + + const v4 = packIp("192.168.1.1").?; + const v6 = packIp("2001:db8::1").?; + + { + var cache = try Cache.init(allocator, io, path); + defer cache.deinit(); + + // A name with commas is the interesting case: compact SRF delimits on + // commas, so this only survives via length prefixing. + try cache.put(v4.key, v4.family, .{ + .allocator = allocator, + .name = "San Francisco, California, United States", + .coords = .{ .latitude = 37.5, .longitude = -122.5 }, + }); + try cache.put(v6.key, v6.family, .{ + .allocator = allocator, + .name = "London, United Kingdom", + .coords = .{ .latitude = 51.5, .longitude = -0.1 }, + }); + } + + var reopened = try Cache.init(allocator, io, path); + defer reopened.deinit(); + + const got4 = reopened.get(v4.key) orelse return error.TestUnexpectedResult; + defer got4.deinit(); + try std.testing.expectEqualStrings("San Francisco, California, United States", got4.name); + try std.testing.expectEqual(@as(f64, 37.5), got4.coords.latitude); + try std.testing.expectEqual(@as(f64, -122.5), got4.coords.longitude); + + const got6 = reopened.get(v6.key) orelse return error.TestUnexpectedResult; + defer got6.deinit(); + try std.testing.expectEqualStrings("London, United Kingdom", got6.name); + try std.testing.expectEqual(@as(f64, 51.5), got6.coords.latitude); } -test "parseCacheLine: valid IPv6 line" { +test "Cache: a legacy or corrupt file starts empty instead of failing" { const allocator = std.testing.allocator; - const line = "2001:db8::1,51.5,-0.1,London, United Kingdom"; - const entry = try Cache.parseCacheLine(allocator, line); - defer allocator.free(entry.location.name); + const io = std.testing.io; - try std.testing.expect(entry.ip > 0); - try std.testing.expectEqual(@as(f64, 51.5), entry.location.coords.latitude); - try std.testing.expectEqual(@as(f64, -0.1), entry.location.coords.longitude); - try std.testing.expectEqualStrings("London, United Kingdom", entry.location.name); + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const dir_len = try tmp.dir.realPath(io, &path_buf); + const path = try std.fmt.allocPrint(allocator, "{s}/ip.cache", .{path_buf[0..dir_len]}); + defer allocator.free(path); + + // The previous hand-rolled format. Deliberately not migrated: the entries are + // re-fetchable and losing them costs a handful of API calls. + try std.Io.Dir.cwd().writeFile(io, .{ + .sub_path = path, + .data = "#Ip2location:v2\n192.168.1.1,37.5,-122.5,San Francisco\n", + }); + + var cache = try Cache.init(allocator, io, path); + defer cache.deinit(); + + try std.testing.expectEqual(@as(usize, 0), cache.entries.count()); + + // Still usable afterwards, which is the point of not treating it as fatal. + const v4 = packIp("10.0.0.1").?; + try cache.put(v4.key, v4.family, .{ + .allocator = allocator, + .name = "Somewhere", + .coords = .{ .latitude = 1, .longitude = 2 }, + }); + const got = cache.get(v4.key) orelse return error.TestUnexpectedResult; + defer got.deinit(); + try std.testing.expectEqualStrings("Somewhere", got.name); } -test "parseCacheLine: empty name" { - const allocator = std.testing.allocator; - const line = "10.0.0.1,0.0,0.0,"; - const entry = try Cache.parseCacheLine(allocator, line); - defer allocator.free(entry.location.name); +test "Cache: formatKey renders both families without a port" { + var buf: [64]u8 = undefined; + const v4 = packIp("12.94.132.170").?; + try std.testing.expectEqualStrings("12.94.132.170", try Cache.formatKey(v4.key, v4.family, &buf)); - try std.testing.expectEqualStrings("", entry.location.name); -} - -test "parseCacheLine: missing fields" { - const allocator = std.testing.allocator; - const line = "192.168.1.1,37.5"; - try std.testing.expectError(error.InvalidFormat, Cache.parseCacheLine(allocator, line)); -} - -test "parseCacheLine: invalid IP" { - const allocator = std.testing.allocator; - const line = "not.an.ip,37.5,-122.5,Test"; - try std.testing.expectError(error.InvalidCharacter, Cache.parseCacheLine(allocator, line)); -} - -test "parseCacheLine: invalid latitude" { - const allocator = std.testing.allocator; - const line = "192.168.1.1,invalid,-122.5,Test"; - try std.testing.expectError(error.InvalidCharacter, Cache.parseCacheLine(allocator, line)); -} - -test "parseCacheLine: invalid longitude" { - const allocator = std.testing.allocator; - const line = "192.168.1.1,37.5,invalid,Test"; - try std.testing.expectError(error.InvalidCharacter, Cache.parseCacheLine(allocator, line)); + const v6 = packIp("2001:db8::1").?; + try std.testing.expectEqualStrings("2001:db8::1", try Cache.formatKey(v6.key, v6.family, &buf)); }