move to srf for all cache file formats
This commit is contained in:
parent
11364b10e7
commit
16319a9f47
GPG key ID:
A7B62D657EF764F8
4 changed files with 382 additions and 208 deletions
|
|
@ -110,7 +110,7 @@ pub fn load(allocator: std.mem.Allocator, env: *const std.process.Environ.Map) !
|
|||
if (env.get("WTTR_GEOCACHE_FILE")) |v| {
|
||||
break :blk try allocator.dupe(u8, v);
|
||||
}
|
||||
break :blk try std.fmt.allocPrint(allocator, "{s}/geocache.json", .{
|
||||
break :blk try std.fmt.allocPrint(allocator, "{s}/geocache.srf", .{
|
||||
env.get("WTTR_CACHE_DIR") orelse default_cache_dir,
|
||||
});
|
||||
},
|
||||
|
|
@ -125,7 +125,7 @@ pub fn load(allocator: std.mem.Allocator, env: *const std.process.Environ.Map) !
|
|||
if (env.get("IP2LOCATION_CACHE_FILE")) |v| {
|
||||
break :blk try allocator.dupe(u8, v);
|
||||
}
|
||||
break :blk try std.fmt.allocPrint(allocator, "{s}/ip2location.cache", .{env.get("WTTR_CACHE_DIR") orelse default_cache_dir});
|
||||
break :blk try std.fmt.allocPrint(allocator, "{s}/ip2location.srf", .{env.get("WTTR_CACHE_DIR") orelse default_cache_dir});
|
||||
},
|
||||
.pins_file = blk: {
|
||||
if (env.get("WTTR_PINS_FILE")) |v| {
|
||||
|
|
@ -137,7 +137,7 @@ pub fn load(allocator: std.mem.Allocator, env: *const std.process.Environ.Map) !
|
|||
if (env.get("IPWHOIS_CACHE_FILE")) |v| {
|
||||
break :blk try allocator.dupe(u8, v);
|
||||
}
|
||||
break :blk try std.fmt.allocPrint(allocator, "{s}/ipwhois.cache", .{env.get("WTTR_CACHE_DIR") orelse default_cache_dir});
|
||||
break :blk try std.fmt.allocPrint(allocator, "{s}/ipwhois.srf", .{env.get("WTTR_CACHE_DIR") orelse default_cache_dir});
|
||||
},
|
||||
};
|
||||
}
|
||||
|
|
|
|||
116
src/cache/Cache.zig
vendored
116
src/cache/Cache.zig
vendored
|
|
@ -1,5 +1,6 @@
|
|||
const std = @import("std");
|
||||
const Lru = @import("Lru.zig");
|
||||
const srf = @import("srf");
|
||||
|
||||
const Cache = @This();
|
||||
|
||||
|
|
@ -83,11 +84,15 @@ pub fn deinit(self: *Cache) void {
|
|||
self.allocator.destroy(self);
|
||||
}
|
||||
|
||||
/// Extension for L2 entries. Named so `loadFromDir` can tell weather entries
|
||||
/// apart from the other state living in the same directory.
|
||||
const cache_file_extension = ".srf";
|
||||
|
||||
fn getCacheFilename(self: *Cache, key: []const u8) ![]const u8 {
|
||||
var hasher = std.hash.Wyhash.init(0);
|
||||
hasher.update(key);
|
||||
const hash = hasher.final();
|
||||
return std.fmt.allocPrint(self.allocator, "{x}.json", .{hash});
|
||||
return std.fmt.allocPrint(self.allocator, "{x}" ++ cache_file_extension, .{hash});
|
||||
}
|
||||
|
||||
const CacheEntry = struct {
|
||||
|
|
@ -143,25 +148,23 @@ fn loadFromFilePath(self: *Cache, file_path: []const u8) !CacheEntry {
|
|||
|
||||
fn serialize(writer: *std.Io.Writer, key: []const u8, value: []const u8, expires: i64) !void {
|
||||
const entry = CacheEntry{ .key = key, .value = value, .expires = expires };
|
||||
try writer.print("{f}", .{std.json.fmt(entry, .{})});
|
||||
// Long format: the cached payload is a multi-kilobyte provider response, so
|
||||
// one field per line keeps the file skimmable. SRF length-prefixes the value
|
||||
// because it contains newlines, so the payload needs no escaping.
|
||||
try writer.print("{f}", .{srf.fmt(CacheEntry, &.{entry}, .{ .long_format = true })});
|
||||
}
|
||||
|
||||
fn deserialize(allocator: std.mem.Allocator, reader: *std.Io.Reader) !CacheEntry {
|
||||
var json_reader = std.json.Reader.init(allocator, reader);
|
||||
defer json_reader.deinit();
|
||||
var records = try srf.iterator(reader, allocator, .{});
|
||||
defer records.deinit();
|
||||
|
||||
const parsed = try std.json.parseFromTokenSource(
|
||||
CacheEntry,
|
||||
allocator,
|
||||
&json_reader,
|
||||
.{},
|
||||
);
|
||||
defer parsed.deinit();
|
||||
const fields = try records.next() orelse return error.EmptyCacheEntry;
|
||||
const entry = try fields.to(CacheEntry, .{});
|
||||
|
||||
return .{
|
||||
.key = try allocator.dupe(u8, parsed.value.key),
|
||||
.value = try allocator.dupe(u8, parsed.value.value),
|
||||
.expires = parsed.value.expires,
|
||||
.key = try allocator.dupe(u8, entry.key),
|
||||
.value = try allocator.dupe(u8, entry.value),
|
||||
.expires = entry.expires,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -184,6 +187,22 @@ fn saveToFile(self: *Cache, key: []const u8, value: []const u8, expires: i64) !v
|
|||
try writer.flush();
|
||||
}
|
||||
|
||||
/// Whether a directory entry is one of our L2 entries.
|
||||
///
|
||||
/// Entry names are a hash rendered as hex plus the extension. Matching the shape
|
||||
/// rather than just the extension keeps the descriptively-named state files in
|
||||
/// the same directory from being mistaken for cache entries now that they share
|
||||
/// a format.
|
||||
fn isCacheFileName(name: []const u8) bool {
|
||||
if (!std.mem.endsWith(u8, name, cache_file_extension)) return false;
|
||||
const stem = name[0 .. name.len - cache_file_extension.len];
|
||||
if (stem.len == 0) return false;
|
||||
for (stem) |ch| {
|
||||
if (!std.ascii.isHex(ch)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
fn loadFromDir(self: *Cache) !void {
|
||||
if (self.cache_dir == null) return error.NoCacheDir;
|
||||
|
||||
|
|
@ -193,6 +212,11 @@ fn loadFromDir(self: *Cache) !void {
|
|||
var it = dir.iterate();
|
||||
while (try it.next(self.io)) |entry| {
|
||||
if (entry.kind != .file) continue;
|
||||
// The cache directory also holds the GeoLite2 database and the named
|
||||
// state files (pins, geocache, provider caches). Without this filter every
|
||||
// startup opened and tried to parse all of them, including reading a
|
||||
// megabyte of the 60+ MB database.
|
||||
if (!isCacheFileName(entry.name)) continue;
|
||||
|
||||
const file_path = try std.fs.path.join(self.allocator, &.{ self.cache_dir.?, entry.name });
|
||||
defer self.allocator.free(file_path);
|
||||
|
|
@ -233,7 +257,12 @@ test "serialize and deserialize" {
|
|||
try fixed_writer.flush();
|
||||
|
||||
const serialized = buffer[0..fixed_writer.end];
|
||||
try std.testing.expectEqualStrings("{\"key\":\"test_key\",\"value\":\"test_value\",\"expires\":1234567890}", serialized);
|
||||
// Assert the shape as well as the round trip, so a change of format is a
|
||||
// deliberate edit here rather than a silent difference on disk.
|
||||
try std.testing.expect(std.mem.indexOf(u8, serialized, "#!srfv1") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, serialized, "#!long") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, serialized, "key::test_key") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, serialized, "value::test_value") != null);
|
||||
|
||||
var fixed_reader = std.Io.Reader.fixed(serialized);
|
||||
|
||||
|
|
@ -245,19 +274,68 @@ test "serialize and deserialize" {
|
|||
try std.testing.expectEqual(expires, cached.expires);
|
||||
}
|
||||
|
||||
test "deserialize handles integer expires" {
|
||||
test "deserialize preserves a millisecond timestamp exactly" {
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
const json = "{\"key\":\"k\",\"value\":\"v\",\"expires\":9999999999999}";
|
||||
// SRF carries numbers as f64. Millisecond timestamps are around 1.7e12, well
|
||||
// inside the 2^53 range f64 represents exactly, so no precision is lost --
|
||||
// but that is worth pinning down rather than assuming.
|
||||
const expires: i64 = 9999999999999;
|
||||
|
||||
var fixed_reader = std.Io.Reader.fixed(json);
|
||||
var buffer: [1024]u8 = undefined;
|
||||
var fixed_writer = std.Io.Writer.fixed(&buffer);
|
||||
try serialize(&fixed_writer, "k", "v", expires);
|
||||
try fixed_writer.flush();
|
||||
|
||||
var fixed_reader = std.Io.Reader.fixed(buffer[0..fixed_writer.end]);
|
||||
const cached = try deserialize(allocator, &fixed_reader);
|
||||
defer cached.deinit(allocator);
|
||||
|
||||
try std.testing.expectEqualStrings("k", cached.key);
|
||||
try std.testing.expectEqualStrings("v", cached.value);
|
||||
try std.testing.expectEqual(9999999999999, cached.expires);
|
||||
try std.testing.expectEqual(expires, cached.expires);
|
||||
}
|
||||
|
||||
test "serialize round-trips a payload containing newlines and commas" {
|
||||
const allocator = std.testing.allocator;
|
||||
|
||||
// The real payload is a provider JSON response: multi-line, comma-heavy, and
|
||||
// quoted. It survives only because SRF length-prefixes such values.
|
||||
const value =
|
||||
\\{"properties":{"timeseries":[
|
||||
\\ {"time":"2026-08-05T00:00:00Z","data":{"instant":1.5}},
|
||||
\\ {"time":"2026-08-05T01:00:00Z","data":{"instant":2.5}}
|
||||
\\]}}
|
||||
;
|
||||
|
||||
var buffer: [4096]u8 = undefined;
|
||||
var fixed_writer = std.Io.Writer.fixed(&buffer);
|
||||
try serialize(&fixed_writer, "59.9,10.7", value, 9999999999999);
|
||||
try fixed_writer.flush();
|
||||
|
||||
var fixed_reader = std.Io.Reader.fixed(buffer[0..fixed_writer.end]);
|
||||
const cached = try deserialize(allocator, &fixed_reader);
|
||||
defer cached.deinit(allocator);
|
||||
|
||||
try std.testing.expectEqualStrings("59.9,10.7", cached.key);
|
||||
try std.testing.expectEqualStrings(value, cached.value);
|
||||
}
|
||||
|
||||
test "isCacheFileName distinguishes entries from the other state files" {
|
||||
// Entries are a hex hash plus the extension.
|
||||
try std.testing.expect(isCacheFileName("1f44d18884c40d0a.srf"));
|
||||
try std.testing.expect(isCacheFileName("abcdef.srf"));
|
||||
|
||||
// Everything else sharing the directory must be left alone, including the
|
||||
// state files that now use the same format.
|
||||
try std.testing.expect(!isCacheFileName("pins.srf"));
|
||||
try std.testing.expect(!isCacheFileName("geocache.srf"));
|
||||
try std.testing.expect(!isCacheFileName("ipwhois.srf"));
|
||||
try std.testing.expect(!isCacheFileName("ip2location.srf"));
|
||||
try std.testing.expect(!isCacheFileName("GeoLite2-City.mmdb"));
|
||||
try std.testing.expect(!isCacheFileName("GeoLite2-City.mmdb.download"));
|
||||
try std.testing.expect(!isCacheFileName(".srf"));
|
||||
try std.testing.expect(!isCacheFileName("1f44d18884c40d0a.json"));
|
||||
}
|
||||
|
||||
test "L1/L2 cache flow" {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
const std = @import("std");
|
||||
const Coordinates = @import("../Coordinates.zig");
|
||||
const srf = @import("srf");
|
||||
|
||||
const GeoCache = @This();
|
||||
|
||||
|
|
@ -91,31 +92,48 @@ pub fn saveIfNeeded(self: *GeoCache) void {
|
|||
self.last_save = now;
|
||||
}
|
||||
|
||||
/// On-disk shape of one geocoded place.
|
||||
///
|
||||
/// The query is stored alongside the result rather than used as an object key,
|
||||
/// because SRF records are flat field lists rather than a nested map.
|
||||
const Record = struct {
|
||||
query: []const u8,
|
||||
name: []const u8,
|
||||
lat: f64,
|
||||
lon: f64,
|
||||
};
|
||||
|
||||
fn load(allocator: std.mem.Allocator, cache: *std.StringHashMap(CachedLocation), content: []const u8) !void {
|
||||
const CacheData = struct {
|
||||
name: []const u8,
|
||||
latitude: f64,
|
||||
longitude: f64,
|
||||
};
|
||||
if (std.mem.trim(u8, content, " \r\n\t").len == 0) return;
|
||||
|
||||
const parsed = try std.json.parseFromSlice(
|
||||
std.json.ArrayHashMap(CacheData),
|
||||
allocator,
|
||||
content,
|
||||
.{},
|
||||
);
|
||||
defer parsed.deinit();
|
||||
var reader = std.Io.Reader.fixed(content);
|
||||
var records = try srf.iterator(&reader, allocator, .{});
|
||||
defer records.deinit();
|
||||
|
||||
for (parsed.value.map.keys(), parsed.value.map.values()) |key, value| {
|
||||
const cache_key = try allocator.dupe(u8, key);
|
||||
const cache_value = CachedLocation{
|
||||
.name = try allocator.dupe(u8, value.name),
|
||||
.coords = .{
|
||||
.latitude = value.latitude,
|
||||
.longitude = value.longitude,
|
||||
},
|
||||
};
|
||||
try cache.put(cache_key, cache_value);
|
||||
var index: usize = 0;
|
||||
while (records.next() catch |err| {
|
||||
log.warn("stopped reading geocache after {d} entr(ies): {t}", .{ index, err });
|
||||
return;
|
||||
}) |fields| {
|
||||
index += 1;
|
||||
const record = fields.to(Record, .{}) catch continue;
|
||||
if (record.query.len == 0) continue;
|
||||
|
||||
const cache_key = try allocator.dupe(u8, record.query);
|
||||
errdefer allocator.free(cache_key);
|
||||
const name_copy = try allocator.dupe(u8, record.name);
|
||||
errdefer allocator.free(name_copy);
|
||||
|
||||
// A duplicate query in the file would otherwise leak the key and name
|
||||
// already stored under it.
|
||||
const existing = try cache.fetchPut(cache_key, .{
|
||||
.name = name_copy,
|
||||
.coords = .{ .latitude = record.lat, .longitude = record.lon },
|
||||
});
|
||||
if (existing) |old| {
|
||||
allocator.free(old.key);
|
||||
allocator.free(old.value.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -127,25 +145,24 @@ fn loadFromFile(allocator: std.mem.Allocator, io: std.Io, cache: *std.StringHash
|
|||
}
|
||||
|
||||
fn save(self: *GeoCache, writer: *std.Io.Writer) !void {
|
||||
try writer.writeAll("{\n");
|
||||
// Rewritten in full rather than appended to, because entries are replaced in
|
||||
// place; SRF long format keeps the result legible for a file an operator may
|
||||
// want to inspect or prune.
|
||||
var records = try self.allocator.alloc(Record, self.cache.count());
|
||||
defer self.allocator.free(records);
|
||||
|
||||
var it = self.cache.iterator();
|
||||
var first = true;
|
||||
while (it.next()) |entry| {
|
||||
if (!first) try writer.writeAll(",\n");
|
||||
first = false;
|
||||
|
||||
try writer.print(" {f}: {f}", .{
|
||||
std.json.fmt(entry.key_ptr.*, .{}),
|
||||
std.json.fmt(.{
|
||||
.name = entry.value_ptr.name,
|
||||
.latitude = entry.value_ptr.coords.latitude,
|
||||
.longitude = entry.value_ptr.coords.longitude,
|
||||
}, .{}),
|
||||
});
|
||||
var i: usize = 0;
|
||||
while (it.next()) |entry| : (i += 1) {
|
||||
records[i] = .{
|
||||
.query = entry.key_ptr.*,
|
||||
.name = entry.value_ptr.name,
|
||||
.lat = entry.value_ptr.coords.latitude,
|
||||
.lon = entry.value_ptr.coords.longitude,
|
||||
};
|
||||
}
|
||||
|
||||
try writer.writeAll("\n}\n");
|
||||
try writer.print("{f}", .{srf.fmt(Record, records, .{ .long_format = true })});
|
||||
}
|
||||
|
||||
fn saveToFile(self: *GeoCache, file_path: []const u8) !void {
|
||||
|
|
@ -189,7 +206,7 @@ test "GeoCache miss returns null" {
|
|||
try std.testing.expect(result == null);
|
||||
}
|
||||
|
||||
test "save produces valid JSON" {
|
||||
test "save produces SRF an operator can read" {
|
||||
const allocator = std.testing.allocator;
|
||||
var cache = try GeoCache.init(allocator, std.testing.io, null);
|
||||
defer cache.deinit();
|
||||
|
|
@ -208,12 +225,16 @@ test "save produces valid JSON" {
|
|||
try cache.save(&writer);
|
||||
|
||||
const output = buffer[0..writer.end];
|
||||
try std.testing.expect(std.mem.indexOf(u8, output, "London") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, output, "Paris") != null);
|
||||
// Assert the shape, not just that the values appear: a silent switch to
|
||||
// compact output would still round-trip while making the file harder to read.
|
||||
try std.testing.expect(std.mem.indexOf(u8, output, "#!srfv1") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, output, "#!long") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, output, "query::London") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, output, "query::Paris") != null);
|
||||
try std.testing.expect(std.mem.indexOf(u8, output, "51.5074") != null);
|
||||
}
|
||||
|
||||
test "load parses valid JSON" {
|
||||
test "load parses SRF records" {
|
||||
const allocator = std.testing.allocator;
|
||||
var cache_map = std.StringHashMap(CachedLocation).init(allocator);
|
||||
defer {
|
||||
|
|
@ -225,22 +246,53 @@ test "load parses valid JSON" {
|
|||
cache_map.deinit();
|
||||
}
|
||||
|
||||
const json =
|
||||
const content =
|
||||
\\#!srfv1
|
||||
\\#!long
|
||||
\\query::London
|
||||
\\name::London, UK
|
||||
\\lat:num:51.5074
|
||||
\\lon:num:-0.1278
|
||||
\\
|
||||
\\query::Paris
|
||||
\\name::Paris, France
|
||||
\\lat:num:48.8566
|
||||
\\lon:num:2.3522
|
||||
\\
|
||||
;
|
||||
|
||||
try load(allocator, &cache_map, content);
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 2), cache_map.count());
|
||||
const london = cache_map.get("London") orelse return error.TestUnexpectedResult;
|
||||
try std.testing.expectEqualStrings("London, UK", london.name);
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 51.5074), london.coords.latitude, 0.0001);
|
||||
const paris = cache_map.get("Paris") orelse return error.TestUnexpectedResult;
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 2.3522), paris.coords.longitude, 0.0001);
|
||||
}
|
||||
|
||||
test "load ignores a legacy JSON file rather than failing" {
|
||||
const allocator = std.testing.allocator;
|
||||
var cache_map = std.StringHashMap(CachedLocation).init(allocator);
|
||||
defer {
|
||||
var it = cache_map.iterator();
|
||||
while (it.next()) |entry| {
|
||||
allocator.free(entry.key_ptr.*);
|
||||
allocator.free(entry.value_ptr.name);
|
||||
}
|
||||
cache_map.deinit();
|
||||
}
|
||||
|
||||
// The previous format. Entries are re-geocodable, so the file is dropped
|
||||
// rather than migrated; what matters is that this is not fatal.
|
||||
const legacy =
|
||||
\\{
|
||||
\\ "London": {"name": "London, UK", "latitude": 51.5074, "longitude": -0.1278},
|
||||
\\ "Paris": {"name": "Paris, France", "latitude": 48.8566, "longitude": 2.3522}
|
||||
\\ "London": {"name": "London, UK", "latitude": 51.5074, "longitude": -0.1278}
|
||||
\\}
|
||||
;
|
||||
|
||||
try load(allocator, &cache_map, json);
|
||||
|
||||
const london = cache_map.get("London");
|
||||
try std.testing.expect(london != null);
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 51.5074), london.?.coords.latitude, 0.0001);
|
||||
|
||||
const paris = cache_map.get("Paris");
|
||||
try std.testing.expect(paris != null);
|
||||
try std.testing.expectApproxEqAbs(@as(f64, 48.8566), paris.?.coords.latitude, 0.0001);
|
||||
load(allocator, &cache_map, legacy) catch {};
|
||||
try std.testing.expectEqual(@as(usize, 0), cache_map.count());
|
||||
}
|
||||
|
||||
test "save and load round-trip" {
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
const std = @import("std");
|
||||
const Allocator = std.mem.Allocator;
|
||||
const Location = @import("resolver.zig").Location;
|
||||
const srf = @import("srf");
|
||||
|
||||
const Self = @This();
|
||||
|
||||
|
|
@ -174,6 +175,15 @@ inline fn getString(obj: std.json.ObjectMap, key: []const u8) []const u8 {
|
|||
return maybe_val.?.string;
|
||||
}
|
||||
|
||||
/// Permanent IP-to-location cache, shared by the online providers.
|
||||
///
|
||||
/// Append-only: an online lookup for an address is answered once and the result
|
||||
/// is kept forever, because an address's city does not meaningfully change and
|
||||
/// every avoided request is one that does not count against a rate limit.
|
||||
///
|
||||
/// Stored as SRF in compact form, one record per line. Compact is safe for place
|
||||
/// names containing commas because SRF length-prefixes any string containing the
|
||||
/// field delimiter, so no escaping rules are needed here.
|
||||
pub const Cache = struct {
|
||||
allocator: Allocator,
|
||||
io: std.Io,
|
||||
|
|
@ -181,6 +191,17 @@ pub const Cache = struct {
|
|||
entries: std.AutoHashMap(u128, Location),
|
||||
file: ?std.Io.File,
|
||||
|
||||
/// On-disk shape of one entry.
|
||||
///
|
||||
/// The address is stored as text rather than the `u128` used for the in-memory
|
||||
/// key, so the file stays legible to whoever has to look at it.
|
||||
const Record = struct {
|
||||
ip: []const u8,
|
||||
lat: f64,
|
||||
lon: f64,
|
||||
name: []const u8,
|
||||
};
|
||||
|
||||
pub fn init(allocator: Allocator, io: std.Io, path: []const u8) !Cache {
|
||||
var cache = Cache{
|
||||
.allocator = allocator,
|
||||
|
|
@ -190,19 +211,22 @@ pub const Cache = struct {
|
|||
.file = null,
|
||||
};
|
||||
errdefer allocator.free(cache.path);
|
||||
errdefer cache.entries.deinit();
|
||||
|
||||
// Try to open existing cache file
|
||||
if (std.Io.Dir.openFileAbsolute(io, path, .{ .mode = .read_write })) |file| {
|
||||
cache.file = file;
|
||||
try cache.load();
|
||||
cache.load() catch |err| {
|
||||
// A cache that cannot be read is worth strictly less than the
|
||||
// service staying up: start empty and let it refill.
|
||||
log.warn("could not read cache {s} ({t}); starting empty", .{ path, err });
|
||||
};
|
||||
} else |err| switch (err) {
|
||||
error.FileNotFound => {
|
||||
// Create new cache file
|
||||
const dir = std.fs.path.dirname(path) orelse return error.InvalidPath;
|
||||
try std.Io.Dir.cwd().createDirPath(io, dir);
|
||||
cache.file = try std.Io.Dir.createFileAbsolute(io, path, .{ .read = true });
|
||||
// Write header
|
||||
try cache.file.?.writePositionalAll(io, "#Ip2location:v2\n", 0);
|
||||
try cache.writeDirectives();
|
||||
},
|
||||
else => return err,
|
||||
}
|
||||
|
|
@ -210,6 +234,17 @@ pub const Cache = struct {
|
|||
return cache;
|
||||
}
|
||||
|
||||
/// Writes the SRF front matter that opens a new cache file.
|
||||
///
|
||||
/// Emitted once at creation; `put` appends bare records afterwards.
|
||||
fn writeDirectives(self: *Cache) !void {
|
||||
const file = self.file orelse return;
|
||||
var buf: [64]u8 = undefined;
|
||||
var fw = file.writer(self.io, &buf);
|
||||
try fw.interface.print("{f}", .{srf.fmt(Record, &.{}, .{})});
|
||||
try fw.interface.flush();
|
||||
}
|
||||
|
||||
pub fn deinit(self: *Cache) void {
|
||||
if (self.file) |f| f.close(self.io);
|
||||
var it = self.entries.valueIterator();
|
||||
|
|
@ -236,61 +271,30 @@ pub const Cache = struct {
|
|||
);
|
||||
defer self.allocator.free(content);
|
||||
|
||||
var lines = std.mem.splitScalar(u8, content, '\n');
|
||||
var reader = std.Io.Reader.fixed(content);
|
||||
var records = try srf.iterator(&reader, self.allocator, .{});
|
||||
defer records.deinit();
|
||||
|
||||
// Check for header magic string
|
||||
if (lines.next()) |first_line| {
|
||||
if (!std.mem.eql(u8, first_line, "#Ip2location:v2")) {
|
||||
log.warn("Cache file missing or invalid header, discarding", .{});
|
||||
file.close(self.io);
|
||||
self.file = null;
|
||||
std.Io.Dir.deleteFileAbsolute(self.io, self.path) catch |e| {
|
||||
log.err("error deleting {s}: {}", .{ self.path, e });
|
||||
};
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
return; // Empty file
|
||||
}
|
||||
var index: usize = 0;
|
||||
while (records.next() catch |err| {
|
||||
log.warn("stopped reading {s} after {d} entr(ies): {t}", .{ self.path, index, err });
|
||||
return;
|
||||
}) |fields| {
|
||||
index += 1;
|
||||
const record = fields.to(Record, .{}) catch continue;
|
||||
const parsed = packIp(record.ip) orelse continue;
|
||||
|
||||
while (lines.next()) |line| {
|
||||
if (line.len == 0) continue;
|
||||
const entry = parseCacheLine(self.allocator, line) catch continue;
|
||||
try self.entries.put(entry.ip, entry.location);
|
||||
}
|
||||
}
|
||||
const name_copy = try self.allocator.dupe(u8, record.name);
|
||||
errdefer self.allocator.free(name_copy);
|
||||
|
||||
const CacheEntry = struct {
|
||||
ip: u128,
|
||||
location: Location,
|
||||
};
|
||||
|
||||
fn parseCacheLine(allocator: Allocator, line: []const u8) !CacheEntry {
|
||||
// Parse: ip,lat,lon,name
|
||||
var parts = std.mem.splitScalar(u8, line, ',');
|
||||
const ip_str = parts.next() orelse return error.InvalidFormat;
|
||||
const lat_str = parts.next() orelse return error.InvalidFormat;
|
||||
const lon_str = parts.next() orelse return error.InvalidFormat;
|
||||
const name = parts.rest();
|
||||
|
||||
const lat = try std.fmt.parseFloat(f64, lat_str);
|
||||
const lon = try std.fmt.parseFloat(f64, lon_str);
|
||||
|
||||
// Try parsing as IP address first, fall back to u128
|
||||
const ip_u128 = if (packIp(ip_str)) |parsed|
|
||||
parsed.key
|
||||
else
|
||||
try std.fmt.parseInt(u128, ip_str, 10);
|
||||
|
||||
const name_copy = try allocator.dupe(u8, name);
|
||||
return .{
|
||||
.ip = ip_u128,
|
||||
.location = .{
|
||||
.allocator = allocator,
|
||||
// Replacing an existing key would leak the name it already owns.
|
||||
const existing = try self.entries.fetchPut(parsed.key, .{
|
||||
.allocator = self.allocator,
|
||||
.name = name_copy,
|
||||
.coords = .{ .latitude = lat, .longitude = lon },
|
||||
},
|
||||
};
|
||||
.coords = .{ .latitude = record.lat, .longitude = record.lon },
|
||||
});
|
||||
if (existing) |old| self.allocator.free(old.value.name);
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get(self: *Cache, ip: u128) ?Location {
|
||||
|
|
@ -304,101 +308,141 @@ pub const Cache = struct {
|
|||
|
||||
pub fn put(self: *Cache, ip: u128, family: u8, loc: Location) !void {
|
||||
const name_copy = try self.allocator.dupe(u8, loc.name);
|
||||
try self.entries.put(ip, .{
|
||||
errdefer self.allocator.free(name_copy);
|
||||
|
||||
const existing = try self.entries.fetchPut(ip, .{
|
||||
.allocator = self.allocator,
|
||||
.name = name_copy,
|
||||
.coords = loc.coords,
|
||||
});
|
||||
if (existing) |old| self.allocator.free(old.value.name);
|
||||
|
||||
// Append to file: ip,lat,lon,name
|
||||
if (self.file) |file| {
|
||||
// 0.16 has no seek+write; append at the current end offset.
|
||||
const end = try file.length(self.io);
|
||||
// Format IP as string for file
|
||||
var buf: [64]u8 = undefined;
|
||||
const ip_str = if (family == 4)
|
||||
try std.fmt.bufPrint(&buf, "{}.{}.{}.{}", .{
|
||||
@as(u8, @truncate(ip >> 24)),
|
||||
@as(u8, @truncate(ip >> 16)),
|
||||
@as(u8, @truncate(ip >> 8)),
|
||||
@as(u8, @truncate(ip)),
|
||||
})
|
||||
else
|
||||
try std.fmt.bufPrint(&buf, "{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}:{x:0>4}", .{
|
||||
@as(u16, @truncate(ip >> 112)),
|
||||
@as(u16, @truncate(ip >> 96)),
|
||||
@as(u16, @truncate(ip >> 80)),
|
||||
@as(u16, @truncate(ip >> 64)),
|
||||
@as(u16, @truncate(ip >> 48)),
|
||||
@as(u16, @truncate(ip >> 32)),
|
||||
@as(u16, @truncate(ip >> 16)),
|
||||
@as(u16, @truncate(ip)),
|
||||
});
|
||||
const line = try std.fmt.allocPrint(self.allocator, "{s},{d},{d},{s}\n", .{
|
||||
ip_str,
|
||||
loc.coords.latitude,
|
||||
loc.coords.longitude,
|
||||
loc.name,
|
||||
const file = self.file orelse return;
|
||||
|
||||
var ip_buf: [64]u8 = undefined;
|
||||
const ip_str = try formatKey(ip, family, &ip_buf);
|
||||
|
||||
// Append one record with no front matter; the directives were written
|
||||
// when the file was created.
|
||||
var line_buf: [512]u8 = undefined;
|
||||
const line = try std.fmt.bufPrint(&line_buf, "{f}", .{srf.fmt(Record, &.{.{
|
||||
.ip = ip_str,
|
||||
.lat = loc.coords.latitude,
|
||||
.lon = loc.coords.longitude,
|
||||
.name = loc.name,
|
||||
}}, .{ .emit_directives = false })});
|
||||
|
||||
// 0.16 has no seek+write; append at the current end offset.
|
||||
const end = try file.length(self.io);
|
||||
try file.writePositionalAll(self.io, line, end);
|
||||
}
|
||||
|
||||
/// Renders a packed key back to text for storage.
|
||||
fn formatKey(ip: u128, family: u8, buf: []u8) ![]const u8 {
|
||||
if (family == 4) {
|
||||
return std.fmt.bufPrint(buf, "{d}.{d}.{d}.{d}", .{
|
||||
@as(u8, @truncate(ip >> 24)),
|
||||
@as(u8, @truncate(ip >> 16)),
|
||||
@as(u8, @truncate(ip >> 8)),
|
||||
@as(u8, @truncate(ip)),
|
||||
});
|
||||
defer self.allocator.free(line);
|
||||
try file.writePositionalAll(self.io, line, end);
|
||||
}
|
||||
var bytes: [16]u8 = undefined;
|
||||
std.mem.writeInt(u128, &bytes, ip, .big);
|
||||
// `Ip6Address.format` would append ":port" and bracket the address;
|
||||
// `Unresolved` is the bare-address formatter.
|
||||
const bare: std.Io.net.Ip6Address.Unresolved = .{ .bytes = bytes, .interface_name = null };
|
||||
return std.fmt.bufPrint(buf, "{f}", .{&bare});
|
||||
}
|
||||
};
|
||||
|
||||
test "parseCacheLine: valid IPv4 line" {
|
||||
test "Cache: round-trips IPv4 and IPv6 entries through the file" {
|
||||
const allocator = std.testing.allocator;
|
||||
const line = "192.168.1.1,37.5,-122.5,San Francisco, California, United States";
|
||||
const entry = try Cache.parseCacheLine(allocator, line);
|
||||
defer allocator.free(entry.location.name);
|
||||
const io = std.testing.io;
|
||||
|
||||
try std.testing.expectEqual(@as(u128, 3232235777), entry.ip); // 192.168.1.1 as u128
|
||||
try std.testing.expectEqual(@as(f64, 37.5), entry.location.coords.latitude);
|
||||
try std.testing.expectEqual(@as(f64, -122.5), entry.location.coords.longitude);
|
||||
try std.testing.expectEqualStrings("San Francisco, California, United States", entry.location.name);
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
var path_buf: [std.fs.max_path_bytes]u8 = undefined;
|
||||
const dir_len = try tmp.dir.realPath(io, &path_buf);
|
||||
const path = try std.fmt.allocPrint(allocator, "{s}/ip.cache", .{path_buf[0..dir_len]});
|
||||
defer allocator.free(path);
|
||||
|
||||
const v4 = packIp("192.168.1.1").?;
|
||||
const v6 = packIp("2001:db8::1").?;
|
||||
|
||||
{
|
||||
var cache = try Cache.init(allocator, io, path);
|
||||
defer cache.deinit();
|
||||
|
||||
// A name with commas is the interesting case: compact SRF delimits on
|
||||
// commas, so this only survives via length prefixing.
|
||||
try cache.put(v4.key, v4.family, .{
|
||||
.allocator = allocator,
|
||||
.name = "San Francisco, California, United States",
|
||||
.coords = .{ .latitude = 37.5, .longitude = -122.5 },
|
||||
});
|
||||
try cache.put(v6.key, v6.family, .{
|
||||
.allocator = allocator,
|
||||
.name = "London, United Kingdom",
|
||||
.coords = .{ .latitude = 51.5, .longitude = -0.1 },
|
||||
});
|
||||
}
|
||||
|
||||
var reopened = try Cache.init(allocator, io, path);
|
||||
defer reopened.deinit();
|
||||
|
||||
const got4 = reopened.get(v4.key) orelse return error.TestUnexpectedResult;
|
||||
defer got4.deinit();
|
||||
try std.testing.expectEqualStrings("San Francisco, California, United States", got4.name);
|
||||
try std.testing.expectEqual(@as(f64, 37.5), got4.coords.latitude);
|
||||
try std.testing.expectEqual(@as(f64, -122.5), got4.coords.longitude);
|
||||
|
||||
const got6 = reopened.get(v6.key) orelse return error.TestUnexpectedResult;
|
||||
defer got6.deinit();
|
||||
try std.testing.expectEqualStrings("London, United Kingdom", got6.name);
|
||||
try std.testing.expectEqual(@as(f64, 51.5), got6.coords.latitude);
|
||||
}
|
||||
|
||||
test "parseCacheLine: valid IPv6 line" {
|
||||
test "Cache: a legacy or corrupt file starts empty instead of failing" {
|
||||
const allocator = std.testing.allocator;
|
||||
const line = "2001:db8::1,51.5,-0.1,London, United Kingdom";
|
||||
const entry = try Cache.parseCacheLine(allocator, line);
|
||||
defer allocator.free(entry.location.name);
|
||||
const io = std.testing.io;
|
||||
|
||||
try std.testing.expect(entry.ip > 0);
|
||||
try std.testing.expectEqual(@as(f64, 51.5), entry.location.coords.latitude);
|
||||
try std.testing.expectEqual(@as(f64, -0.1), entry.location.coords.longitude);
|
||||
try std.testing.expectEqualStrings("London, United Kingdom", entry.location.name);
|
||||
var tmp = std.testing.tmpDir(.{});
|
||||
defer tmp.cleanup();
|
||||
var path_buf: [std.fs.max_path_bytes]u8 = undefined;
|
||||
const dir_len = try tmp.dir.realPath(io, &path_buf);
|
||||
const path = try std.fmt.allocPrint(allocator, "{s}/ip.cache", .{path_buf[0..dir_len]});
|
||||
defer allocator.free(path);
|
||||
|
||||
// The previous hand-rolled format. Deliberately not migrated: the entries are
|
||||
// re-fetchable and losing them costs a handful of API calls.
|
||||
try std.Io.Dir.cwd().writeFile(io, .{
|
||||
.sub_path = path,
|
||||
.data = "#Ip2location:v2\n192.168.1.1,37.5,-122.5,San Francisco\n",
|
||||
});
|
||||
|
||||
var cache = try Cache.init(allocator, io, path);
|
||||
defer cache.deinit();
|
||||
|
||||
try std.testing.expectEqual(@as(usize, 0), cache.entries.count());
|
||||
|
||||
// Still usable afterwards, which is the point of not treating it as fatal.
|
||||
const v4 = packIp("10.0.0.1").?;
|
||||
try cache.put(v4.key, v4.family, .{
|
||||
.allocator = allocator,
|
||||
.name = "Somewhere",
|
||||
.coords = .{ .latitude = 1, .longitude = 2 },
|
||||
});
|
||||
const got = cache.get(v4.key) orelse return error.TestUnexpectedResult;
|
||||
defer got.deinit();
|
||||
try std.testing.expectEqualStrings("Somewhere", got.name);
|
||||
}
|
||||
|
||||
test "parseCacheLine: empty name" {
|
||||
const allocator = std.testing.allocator;
|
||||
const line = "10.0.0.1,0.0,0.0,";
|
||||
const entry = try Cache.parseCacheLine(allocator, line);
|
||||
defer allocator.free(entry.location.name);
|
||||
test "Cache: formatKey renders both families without a port" {
|
||||
var buf: [64]u8 = undefined;
|
||||
const v4 = packIp("12.94.132.170").?;
|
||||
try std.testing.expectEqualStrings("12.94.132.170", try Cache.formatKey(v4.key, v4.family, &buf));
|
||||
|
||||
try std.testing.expectEqualStrings("", entry.location.name);
|
||||
}
|
||||
|
||||
test "parseCacheLine: missing fields" {
|
||||
const allocator = std.testing.allocator;
|
||||
const line = "192.168.1.1,37.5";
|
||||
try std.testing.expectError(error.InvalidFormat, Cache.parseCacheLine(allocator, line));
|
||||
}
|
||||
|
||||
test "parseCacheLine: invalid IP" {
|
||||
const allocator = std.testing.allocator;
|
||||
const line = "not.an.ip,37.5,-122.5,Test";
|
||||
try std.testing.expectError(error.InvalidCharacter, Cache.parseCacheLine(allocator, line));
|
||||
}
|
||||
|
||||
test "parseCacheLine: invalid latitude" {
|
||||
const allocator = std.testing.allocator;
|
||||
const line = "192.168.1.1,invalid,-122.5,Test";
|
||||
try std.testing.expectError(error.InvalidCharacter, Cache.parseCacheLine(allocator, line));
|
||||
}
|
||||
|
||||
test "parseCacheLine: invalid longitude" {
|
||||
const allocator = std.testing.allocator;
|
||||
const line = "192.168.1.1,37.5,invalid,Test";
|
||||
try std.testing.expectError(error.InvalidCharacter, Cache.parseCacheLine(allocator, line));
|
||||
const v6 = packIp("2001:db8::1").?;
|
||||
try std.testing.expectEqualStrings("2001:db8::1", try Cache.formatKey(v6.key, v6.family, &buf));
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Reference in a new issue