lobo/srf
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Compare commits

base: lobo:1a47ad0ad2cf2d8e40cff876d16cc81bcd730e40
Branches Tags
lobo:master
...
compare: lobo:8e12b7396afc1bcbc4e2a3f19d8725a82b71b27e
Branches Tags
lobo:master

3 commits
1a47ad0ad2 ... 8e12b7396a

Author SHA1 Message Date
Emil Lerch
8e12b7396a
introduce iterator version of to()
All checks were successful
Generic zig build / build (push) Successful in 34s
Details
2026-03-09 09:45:21 -07:00
Emil Lerch
bb4a5e0bf3
handle multiple recorditerator.next() without processing fields 2026-03-09 09:45:05 -07:00
Emil Lerch
3f7953be74
cleanup and optimization 2026-03-07 17:39:11 -08:00
1 changed files with 223 additions and 145 deletions
Show stats Expand all files Collapse all files

302
src/srf.zig
View file

@ -87,11 +87,9 @@ pub const Value = union(enum) {
// } // }
// } // }
pub fn parse(allocator: std.mem.Allocator, str: []const u8, state: *RecordIterator.State, delimiter: u8) ParseError!ValueWithMetaData { pub fn parse(allocator: std.mem.Allocator, str: []const u8, state: *RecordIterator.State, delimiter: u8) ParseError!ValueWithMetaData {
const debug = str.len > 2 and str[0] == '1' and str[1] == '1';
if (debug) log.debug("parsing {s}", .{str});
const type_val_sep_raw = std.mem.indexOfScalar(u8, str, ':'); const type_val_sep_raw = std.mem.indexOfScalar(u8, str, ':');
if (type_val_sep_raw == null) { if (type_val_sep_raw == null) {
try parseError(allocator, "no type data or value after key", state.*); try parseError(allocator, "no type data or value after key", state);
return ParseError.ParseFailed; return ParseError.ParseFailed;
} }
@ -121,7 +119,7 @@ pub const Value = union(enum) {
state.partial_line_column += total_chars; state.partial_line_column += total_chars;
const Decoder = std.base64.standard.Decoder; const Decoder = std.base64.standard.Decoder;
const size = Decoder.calcSizeForSlice(val) catch { const size = Decoder.calcSizeForSlice(val) catch {
try parseError(allocator, "error parsing base64 value", state.*); try parseError(allocator, "error parsing base64 value", state);
return .{ return .{
.item_value = null, .item_value = null,
.error_parsing = true, .error_parsing = true,
@ -130,7 +128,7 @@ pub const Value = union(enum) {
const data = try allocator.alloc(u8, size); const data = try allocator.alloc(u8, size);
errdefer allocator.free(data); errdefer allocator.free(data);
Decoder.decode(data, val) catch { Decoder.decode(data, val) catch {
try parseError(allocator, "error parsing base64 value", state.*); try parseError(allocator, "error parsing base64 value", state);
allocator.free(data); allocator.free(data);
return .{ return .{
.item_value = null, .item_value = null,
@ -151,7 +149,7 @@ pub const Value = union(enum) {
state.partial_line_column += total_chars; state.partial_line_column += total_chars;
const val_trimmed = std.mem.trim(u8, val, &std.ascii.whitespace); const val_trimmed = std.mem.trim(u8, val, &std.ascii.whitespace);
const number = std.fmt.parseFloat(@FieldType(Value, "number"), val_trimmed) catch { const number = std.fmt.parseFloat(@FieldType(Value, "number"), val_trimmed) catch {
try parseError(allocator, "error parsing numeric value", state.*); try parseError(allocator, "error parsing numeric value", state);
return .{ return .{
.item_value = null, .item_value = null,
.error_parsing = true, .error_parsing = true,
@ -173,7 +171,7 @@ pub const Value = union(enum) {
if (std.mem.eql(u8, "false", val_trimmed)) break :blk false; if (std.mem.eql(u8, "false", val_trimmed)) break :blk false;
if (std.mem.eql(u8, "true", val_trimmed)) break :blk true; if (std.mem.eql(u8, "true", val_trimmed)) break :blk true;
try parseError(allocator, "error parsing boolean value", state.*); try parseError(allocator, "error parsing boolean value", state);
return .{ return .{
.item_value = null, .item_value = null,
.error_parsing = true, .error_parsing = true,
@ -200,18 +198,16 @@ pub const Value = union(enum) {
state.partial_line_column += total_metadata_chars; state.partial_line_column += total_metadata_chars;
const size = std.fmt.parseInt(usize, trimmed_meta, 0) catch { const size = std.fmt.parseInt(usize, trimmed_meta, 0) catch {
log.debug("parseInt fail, trimmed_data: '{s}'", .{trimmed_meta}); log.debug("parseInt fail, trimmed_data: '{s}'", .{trimmed_meta});
try parseError(allocator, "unrecognized metadata for key", state.*); try parseError(allocator, "unrecognized metadata for key", state);
return .{ return .{
.item_value = null, .item_value = null,
.error_parsing = true, .error_parsing = true,
}; };
}; };
if (debug) log.debug("found fixed string size {d}. State {f}", .{ size, state });
// Update again for number of bytes. All failures beyond this point are // Update again for number of bytes. All failures beyond this point are
// fatal, so this is safe. // fatal, so this is safe.
state.column += size; state.column += size;
state.partial_line_column += size; state.partial_line_column += size;
if (debug) log.debug("New state {f}", .{state});
// If we are being asked specifically for bytes, we no longer care about // If we are being asked specifically for bytes, we no longer care about
// delimiters. We just want raw bytes. This might adjust our line/column // delimiters. We just want raw bytes. This might adjust our line/column
@ -220,41 +216,29 @@ pub const Value = union(enum) {
if (rest_of_data.len >= size) { if (rest_of_data.len >= size) {
// We fit on this line, everything is "normal" // We fit on this line, everything is "normal"
const val = rest_of_data[0..size]; const val = rest_of_data[0..size];
if (debug) log.debug("val {s}", .{val});
return .{ return .{
.item_value = .{ .string = val }, .item_value = .{ .string = val },
}; };
} }
// This is not enough, we need more data from the reader // This is not enough, we need more data from the reader
log.debug("item value includes newlines {f}", .{state}); const buf = try allocator.alloc(u8, size);
// We need to advance the reader, so we need a copy of what we have so fa errdefer allocator.free(buf);
const start = try dupe(allocator, state.options, rest_of_data); @memcpy(buf[0..rest_of_data.len], rest_of_data);
defer allocator.free(start); // add back the newline we are skipping
buf[rest_of_data.len] = '\n';
// We won't do a parseError here. If we have an allocation error, read // We won't do a parseError here. If we have an allocation error, read
// error, or end of stream, all of these are fatal. Our reader is currently // error, or end of stream, all of these are fatal. Our reader is currently
// past the newline, so we have to remove a character from size to account. // past the newline, so we have to remove a character from size to account.
const end = try state.reader.readAlloc(allocator, size - rest_of_data.len - 1); try state.reader.readSliceAll(buf[rest_of_data.len + 1 ..]);
// However, we want to be past the end of the *next* newline too (in long // However, we want to be past the end of the *next* newline too (in long
// format mode) // format mode)
if (delimiter == '\n') state.reader.toss(1); if (delimiter == '\n') state.reader.toss(1);
defer allocator.free(end);
// This \n is because the reader state will have advanced beyond the next newline, so end
// really should start with the newline. This only applies to long mode, because otherwise the
// entire record is a single line
const final = try std.mem.concat(allocator, u8, &.{ start, "\n", end });
// const final = if (delimiter == '\n')
// try std.mem.concat(allocator, u8, &.{ start, "\n", end })
// else
// try std.mem.concat(allocator, u8, &.{ start, end });
errdefer allocator.free(final);
// log.debug("full val: {s}", .{final});
std.debug.assert(final.len == size);
// Because we've now advanced the line, we need to reset everything // Because we've now advanced the line, we need to reset everything
state.line += std.mem.count(u8, final, "\n"); state.line += std.mem.count(u8, buf, "\n");
state.column = final.len - std.mem.lastIndexOf(u8, final, "\n").?; state.column = buf.len - std.mem.lastIndexOf(u8, buf, "\n").?;
state.partial_line_column = state.column; state.partial_line_column = state.column;
return .{ return .{
.item_value = .{ .string = final }, .item_value = .{ .string = buf },
.reader_advanced = true, .reader_advanced = true,
}; };
} }
@ -266,30 +250,7 @@ pub const Field = struct {
value: ?Value, value: ?Value,
}; };
// A record has a list of fields, with no assumptions regarding duplication, fn coerce(name: []const u8, comptime T: type, val: ?Value) !T {
// etc. This is for parsing speed, but also for more flexibility in terms of
// use cases. One can make a defacto array out of this structure by having
// something like:
//
// arr:string:foo
// arr:string:bar
//
// and when you coerce to zig struct have an array .arr that gets populated
// with strings "foo" and "bar".
pub const Record = struct {
fields: []const Field,
pub fn fmt(value: Record, options: FormatOptions) RecordFormatter {
return .{ .value = value, .options = options };
}
pub fn firstFieldByName(self: Record, field_name: []const u8) ?Field {
for (self.fields) |f|
if (std.mem.eql(u8, f.key, field_name)) return f;
return null;
}
fn coerce(name: []const u8, comptime T: type, val: ?Value) !T {
// Here's the deduplicated set of field types that coerce needs to handle: // Here's the deduplicated set of field types that coerce needs to handle:
// Direct from SRF values: // Direct from SRF values:
// Need parsing from string: // Need parsing from string:
@ -344,6 +305,29 @@ pub const Record = struct {
}, },
} }
return null; return null;
}
// A record has a list of fields, with no assumptions regarding duplication,
// etc. This is for parsing speed, but also for more flexibility in terms of
// use cases. One can make a defacto array out of this structure by having
// something like:
//
// arr:string:foo
// arr:string:bar
//
// and when you coerce to zig struct have an array .arr that gets populated
// with strings "foo" and "bar".
pub const Record = struct {
fields: []const Field,
pub fn fmt(value: Record, options: FormatOptions) RecordFormatter {
return .{ .value = value, .options = options };
}
pub fn firstFieldByName(self: Record, field_name: []const u8) ?Field {
for (self.fields) |f|
if (std.mem.eql(u8, f.key, field_name)) return f;
return null;
} }
fn maxFields(comptime T: type) usize { fn maxFields(comptime T: type) usize {
@ -605,6 +589,7 @@ pub const RecordIterator = struct {
field_delimiter: u8 = ',', field_delimiter: u8 = ',',
end_of_record_reached: bool = false, end_of_record_reached: bool = false,
field_iterator: ?FieldIterator = null,
/// Takes the next line, trimming leading whitespace and ignoring comments /// Takes the next line, trimming leading whitespace and ignoring comments
/// Directives (comments starting with #!) are preserved /// Directives (comments starting with #!) are preserved
@ -630,6 +615,11 @@ pub const RecordIterator = struct {
// TODO: we need to capture the fieldIterator here and make sure it's run // TODO: we need to capture the fieldIterator here and make sure it's run
// to the ground to keep our state intact // to the ground to keep our state intact
const state = self.state; const state = self.state;
if (state.field_iterator) |f| {
// We need to finish the fields on the previous record
while (try f.next()) |_| {}
state.field_iterator = null;
}
if (state.current_line == null) { if (state.current_line == null) {
if (state.options.diagnostics) |d| if (state.options.diagnostics) |d|
if (d.errors.items.len > 0) return ParseError.ParseFailed; if (d.errors.items.len > 0) return ParseError.ParseFailed;
@ -646,12 +636,12 @@ pub const RecordIterator = struct {
if (state.current_line == null) return self.next(); if (state.current_line == null) return self.next();
} }
// non-blank line, but we could have an eof marker // non-blank line, but we could have an eof marker
if (try Directive.parse(self.arena.allocator(), state.current_line.?, state.*)) |d| { if (try Directive.parse(self.arena.allocator(), state.current_line.?, state)) |d| {
switch (d) { switch (d) {
.eof => { .eof => {
// there needs to be an eof then // there needs to be an eof then
if (state.nextLine()) |_| { if (state.nextLine()) |_| {
try parseError(self.arena.allocator(), "Data found after #!eof", state.*); try parseError(self.arena.allocator(), "Data found after #!eof", state);
return ParseError.ParseFailed; // this is terminal return ParseError.ParseFailed; // this is terminal
} else { } else {
state.eof_found = true; state.eof_found = true;
@ -660,7 +650,7 @@ pub const RecordIterator = struct {
} }
}, },
else => { else => {
try parseError(self.arena.allocator(), "Directive found after data started", state.*); try parseError(self.arena.allocator(), "Directive found after data started", state);
state.current_line = state.nextLine(); state.current_line = state.nextLine();
// TODO: This runs the risk of a malicious file creating // TODO: This runs the risk of a malicious file creating
// a stackoverflow by using many non-eof directives // a stackoverflow by using many non-eof directives
@ -669,7 +659,8 @@ pub const RecordIterator = struct {
} }
} }
state.end_of_record_reached = false; state.end_of_record_reached = false;
return .{ .ri = self }; state.field_iterator = .{ .ri = self };
return state.field_iterator.?;
} }
pub const FieldIterator = struct { pub const FieldIterator = struct {
@ -677,6 +668,7 @@ pub const RecordIterator = struct {
pub fn next(self: FieldIterator) !?Field { pub fn next(self: FieldIterator) !?Field {
const state = self.ri.state; const state = self.ri.state;
const aa = self.ri.arena.allocator();
// Main parsing. We already have the first line of data, which could // Main parsing. We already have the first line of data, which could
// be a record (compact format) or a key/value pair (long format) // be a record (compact format) or a key/value pair (long format)
@ -686,12 +678,12 @@ pub const RecordIterator = struct {
if (state.end_of_record_reached) return null; if (state.end_of_record_reached) return null;
// non-blank line, but we could have an eof marker // non-blank line, but we could have an eof marker
// TODO: deduplicate this code // TODO: deduplicate this code
if (try Directive.parse(self.ri.arena.allocator(), state.current_line.?, state.*)) |d| { if (try Directive.parse(aa, state.current_line.?, state)) |d| {
switch (d) { switch (d) {
.eof => { .eof => {
// there needs to be an eof then // there needs to be an eof then
if (state.nextLine()) |_| { if (state.nextLine()) |_| {
try parseError(self.ri.arena.allocator(), "Data found after #!eof", state.*); try parseError(aa, "Data found after #!eof", state);
return ParseError.ParseFailed; // this is terminal return ParseError.ParseFailed; // this is terminal
} else { } else {
state.eof_found = true; state.eof_found = true;
@ -700,7 +692,7 @@ pub const RecordIterator = struct {
} }
}, },
else => { else => {
try parseError(self.ri.arena.allocator(), "Directive found after data started", state.*); try parseError(aa, "Directive found after data started", state);
state.current_line = state.nextLine(); state.current_line = state.nextLine();
// TODO: This runs the risk of a malicious file creating // TODO: This runs the risk of a malicious file creating
// a stackoverflow by using many non-eof directives // a stackoverflow by using many non-eof directives
@ -717,7 +709,7 @@ pub const RecordIterator = struct {
state.column += key.len + 1; state.column += key.len + 1;
state.partial_line_column += key.len + 1; state.partial_line_column += key.len + 1;
const value = try Value.parse( const value = try Value.parse(
self.ri.arena.allocator(), aa,
it.rest(), it.rest(),
state, state,
state.field_delimiter, state.field_delimiter,
@ -725,7 +717,7 @@ pub const RecordIterator = struct {
var field: ?Field = null; var field: ?Field = null;
if (!value.error_parsing) { if (!value.error_parsing) {
field = .{ .key = try dupe(self.ri.arena.allocator(), state.options, key), .value = value.item_value }; field = .{ .key = try dupe(aa, state.options, key), .value = value.item_value };
} }
if (value.reader_advanced and state.field_delimiter == ',') { if (value.reader_advanced and state.field_delimiter == ',') {
@ -774,8 +766,84 @@ pub const RecordIterator = struct {
} }
return field; return field;
} }
};
/// Coerce Record to a type. Does not handle fields with arrays
pub fn to(self: FieldIterator, comptime T: type) !T {
const ti = @typeInfo(T);
switch (ti) {
.@"struct" => {
// What is this magic? The FieldEnum creates a type (an enum)
// where each enum member has the name of a field in the struct
//
// So... struct { a: u8, b: u8 } will yield enum { a, b }
const FieldEnum = std.meta.FieldEnum(T);
// Then...EnumFieldStruct will create a struct from this, where
// each enum value becomes a field. We will specify the field
// type, and the default value. Combining these two calls gets
// us a struct with all the same field names, but we get a chance
// to make all the fields boolean, so we can use it to track
// which fields have been set
var found: std.enums.EnumFieldStruct(FieldEnum, bool, false) = .{};
// SAFETY: all fields updated below or error is returned
var obj: T = undefined;
while (try self.next()) |f| {
inline for (std.meta.fields(T)) |type_field| {
// To replicate the behavior of the record version of to,
// we need to only take the first version of the field,
// so if it's specified twice in the data, we will ignore
// all but the first instance
if (std.mem.eql(u8, f.key, type_field.name) and
!@field(found, type_field.name))
{
@field(obj, type_field.name) =
try coerce(type_field.name, type_field.type, f.value);
// Now account for this in our magic found struct...
@field(found, type_field.name) = true;
}
}
}
// Fill in the defaults for remaining fields. Throw if anything
// is missing both value (from above) and default (from here)
inline for (std.meta.fields(T)) |type_field| {
if (!@field(found, type_field.name)) {
// We did not find this field above...revert to default value
if (type_field.default_value_ptr) |ptr| {
@field(obj, type_field.name) = @as(*const type_field.type, @ptrCast(@alignCast(ptr))).*;
} else {
log.debug("Record could not be coerced. Field {s} not found on srf data, and no default value exists on the type", .{type_field.name});
return error.FieldNotFoundOnFieldWithoutDefaultValue;
}
}
}
return obj;
},
.@"union" => {
const active_tag_name = if (@hasDecl(T, "srf_tag_field"))
T.srf_tag_field
else
"active_tag";
const first_try = try self.next();
if (first_try == null) return error.ActiveTagFieldNotFound;
const f = first_try.?;
if (!std.mem.eql(u8, f.key, active_tag_name))
return error.ActiveTagNotFirstField; // required here, but not on the Record version of to
if (f.value == null or f.value.? != .string)
return error.ActiveTagValueMustBeAString;
const active_tag = f.value.?.string;
inline for (std.meta.fields(T)) |field_type| {
if (std.mem.eql(u8, active_tag, field_type.name)) {
return @unionInit(T, field_type.name, try self.to(field_type.type));
}
}
return error.ActiveTagDoesNotExist;
},
else => @compileError("Deserialization not supported on " ++ @tagName(ti) ++ " types"),
}
return error.CoercionNotPossible;
}
};
pub fn deinit(self: RecordIterator) void { pub fn deinit(self: RecordIterator) void {
const child_allocator = self.arena.child_allocator; const child_allocator = self.arena.child_allocator;
self.arena.deinit(); self.arena.deinit();
@ -809,7 +877,7 @@ const Directive = union(enum) {
eof, eof,
expires: i64, expires: i64,
pub fn parse(allocator: std.mem.Allocator, str: []const u8, state: RecordIterator.State) ParseError!?Directive { pub fn parse(allocator: std.mem.Allocator, str: []const u8, state: *RecordIterator.State) ParseError!?Directive {
if (!std.mem.startsWith(u8, str, "#!")) return null; if (!std.mem.startsWith(u8, str, "#!")) return null;
// strip any comments off // strip any comments off
var it = std.mem.splitScalar(u8, str[2..], '#'); var it = std.mem.splitScalar(u8, str[2..], '#');
@ -962,8 +1030,7 @@ pub const RecordFormatter = struct {
}; };
pub const Parsed = struct { pub const Parsed = struct {
// TODO: rip this down and return an array from parse records: []Record,
records: std.ArrayList(Record),
arena: *std.heap.ArenaAllocator, arena: *std.heap.ArenaAllocator,
expires: ?i64, expires: ?i64,
@ -974,35 +1041,29 @@ pub const Parsed = struct {
} }
}; };
/// parse function. Prefer iterator over this function. Note that this function will /// parse function
/// change soon
pub fn parse(reader: *std.Io.Reader, allocator: std.mem.Allocator, options: ParseOptions) ParseError!Parsed { pub fn parse(reader: *std.Io.Reader, allocator: std.mem.Allocator, options: ParseOptions) ParseError!Parsed {
var records = std.ArrayList(Record).empty; var records = std.ArrayList(Record).empty;
var it = try iterator(reader, allocator, options); var it = try iterator(reader, allocator, options);
errdefer it.deinit(); errdefer it.deinit();
const aa = it.arena.allocator(); const aa = it.arena.allocator();
var field_count: usize = 1;
while (try it.next()) |fi| { while (try it.next()) |fi| {
var al = std.ArrayList(Field).empty; var al = try std.ArrayList(Field).initCapacity(aa, field_count);
while (try fi.next()) |f| { while (try fi.next()) |f| {
const val = if (f.value != null)
switch (f.value.?) {
.string => Value{ .string = try aa.dupe(u8, f.value.?.string) },
.bytes => Value{ .bytes = try aa.dupe(u8, f.value.?.bytes) },
else => f.value,
}
else
f.value;
try al.append(aa, .{ try al.append(aa, .{
.key = try aa.dupe(u8, f.key), .key = f.key,
.value = val, .value = f.value,
}); });
} }
// assume that most records are same number of fields
field_count = @max(field_count, al.items.len);
try records.append(aa, .{ try records.append(aa, .{
.fields = try al.toOwnedSlice(aa), .fields = try al.toOwnedSlice(aa),
}); });
} }
return .{ return .{
.records = records, .records = try records.toOwnedSlice(aa),
.arena = it.arena, .arena = it.arena,
.expires = it.expires, .expires = it.expires,
}; };
@ -1032,16 +1093,16 @@ pub fn iterator(reader: *std.Io.Reader, allocator: std.mem.Allocator, options: P
}; };
const first_line = it.state.nextLine() orelse return ParseError.ParseFailed; const first_line = it.state.nextLine() orelse return ParseError.ParseFailed;
if (try Directive.parse(aa, first_line, it.state.*)) |d| { if (try Directive.parse(aa, first_line, it.state)) |d| {
if (d != .magic) try parseError(aa, "Magic header not found on first line", it.state.*); if (d != .magic) try parseError(aa, "Magic header not found on first line", it.state);
} else try parseError(aa, "Magic header not found on first line", it.state.*); } else try parseError(aa, "Magic header not found on first line", it.state);
// Loop through the header material and configure our main parsing // Loop through the header material and configure our main parsing
it.state.current_line = blk: { it.state.current_line = blk: {
while (it.state.nextLine()) |line| { while (it.state.nextLine()) |line| {
if (try Directive.parse(aa, line, it.state.*)) |d| { if (try Directive.parse(aa, line, it.state)) |d| {
switch (d) { switch (d) {
.magic => try parseError(aa, "Found a duplicate magic header", it.state.*), .magic => try parseError(aa, "Found a duplicate magic header", it.state),
.long_format => it.state.field_delimiter = '\n', .long_format => it.state.field_delimiter = '\n',
.compact_format => it.state.field_delimiter = ',', // what if we have both? .compact_format => it.state.field_delimiter = ',', // what if we have both?
.require_eof => it.state.require_eof = true, .require_eof => it.state.require_eof = true,
@ -1049,7 +1110,7 @@ pub fn iterator(reader: *std.Io.Reader, allocator: std.mem.Allocator, options: P
.eof => { .eof => {
// there needs to be an eof then // there needs to be an eof then
if (it.state.nextLine()) |_| { if (it.state.nextLine()) |_| {
try parseError(aa, "Data found after #!eof", it.state.*); try parseError(aa, "Data found after #!eof", it.state);
return ParseError.ParseFailed; // this is terminal return ParseError.ParseFailed; // this is terminal
} else return it; } else return it;
}, },
@ -1066,7 +1127,7 @@ inline fn dupe(allocator: std.mem.Allocator, options: ParseOptions, data: []cons
return try allocator.dupe(u8, data); return try allocator.dupe(u8, data);
return data; return data;
} }
inline fn parseError(allocator: std.mem.Allocator, message: []const u8, state: RecordIterator.State) ParseError!void { inline fn parseError(allocator: std.mem.Allocator, message: []const u8, state: *RecordIterator.State) ParseError!void {
log.debug("Parse error. Parse state {f}, message: {s}", .{ state, message }); log.debug("Parse error. Parse state {f}, message: {s}", .{ state, message });
if (state.options.diagnostics) |d| { if (state.options.diagnostics) |d| {
try d.addError(allocator, .{ try d.addError(allocator, .{
@ -1093,9 +1154,9 @@ test "long format single record, no eof" {
var reader = std.Io.Reader.fixed(data); var reader = std.Io.Reader.fixed(data);
const records = try parse(&reader, allocator, .{}); const records = try parse(&reader, allocator, .{});
defer records.deinit(); defer records.deinit();
try std.testing.expectEqual(@as(usize, 1), records.records.items.len); try std.testing.expectEqual(@as(usize, 1), records.records.len);
try std.testing.expectEqual(@as(usize, 1), records.records.items[0].fields.len); try std.testing.expectEqual(@as(usize, 1), records.records[0].fields.len);
const kvps = records.records.items[0].fields; const kvps = records.records[0].fields;
try std.testing.expectEqualStrings("key", kvps[0].key); try std.testing.expectEqualStrings("key", kvps[0].key);
try std.testing.expectEqualStrings("string value, with any data except a \\n. an optional string length between the colons", kvps[0].value.?.string); try std.testing.expectEqualStrings("string value, with any data except a \\n. an optional string length between the colons", kvps[0].value.?.string);
} }
@ -1115,7 +1176,7 @@ test "long format from README - generic data structures, first record only" {
var reader = std.Io.Reader.fixed(data); var reader = std.Io.Reader.fixed(data);
const records = try parse(&reader, allocator, .{}); const records = try parse(&reader, allocator, .{});
defer records.deinit(); defer records.deinit();
try std.testing.expectEqual(@as(usize, 1), records.records.items.len); try std.testing.expectEqual(@as(usize, 1), records.records.len);
} }
test "long format from README - generic data structures" { test "long format from README - generic data structures" {
@ -1147,8 +1208,8 @@ test "long format from README - generic data structures" {
var reader = std.Io.Reader.fixed(data); var reader = std.Io.Reader.fixed(data);
const records = try parse(&reader, allocator, .{}); const records = try parse(&reader, allocator, .{});
defer records.deinit(); defer records.deinit();
try std.testing.expectEqual(@as(usize, 2), records.records.items.len); try std.testing.expectEqual(@as(usize, 2), records.records.len);
const first = records.records.items[0]; const first = records.records[0];
try std.testing.expectEqual(@as(usize, 6), first.fields.len); try std.testing.expectEqual(@as(usize, 6), first.fields.len);
try std.testing.expectEqualStrings("key", first.fields[0].key); try std.testing.expectEqualStrings("key", first.fields[0].key);
try std.testing.expectEqualStrings("string value, with any data except a \\n. an optional string length between the colons", first.fields[0].value.?.string); try std.testing.expectEqualStrings("string value, with any data except a \\n. an optional string length between the colons", first.fields[0].value.?.string);
@ -1163,7 +1224,7 @@ test "long format from README - generic data structures" {
try std.testing.expectEqualStrings("boolean value", first.fields[5].key); try std.testing.expectEqualStrings("boolean value", first.fields[5].key);
try std.testing.expect(!first.fields[5].value.?.boolean); try std.testing.expect(!first.fields[5].value.?.boolean);
const second = records.records.items[1]; const second = records.records[1];
try std.testing.expectEqual(@as(usize, 5), second.fields.len); try std.testing.expectEqual(@as(usize, 5), second.fields.len);
try std.testing.expectEqualStrings("key", second.fields[0].key); try std.testing.expectEqualStrings("key", second.fields[0].key);
try std.testing.expectEqualStrings("this is the second record", second.fields[0].value.?.string); try std.testing.expectEqualStrings("this is the second record", second.fields[0].value.?.string);
@ -1190,8 +1251,8 @@ test "compact format from README - generic data structures" {
// We want "parse" and "parseLeaky" probably. Second parameter is a diagnostics // We want "parse" and "parseLeaky" probably. Second parameter is a diagnostics
const records = try parse(&reader, allocator, .{}); const records = try parse(&reader, allocator, .{});
defer records.deinit(); defer records.deinit();
try std.testing.expectEqual(@as(usize, 2), records.records.items.len); try std.testing.expectEqual(@as(usize, 2), records.records.len);
const first = records.records.items[0]; const first = records.records[0];
try std.testing.expectEqual(@as(usize, 6), first.fields.len); try std.testing.expectEqual(@as(usize, 6), first.fields.len);
try std.testing.expectEqualStrings("key", first.fields[0].key); try std.testing.expectEqualStrings("key", first.fields[0].key);
try std.testing.expectEqualStrings("string value must have a length between colons or end with a comma", first.fields[0].value.?.string); try std.testing.expectEqualStrings("string value must have a length between colons or end with a comma", first.fields[0].value.?.string);
@ -1206,7 +1267,7 @@ test "compact format from README - generic data structures" {
try std.testing.expectEqualStrings("boolean value", first.fields[5].key); try std.testing.expectEqualStrings("boolean value", first.fields[5].key);
try std.testing.expect(!first.fields[5].value.?.boolean); try std.testing.expect(!first.fields[5].value.?.boolean);
const second = records.records.items[1]; const second = records.records[1];
try std.testing.expectEqual(@as(usize, 1), second.fields.len); try std.testing.expectEqual(@as(usize, 1), second.fields.len);
try std.testing.expectEqualStrings("key", second.fields[0].key); try std.testing.expectEqualStrings("key", second.fields[0].key);
try std.testing.expectEqualStrings("this is the second record", second.fields[0].value.?.string); try std.testing.expectEqualStrings("this is the second record", second.fields[0].value.?.string);
@ -1273,7 +1334,7 @@ test "format all the things" {
var formatted_reader = std.Io.Reader.fixed(formatted); var formatted_reader = std.Io.Reader.fixed(formatted);
const parsed = try parse(&formatted_reader, std.testing.allocator, .{}); const parsed = try parse(&formatted_reader, std.testing.allocator, .{});
defer parsed.deinit(); defer parsed.deinit();
try std.testing.expectEqualDeep(records, parsed.records.items); try std.testing.expectEqualDeep(records, parsed.records);
const compact = try std.fmt.bufPrint( const compact = try std.fmt.bufPrint(
&buf, &buf,
@ -1290,7 +1351,7 @@ test "format all the things" {
var compact_reader = std.Io.Reader.fixed(compact); var compact_reader = std.Io.Reader.fixed(compact);
const parsed_compact = try parse(&compact_reader, std.testing.allocator, .{}); const parsed_compact = try parse(&compact_reader, std.testing.allocator, .{});
defer parsed_compact.deinit(); defer parsed_compact.deinit();
try std.testing.expectEqualDeep(records, parsed_compact.records.items); try std.testing.expectEqualDeep(records, parsed_compact.records);
const expected_expires: i64 = 1772589213; const expected_expires: i64 = 1772589213;
const compact_expires = try std.fmt.bufPrint( const compact_expires = try std.fmt.bufPrint(
@ -1309,7 +1370,7 @@ test "format all the things" {
var expires_reader = std.Io.Reader.fixed(compact_expires); var expires_reader = std.Io.Reader.fixed(compact_expires);
const parsed_expires = try parse(&expires_reader, std.testing.allocator, .{}); const parsed_expires = try parse(&expires_reader, std.testing.allocator, .{});
defer parsed_expires.deinit(); defer parsed_expires.deinit();
try std.testing.expectEqualDeep(records, parsed_expires.records.items); try std.testing.expectEqualDeep(records, parsed_expires.records);
try std.testing.expectEqual(expected_expires, parsed_expires.expires.?); try std.testing.expectEqual(expected_expires, parsed_expires.expires.?);
} }
test "serialize/deserialize" { test "serialize/deserialize" {
@ -1355,17 +1416,34 @@ test "serialize/deserialize" {
const parsed = try parse(&compact_reader, std.testing.allocator, .{}); const parsed = try parse(&compact_reader, std.testing.allocator, .{});
defer parsed.deinit(); defer parsed.deinit();
const rec1 = try parsed.records.items[0].to(Data); const rec1 = try parsed.records[0].to(Data);
try std.testing.expectEqualStrings("bar", rec1.foo); try std.testing.expectEqualStrings("bar", rec1.foo);
try std.testing.expectEqual(@as(u8, 42), rec1.bar); try std.testing.expectEqual(@as(u8, 42), rec1.bar);
try std.testing.expectEqual(@as(RecType, .foo), rec1.qux); try std.testing.expectEqual(@as(RecType, .foo), rec1.qux);
const rec4 = try parsed.records.items[3].to(Data); const rec4 = try parsed.records[3].to(Data);
try std.testing.expectEqualStrings("bar", rec4.foo); try std.testing.expectEqualStrings("bar", rec4.foo);
try std.testing.expectEqual(@as(u8, 42), rec4.bar); try std.testing.expectEqual(@as(u8, 42), rec4.bar);
try std.testing.expectEqual(@as(RecType, .bar), rec4.qux.?); try std.testing.expectEqual(@as(RecType, .bar), rec4.qux.?);
try std.testing.expectEqual(true, rec4.b); try std.testing.expectEqual(true, rec4.b);
try std.testing.expectEqual(@as(f32, 6.9), rec4.f); try std.testing.expectEqual(@as(f32, 6.9), rec4.f);
// Now we'll do it with the iterator version
var it_reader = std.Io.Reader.fixed(compact);
const ri = try iterator(&it_reader, std.testing.allocator, .{});
defer ri.deinit();
const rec1_it = try (try ri.next()).?.to(Data);
try std.testing.expectEqualStrings("bar", rec1_it.foo);
try std.testing.expectEqual(@as(u8, 42), rec1_it.bar);
try std.testing.expectEqual(@as(RecType, .foo), rec1_it.qux);
_ = try ri.next();
_ = try ri.next();
const rec4_it = try (try ri.next()).?.to(Data);
try std.testing.expectEqualStrings("bar", rec4_it.foo);
try std.testing.expectEqual(@as(u8, 42), rec4_it.bar);
try std.testing.expectEqual(@as(RecType, .bar), rec4_it.qux.?);
try std.testing.expectEqual(true, rec4_it.b);
try std.testing.expectEqual(@as(f32, 6.9), rec4_it.f);
const alloc = std.testing.allocator; const alloc = std.testing.allocator;
var owned_record_1 = try Record.from(Data, alloc, rec1); var owned_record_1 = try Record.from(Data, alloc, rec1);
defer owned_record_1.deinit(); defer owned_record_1.deinit();
@ -1443,9 +1521,9 @@ test "unions" {
const parsed = try parse(&compact_reader, std.testing.allocator, .{}); const parsed = try parse(&compact_reader, std.testing.allocator, .{});
defer parsed.deinit(); defer parsed.deinit();
const rec1 = try parsed.records.items[0].to(MixedData); const rec1 = try parsed.records[0].to(MixedData);
try std.testing.expectEqualDeep(data[0], rec1); try std.testing.expectEqualDeep(data[0], rec1);
const rec2 = try parsed.records.items[1].to(MixedData); const rec2 = try parsed.records[1].to(MixedData);
try std.testing.expectEqualDeep(data[1], rec2); try std.testing.expectEqualDeep(data[1], rec2);
} }
test "enums" { test "enums" {
@ -1488,9 +1566,9 @@ test "enums" {
const parsed = try parse(&compact_reader, std.testing.allocator, .{}); const parsed = try parse(&compact_reader, std.testing.allocator, .{});
defer parsed.deinit(); defer parsed.deinit();
const rec1 = try parsed.records.items[0].to(Data); const rec1 = try parsed.records[0].to(Data);
try std.testing.expectEqualDeep(data[0], rec1); try std.testing.expectEqualDeep(data[0], rec1);
const rec2 = try parsed.records.items[1].to(Data); const rec2 = try parsed.records[1].to(Data);
try std.testing.expectEqualDeep(data[1], rec2); try std.testing.expectEqualDeep(data[1], rec2);
const missing_tag = const missing_tag =
@ -1501,10 +1579,10 @@ test "enums" {
var mt_reader = std.Io.Reader.fixed(missing_tag); var mt_reader = std.Io.Reader.fixed(missing_tag);
const mt_parsed = try parse(&mt_reader, std.testing.allocator, .{}); const mt_parsed = try parse(&mt_reader, std.testing.allocator, .{});
defer mt_parsed.deinit(); defer mt_parsed.deinit();
const mt_rec1 = try mt_parsed.records.items[0].to(Data); const mt_rec1 = try mt_parsed.records[0].to(Data);
try std.testing.expect(mt_rec1.data_type == null); try std.testing.expect(mt_rec1.data_type == null);
const mt_rec1_dt2 = try mt_parsed.records.items[0].to(Data2); const mt_rec1_dt2 = try mt_parsed.records[0].to(Data2);
try std.testing.expect(mt_rec1_dt2.data_type == .bar); try std.testing.expect(mt_rec1_dt2.data_type == .bar);
} }
test "compact format length-prefixed string as last field" { test "compact format length-prefixed string as last field" {
@ -1520,8 +1598,8 @@ test "compact format length-prefixed string as last field" {
var reader = std.Io.Reader.fixed(data); var reader = std.Io.Reader.fixed(data);
const records = try parse(&reader, allocator, .{}); const records = try parse(&reader, allocator, .{});
defer records.deinit(); defer records.deinit();
try std.testing.expectEqual(@as(usize, 1), records.records.items.len); try std.testing.expectEqual(@as(usize, 1), records.records.len);
const rec = records.records.items[0]; const rec = records.records[0];
try std.testing.expectEqual(@as(usize, 2), rec.fields.len); try std.testing.expectEqual(@as(usize, 2), rec.fields.len);
try std.testing.expectEqualStrings("name", rec.fields[0].key); try std.testing.expectEqualStrings("name", rec.fields[0].key);
try std.testing.expectEqualStrings("alice", rec.fields[0].value.?.string); try std.testing.expectEqualStrings("alice", rec.fields[0].value.?.string);