introduce iterator version of to()

src/srf.zig

@@ -87,11 +87,9 @@ pub const Value = union(enum) {
     //     }
     // }
     pub fn parse(allocator: std.mem.Allocator, str: []const u8, state: *RecordIterator.State, delimiter: u8) ParseError!ValueWithMetaData {
-        const debug = str.len > 2 and str[0] == '1' and str[1] == '1';
-        if (debug) log.debug("parsing {s}", .{str});
         const type_val_sep_raw = std.mem.indexOfScalar(u8, str, ':');
         if (type_val_sep_raw == null) {
-            try parseError(allocator, "no type data or value after key", state.*);
+            try parseError(allocator, "no type data or value after key", state);
             return ParseError.ParseFailed;
         }
 
@@ -121,7 +119,7 @@ pub const Value = union(enum) {
             state.partial_line_column += total_chars;
             const Decoder = std.base64.standard.Decoder;
             const size = Decoder.calcSizeForSlice(val) catch {
-                try parseError(allocator, "error parsing base64 value", state.*);
+                try parseError(allocator, "error parsing base64 value", state);
                 return .{
                     .item_value = null,
                     .error_parsing = true,
@@ -130,7 +128,7 @@ pub const Value = union(enum) {
             const data = try allocator.alloc(u8, size);
             errdefer allocator.free(data);
             Decoder.decode(data, val) catch {
-                try parseError(allocator, "error parsing base64 value", state.*);
+                try parseError(allocator, "error parsing base64 value", state);
                 allocator.free(data);
                 return .{
                     .item_value = null,
@@ -151,7 +149,7 @@ pub const Value = union(enum) {
             state.partial_line_column += total_chars;
             const val_trimmed = std.mem.trim(u8, val, &std.ascii.whitespace);
             const number = std.fmt.parseFloat(@FieldType(Value, "number"), val_trimmed) catch {
-                try parseError(allocator, "error parsing numeric value", state.*);
+                try parseError(allocator, "error parsing numeric value", state);
                 return .{
                     .item_value = null,
                     .error_parsing = true,
@@ -173,7 +171,7 @@ pub const Value = union(enum) {
                 if (std.mem.eql(u8, "false", val_trimmed)) break :blk false;
                 if (std.mem.eql(u8, "true", val_trimmed)) break :blk true;
 
-                try parseError(allocator, "error parsing boolean value", state.*);
+                try parseError(allocator, "error parsing boolean value", state);
                 return .{
                     .item_value = null,
                     .error_parsing = true,
@@ -200,18 +198,16 @@ pub const Value = union(enum) {
         state.partial_line_column += total_metadata_chars;
         const size = std.fmt.parseInt(usize, trimmed_meta, 0) catch {
             log.debug("parseInt fail, trimmed_data: '{s}'", .{trimmed_meta});
-            try parseError(allocator, "unrecognized metadata for key", state.*);
+            try parseError(allocator, "unrecognized metadata for key", state);
             return .{
                 .item_value = null,
                 .error_parsing = true,
             };
         };
-        if (debug) log.debug("found fixed string size {d}. State {f}", .{ size, state });
         // Update again for number of bytes. All failures beyond this point are
         // fatal, so this is safe.
         state.column += size;
         state.partial_line_column += size;
-        if (debug) log.debug("New state {f}", .{state});
 
         // If we are being asked specifically for bytes, we no longer care about
         // delimiters. We just want raw bytes. This might adjust our line/column
@@ -220,41 +216,29 @@ pub const Value = union(enum) {
         if (rest_of_data.len >= size) {
             // We fit on this line, everything is "normal"
             const val = rest_of_data[0..size];
-            if (debug) log.debug("val {s}", .{val});
             return .{
                 .item_value = .{ .string = val },
             };
         }
         // This is not enough, we need more data from the reader
-        log.debug("item value includes newlines {f}", .{state});
-        // We need to advance the reader, so we need a copy of what we have so fa
-        const start = try dupe(allocator, state.options, rest_of_data);
-        defer allocator.free(start);
+        const buf = try allocator.alloc(u8, size);
+        errdefer allocator.free(buf);
+        @memcpy(buf[0..rest_of_data.len], rest_of_data);
+        // add back the newline we are skipping
+        buf[rest_of_data.len] = '\n';
         // We won't do a parseError here. If we have an allocation error, read
         // error, or end of stream, all of these are fatal. Our reader is currently
         // past the newline, so we have to remove a character from size to account.
-        const end = try state.reader.readAlloc(allocator, size - rest_of_data.len - 1);
+        try state.reader.readSliceAll(buf[rest_of_data.len + 1 ..]);
         // However, we want to be past the end of the *next* newline too (in long
         // format mode)
         if (delimiter == '\n') state.reader.toss(1);
-        defer allocator.free(end);
-        // This \n is because the reader state will have advanced beyond the next newline, so end
-        // really should start with the newline. This only applies to long mode, because otherwise the
-        // entire record is a single line
-        const final = try std.mem.concat(allocator, u8, &.{ start, "\n", end });
-        // const final = if (delimiter == '\n')
-        //     try std.mem.concat(allocator, u8, &.{ start, "\n", end })
-        // else
-        //     try std.mem.concat(allocator, u8, &.{ start, end });
-        errdefer allocator.free(final);
-        // log.debug("full val: {s}", .{final});
-        std.debug.assert(final.len == size);
         // Because we've now advanced the line, we need to reset everything
-        state.line += std.mem.count(u8, final, "\n");
-        state.column = final.len - std.mem.lastIndexOf(u8, final, "\n").?;
+        state.line += std.mem.count(u8, buf, "\n");
+        state.column = buf.len - std.mem.lastIndexOf(u8, buf, "\n").?;
         state.partial_line_column = state.column;
         return .{
-            .item_value = .{ .string = final },
+            .item_value = .{ .string = buf },
             .reader_advanced = true,
         };
     }
@@ -266,6 +250,63 @@ pub const Field = struct {
     value: ?Value,
 };
 
+fn coerce(name: []const u8, comptime T: type, val: ?Value) !T {
+    // Here's the deduplicated set of field types that coerce needs to handle:
+    // Direct from SRF values:
+    // Need parsing from string:
+    // - Date, ?Date -- Date.parse(string)
+    //
+    // Won't work with Record.to(T) generically:
+    // - []const OptionContract -- nested sub-records (OptionsChain has calls/puts arrays)
+    // - ?[]const Holding, ?[]const SectorWeight -- nested sub-records in EtfProfile
+    //
+    const ti = @typeInfo(T);
+    if (val == null and ti != .optional)
+        return error.NullValueCannotBeAssignedToNonNullField;
+
+    // []const u8 is classified as a pointer
+    switch (ti) {
+        .optional => |o| if (val) |_|
+            return try coerce(name, o.child, val)
+        else
+            return null,
+        .pointer => |p| {
+            // We don't have an allocator, so the only thing we can do
+            // here is manage []const u8 or []u8
+            if (p.size != .slice or p.child != u8)
+                return error.CoercionNotPossible;
+            if (val.? != .string and val.? != .bytes)
+                return error.CoercionNotPossible;
+            if (val.? == .string)
+                return val.?.string;
+            return val.?.bytes;
+        },
+        .type, .void, .noreturn => return error.CoercionNotPossible,
+        .comptime_float, .comptime_int, .undefined, .null, .error_union => return error.CoercionNotPossible,
+        .error_set, .@"fn", .@"opaque", .frame => return error.CoercionNotPossible,
+        .@"anyframe", .vector, .enum_literal => return error.CoercionNotPossible,
+        .int => return @as(T, @intFromFloat(val.?.number)),
+        .float => return @as(T, @floatCast(val.?.number)),
+        .bool => return val.?.boolean,
+        .@"enum" => return std.meta.stringToEnum(T, val.?.string).?,
+        .array => return error.NotImplemented,
+        .@"struct", .@"union" => {
+            if (std.meta.hasMethod(T, "srfParse")) {
+                if (val.? == .string)
+                    return T.srfParse(val.?.string) catch |e| {
+                        log.err(
+                            "custom parse of value {s} failed : {}",
+                            .{ val.?.string, e },
+                        );
+                        return error.CustomParseFailed;
+                    };
+            }
+            return error.CoercionNotPossible;
+        },
+    }
+    return null;
+}
+
 // A record has a list of fields, with no assumptions regarding duplication,
 // etc. This is for parsing speed, but also for more flexibility in terms of
 // use cases. One can make a defacto array out of this structure by having
@@ -289,63 +330,6 @@ pub const Record = struct {
         return null;
     }
 
-    fn coerce(name: []const u8, comptime T: type, val: ?Value) !T {
-        // Here's the deduplicated set of field types that coerce needs to handle:
-        // Direct from SRF values:
-        // Need parsing from string:
-        // - Date, ?Date -- Date.parse(string)
-        //
-        // Won't work with Record.to(T) generically:
-        // - []const OptionContract -- nested sub-records (OptionsChain has calls/puts arrays)
-        // - ?[]const Holding, ?[]const SectorWeight -- nested sub-records in EtfProfile
-        //
-        const ti = @typeInfo(T);
-        if (val == null and ti != .optional)
-            return error.NullValueCannotBeAssignedToNonNullField;
-
-        // []const u8 is classified as a pointer
-        switch (ti) {
-            .optional => |o| if (val) |_|
-                return try coerce(name, o.child, val)
-            else
-                return null,
-            .pointer => |p| {
-                // We don't have an allocator, so the only thing we can do
-                // here is manage []const u8 or []u8
-                if (p.size != .slice or p.child != u8)
-                    return error.CoercionNotPossible;
-                if (val.? != .string and val.? != .bytes)
-                    return error.CoercionNotPossible;
-                if (val.? == .string)
-                    return val.?.string;
-                return val.?.bytes;
-            },
-            .type, .void, .noreturn => return error.CoercionNotPossible,
-            .comptime_float, .comptime_int, .undefined, .null, .error_union => return error.CoercionNotPossible,
-            .error_set, .@"fn", .@"opaque", .frame => return error.CoercionNotPossible,
-            .@"anyframe", .vector, .enum_literal => return error.CoercionNotPossible,
-            .int => return @as(T, @intFromFloat(val.?.number)),
-            .float => return @as(T, @floatCast(val.?.number)),
-            .bool => return val.?.boolean,
-            .@"enum" => return std.meta.stringToEnum(T, val.?.string).?,
-            .array => return error.NotImplemented,
-            .@"struct", .@"union" => {
-                if (std.meta.hasMethod(T, "srfParse")) {
-                    if (val.? == .string)
-                        return T.srfParse(val.?.string) catch |e| {
-                            log.err(
-                                "custom parse of value {s} failed : {}",
-                                .{ val.?.string, e },
-                            );
-                            return error.CustomParseFailed;
-                        };
-                }
-                return error.CoercionNotPossible;
-            },
-        }
-        return null;
-    }
-
     fn maxFields(comptime T: type) usize {
         const ti = @typeInfo(T);
         if (ti != .@"union") return std.meta.fields(T).len;
@@ -605,6 +589,7 @@ pub const RecordIterator = struct {
 
         field_delimiter: u8 = ',',
         end_of_record_reached: bool = false,
+        field_iterator: ?FieldIterator = null,
 
         /// Takes the next line, trimming leading whitespace and ignoring comments
         /// Directives (comments starting with #!) are preserved
@@ -630,6 +615,11 @@ pub const RecordIterator = struct {
         // TODO: we need to capture the fieldIterator here and make sure it's run
         // to the ground to keep our state intact
         const state = self.state;
+        if (state.field_iterator) |f| {
+            // We need to finish the fields on the previous record
+            while (try f.next()) |_| {}
+            state.field_iterator = null;
+        }
         if (state.current_line == null) {
             if (state.options.diagnostics) |d|
                 if (d.errors.items.len > 0) return ParseError.ParseFailed;
@@ -646,12 +636,12 @@ pub const RecordIterator = struct {
             if (state.current_line == null) return self.next();
         }
         // non-blank line, but we could have an eof marker
-        if (try Directive.parse(self.arena.allocator(), state.current_line.?, state.*)) |d| {
+        if (try Directive.parse(self.arena.allocator(), state.current_line.?, state)) |d| {
             switch (d) {
                 .eof => {
                     // there needs to be an eof then
                     if (state.nextLine()) |_| {
-                        try parseError(self.arena.allocator(), "Data found after #!eof", state.*);
+                        try parseError(self.arena.allocator(), "Data found after #!eof", state);
                         return ParseError.ParseFailed; // this is terminal
                     } else {
                         state.eof_found = true;
@@ -660,7 +650,7 @@ pub const RecordIterator = struct {
                     }
                 },
                 else => {
-                    try parseError(self.arena.allocator(), "Directive found after data started", state.*);
+                    try parseError(self.arena.allocator(), "Directive found after data started", state);
                     state.current_line = state.nextLine();
                     // TODO: This runs the risk of a malicious file creating
                     // a stackoverflow by using many non-eof directives
@@ -669,7 +659,8 @@ pub const RecordIterator = struct {
             }
         }
         state.end_of_record_reached = false;
-        return .{ .ri = self };
+        state.field_iterator = .{ .ri = self };
+        return state.field_iterator.?;
     }
 
     pub const FieldIterator = struct {
@@ -677,6 +668,7 @@ pub const RecordIterator = struct {
 
         pub fn next(self: FieldIterator) !?Field {
             const state = self.ri.state;
+            const aa = self.ri.arena.allocator();
             // Main parsing. We already have the first line of data, which could
             // be a record (compact format) or a key/value pair (long format)
 
@@ -686,12 +678,12 @@ pub const RecordIterator = struct {
             if (state.end_of_record_reached) return null;
             // non-blank line, but we could have an eof marker
             // TODO: deduplicate this code
-            if (try Directive.parse(self.ri.arena.allocator(), state.current_line.?, state.*)) |d| {
+            if (try Directive.parse(aa, state.current_line.?, state)) |d| {
                 switch (d) {
                     .eof => {
                         // there needs to be an eof then
                         if (state.nextLine()) |_| {
-                            try parseError(self.ri.arena.allocator(), "Data found after #!eof", state.*);
+                            try parseError(aa, "Data found after #!eof", state);
                             return ParseError.ParseFailed; // this is terminal
                         } else {
                             state.eof_found = true;
@@ -700,7 +692,7 @@ pub const RecordIterator = struct {
                         }
                     },
                     else => {
-                        try parseError(self.ri.arena.allocator(), "Directive found after data started", state.*);
+                        try parseError(aa, "Directive found after data started", state);
                         state.current_line = state.nextLine();
                         // TODO: This runs the risk of a malicious file creating
                         // a stackoverflow by using many non-eof directives
@@ -717,7 +709,7 @@ pub const RecordIterator = struct {
             state.column += key.len + 1;
             state.partial_line_column += key.len + 1;
             const value = try Value.parse(
-                self.ri.arena.allocator(),
+                aa,
                 it.rest(),
                 state,
                 state.field_delimiter,
@@ -725,7 +717,7 @@ pub const RecordIterator = struct {
 
             var field: ?Field = null;
             if (!value.error_parsing) {
-                field = .{ .key = try dupe(self.ri.arena.allocator(), state.options, key), .value = value.item_value };
+                field = .{ .key = try dupe(aa, state.options, key), .value = value.item_value };
             }
 
             if (value.reader_advanced and state.field_delimiter == ',') {
@@ -774,8 +766,84 @@ pub const RecordIterator = struct {
             }
             return field;
         }
-    };
 
+        /// Coerce Record to a type. Does not handle fields with arrays
+        pub fn to(self: FieldIterator, comptime T: type) !T {
+            const ti = @typeInfo(T);
+
+            switch (ti) {
+                .@"struct" => {
+                    // What is this magic? The FieldEnum creates a type (an enum)
+                    // where each enum member has the name of a field in the struct
+                    //
+                    // So... struct { a: u8, b: u8 } will yield enum { a, b }
+                    const FieldEnum = std.meta.FieldEnum(T);
+                    // Then...EnumFieldStruct will create a struct from this, where
+                    // each enum value becomes a field. We will specify the field
+                    // type, and the default value. Combining these two calls gets
+                    // us a struct with all the same field names, but we get a chance
+                    // to make all the fields boolean, so we can use it to track
+                    // which fields have been set
+                    var found: std.enums.EnumFieldStruct(FieldEnum, bool, false) = .{};
+                    // SAFETY: all fields updated below or error is returned
+                    var obj: T = undefined;
+
+                    while (try self.next()) |f| {
+                        inline for (std.meta.fields(T)) |type_field| {
+                            // To replicate the behavior of the record version of to,
+                            // we need to only take the first version of the field,
+                            // so if it's specified twice in the data, we will ignore
+                            // all but the first instance
+                            if (std.mem.eql(u8, f.key, type_field.name) and
+                                !@field(found, type_field.name))
+                            {
+                                @field(obj, type_field.name) =
+                                    try coerce(type_field.name, type_field.type, f.value);
+                                // Now account for this in our magic found struct...
+                                @field(found, type_field.name) = true;
+                            }
+                        }
+                    }
+                    // Fill in the defaults for remaining fields. Throw if anything
+                    // is missing both value (from above) and default (from here)
+                    inline for (std.meta.fields(T)) |type_field| {
+                        if (!@field(found, type_field.name)) {
+                            // We did not find this field above...revert to default value
+                            if (type_field.default_value_ptr) |ptr| {
+                                @field(obj, type_field.name) = @as(*const type_field.type, @ptrCast(@alignCast(ptr))).*;
+                            } else {
+                                log.debug("Record could not be coerced. Field {s} not found on srf data, and no default value exists on the type", .{type_field.name});
+                                return error.FieldNotFoundOnFieldWithoutDefaultValue;
+                            }
+                        }
+                    }
+                    return obj;
+                },
+                .@"union" => {
+                    const active_tag_name = if (@hasDecl(T, "srf_tag_field"))
+                        T.srf_tag_field
+                    else
+                        "active_tag";
+                    const first_try = try self.next();
+                    if (first_try == null) return error.ActiveTagFieldNotFound;
+                    const f = first_try.?;
+                    if (!std.mem.eql(u8, f.key, active_tag_name))
+                        return error.ActiveTagNotFirstField; // required here, but not on the Record version of to
+                    if (f.value == null or f.value.? != .string)
+                        return error.ActiveTagValueMustBeAString;
+                    const active_tag = f.value.?.string;
+                    inline for (std.meta.fields(T)) |field_type| {
+                        if (std.mem.eql(u8, active_tag, field_type.name)) {
+                            return @unionInit(T, field_type.name, try self.to(field_type.type));
+                        }
+                    }
+                    return error.ActiveTagDoesNotExist;
+                },
+                else => @compileError("Deserialization not supported on " ++ @tagName(ti) ++ " types"),
+            }
+            return error.CoercionNotPossible;
+        }
+    };
     pub fn deinit(self: RecordIterator) void {
         const child_allocator = self.arena.child_allocator;
         self.arena.deinit();
@@ -809,7 +877,7 @@ const Directive = union(enum) {
     eof,
     expires: i64,
 
-    pub fn parse(allocator: std.mem.Allocator, str: []const u8, state: RecordIterator.State) ParseError!?Directive {
+    pub fn parse(allocator: std.mem.Allocator, str: []const u8, state: *RecordIterator.State) ParseError!?Directive {
         if (!std.mem.startsWith(u8, str, "#!")) return null;
         // strip any comments off
         var it = std.mem.splitScalar(u8, str[2..], '#');
@@ -962,8 +1030,7 @@ pub const RecordFormatter = struct {
 };
 
 pub const Parsed = struct {
-    // TODO: rip this down and return an array from parse
-    records: std.ArrayList(Record),
+    records: []Record,
     arena: *std.heap.ArenaAllocator,
     expires: ?i64,
 
@@ -974,35 +1041,29 @@ pub const Parsed = struct {
     }
 };
 
-/// parse function. Prefer iterator over this function. Note that this function will
-/// change soon
+/// parse function
 pub fn parse(reader: *std.Io.Reader, allocator: std.mem.Allocator, options: ParseOptions) ParseError!Parsed {
     var records = std.ArrayList(Record).empty;
     var it = try iterator(reader, allocator, options);
     errdefer it.deinit();
     const aa = it.arena.allocator();
+    var field_count: usize = 1;
     while (try it.next()) |fi| {
-        var al = std.ArrayList(Field).empty;
+        var al = try std.ArrayList(Field).initCapacity(aa, field_count);
         while (try fi.next()) |f| {
-            const val = if (f.value != null)
-                switch (f.value.?) {
-                    .string => Value{ .string = try aa.dupe(u8, f.value.?.string) },
-                    .bytes => Value{ .bytes = try aa.dupe(u8, f.value.?.bytes) },
-                    else => f.value,
-                }
-            else
-                f.value;
             try al.append(aa, .{
-                .key = try aa.dupe(u8, f.key),
-                .value = val,
+                .key = f.key,
+                .value = f.value,
             });
         }
+        // assume that most records are same number of fields
+        field_count = @max(field_count, al.items.len);
         try records.append(aa, .{
             .fields = try al.toOwnedSlice(aa),
         });
     }
     return .{
-        .records = records,
+        .records = try records.toOwnedSlice(aa),
         .arena = it.arena,
         .expires = it.expires,
     };
@@ -1032,16 +1093,16 @@ pub fn iterator(reader: *std.Io.Reader, allocator: std.mem.Allocator, options: P
     };
     const first_line = it.state.nextLine() orelse return ParseError.ParseFailed;
 
-    if (try Directive.parse(aa, first_line, it.state.*)) |d| {
-        if (d != .magic) try parseError(aa, "Magic header not found on first line", it.state.*);
-    } else try parseError(aa, "Magic header not found on first line", it.state.*);
+    if (try Directive.parse(aa, first_line, it.state)) |d| {
+        if (d != .magic) try parseError(aa, "Magic header not found on first line", it.state);
+    } else try parseError(aa, "Magic header not found on first line", it.state);
 
     // Loop through the header material and configure our main parsing
     it.state.current_line = blk: {
         while (it.state.nextLine()) |line| {
-            if (try Directive.parse(aa, line, it.state.*)) |d| {
+            if (try Directive.parse(aa, line, it.state)) |d| {
                 switch (d) {
-                    .magic => try parseError(aa, "Found a duplicate magic header", it.state.*),
+                    .magic => try parseError(aa, "Found a duplicate magic header", it.state),
                     .long_format => it.state.field_delimiter = '\n',
                     .compact_format => it.state.field_delimiter = ',', // what if we have both?
                     .require_eof => it.state.require_eof = true,
@@ -1049,7 +1110,7 @@ pub fn iterator(reader: *std.Io.Reader, allocator: std.mem.Allocator, options: P
                     .eof => {
                         // there needs to be an eof then
                         if (it.state.nextLine()) |_| {
-                            try parseError(aa, "Data found after #!eof", it.state.*);
+                            try parseError(aa, "Data found after #!eof", it.state);
                             return ParseError.ParseFailed; // this is terminal
                         } else return it;
                     },
@@ -1066,7 +1127,7 @@ inline fn dupe(allocator: std.mem.Allocator, options: ParseOptions, data: []cons
         return try allocator.dupe(u8, data);
     return data;
 }
-inline fn parseError(allocator: std.mem.Allocator, message: []const u8, state: RecordIterator.State) ParseError!void {
+inline fn parseError(allocator: std.mem.Allocator, message: []const u8, state: *RecordIterator.State) ParseError!void {
     log.debug("Parse error. Parse state {f}, message: {s}", .{ state, message });
     if (state.options.diagnostics) |d| {
         try d.addError(allocator, .{
@@ -1093,9 +1154,9 @@ test "long format single record, no eof" {
     var reader = std.Io.Reader.fixed(data);
     const records = try parse(&reader, allocator, .{});
     defer records.deinit();
-    try std.testing.expectEqual(@as(usize, 1), records.records.items.len);
-    try std.testing.expectEqual(@as(usize, 1), records.records.items[0].fields.len);
-    const kvps = records.records.items[0].fields;
+    try std.testing.expectEqual(@as(usize, 1), records.records.len);
+    try std.testing.expectEqual(@as(usize, 1), records.records[0].fields.len);
+    const kvps = records.records[0].fields;
     try std.testing.expectEqualStrings("key", kvps[0].key);
     try std.testing.expectEqualStrings("string value, with any data except a \\n. an optional string length between the colons", kvps[0].value.?.string);
 }
@@ -1115,7 +1176,7 @@ test "long format from README - generic data structures, first record only" {
     var reader = std.Io.Reader.fixed(data);
     const records = try parse(&reader, allocator, .{});
     defer records.deinit();
-    try std.testing.expectEqual(@as(usize, 1), records.records.items.len);
+    try std.testing.expectEqual(@as(usize, 1), records.records.len);
 }
 
 test "long format from README - generic data structures" {
@@ -1147,8 +1208,8 @@ test "long format from README - generic data structures" {
     var reader = std.Io.Reader.fixed(data);
     const records = try parse(&reader, allocator, .{});
     defer records.deinit();
-    try std.testing.expectEqual(@as(usize, 2), records.records.items.len);
-    const first = records.records.items[0];
+    try std.testing.expectEqual(@as(usize, 2), records.records.len);
+    const first = records.records[0];
     try std.testing.expectEqual(@as(usize, 6), first.fields.len);
     try std.testing.expectEqualStrings("key", first.fields[0].key);
     try std.testing.expectEqualStrings("string value, with any data except a \\n. an optional string length between the colons", first.fields[0].value.?.string);
@@ -1163,7 +1224,7 @@ test "long format from README - generic data structures" {
     try std.testing.expectEqualStrings("boolean value", first.fields[5].key);
     try std.testing.expect(!first.fields[5].value.?.boolean);
 
-    const second = records.records.items[1];
+    const second = records.records[1];
     try std.testing.expectEqual(@as(usize, 5), second.fields.len);
     try std.testing.expectEqualStrings("key", second.fields[0].key);
     try std.testing.expectEqualStrings("this is the second record", second.fields[0].value.?.string);
@@ -1190,8 +1251,8 @@ test "compact format from README - generic data structures" {
     // We want "parse" and "parseLeaky" probably. Second parameter is a diagnostics
     const records = try parse(&reader, allocator, .{});
     defer records.deinit();
-    try std.testing.expectEqual(@as(usize, 2), records.records.items.len);
-    const first = records.records.items[0];
+    try std.testing.expectEqual(@as(usize, 2), records.records.len);
+    const first = records.records[0];
     try std.testing.expectEqual(@as(usize, 6), first.fields.len);
     try std.testing.expectEqualStrings("key", first.fields[0].key);
     try std.testing.expectEqualStrings("string value must have a length between colons or end with a comma", first.fields[0].value.?.string);
@@ -1206,7 +1267,7 @@ test "compact format from README - generic data structures" {
     try std.testing.expectEqualStrings("boolean value", first.fields[5].key);
     try std.testing.expect(!first.fields[5].value.?.boolean);
 
-    const second = records.records.items[1];
+    const second = records.records[1];
     try std.testing.expectEqual(@as(usize, 1), second.fields.len);
     try std.testing.expectEqualStrings("key", second.fields[0].key);
     try std.testing.expectEqualStrings("this is the second record", second.fields[0].value.?.string);
@@ -1273,7 +1334,7 @@ test "format all the things" {
     var formatted_reader = std.Io.Reader.fixed(formatted);
     const parsed = try parse(&formatted_reader, std.testing.allocator, .{});
     defer parsed.deinit();
-    try std.testing.expectEqualDeep(records, parsed.records.items);
+    try std.testing.expectEqualDeep(records, parsed.records);
 
     const compact = try std.fmt.bufPrint(
         &buf,
@@ -1290,7 +1351,7 @@ test "format all the things" {
     var compact_reader = std.Io.Reader.fixed(compact);
     const parsed_compact = try parse(&compact_reader, std.testing.allocator, .{});
     defer parsed_compact.deinit();
-    try std.testing.expectEqualDeep(records, parsed_compact.records.items);
+    try std.testing.expectEqualDeep(records, parsed_compact.records);
 
     const expected_expires: i64 = 1772589213;
     const compact_expires = try std.fmt.bufPrint(
@@ -1309,7 +1370,7 @@ test "format all the things" {
     var expires_reader = std.Io.Reader.fixed(compact_expires);
     const parsed_expires = try parse(&expires_reader, std.testing.allocator, .{});
     defer parsed_expires.deinit();
-    try std.testing.expectEqualDeep(records, parsed_expires.records.items);
+    try std.testing.expectEqualDeep(records, parsed_expires.records);
     try std.testing.expectEqual(expected_expires, parsed_expires.expires.?);
 }
 test "serialize/deserialize" {
@@ -1355,17 +1416,34 @@ test "serialize/deserialize" {
     const parsed = try parse(&compact_reader, std.testing.allocator, .{});
     defer parsed.deinit();
 
-    const rec1 = try parsed.records.items[0].to(Data);
+    const rec1 = try parsed.records[0].to(Data);
     try std.testing.expectEqualStrings("bar", rec1.foo);
     try std.testing.expectEqual(@as(u8, 42), rec1.bar);
     try std.testing.expectEqual(@as(RecType, .foo), rec1.qux);
-    const rec4 = try parsed.records.items[3].to(Data);
+    const rec4 = try parsed.records[3].to(Data);
     try std.testing.expectEqualStrings("bar", rec4.foo);
     try std.testing.expectEqual(@as(u8, 42), rec4.bar);
     try std.testing.expectEqual(@as(RecType, .bar), rec4.qux.?);
     try std.testing.expectEqual(true, rec4.b);
     try std.testing.expectEqual(@as(f32, 6.9), rec4.f);
 
+    // Now we'll do it with the iterator version
+    var it_reader = std.Io.Reader.fixed(compact);
+    const ri = try iterator(&it_reader, std.testing.allocator, .{});
+    defer ri.deinit();
+    const rec1_it = try (try ri.next()).?.to(Data);
+    try std.testing.expectEqualStrings("bar", rec1_it.foo);
+    try std.testing.expectEqual(@as(u8, 42), rec1_it.bar);
+    try std.testing.expectEqual(@as(RecType, .foo), rec1_it.qux);
+    _ = try ri.next();
+    _ = try ri.next();
+    const rec4_it = try (try ri.next()).?.to(Data);
+    try std.testing.expectEqualStrings("bar", rec4_it.foo);
+    try std.testing.expectEqual(@as(u8, 42), rec4_it.bar);
+    try std.testing.expectEqual(@as(RecType, .bar), rec4_it.qux.?);
+    try std.testing.expectEqual(true, rec4_it.b);
+    try std.testing.expectEqual(@as(f32, 6.9), rec4_it.f);
+
     const alloc = std.testing.allocator;
     var owned_record_1 = try Record.from(Data, alloc, rec1);
     defer owned_record_1.deinit();
@@ -1443,9 +1521,9 @@ test "unions" {
     const parsed = try parse(&compact_reader, std.testing.allocator, .{});
     defer parsed.deinit();
 
-    const rec1 = try parsed.records.items[0].to(MixedData);
+    const rec1 = try parsed.records[0].to(MixedData);
     try std.testing.expectEqualDeep(data[0], rec1);
-    const rec2 = try parsed.records.items[1].to(MixedData);
+    const rec2 = try parsed.records[1].to(MixedData);
     try std.testing.expectEqualDeep(data[1], rec2);
 }
 test "enums" {
@@ -1488,9 +1566,9 @@ test "enums" {
     const parsed = try parse(&compact_reader, std.testing.allocator, .{});
     defer parsed.deinit();
 
-    const rec1 = try parsed.records.items[0].to(Data);
+    const rec1 = try parsed.records[0].to(Data);
     try std.testing.expectEqualDeep(data[0], rec1);
-    const rec2 = try parsed.records.items[1].to(Data);
+    const rec2 = try parsed.records[1].to(Data);
     try std.testing.expectEqualDeep(data[1], rec2);
 
     const missing_tag =
@@ -1501,10 +1579,10 @@ test "enums" {
     var mt_reader = std.Io.Reader.fixed(missing_tag);
     const mt_parsed = try parse(&mt_reader, std.testing.allocator, .{});
     defer mt_parsed.deinit();
-    const mt_rec1 = try mt_parsed.records.items[0].to(Data);
+    const mt_rec1 = try mt_parsed.records[0].to(Data);
     try std.testing.expect(mt_rec1.data_type == null);
 
-    const mt_rec1_dt2 = try mt_parsed.records.items[0].to(Data2);
+    const mt_rec1_dt2 = try mt_parsed.records[0].to(Data2);
     try std.testing.expect(mt_rec1_dt2.data_type == .bar);
 }
 test "compact format length-prefixed string as last field" {
@@ -1520,8 +1598,8 @@ test "compact format length-prefixed string as last field" {
     var reader = std.Io.Reader.fixed(data);
     const records = try parse(&reader, allocator, .{});
     defer records.deinit();
-    try std.testing.expectEqual(@as(usize, 1), records.records.items.len);
-    const rec = records.records.items[0];
+    try std.testing.expectEqual(@as(usize, 1), records.records.len);
+    const rec = records.records[0];
     try std.testing.expectEqual(@as(usize, 2), rec.fields.len);
     try std.testing.expectEqualStrings("name", rec.fields[0].key);
     try std.testing.expectEqualStrings("alice", rec.fields[0].value.?.string);