3196 lines
133 KiB
Zig
3196 lines
133 KiB
Zig
//!SRF is a minimal data format designed for L2 caches and simple structured storage suitable for simple configuration as well. It provides human-readable key-value records with basic type hints, while avoiding the parsing complexity and escaping requirements of JSON. Current benchmarking with hyperfine demonstrate approximately twice the performance of JSON parsing, though for L2 caches, JSON may be a poor choice. Compared to jsonl, it is approximately 40x faster. Performance also improves by 8% if you instruct the library not to copy strings around (ParseOptions alloc_strings = false).
|
|
//!
|
|
//!**Features:**
|
|
//!- No escaping required - use length-prefixed strings for complex data
|
|
//!- Single-pass parsing with minimal memory allocation
|
|
//!- Basic type system (string, num, bool, null, binary) with explicit type hints
|
|
//!- Compact format for machine generation, long format for human editing
|
|
//!- Built-in corruption detection with optional EOF markers
|
|
//!
|
|
//!**When to use SRF:**
|
|
//!- L2 caches that need occasional human inspection
|
|
//!- Simple configuration files with mixed data types
|
|
//!- Data exchange where JSON escaping is problematic
|
|
//!- Applications requiring fast, predictable parsing
|
|
//!
|
|
//!**When not to use SRF:**
|
|
//!- Complex nested data structures (use JSON/TOML instead)
|
|
//!- Schema validation requirements
|
|
//!- Arrays or object hierarchies (arrays can be managed in the data itself, however)
|
|
const std = @import("std");
|
|
|
|
const log = std.log.scoped(.srf);
|
|
|
|
pub const ParseLineError = struct {
|
|
message: []const u8,
|
|
level: std.log.Level,
|
|
line: usize,
|
|
column: usize,
|
|
};
|
|
pub const Diagnostics = struct {
|
|
ptr: *anyopaque,
|
|
addErrorFn: *const fn (*anyopaque, ParseLineError) ParseError!void,
|
|
has_errors: bool = false,
|
|
|
|
pub fn addError(self: *Diagnostics, err: ParseLineError) ParseError!void {
|
|
try self.addErrorFn(self.ptr, err);
|
|
self.has_errors = true;
|
|
}
|
|
};
|
|
pub fn BoundedDiagnostics(comptime max_errors: usize) type {
|
|
return struct {
|
|
buffer: [max_errors]ParseLineError,
|
|
/// backing buffer for message data. We provide 256 bytes for each message,
|
|
/// which should be fine, and if it's not, we need to fix the code
|
|
msg_buffer: [max_errors * 256]u8,
|
|
fba: std.heap.FixedBufferAllocator,
|
|
allocator: std.mem.Allocator,
|
|
capacity: usize = max_errors,
|
|
error_count: usize = 0,
|
|
|
|
const Self = @This();
|
|
|
|
pub const empty: Self = .{
|
|
// SAFETY: buffer is managed through addError below
|
|
.buffer = undefined,
|
|
// SAFETY: msg_buffer is managed through the FixedBufferAllocator
|
|
.msg_buffer = undefined,
|
|
// SAFETY: fba is established on first use of addError
|
|
.fba = undefined,
|
|
// SAFETY: allocator is established on first use of addError
|
|
.allocator = undefined,
|
|
};
|
|
|
|
pub fn diagnostics(self: *Self) Diagnostics {
|
|
return .{
|
|
.ptr = self,
|
|
.addErrorFn = addDiagnosticsError,
|
|
};
|
|
}
|
|
fn addDiagnosticsError(ptr: *anyopaque, err: ParseLineError) ParseError!void {
|
|
const self: *Self = @ptrCast(@alignCast(ptr));
|
|
try self.addError(err);
|
|
}
|
|
pub fn addError(self: *Self, err: ParseLineError) ParseError!void {
|
|
if (self.error_count == 0) {
|
|
self.fba = std.heap.FixedBufferAllocator.init(&self.msg_buffer);
|
|
self.allocator = self.fba.allocator();
|
|
}
|
|
|
|
if (self.error_count >= self.capacity) {
|
|
return ParseError.ParseFailed;
|
|
}
|
|
self.buffer[self.error_count] = err;
|
|
self.buffer[self.error_count].message = try self.allocator.dupe(u8, err.message);
|
|
self.error_count += 1;
|
|
}
|
|
pub fn errors(self: *const Self) []const ParseLineError {
|
|
return self.buffer[0..self.error_count];
|
|
}
|
|
};
|
|
}
|
|
|
|
pub const ParseError = error{
|
|
ParseFailed,
|
|
ReadFailed,
|
|
StreamTooLong,
|
|
OutOfMemory,
|
|
AllocationRequired,
|
|
EndOfStream,
|
|
};
|
|
|
|
const ValueWithMetaData = struct {
|
|
item_value: ?Value,
|
|
error_parsing: bool = false,
|
|
reader_advanced: bool = false,
|
|
length_prefix_detected_underflow: usize = 0,
|
|
};
|
|
/// A parsed SRF value. Each field in a record has a key and an optional `Value`.
|
|
pub const Value = union(enum) {
|
|
/// A numeric value, parsed from the `num` type hint.
|
|
number: f64,
|
|
|
|
/// Raw bytes decoded from base64, parsed from the `binary` type hint.
|
|
bytes: []const u8,
|
|
|
|
/// A string value, either delimiter-terminated or length-prefixed.
|
|
/// Not transformed during parsing (no escaping/unescaping), but will be
|
|
/// allocated if .alloc_strings = true is passed during parsing, or if
|
|
/// a multi-line string is found in the data
|
|
string: []const u8,
|
|
|
|
/// A boolean value, parsed from the `bool` type hint (`true` or `false`).
|
|
boolean: bool,
|
|
|
|
/// parses a single srf value, without the key. The the whole field is:
|
|
///
|
|
/// SRF Field: 'foo:3:bar'
|
|
///
|
|
/// The value we expect to be sent to this function is:
|
|
///
|
|
/// SRF Value: '3:bar'
|
|
///
|
|
/// The value is allowed to have extra data...for instance, in compact format
|
|
/// the value above can be represented by:
|
|
///
|
|
/// SRF Value: '3:bar,next_field::foobar'
|
|
///
|
|
/// and the next field will be ignored
|
|
///
|
|
/// This function may need to advance the reader in the case of multi-line
|
|
/// strings. It may also allocate data in the case of base64 (binary) values
|
|
/// as well as multi-line strings. Metadata is returned to assist in tracking
|
|
///
|
|
/// This function is intended to be used by the SRF parser
|
|
pub fn parse(str: []const u8, state: *RecordIterator.State, delimiter: u8) ParseError!ValueWithMetaData {
|
|
const type_val_sep_raw = std.mem.indexOfScalar(u8, str, ':');
|
|
if (type_val_sep_raw == null) {
|
|
try parseError("no type data or value after key", state);
|
|
return ParseError.ParseFailed;
|
|
}
|
|
|
|
const type_val_sep = type_val_sep_raw.?;
|
|
const metadata = str[0..type_val_sep];
|
|
const trimmed_meta = std.mem.trim(u8, metadata, &std.ascii.whitespace);
|
|
if (trimmed_meta.len == 0 or std.mem.eql(u8, "string", trimmed_meta)) {
|
|
// delimiter ended string
|
|
var it = std.mem.splitScalar(u8, str[type_val_sep + 1 ..], delimiter);
|
|
const val = it.first();
|
|
// we need to advance the column/partial_line_column of our parsing state
|
|
const total_chars = metadata.len + 1 + val.len;
|
|
state.column += total_chars;
|
|
state.partial_line_column += total_chars;
|
|
return .{
|
|
.item_value = .{ .string = try dupe(
|
|
state.*,
|
|
val,
|
|
.value,
|
|
) },
|
|
};
|
|
}
|
|
if (std.mem.eql(u8, "binary", trimmed_meta)) {
|
|
// binary is base64 encoded, so we need to decode it, but we don't
|
|
// risk delimiter collision, so we don't need a length for this
|
|
var it = std.mem.splitScalar(u8, str[type_val_sep + 1 ..], delimiter);
|
|
const val = it.first();
|
|
// we need to advance the column/partial_line_column of our parsing state
|
|
const total_chars = metadata.len + 1 + val.len;
|
|
state.column += total_chars;
|
|
state.partial_line_column += total_chars;
|
|
const Decoder = std.base64.standard.Decoder;
|
|
const size = Decoder.calcSizeForSlice(val) catch {
|
|
try parseError("error parsing base64 value", state);
|
|
return .{
|
|
.item_value = null,
|
|
.error_parsing = true,
|
|
};
|
|
};
|
|
const alloc = findAllocator(state.*, .value) orelse
|
|
try fallbackAllocatorFor(state);
|
|
const data = try alloc.alloc(u8, size);
|
|
errdefer alloc.free(data);
|
|
Decoder.decode(data, val) catch {
|
|
try parseError("error parsing base64 value", state);
|
|
alloc.free(data);
|
|
return .{
|
|
.item_value = null,
|
|
.error_parsing = true,
|
|
};
|
|
};
|
|
return .{
|
|
.item_value = .{ .bytes = data },
|
|
};
|
|
}
|
|
if (std.mem.eql(u8, "num", trimmed_meta)) {
|
|
var it = std.mem.splitScalar(u8, str[type_val_sep + 1 ..], delimiter);
|
|
const val = it.first();
|
|
// we need to advance the column/partial_line_column of our parsing state
|
|
const total_chars = metadata.len + 1 + val.len;
|
|
// log.debug("num total_chars: {d}", .{total_chars});
|
|
state.column += total_chars;
|
|
state.partial_line_column += total_chars;
|
|
const val_trimmed = std.mem.trim(u8, val, &std.ascii.whitespace);
|
|
const number = (if (state.options.strict_number_parsing)
|
|
std.fmt.parseFloat(@FieldType(Value, "number"), val_trimmed)
|
|
else
|
|
parseFloat(@FieldType(Value, "number"), val_trimmed)) catch {
|
|
try parseError("error parsing numeric value", state);
|
|
return .{
|
|
.item_value = null,
|
|
.error_parsing = true,
|
|
};
|
|
};
|
|
return .{
|
|
.item_value = .{ .number = number },
|
|
};
|
|
}
|
|
if (std.mem.eql(u8, "bool", trimmed_meta)) {
|
|
var it = std.mem.splitScalar(u8, str[type_val_sep + 1 ..], delimiter);
|
|
const val = it.first();
|
|
// we need to advance the column/partial_line_column of our parsing state
|
|
const total_chars = metadata.len + 1 + val.len;
|
|
state.column += total_chars;
|
|
state.partial_line_column += total_chars;
|
|
const val_trimmed = std.mem.trim(u8, val, &std.ascii.whitespace);
|
|
const boolean = blk: {
|
|
if (std.mem.eql(u8, "false", val_trimmed)) break :blk false;
|
|
if (std.mem.eql(u8, "true", val_trimmed)) break :blk true;
|
|
|
|
try parseError("error parsing boolean value", state);
|
|
return .{
|
|
.item_value = null,
|
|
.error_parsing = true,
|
|
};
|
|
};
|
|
return .{
|
|
.item_value = .{ .boolean = boolean },
|
|
};
|
|
}
|
|
if (std.mem.eql(u8, "null", trimmed_meta)) {
|
|
// we need to advance the column/partial_line_column of our parsing state
|
|
const total_chars = metadata.len + 1;
|
|
state.column += total_chars;
|
|
state.partial_line_column += total_chars;
|
|
return .{
|
|
.item_value = null,
|
|
};
|
|
}
|
|
// Last chance...the thing between these colons is a usize indicating
|
|
// the number of bytes to grab for a string. In case parseInt fails,
|
|
// we need to advance the position of our column counters
|
|
const total_metadata_chars = metadata.len + 1;
|
|
state.column += total_metadata_chars;
|
|
state.partial_line_column += total_metadata_chars;
|
|
const size = std.fmt.parseInt(usize, trimmed_meta, 0) catch {
|
|
log.debug("parseInt fail, trimmed_data: '{s}'", .{trimmed_meta});
|
|
try parseError("unrecognized metadata for key", state);
|
|
return .{
|
|
.item_value = null,
|
|
.error_parsing = true,
|
|
};
|
|
};
|
|
// Update again for number of bytes. All failures beyond this point are
|
|
// fatal, so this is safe.
|
|
state.column += size;
|
|
state.partial_line_column += size;
|
|
|
|
// If we are being asked specifically for bytes, we no longer care about
|
|
// delimiters. We just want raw bytes. This might adjust our line/column
|
|
// in the parse state
|
|
const rest_of_data = str[type_val_sep + 1 ..];
|
|
if (rest_of_data.len >= size) {
|
|
// We fit on this line, everything is "normal"
|
|
const val = rest_of_data[0..size];
|
|
var underflow: usize = 0;
|
|
if (val.len < rest_of_data.len) {
|
|
const past_val = rest_of_data[size..];
|
|
underflow = try checkShortPrefix(past_val, size, state, false);
|
|
}
|
|
return .{
|
|
.length_prefix_detected_underflow = underflow,
|
|
.item_value = .{ .string = try dupe(
|
|
state.*,
|
|
val,
|
|
.value,
|
|
) },
|
|
};
|
|
}
|
|
// This is not enough, we need more data from the reader
|
|
const alloc = findAllocator(state.*, .value) orelse
|
|
try fallbackAllocatorFor(state);
|
|
const buf = try alloc.alloc(u8, size);
|
|
errdefer alloc.free(buf);
|
|
@memcpy(buf[0..rest_of_data.len], rest_of_data);
|
|
// add back the newline we are skipping
|
|
buf[rest_of_data.len] = '\n';
|
|
// We won't do a parseError here. If we have an allocation error, read
|
|
// error, or end of stream, all of these are fatal. Our reader is currently
|
|
// past the newline, so we have to remove a character from size to account.
|
|
try state.reader.readSliceAll(buf[rest_of_data.len + 1 ..]);
|
|
// Because we've now advanced the line, we need to reset everything
|
|
state.line += std.mem.count(u8, buf, "\n");
|
|
state.column = buf.len - std.mem.lastIndexOf(u8, buf, "\n").?;
|
|
state.partial_line_column = state.column;
|
|
// However, we want to be past the end of the *next* newline too (in long
|
|
// format mode)
|
|
var long_format_ok = false;
|
|
if (delimiter == '\n' and state.reader.peekByte() catch 'x' == '\n') {
|
|
state.reader.toss(1);
|
|
state.line += 1;
|
|
long_format_ok = true; // in long format, the bytes ended directly on a newline, so we're good
|
|
}
|
|
// If we're short, we need to actually look ahead here to detect that
|
|
const val_to_eol = state.reader.peekDelimiterExclusive('\n') catch "";
|
|
// log.debug("multiline verification. val_to_eol: '{s}', val: '{s}'", .{ val_to_eol, buf });
|
|
var underflow: usize = 0;
|
|
if (!long_format_ok and val_to_eol.len > 0)
|
|
underflow = try checkShortPrefix(val_to_eol, size, state, true);
|
|
return .{
|
|
.length_prefix_detected_underflow = underflow,
|
|
.item_value = .{ .string = buf },
|
|
.reader_advanced = true,
|
|
};
|
|
}
|
|
inline fn checkShortPrefix(past_val: []const u8, declared_size: usize, state: *RecordIterator.State, advance_reader: bool) ParseError!usize {
|
|
// there is a concern now the length specified might be short,
|
|
// we need to check. If this is compact mode, the next byte should
|
|
// be a comma. If in long mode, we could have whitespace + comment
|
|
var buf: [256]u8 = undefined;
|
|
if (state.field_delimiter == '\n') {
|
|
const rest = std.mem.trimStart(u8, past_val, std.ascii.whitespace[0..]);
|
|
if (rest.len > 0 and rest[0] != '#') {
|
|
const msg = std.fmt.bufPrint(
|
|
&buf,
|
|
"line has {} additional bytes after field value. Perhaps length should be restated as {}?",
|
|
.{ past_val.len, past_val.len + declared_size },
|
|
) catch "line has additional bytes after field value. Perhaps length should be restated";
|
|
try parseError(msg, state);
|
|
// Kill the rest of this line
|
|
if (advance_reader) {
|
|
_ = try state.reader.takeDelimiterExclusive('\n');
|
|
return 0;
|
|
}
|
|
return past_val.len;
|
|
}
|
|
} else if (past_val[0] != state.field_delimiter) {
|
|
// compact form
|
|
const extra_bytes = std.mem.indexOfScalar(u8, past_val, state.field_delimiter) orelse past_val.len;
|
|
const msg = std.fmt.bufPrint(
|
|
&buf,
|
|
"field value has {} additional bytes. Perhaps length should be restated as {}?",
|
|
.{ extra_bytes, extra_bytes + declared_size },
|
|
) catch "field value has additional bytes. Perhaps length should be restated";
|
|
// we can try to advance the reader to the next field, but we might be cooked
|
|
try parseError(msg, state);
|
|
return std.mem.indexOfScalar(u8, past_val, state.field_delimiter) orelse past_val.len;
|
|
}
|
|
return 0;
|
|
}
|
|
inline fn fallbackAllocatorFor(state: *RecordIterator.State) !std.mem.Allocator {
|
|
if (state.fallback_arena) |f| return f.allocator();
|
|
if (state.options.parse_allocator == .none) return error.AllocationRequired;
|
|
state.fallback_arena = try state.allocator.create(std.heap.ArenaAllocator);
|
|
state.fallback_arena.?.* = .init(state.allocator);
|
|
return state.fallback_arena.?.allocator();
|
|
}
|
|
};
|
|
|
|
fn parseFloat(comptime T: type, value: []const u8) !T {
|
|
if (std.fmt.parseFloat(T, value)) |f| {
|
|
// clean parse
|
|
return f;
|
|
} else |_| {} // error
|
|
|
|
if (@typeInfo(T) != .float) {
|
|
@compileError("Cannot parse a float into a non-floating point type.");
|
|
}
|
|
// Need a temporary buffer. The maximum number of characters in our float
|
|
// can be calculated. This is apparently out of all the text from
|
|
// https://dl.acm.org/doi/epdf/10.1145/93542.93557
|
|
// and
|
|
// https://dl.acm.org/doi/epdf/10.1145/93548.93559
|
|
// and boils down to:
|
|
// 1 + ceil(p * log10(2))
|
|
//
|
|
// This is a little relevant here because we don't know the exact type,
|
|
// even though it's almost certainly f64 (look at the Value struct)
|
|
const buf_len: usize = 1 + @trunc(std.math.ceil(@as(f64, @typeInfo(T).float.bits) * @log10(@as(f64, 2))));
|
|
var buffer: [buf_len]u8 = undefined;
|
|
var val_inx: usize = 0;
|
|
var buf_inx: usize = 0;
|
|
var state: enum { start, middle, end } = .start;
|
|
|
|
// We need to "clean up" the input here
|
|
while (val_inx < value.len) {
|
|
const c = value[val_inx];
|
|
switch (state) {
|
|
.start => {
|
|
if (isNumberIsh(c, false)) {
|
|
state = .middle;
|
|
// we don't increment val_inx here because we need to add to the buffer...
|
|
continue;
|
|
}
|
|
// We need to have at least one more character in the string
|
|
if (val_inx + 1 >= value.len) return error.InvalidCharacter;
|
|
|
|
if (leadingCurrency(value[val_inx..])) |curr| {
|
|
val_inx += curr.len;
|
|
while (val_inx < value.len and value[val_inx] == ' ') val_inx += 1;
|
|
state = .middle;
|
|
continue;
|
|
}
|
|
return error.InvalidCharacter;
|
|
},
|
|
.middle => {
|
|
if (!isNumberIsh(c, true)) {
|
|
val_inx += 1;
|
|
state = .end;
|
|
continue;
|
|
}
|
|
// add to our buffer if it's not a comma. We aren't dealing
|
|
// with comma/period locale semantics
|
|
if (c == ',') {
|
|
val_inx += 1;
|
|
continue;
|
|
}
|
|
buffer[buf_inx] = c;
|
|
buf_inx += 1;
|
|
val_inx += 1;
|
|
},
|
|
.end => {
|
|
if (value[val_inx] == ' ' and val_inx < value.len + 1) {
|
|
// we don't allow trailing spaces
|
|
val_inx += 1;
|
|
continue;
|
|
}
|
|
if (leadingCurrency(value[val_inx..])) |curr| {
|
|
// we are ok to end with a currency, but nothing else
|
|
if (val_inx + curr.len == value.len) break;
|
|
}
|
|
return error.InvalidCharacter;
|
|
},
|
|
}
|
|
}
|
|
return std.fmt.parseFloat(T, buffer[0..buf_inx]);
|
|
}
|
|
fn isNumberIsh(ch: u8, in_middle: bool) bool {
|
|
if (ch >= '0' and ch <= '9')
|
|
return true;
|
|
if (ch == '-' or ch == '+')
|
|
return true;
|
|
if (in_middle and (ch == '.' or ch == ','))
|
|
return true; // we will allow . and , and allow ordering to the caller
|
|
return false;
|
|
}
|
|
fn leadingCurrency(s: []const u8) ?[]const u8 {
|
|
// Check known single character currency symbols
|
|
const single_byte_currencies = "$KLPQR";
|
|
for (single_byte_currencies) |curr|
|
|
if (s[0] == curr)
|
|
return s[0..1];
|
|
const two_byte_currencies = "£¤¥֏";
|
|
var i: usize = 0;
|
|
while (i < two_byte_currencies.len - 1) : (i += 2) {
|
|
if (two_byte_currencies[i] == s[0] and
|
|
two_byte_currencies[i + 1] == s[1])
|
|
return s[0..2];
|
|
}
|
|
const three_byte_currencies = "৳฿៛₡₦₧₩₪₫€₭₮₱₲₴₸₹₺₼₽₾⃀";
|
|
i = 0;
|
|
while (i < three_byte_currencies.len - 2) : (i += 3) {
|
|
if (three_byte_currencies[i] == s[0] and
|
|
three_byte_currencies[i + 1] == s[1] and
|
|
three_byte_currencies[i + 2] == s[2])
|
|
return s[0..3];
|
|
}
|
|
return null;
|
|
}
|
|
|
|
/// A single key-value pair within a record. The key is always a string.
|
|
/// The value may be `null` (from the `null` type hint) or one of the
|
|
/// `Value` variants. Yielded by `RecordIterator.FieldIterator.next`.
|
|
pub const Field = struct {
|
|
key: []const u8,
|
|
value: ?Value,
|
|
/// raw value is a borrowed value (not parsed into numerics or bools, and
|
|
/// NOT duplicated into a caller-owned buffer). For example:
|
|
///
|
|
/// height:num:55.2
|
|
///
|
|
/// Has a value of .numeric = 55.2, and a raw value "55.2"
|
|
///
|
|
/// It is only set if:
|
|
///
|
|
/// * include_raw is set to true in ParseOptions (default is false)
|
|
/// * the reader was not advanced during parsing
|
|
/// * the input was TRANSFORMED/Parsed. i.e., the value is NOT a string
|
|
///
|
|
/// It's primary use is to enable surgical editing of srf data, so a caller
|
|
/// using this can assume a null value means "cannot edit in place", or
|
|
/// the data is a string.
|
|
/// Strings are out of scope for raw, because that data is already included
|
|
/// in the value, and string parsing is non-trivial due to the possibility
|
|
/// of length-prefixed strings
|
|
raw: ?[]const u8,
|
|
};
|
|
|
|
/// Options for type coercion
|
|
pub const CoercionOptions = struct {
|
|
/// Coerce strings to numbers. This is false for the same reason
|
|
/// that lienient number conversion ParseOptions is off by default.
|
|
/// This library is intended for performant access for cache use cases.
|
|
/// It happens to be a relatively simple format for humans to grok,
|
|
/// so if you want to use this for human-edited files, turn this on
|
|
strings_to_numbers: bool = false,
|
|
};
|
|
|
|
pub fn CoercionResult(T: type) type {
|
|
return struct {
|
|
value: T,
|
|
|
|
/// Set this to true if the original value has been consumed and
|
|
/// is no longer needed. For example the string "89" coerced to int
|
|
/// should be freed by the caller
|
|
require_free_original: bool = false,
|
|
|
|
const Self = @This();
|
|
/// Returns a normal struct - no free required
|
|
pub fn init(value: T) Self {
|
|
return .{ .value = value };
|
|
}
|
|
/// Returns a struct requiring the caller to free original memory
|
|
pub fn initFree(value: T) Self {
|
|
return .{ .value = value, .require_free_original = true };
|
|
}
|
|
};
|
|
}
|
|
|
|
fn coerce(name: []const u8, comptime T: type, val: ?Value, options: CoercionOptions) !CoercionResult(T) {
|
|
const ti = @typeInfo(T);
|
|
if (val == null and ti != .optional)
|
|
return error.NullValueCannotBeAssignedToNonNullField;
|
|
|
|
// []const u8 is classified as a pointer
|
|
switch (ti) {
|
|
.optional => |o| if (val) |_| {
|
|
const child = try coerce(name, o.child, val, options);
|
|
return .{ .value = child.value, .require_free_original = child.require_free_original };
|
|
} else return .{ .value = null },
|
|
.pointer => |p| {
|
|
// We don't have an allocator, so the only thing we can do
|
|
// here is manage []const u8 or []u8
|
|
if (p.size != .slice or p.child != u8)
|
|
return error.CoercionNotPossible;
|
|
if (val.? != .string and val.? != .bytes)
|
|
return error.CoercionNotPossible;
|
|
if (val.? == .string)
|
|
return .init(val.?.string);
|
|
return .init(val.?.bytes);
|
|
},
|
|
.type, .void, .noreturn => return error.CoercionNotPossible,
|
|
.comptime_float, .comptime_int, .undefined, .null, .error_union => return error.CoercionNotPossible,
|
|
.error_set, .@"fn", .@"opaque", .frame => return error.CoercionNotPossible,
|
|
.@"anyframe", .vector, .enum_literal => return error.CoercionNotPossible,
|
|
.int => {
|
|
if (options.strings_to_numbers and val.? == .string) {
|
|
// At this point, we're in lienent mode, so we'll do our lienient
|
|
// parse, then truncate it to the int the user wants
|
|
const float = try parseFloat(f64, val.?.string);
|
|
return .initFree(@as(T, @trunc(float)));
|
|
}
|
|
if (val.? != .number) return error.IntNotNumberType;
|
|
return .init(@as(T, @intFromFloat(val.?.number)));
|
|
},
|
|
.float => {
|
|
if (options.strings_to_numbers and val.? == .string) {
|
|
return .initFree(try parseFloat(T, val.?.string));
|
|
}
|
|
if (val.? != .number) return error.FloatNotNumberType;
|
|
return .init(@as(T, @floatCast(val.?.number)));
|
|
},
|
|
.bool => return switch (val.?) {
|
|
.boolean => |b| .init(b),
|
|
.string => |s| if (std.mem.eql(u8, "true", s))
|
|
.initFree(true)
|
|
else if (std.mem.eql(u8, "false", s))
|
|
.initFree(false)
|
|
else
|
|
return error.StringValueOfBooleanMustBetrueOrfalse,
|
|
else => return error.BooleanNotBooleanOrString,
|
|
},
|
|
.@"enum" => {
|
|
if (val.? != .string) return error.EnumValueNotString;
|
|
return .initFree(std.meta.stringToEnum(T, val.?.string) orelse return error.StringValueNotValidEnumMember);
|
|
},
|
|
.array => return error.NotImplemented,
|
|
.@"struct", .@"union" => {
|
|
if (std.meta.hasMethod(T, "srfParse")) {
|
|
if (val.? == .string)
|
|
return T.srfParse(val.?.string) catch |e| {
|
|
log.err(
|
|
"custom parse of value {s} failed : {}",
|
|
.{ val.?.string, e },
|
|
);
|
|
return error.CustomParseFailed;
|
|
};
|
|
}
|
|
return error.CoercionNotPossible;
|
|
},
|
|
}
|
|
return null;
|
|
}
|
|
|
|
pub const Tombstone = struct {
|
|
buf: [256]u8,
|
|
reason_len: ?usize,
|
|
until: i64,
|
|
|
|
pub fn isForever(self: *const Tombstone) bool {
|
|
return self.until == std.math.maxInt(i64);
|
|
}
|
|
|
|
pub fn reason(self: *const Tombstone) ?[]const u8 {
|
|
if (self.reason_len) |l|
|
|
return self.buf[0..l];
|
|
return null;
|
|
}
|
|
|
|
pub fn parse(val: []const u8) !Tombstone {
|
|
var tomb_it = std.mem.splitScalar(u8, val, ' ');
|
|
const until_str = tomb_it.next().?;
|
|
const reason_str = tomb_it.rest();
|
|
|
|
const until = if (std.mem.eql(u8, until_str, "forever"))
|
|
std.math.maxInt(i64)
|
|
else
|
|
std.fmt.parseInt(i64, until_str, 10) catch return ParseError.ParseFailed;
|
|
|
|
var rc: Tombstone = .{
|
|
// SAFETY: this is the reason buffer to avoid allocation, and is
|
|
// set below, gated by the reason_len
|
|
.buf = undefined,
|
|
.until = until,
|
|
.reason_len = null,
|
|
};
|
|
if (reason_str.len > 0) {
|
|
if (reason_str.len > 256) return ParseError.ParseFailed;
|
|
@memcpy(rc.buf[0..reason_str.len], reason_str);
|
|
rc.reason_len = reason_str.len;
|
|
}
|
|
return rc;
|
|
}
|
|
};
|
|
/// A streaming record iterator for parsing SRF data. This is the preferred
|
|
/// parsing API because it avoids collecting all records and fields into memory
|
|
/// at once. Created by calling `iterator`.
|
|
///
|
|
/// Each call to `next` yields a `FieldIterator` for one record. Fields within
|
|
/// that record are consumed lazily via `FieldIterator.next` or coerced directly
|
|
/// into a Zig type via `FieldIterator.to`. All allocations go through an
|
|
/// internal arena; call `deinit` to release everything when done.
|
|
///
|
|
/// If `RecordIterator.next` is called before the previous `FieldIterator` has
|
|
/// been fully consumed, the remaining fields are automatically drained to keep
|
|
/// the parser state consistent.
|
|
pub const RecordIterator = struct {
|
|
arena: *std.heap.ArenaAllocator,
|
|
/// optional expiry time for the data. Useful for caching
|
|
/// Note that on a parse, data will always be returned and it will be up
|
|
/// to the caller to check is_fresh and determine the right thing to do
|
|
expires: ?i64,
|
|
|
|
/// optional created time for the data. This library does nothing with
|
|
/// this data, but will be tracked and available immediately after calling
|
|
/// `iterator` if needed/provided
|
|
created: ?i64,
|
|
|
|
/// optional modified time for the data. This library does nothing with
|
|
/// this data, but will be tracked and available immediately after calling
|
|
/// `iterator` if needed/provided
|
|
modified: ?i64,
|
|
|
|
/// optional tombstone time for the data. This indicates that the data
|
|
/// is not valid at all. We assume that the data, when written, was
|
|
/// ABSENT. By providing a date here, we allow for a mechanism for a far
|
|
/// future retry, in effect saying "don't constantly retry, but go ahead
|
|
/// and retry a month from now"
|
|
tombstoned: ?Tombstone,
|
|
|
|
state: *State,
|
|
|
|
pub const State = struct {
|
|
line: usize = 0,
|
|
column: usize = 0,
|
|
partial_line_column: usize = 0,
|
|
reader: *std.Io.Reader,
|
|
options: ParseOptions,
|
|
|
|
require_eof: bool = false,
|
|
eof_found: bool = false,
|
|
current_line: ?[]const u8,
|
|
|
|
field_delimiter: u8 = ',',
|
|
end_of_record_reached: bool = false,
|
|
field_iterator: ?FieldIterator = null,
|
|
|
|
aa: std.mem.Allocator,
|
|
allocator: std.mem.Allocator,
|
|
fallback_arena: ?*std.heap.ArenaAllocator = null,
|
|
|
|
detailed_output: bool = false,
|
|
/// Takes the next line, trimming leading whitespace and ignoring comments
|
|
/// Directives (comments starting with #!) are preserved
|
|
pub fn nextLine(state: *State) ?[]const u8 {
|
|
while (true) {
|
|
state.line += 1;
|
|
state.column = 1; // column is human indexed (one-based)
|
|
state.partial_line_column = 0; // partial_line_column is zero indexed for computers
|
|
const raw_line = (state.reader.takeDelimiter('\n') catch return null) orelse return null;
|
|
// we don't want to trim the end, as there might be a key/value field
|
|
// with a string including important trailing whitespace
|
|
const trimmed_line = std.mem.trimStart(u8, raw_line, &std.ascii.whitespace);
|
|
state.column += raw_line.len - trimmed_line.len;
|
|
if (std.mem.startsWith(u8, trimmed_line, "#") and !std.mem.startsWith(u8, trimmed_line, "#!")) continue;
|
|
return trimmed_line;
|
|
}
|
|
}
|
|
pub fn format(self: State, writer: *std.Io.Writer) std.Io.Writer.Error!void {
|
|
if (!self.detailed_output) {
|
|
try writer.print("line: {}, col: {}", .{ self.line, self.column });
|
|
return;
|
|
}
|
|
|
|
try writer.print("\t line: {}\n", .{self.line});
|
|
try writer.print("\t column: {}\n", .{self.column});
|
|
try writer.print("\t partial_line_column: {}\n", .{self.partial_line_column});
|
|
// try writer.print("reader: {}\n", .{self.reader});
|
|
// try writer.print("options: {}\n", .{self.options});
|
|
|
|
try writer.print("\t require_eof: {}\n", .{self.require_eof});
|
|
try writer.print("\t eof_found: {}\n", .{self.eof_found});
|
|
try writer.print("\t current_line: {?s}\n", .{self.current_line});
|
|
|
|
try writer.print("\t field_delimiter: 0x{x}\n", .{self.field_delimiter});
|
|
try writer.print("\t long_form (field_delimeter \\n): {}\n", .{self.field_delimiter == '\n'});
|
|
try writer.print("\t end_of_record_reached: {}\n", .{self.end_of_record_reached});
|
|
// try writer.print("field_iterator: {}\n", .{self.field_iterator});
|
|
|
|
// try writer.print("aa: {}\n", .{self.aa});
|
|
// try writer.print("allocator: {}\n", .{self.allocator});
|
|
// try writer.print("fallback_arena: {}\n", .{self.fallback_arena});
|
|
}
|
|
};
|
|
|
|
/// Advances to the next record in the stream, returning a `FieldIterator`
|
|
/// for accessing its fields. Returns `null` when all records have been
|
|
/// consumed.
|
|
///
|
|
/// If the previous `FieldIterator` was not fully drained, its remaining
|
|
/// fields are consumed automatically to keep the reader positioned
|
|
/// correctly. It is safe (but unnecessary) to fully consume the
|
|
/// `FieldIterator` before calling `next` again.
|
|
///
|
|
/// Note that all state is stored in a shared area accessible to both
|
|
/// the `RecordIterator` and the `FieldIterator`, so there is no need to
|
|
/// store the return value as a variable
|
|
pub fn next(self: RecordIterator) !?FieldIterator {
|
|
const state = self.state;
|
|
if (state.field_iterator) |f| {
|
|
// We need to finish the fields on the previous record
|
|
while (try f.next()) |_| {}
|
|
state.field_iterator = null;
|
|
}
|
|
if (state.current_line == null) {
|
|
if (state.options.diagnostics) |d|
|
|
if (d.has_errors) return ParseError.ParseFailed;
|
|
if (state.require_eof and !state.eof_found) return ParseError.ParseFailed;
|
|
return null;
|
|
}
|
|
while (std.mem.trim(u8, state.current_line.?, &std.ascii.whitespace).len == 0) {
|
|
// empty lines can be signficant (to indicate a new record, but only once
|
|
// a record is processed, which requires data first. That record processing
|
|
// is at the bottom of the loop, so if an empty line is detected here, we can
|
|
// safely ignore it
|
|
state.current_line = state.nextLine();
|
|
// by calling recursively we get the error handling above
|
|
if (state.current_line == null) return self.next();
|
|
}
|
|
// non-blank line, but we could have an eof marker
|
|
if (try Directive.parse(state.current_line.?, state)) |d| {
|
|
switch (d) {
|
|
.eof => {
|
|
// there needs to be an eof then
|
|
if (state.nextLine()) |_| {
|
|
try parseError("Data found after #!eof", state);
|
|
return ParseError.ParseFailed; // this is terminal
|
|
} else {
|
|
state.eof_found = true;
|
|
state.current_line = null;
|
|
return null; // all is good, we're done
|
|
}
|
|
},
|
|
else => {
|
|
try parseError("Directive found after data started", state);
|
|
state.current_line = state.nextLine();
|
|
// TODO: This runs the risk of a malicious file creating
|
|
// a stackoverflow by using many non-eof directives
|
|
return self.next();
|
|
},
|
|
}
|
|
}
|
|
state.end_of_record_reached = false;
|
|
state.field_iterator = .{
|
|
.state = self.state,
|
|
.arena = self.arena,
|
|
};
|
|
return state.field_iterator.?;
|
|
}
|
|
|
|
/// Iterates over the fields within a single record. Yielded by
|
|
/// `RecordIterator.next`. Each call to `next` returns the next `Field`
|
|
/// in the record, or `null` when the record boundary is reached.
|
|
///
|
|
/// For direct type coercion without manually iterating fields, use `to`.
|
|
pub const FieldIterator = struct {
|
|
state: *State,
|
|
arena: *std.heap.ArenaAllocator,
|
|
|
|
/// Returns the next `Field` in the current record, or `null` when
|
|
/// the record boundary has been reached. After `null` is returned,
|
|
/// subsequent calls continue to return `null`.
|
|
pub fn next(self: FieldIterator) !?Field {
|
|
const state = self.state;
|
|
// Main parsing. We already have the first line of data, which could
|
|
// be a record (compact format) or a key/value pair (long format)
|
|
|
|
if (state.current_line == null) return null;
|
|
if (state.end_of_record_reached) return null;
|
|
// non-blank line, but we could have an eof marker
|
|
// TODO: deduplicate this code
|
|
if (try Directive.parse(state.current_line.?, state)) |d| {
|
|
switch (d) {
|
|
.eof => {
|
|
// there needs to be an eof then
|
|
if (state.nextLine()) |_| {
|
|
try parseError("Data found after #!eof", state);
|
|
return ParseError.ParseFailed; // this is terminal
|
|
} else {
|
|
state.eof_found = true;
|
|
state.current_line = null;
|
|
return null; // all is good, we're done
|
|
}
|
|
},
|
|
else => {
|
|
try parseError("Directive found after data started", state);
|
|
state.current_line = state.nextLine();
|
|
// TODO: This runs the risk of a malicious file creating
|
|
// a stackoverflow by using many non-eof directives
|
|
return self.next();
|
|
},
|
|
}
|
|
}
|
|
|
|
// Whatever the format, the beginning will always be the key data
|
|
// key:stuff:value
|
|
var it = std.mem.splitScalar(u8, state.current_line.?, ':');
|
|
const key = it.next().?; // first one we get for free
|
|
if (key.len > 0) std.debug.assert(key[0] != state.field_delimiter);
|
|
state.column += key.len + 1;
|
|
state.partial_line_column += key.len + 1;
|
|
const rest = it.rest();
|
|
const value = try Value.parse(
|
|
rest,
|
|
state,
|
|
state.field_delimiter,
|
|
);
|
|
// Get raw untouched bytes - useful for surgical editing. We only
|
|
// provide this value if the scenario is simple and safe
|
|
// (binary looks like string, but is safe here because it's never
|
|
// length prefixed, and it goes through a transformation process,
|
|
// so we actually want to set raw in this case)
|
|
const raw = if (!state.options.include_raw or
|
|
value.error_parsing or
|
|
value.reader_advanced or
|
|
value.item_value == null or
|
|
value.item_value.? == .string) null else blk: {
|
|
// Basic parse to find raw value
|
|
const sep = std.mem.indexOfScalar(u8, rest, ':') orelse break :blk null;
|
|
const after = rest[sep + 1 ..];
|
|
const end = std.mem.indexOfScalar(u8, after, state.field_delimiter) orelse after.len;
|
|
break :blk after[0..end];
|
|
};
|
|
|
|
var field: ?Field = null;
|
|
if (!value.error_parsing) {
|
|
field = .{
|
|
.key = try dupe(state.*, key, .key),
|
|
.value = value.item_value,
|
|
.raw = raw,
|
|
};
|
|
}
|
|
|
|
if (value.reader_advanced and state.field_delimiter == ',') {
|
|
// log.debug("advanced", .{});
|
|
// In compact format we'll stay on the same line
|
|
const real_column = state.column;
|
|
state.current_line = state.nextLine();
|
|
// Reset line and column position, because we're actually staying on the same line now
|
|
state.line -= 1;
|
|
state.column = real_column + 1;
|
|
state.partial_line_column = 0;
|
|
}
|
|
|
|
// The difference between compact and line here is that compact we will instead of
|
|
// line = try nextLine, we will do something like line = line[42..]
|
|
if (state.field_delimiter == '\n') {
|
|
state.current_line = state.nextLine();
|
|
if (state.current_line == null) {
|
|
state.end_of_record_reached = true;
|
|
return field;
|
|
}
|
|
// close out record, return
|
|
if (state.current_line.?.len == 0) {
|
|
// End of record
|
|
state.end_of_record_reached = true;
|
|
state.current_line = state.nextLine();
|
|
return field;
|
|
}
|
|
} else {
|
|
// We should be on a delimiter, otherwise, we should be at the end
|
|
if (state.current_line == null) {
|
|
try parseError("current line is null", state);
|
|
return error.ParseFailed; // fatal
|
|
}
|
|
// This usually means I screwed up and advanced the reader in value.parse...
|
|
std.debug.assert(state.partial_line_column <= state.current_line.?.len);
|
|
state.current_line = state.current_line.?[state.partial_line_column..]; // can't use l here because line may have been reassigned
|
|
state.partial_line_column = 0;
|
|
if (state.current_line.?.len == 0) {
|
|
// close out record
|
|
state.current_line = state.nextLine();
|
|
state.partial_line_column = 0;
|
|
state.end_of_record_reached = true;
|
|
return field;
|
|
} else {
|
|
if (state.current_line.?[0] != state.field_delimiter) {
|
|
if (value.length_prefix_detected_underflow > 0) {
|
|
// We're going to try to recover from this situation,
|
|
// but we're dealing with invalid srf at this point,
|
|
// so no guarantees. Note that our line/column counter
|
|
// is already advanced in this situation, as this
|
|
// branch should only trigger in compact mode when
|
|
// length is short
|
|
|
|
log.info(
|
|
"attempting recovery from invalid srf (length prefix underflow of {d}). check diagnostics for more information",
|
|
.{value.length_prefix_detected_underflow},
|
|
);
|
|
state.current_line = if (std.mem.indexOfScalar(u8, state.current_line.?, state.field_delimiter)) |i|
|
|
state.current_line.?[i + 1 ..]
|
|
else
|
|
// Nothing on this line to resync to. The diagnostic is already recorded, so
|
|
// close the record out and let the next one be judged on its own merits.
|
|
null;
|
|
if (state.current_line == null) {
|
|
// close out record
|
|
state.current_line = state.nextLine();
|
|
state.partial_line_column = 0;
|
|
state.end_of_record_reached = true;
|
|
return field;
|
|
}
|
|
}
|
|
} else {
|
|
state.current_line = state.current_line.?[1..];
|
|
}
|
|
}
|
|
}
|
|
return field;
|
|
}
|
|
|
|
/// Coerce a record's remaining fields into a value of type T. Walks the
|
|
/// parsed fields, matches them against T's field set at comptime,
|
|
/// and dispatches each value through `coerce()`.
|
|
///
|
|
/// For structs, fields are matched by name. Only the first occurrence
|
|
/// of each field name is used; duplicates are ignored. Fields in `T`
|
|
/// that have default values and are not present in the data use those
|
|
/// defaults. Missing fields without defaults return an error.
|
|
///
|
|
/// For tagged unions, the active tag field must appear first in the
|
|
/// stream. The tag field name defaults to `"type"` or `T.srf_tag_field` if
|
|
/// declared.
|
|
///
|
|
/// **Performance note.** `to(T)` is a generalized coercer. It
|
|
/// works correctly for any struct or tagged union, but the
|
|
/// per-field abstraction overhead — the `coerce()` call boundary,
|
|
/// bookkeeping, etc, imposes a fixed cost per parsed field that the
|
|
/// optimizer can't fully eliminate. On a 7-field record in
|
|
/// ReleaseFast, that cost is roughly 320 ns/record on an i9-14900K.
|
|
///
|
|
/// For most callers, this should be fine. If, however, you are
|
|
/// parsing millions of records of a single fixed-shape type, and
|
|
/// parse speed dominates, a hand-written coercer
|
|
/// specialized for that T can be 25-50x faster than `to()`.
|
|
/// Here is an example using stock market candle data:
|
|
///
|
|
/// ```zig
|
|
/// fn coerceCandle(fields: srf.RecordIterator.FieldIterator) !Candle {
|
|
/// var c: Candle = .{ .date = ..., .open = 0, ..., .volume = 0 };
|
|
/// while (try fields.next()) |f| {
|
|
/// const val = f.value orelse continue;
|
|
/// // Switch on a unique prefix of the field name.
|
|
/// // For Candle, the first byte alone disambiguates.
|
|
/// switch (f.key[0]) {
|
|
/// 'd' => if (val == .string) c.date = try Date.parse(val.string),
|
|
/// 'o' => if (val == .number) c.open = val.number,
|
|
/// 'h' => if (val == .number) c.high = val.number,
|
|
/// 'l' => if (val == .number) c.low = val.number,
|
|
/// 'c' => if (val == .number) c.close = val.number,
|
|
/// 'a' => if (val == .number) c.adj_close = val.number,
|
|
/// 'v' => if (val == .number) c.volume = @intFromFloat(val.number),
|
|
/// else => {},
|
|
/// }
|
|
/// }
|
|
/// return c;
|
|
/// }
|
|
/// ```
|
|
///
|
|
/// The speedup comes from skipping the framework cost — direct
|
|
/// struct assignment, no `coerce()` call, no found-bitmap, no
|
|
/// per-field bookkeeping. The trade-off is that you give up
|
|
/// `to()`'s correctness guarantees for arbitrary T (default
|
|
/// values, missing-field detection, custom-parse hooks) and
|
|
/// take responsibility for those yourself. You will notice in the
|
|
/// example above, a key of "dinglebat" would happily be parsed as
|
|
/// candle date. ;) to() would appropriately skip this record
|
|
///
|
|
/// Use `to()` unless profiling shows it's a bottleneck.
|
|
pub fn to(self: FieldIterator, comptime T: type, options: CoercionOptions) !T {
|
|
const ti = @typeInfo(T);
|
|
|
|
switch (ti) {
|
|
.@"struct" => {
|
|
// What is this magic? The FieldEnum creates a type (an enum)
|
|
// where each enum member has the name of a field in the struct
|
|
//
|
|
// So... struct { a: u8, b: u8 } will yield enum { a, b }
|
|
const FieldEnum = std.meta.FieldEnum(T);
|
|
// Then...EnumFieldStruct will create a struct from this, where
|
|
// each enum value becomes a field. We will specify the field
|
|
// type, and the default value. Combining these two calls gets
|
|
// us a struct with all the same field names, but we get a chance
|
|
// to make all the fields boolean, so we can use it to track
|
|
// which fields have been set
|
|
var found: std.enums.EnumFieldStruct(FieldEnum, bool, false) = .{};
|
|
// SAFETY: all fields updated below or error is returned
|
|
var obj: T = undefined;
|
|
|
|
while (try self.next()) |f| {
|
|
// Linear scan fallback
|
|
var field_match = false;
|
|
inline for (std.meta.fields(T)) |type_field| {
|
|
// To replicate the behavior of the record version of to,
|
|
// we need to only take the first version of the field,
|
|
// so if it's specified twice in the data, we will ignore
|
|
// all but the first instance
|
|
if (!field_match and std.mem.eql(u8, type_field.name, f.key) and
|
|
!@field(found, type_field.name))
|
|
{
|
|
field_match = true;
|
|
|
|
const result = try coerce(type_field.name, type_field.type, f.value, options);
|
|
@field(obj, type_field.name) = result.value;
|
|
// Now account for this in our magic found struct...
|
|
@field(found, type_field.name) = true;
|
|
if (result.require_free_original) {
|
|
if (findAllocator(self.state.*, .value)) |alloc|
|
|
switch (f.value.?) {
|
|
.string => |s| alloc.free(s),
|
|
.bytes => |b| alloc.free(b),
|
|
else => unreachable,
|
|
};
|
|
// This is not actually true. Without specific
|
|
// allocator, the fallback arena will be used, so we
|
|
// don't really need to free anything in this
|
|
// circumstance
|
|
// else return error.AllocatorRequired;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
// Fill in the defaults for remaining fields. Throw if anything
|
|
// is missing both value (from above) and default (from here)
|
|
inline for (std.meta.fields(T)) |type_field| {
|
|
if (!@field(found, type_field.name)) {
|
|
// We did not find this field above...revert to default value
|
|
if (type_field.default_value_ptr) |ptr| {
|
|
@field(obj, type_field.name) = @as(*const type_field.type, @ptrCast(@alignCast(ptr))).*;
|
|
} else {
|
|
log.debug("Record could not be coerced. Field {s} not found on srf data, and no default value exists on the type", .{type_field.name});
|
|
return error.FieldNotFoundOnFieldWithoutDefaultValue;
|
|
}
|
|
}
|
|
}
|
|
return obj;
|
|
},
|
|
.@"union" => {
|
|
const active_tag_name = if (@hasDecl(T, "srf_tag_field"))
|
|
T.srf_tag_field
|
|
else
|
|
"type";
|
|
const first_try = try self.next();
|
|
if (first_try == null) return error.ActiveTagFieldNotFound;
|
|
const f = first_try.?;
|
|
if (!std.mem.eql(u8, f.key, active_tag_name))
|
|
return error.ActiveTagNotFirstField; // required here, but not on the Record version of to
|
|
if (f.value == null or f.value.? != .string)
|
|
return error.ActiveTagValueMustBeAString;
|
|
const active_tag = f.value.?.string;
|
|
// We're done with the tag. Need to free it in case a custom
|
|
// allocator is in use
|
|
defer if (findAllocator(self.state.*, .value)) |a| a.free(active_tag);
|
|
inline for (std.meta.fields(T)) |field_type| {
|
|
if (std.mem.eql(u8, active_tag, field_type.name)) {
|
|
return @unionInit(T, field_type.name, try self.to(field_type.type, options));
|
|
}
|
|
}
|
|
return error.ActiveTagDoesNotExist;
|
|
},
|
|
else => @compileError("Deserialization not supported on " ++ @tagName(ti) ++ " types"),
|
|
}
|
|
return error.CoercionNotPossible;
|
|
}
|
|
test to {
|
|
// Example: coerce fields directly into a Zig struct from the iterator
|
|
const Data = struct {
|
|
name: []const u8,
|
|
score: u8,
|
|
active: bool = true,
|
|
};
|
|
const data =
|
|
\\#!srfv1
|
|
\\name::alice,score:num:99
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, allocator, .{});
|
|
defer ri.deinit();
|
|
|
|
const result = try (try ri.next()).?.to(Data, .{});
|
|
try std.testing.expectEqualStrings("alice", result.name);
|
|
try std.testing.expectEqual(@as(u8, 99), result.score);
|
|
// `active` was not in the data, so the default value is used
|
|
try std.testing.expect(result.active);
|
|
}
|
|
};
|
|
/// Releases all memory owned by this iterator. This frees the internal
|
|
/// arena (and all parsed data allocated from it), then frees the arena
|
|
/// struct itself. After calling `deinit`, any slices or string pointers
|
|
/// obtained from `FieldIterator.next` or `FieldIterator.to` are invalid.
|
|
pub fn deinit(self: RecordIterator) void {
|
|
self.toOwnedFallback().deinit();
|
|
}
|
|
|
|
pub fn toOwnedFallback(self: RecordIterator) FallbackArena {
|
|
const ca = self.arena.child_allocator;
|
|
const fb = self.state.fallback_arena;
|
|
self.arena.deinit();
|
|
ca.destroy(self.arena);
|
|
return .{ .fallback_arena = fb };
|
|
}
|
|
|
|
pub const FallbackArena = struct {
|
|
fallback_arena: ?*std.heap.ArenaAllocator,
|
|
|
|
pub fn deinit(self: FallbackArena) void {
|
|
if (self.fallback_arena) |f| {
|
|
f.deinit();
|
|
f.child_allocator.destroy(f);
|
|
}
|
|
}
|
|
};
|
|
|
|
/// Returns `true` if the data has not expired based on the `#!expires`
|
|
/// directive. If no expiry was specified, the data is always considered
|
|
/// fresh. Callers should check this after parsing to decide whether to
|
|
/// use or refresh cached data. Note that data will be returned by parse/
|
|
/// iterator regardless of freshness. This enables callers to use cached
|
|
/// data temporarily while refreshing it.
|
|
///
|
|
/// With tombstone functionality, isFresh should be consulted as well
|
|
/// as `dataAvailable`, which will tell the client if data is available at
|
|
/// all. The code looks something like this:
|
|
///
|
|
/// if (!iterator.isFresh()) iterator = refreshData(); // refresh data
|
|
/// if (!iterator.dataAvailable()) fallback();
|
|
pub fn isFresh(self: *const RecordIterator, io: std.Io) bool {
|
|
const now = std.Io.Timestamp.now(io, .real).toSeconds();
|
|
if (self.tombstoned) |t|
|
|
// If we are tombstoned here, we want the client to believe data
|
|
// is fresh until the effective recheck time. They should then
|
|
// call dataAvailable() to see if data is actually available
|
|
return now < t.until;
|
|
|
|
if (self.expires) |exp|
|
|
return now < exp;
|
|
|
|
// no expiry: always fresh, never frozen
|
|
return true;
|
|
}
|
|
test isFresh {
|
|
// Example: check expiry on parsed data. Data without an #!expires
|
|
// directive is always considered fresh.
|
|
const data =
|
|
\\#!srfv1
|
|
\\key::value
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, allocator, .{});
|
|
defer ri.deinit();
|
|
|
|
// No expiry set, so always fresh
|
|
try std.testing.expect(ri.isFresh(std.testing.io));
|
|
}
|
|
/// Returns `false` if any tombstone directive is found.
|
|
///
|
|
/// If a tombstone directive exists, there should be no usable data. Note
|
|
/// that srf itself does not enforce this, it is up to how you use it
|
|
///
|
|
/// Even if the tombstone is set in the past, we assume that the data
|
|
/// has not been re-checked due to the presence of the directive
|
|
pub fn dataAvailable(self: *const RecordIterator) bool {
|
|
if (self.tombstoned) |_| return false;
|
|
|
|
// no tombstone - data is available
|
|
return true;
|
|
}
|
|
};
|
|
|
|
/// Options controlling SRF parsing behavior. Passed to both `iterator` and
|
|
/// `parse`.
|
|
pub const ParseOptions = struct {
|
|
/// diagnostics should be a struct provided by Diagnostics function
|
|
diagnostics: ?*Diagnostics = null,
|
|
|
|
/// By default, the parser will copy data so it is safe to free the original
|
|
/// buffer or use with streaming readers. This will impose about 8% overhead,
|
|
/// and ties the lifetime of any strings to the deinit() method. For
|
|
/// fixed buffer parsing, consider using .none, which will not allocate
|
|
/// strings. More complex use cases can use their own allocator for control
|
|
/// over string lifetime
|
|
parse_allocator: ParseAllocator = .parse_arena,
|
|
|
|
/// Strict number parsing. In strict number parsing, numbers will fail
|
|
/// to parse if `std.fmt.parseFloat` fails. Turn this off to be more lenient.
|
|
///
|
|
/// Turning it off will have a slight performance impact, but helps support
|
|
/// scenarios where srf data is more user facing (think config, not cache).
|
|
/// For example, commas will be tolerated as will leading currency symbols
|
|
strict_number_parsing: bool = true,
|
|
|
|
/// Include raw value data. If this is set to false, only parsed data
|
|
/// will be included for numbers/bools. False is usually what you want,
|
|
/// but setting this to true can be useful to make surgical edits to the
|
|
/// stream while preserving all comments, etc. that may exist. There is
|
|
/// a negligible runtime cost, as the raw slice is ALWAYS borrowed from
|
|
/// the underlying stream, even if allocating during the parse.
|
|
include_raw: bool = false,
|
|
};
|
|
|
|
/// Allocator to use for parsing data
|
|
pub const ParseAllocator = union(enum) {
|
|
/// Absolutely no allocation allowed. This will fail with AllocationRequired under the following circumstances:
|
|
///
|
|
/// * binary data is encountered (needs decoding)
|
|
/// * multi-line string literals are encountered (allocation needed to accomodate streaming readers)
|
|
none,
|
|
/// No allocator. Lifetime of any data parsed is tied to the underlying
|
|
/// data passed to the reader. This is most appropriate when the caller
|
|
/// uses a fixed buffer, and is equivalent of the "Leaky" parsing
|
|
/// functions of std.json. IMPORTANT: This will NOT avoid all allocations.
|
|
/// Specifically binary data is base64 encoded per the spec and we need
|
|
/// to allocate space for the decode. Also, multi-line data can not be
|
|
/// assumed to be available post-reader advance, and therefore allocation
|
|
/// is performed in that case.
|
|
///
|
|
/// For ABSOLUTELY NO ALLOCATION, use none. Otherwise, the Parsed
|
|
/// struct has a deinit function that frees everything, and toOwnedFallback
|
|
/// which will deinit the arena for parsing and return the fallback arena
|
|
/// that can be released at a later time
|
|
none_with_fallback,
|
|
/// Use the arena allocator created by the parser to copy any strings.
|
|
/// This ties the lifetime of any data parsed to the parser deinit()
|
|
/// function. Imposes about 8% overhead compared to "none".
|
|
parse_arena,
|
|
/// Parser will use the caller-supplied allocator, providing the most
|
|
/// flexibility over lifetime. Overhead will be contingent on the allocator
|
|
/// used. If the allocator is an arena allocator, assume 8% overhead over
|
|
/// "none". It is likely a fixed buffer allocator would be somewhat less.
|
|
custom: CustomParseAllocator,
|
|
};
|
|
|
|
/// Allocator to use for a specific scope (either keys or values). Different
|
|
/// from parseAllocator because the custom variant here has to be a std.mem.Allocator
|
|
pub const ScopeAllocator = union(enum) {
|
|
/// Absolutely no allocation allowed. This will fail with AllocationRequired under the following circumstances:
|
|
///
|
|
/// * binary data is encountered (needs decoding)
|
|
/// * multi-line string literals are encountered (allocation needed to accomodate streaming readers)
|
|
none,
|
|
/// No allocator. Lifetime of any data parsed is tied to the underlying
|
|
/// data passed to the reader. This is most appropriate when the caller
|
|
/// uses a fixed buffer, and is equivalent of the "Leaky" parsing
|
|
/// functions of std.json. IMPORTANT: This will NOT avoid all allocations.
|
|
/// Specifically binary data is base64 encoded per the spec and we need
|
|
/// to allocate space for the decode. Also, multi-line data can not be
|
|
/// assumed to be available post-reader advance, and therefore allocation
|
|
/// is performed in that case.
|
|
///
|
|
/// For ABSOLUTELY NO ALLOCATION, use none. Otherwise, the Parsed
|
|
/// struct has a deinit function that frees everything, and toOwnedFallback
|
|
/// which will deinit the arena for parsing and return the fallback arena
|
|
/// that can be released at a later time
|
|
none_with_fallback,
|
|
/// Use the arena allocator created by the parser to copy any strings.
|
|
/// This ties the lifetime of any data parsed to the parser deinit()
|
|
/// function. Imposes about 8% overhead compared to "none".
|
|
parse_arena,
|
|
/// Parser will use the caller-supplied allocator, providing the most
|
|
/// flexibility over lifetime. Overhead will be contingent on the allocator
|
|
/// used. If the allocator is an arena allocator, assume 8% overhead over
|
|
/// "none". It is likely a fixed buffer allocator would be somewhat less.
|
|
allocator: std.mem.Allocator,
|
|
};
|
|
pub const CustomParseAllocator = struct {
|
|
key_allocator: ScopeAllocator,
|
|
value_allocator: ScopeAllocator,
|
|
|
|
/// Initializes a custom parse allocator suitable for use in common workflows
|
|
/// where you iterate each record, then iterate through fields with full control
|
|
pub fn initIterator(allocator: std.mem.Allocator) CustomParseAllocator {
|
|
return .{
|
|
.key_allocator = .{ .allocator = allocator },
|
|
.value_allocator = .{ .allocator = allocator },
|
|
};
|
|
}
|
|
|
|
/// Initializes a custom parse allocator suitable for use in common workflows
|
|
/// where you iterate each record and call RecordIterator.to() on the result
|
|
pub fn initTo(allocator: std.mem.Allocator) CustomParseAllocator {
|
|
return .{
|
|
.key_allocator = .{ .none = {} },
|
|
.value_allocator = .{ .allocator = allocator },
|
|
};
|
|
}
|
|
};
|
|
|
|
const Directive = union(enum) {
|
|
magic,
|
|
long_format,
|
|
compact_format,
|
|
require_eof,
|
|
eof,
|
|
tombstoned: Tombstone,
|
|
expires: i64,
|
|
created: i64,
|
|
modified: i64,
|
|
|
|
/// Parses a Directive. The only reason the allocator is used here is because
|
|
/// a parse error may be logged, so this function should *NOT* be called
|
|
/// with an arena allocator
|
|
pub fn parse(str: []const u8, state: *RecordIterator.State) ParseError!?Directive {
|
|
if (!std.mem.startsWith(u8, str, "#!")) return null;
|
|
// strip any comments off
|
|
var it = std.mem.splitScalar(u8, str[2..], '#');
|
|
const line = std.mem.trimEnd(u8, it.first(), &std.ascii.whitespace);
|
|
if (std.mem.eql(u8, "srfv1", line)) return .magic;
|
|
if (std.mem.eql(u8, "requireeof", line)) return .require_eof;
|
|
if (std.mem.eql(u8, "requireof", line)) {
|
|
try parseError("#!requireof found. Did you mean #!requireeof?", state);
|
|
return null;
|
|
}
|
|
if (std.mem.eql(u8, "eof", line)) return .eof;
|
|
if (std.mem.eql(u8, "compact", line)) return .compact_format;
|
|
if (std.mem.eql(u8, "long", line)) return .long_format;
|
|
if (std.mem.startsWith(u8, line, "tombstoned="))
|
|
return .{ .tombstoned = try Tombstone.parse(line["tombstoned=".len..]) };
|
|
if (std.mem.startsWith(u8, line, "expires=")) {
|
|
return .{ .expires = std.fmt.parseInt(i64, line["expires=".len..], 10) catch return ParseError.ParseFailed };
|
|
}
|
|
if (std.mem.startsWith(u8, line, "created=")) {
|
|
return .{ .created = std.fmt.parseInt(i64, line["created=".len..], 10) catch return ParseError.ParseFailed };
|
|
}
|
|
if (std.mem.startsWith(u8, line, "modified=")) {
|
|
return .{ .modified = std.fmt.parseInt(i64, line["modified=".len..], 10) catch return ParseError.ParseFailed };
|
|
}
|
|
return null;
|
|
}
|
|
};
|
|
/// Options controlling SRF output formatting. Used by `fmt`
|
|
pub const FormatOptions = struct {
|
|
/// When `true`, fields are separated by newlines and records by blank
|
|
/// lines (`#!long` format). When `false` (default), fields are
|
|
/// comma-separated and records are newline-separated (compact format).
|
|
long_format: bool = false,
|
|
|
|
/// Will emit the eof directive as well as requireeof
|
|
emit_eof: bool = false,
|
|
|
|
/// Specify a tombstoned time for the data being written
|
|
tombstoned: ?Tombstone = null,
|
|
|
|
/// Specify an expiration time for the data being written
|
|
expires: ?i64 = null,
|
|
|
|
/// Specify a created time for the data being written
|
|
created: ?i64 = null,
|
|
|
|
/// Specify a modified time for the data being written
|
|
modified: ?i64 = null,
|
|
|
|
/// By setting this to false, you can avoid writing any header/footer data
|
|
/// and just format the record. This is useful for appending to an existing
|
|
/// srf file rather than overwriting all the data
|
|
emit_directives: bool = true,
|
|
|
|
/// When set to true, this will output all values, even if they are the default values
|
|
emit_default_values: bool = false,
|
|
};
|
|
|
|
/// Returns a formatter for writing typed Zig values directly to SRF format.
|
|
/// Custom serialization is supported via the `srfFormat` method convention
|
|
/// on struct/union fields.
|
|
pub fn fmt(comptime T: type, items: []const T, options: FormatOptions) FromFormatter(T) {
|
|
return .{ .items = items, .options = options };
|
|
}
|
|
pub fn FromFormatter(comptime T: type) type {
|
|
return struct {
|
|
items: []const T,
|
|
options: FormatOptions,
|
|
|
|
const Self = @This();
|
|
|
|
pub fn format(self: Self, writer: *std.Io.Writer) std.Io.Writer.Error!void {
|
|
try frontMatter(writer, self.options);
|
|
var first = true;
|
|
for (self.items) |item| {
|
|
if (!first and self.options.long_format) try writer.writeByte('\n');
|
|
first = false;
|
|
try self.formatItem(T, item, writer);
|
|
}
|
|
try epilogue(writer, self.options);
|
|
}
|
|
|
|
/// Formats a single item from the overall array. Analagous to a record
|
|
/// We take a type value here specifically for unions, because we need
|
|
/// to serialize the payload value of unions as well. Normally the
|
|
/// type value is simply "T".
|
|
fn formatItem(self: Self, comptime I: type, value: I, writer: *std.Io.Writer) std.Io.Writer.Error!void {
|
|
const ti = @typeInfo(I);
|
|
const delimiter: u8 = if (self.options.long_format) '\n' else ',';
|
|
switch (ti) {
|
|
.@"struct" => |info| {
|
|
var first = true;
|
|
inline for (info.fields) |f| {
|
|
const field_val = @field(value, f.name);
|
|
const is_default_val = if (f.defaultValue()) |d|
|
|
std.meta.eql(field_val, d)
|
|
else
|
|
false;
|
|
if (self.options.emit_default_values or !is_default_val) {
|
|
if (!first) {
|
|
try writer.writeByte(delimiter);
|
|
}
|
|
try self.formatField(f.type, f.name, field_val, writer);
|
|
first = false;
|
|
}
|
|
}
|
|
try writer.writeByte('\n');
|
|
},
|
|
.@"union" => {
|
|
const active_tag_name = @tagName(value);
|
|
const key = if (@hasDecl(T, "srf_tag_field"))
|
|
T.srf_tag_field
|
|
else
|
|
"type";
|
|
// We need to serialize the active tag name first, then the payload
|
|
try self.formatField([]const u8, key, active_tag_name, writer);
|
|
try writer.writeByte(delimiter);
|
|
// Now to serialize the payload itself
|
|
switch (value) {
|
|
inline else => |payload| {
|
|
if (@typeInfo(@TypeOf(payload)) == .@"union")
|
|
@compileError("Nested unions not supported for srf serialization");
|
|
return self.formatItem(@TypeOf(payload), payload, writer);
|
|
},
|
|
}
|
|
},
|
|
.@"enum" => |info| {
|
|
_ = info;
|
|
@compileError("Enum not implemented");
|
|
// TODO: I do not believe this is correct
|
|
// inline for (info.fields) |f|
|
|
// inx = try self.setField(inx, f.name, self.SourceType, null, val);
|
|
},
|
|
.error_set => return error.ErrorSetNotSupported,
|
|
else => @compileError("Expected struct, union, error set or enum type, found '" ++ @typeName(T) ++ "'"),
|
|
}
|
|
}
|
|
|
|
/// Formats a field from the item (struct/union). Analagous to a field in a record
|
|
fn formatField(self: Self, comptime F: type, comptime field_name: []const u8, val: F, writer: *std.Io.Writer) std.Io.Writer.Error!void {
|
|
const ti = @typeInfo(F);
|
|
switch (ti) {
|
|
.optional => |o| {
|
|
if (val) |v|
|
|
return self.formatField(o.child, field_name, v, writer);
|
|
try self.formatValue(field_name, null, writer);
|
|
},
|
|
.pointer => |p| {
|
|
// We don't have an allocator, so the only thing we can do
|
|
// here is manage []const u8 or []u8
|
|
if (p.size != .slice or p.child != u8)
|
|
return error.WriteFailed;
|
|
try self.formatValue(field_name, .{ .string = val }, writer);
|
|
},
|
|
.type, .void, .noreturn => return error.WriteFailed,
|
|
.comptime_float, .comptime_int, .undefined, .null, .error_union => return error.WriteFailed,
|
|
.error_set, .@"fn", .@"opaque", .frame => return error.WriteFailed,
|
|
.@"anyframe", .vector, .enum_literal => return error.WriteFailed,
|
|
.int => try self.formatValue(field_name, .{ .number = @floatFromInt(val) }, writer),
|
|
.float => try self.formatValue(field_name, .{ .number = @floatCast(val) }, writer),
|
|
.bool => try self.formatValue(field_name, .{ .boolean = val }, writer),
|
|
.@"enum" => try self.formatValue(field_name, .{ .string = @tagName(val) }, writer),
|
|
.array => return error.WriteFailed,
|
|
.@"struct", .@"union" => {
|
|
if (std.meta.hasMethod(F, "srfFormat"))
|
|
return val.srfFormat(field_name, writer);
|
|
@compileError("struct/union formatting requires pub fn srfFormat(self: Self, comptime key, writer: *std.Io.Writer) std.Io.Writer.Error!void. No function found on type: " ++ F);
|
|
},
|
|
}
|
|
}
|
|
|
|
/// Formats a single key/value pair
|
|
fn formatValue(self: Self, key: []const u8, value: ?Value, writer: *std.Io.Writer) std.Io.Writer.Error!void {
|
|
try writer.writeAll(key);
|
|
try writer.writeByte(':');
|
|
if (value) |v|
|
|
switch (v) {
|
|
.string => |s| {
|
|
const newlines = std.mem.containsAtLeastScalar(u8, s, 1, '\n');
|
|
const commas = !self.options.long_format and std.mem.containsAtLeastScalar(u8, s, 1, ',');
|
|
// Output the count if newlines exist
|
|
const count = if (newlines or commas) s.len else null;
|
|
if (count) |c| try writer.print("{d}", .{c});
|
|
try writer.writeByte(':');
|
|
try writer.writeAll(s);
|
|
},
|
|
.number => |n| try writer.print("num:{d}", .{@as(f64, @floatCast(n))}),
|
|
.boolean => |b| try writer.print("bool:{}", .{b}),
|
|
.bytes => |b| try writer.print("binary:{b64}", .{b}),
|
|
}
|
|
else
|
|
try writer.writeAll(":null:");
|
|
}
|
|
};
|
|
}
|
|
fn frontMatter(writer: *std.Io.Writer, options: FormatOptions) !void {
|
|
if (!options.emit_directives) return;
|
|
try writer.writeAll("#!srfv1\n");
|
|
if (options.long_format)
|
|
try writer.writeAll("#!long\n");
|
|
if (options.emit_eof)
|
|
try writer.writeAll("#!requireeof\n");
|
|
if (options.tombstoned) |t| {
|
|
if (t.isForever())
|
|
try writer.writeAll("#!tombstoned=forever")
|
|
else
|
|
try writer.print("#!tombstoned={d}", .{t.until});
|
|
if (t.reason()) |r|
|
|
try writer.print(" {s}", .{r});
|
|
try writer.writeByte('\n');
|
|
}
|
|
if (options.expires) |e|
|
|
try writer.print("#!expires={d}\n", .{e});
|
|
if (options.created) |e|
|
|
try writer.print("#!created={d}\n", .{e});
|
|
if (options.modified) |e|
|
|
try writer.print("#!modified={d}\n", .{e});
|
|
}
|
|
fn epilogue(writer: *std.Io.Writer, options: FormatOptions) !void {
|
|
if (!options.emit_directives) return;
|
|
if (options.emit_eof)
|
|
try writer.writeAll("#!eof\n");
|
|
}
|
|
|
|
/// Creates a streaming `RecordIterator` for the given reader. This is the
|
|
/// preferred entry point for parsing SRF data, as it yields records and
|
|
/// fields lazily without collecting them into slices.
|
|
///
|
|
/// The returned iterator owns an arena allocator that holds all parsed data
|
|
/// (string values, keys, etc.). Call `RecordIterator.deinit` to free
|
|
/// everything when done. Parsed field data remains valid until `deinit` is
|
|
/// called.
|
|
///
|
|
/// The iterator handles SRF header directives (`#!srfv1`, `#!long`,
|
|
/// `#!compact`, `#!requireeof`, `#!expires`) automatically during
|
|
/// construction. Notably this means you can check isFresh() immediately.
|
|
///
|
|
/// Also note that as state is allocated and stored within the recorditerator,
|
|
/// callers can assign the return value to a constant
|
|
pub fn iterator(reader: *std.Io.Reader, allocator: std.mem.Allocator, options: ParseOptions) ParseError!RecordIterator {
|
|
|
|
// The arena and state are heap-allocated because RecordIterator is returned
|
|
// by value. Both RecordIterator and FieldIterator must share mutable state,
|
|
// so State is held by pointer to ensure mutations propagate across copies.
|
|
// The arena pointer serves the same purpose -- an inline arena would be
|
|
// duplicated on copy, creating dangling pointers. These are O(1) per parse
|
|
// session (not per-record or per-field), so the cost is negligible.
|
|
|
|
// create an arena allocator for everytyhing related to parsing
|
|
const arena: *std.heap.ArenaAllocator = try allocator.create(std.heap.ArenaAllocator);
|
|
errdefer allocator.destroy(arena);
|
|
arena.* = .init(allocator);
|
|
errdefer arena.deinit();
|
|
const state = try arena.allocator().create(RecordIterator.State);
|
|
state.* = .{
|
|
.reader = reader,
|
|
.current_line = null,
|
|
.options = options,
|
|
.aa = arena.allocator(),
|
|
.allocator = allocator,
|
|
};
|
|
var it: RecordIterator = .{
|
|
.arena = arena,
|
|
.tombstoned = null,
|
|
.expires = null,
|
|
.created = null,
|
|
.modified = null,
|
|
.state = state,
|
|
};
|
|
const first_line = it.state.nextLine() orelse return ParseError.ParseFailed;
|
|
|
|
if (try Directive.parse(first_line, it.state)) |d| {
|
|
if (d != .magic) try parseError("Magic header not found on first line", it.state);
|
|
} else try parseError("Magic header not found on first line", it.state);
|
|
|
|
// Loop through the header material and configure our main parsing
|
|
it.state.current_line = blk: {
|
|
while (it.state.nextLine()) |line| {
|
|
if (try Directive.parse(line, it.state)) |d| {
|
|
switch (d) {
|
|
.magic => try parseError("Found a duplicate magic header", it.state),
|
|
.long_format => it.state.field_delimiter = '\n',
|
|
.compact_format => it.state.field_delimiter = ',', // what if we have both?
|
|
.require_eof => it.state.require_eof = true,
|
|
.tombstoned => |exp| it.tombstoned = exp,
|
|
.expires => |exp| it.expires = exp,
|
|
.created => |exp| it.created = exp,
|
|
.modified => |exp| it.modified = exp,
|
|
.eof => {
|
|
// there needs to be an eof then
|
|
if (it.state.nextLine()) |_| {
|
|
try parseError("Data found after #!eof", it.state);
|
|
return ParseError.ParseFailed; // this is terminal
|
|
} else return it;
|
|
},
|
|
}
|
|
} else break :blk line;
|
|
}
|
|
return it; //without current_line - we're at the end of file
|
|
};
|
|
return it; // with current_line
|
|
}
|
|
const DataScope = enum {
|
|
key,
|
|
value,
|
|
};
|
|
inline fn dupe(state: RecordIterator.State, data: []const u8, scope: DataScope) ParseError![]const u8 {
|
|
if (findAllocator(state, scope)) |a|
|
|
return try a.dupe(u8, data);
|
|
return data;
|
|
}
|
|
inline fn findAllocator(state: RecordIterator.State, scope: DataScope) ?std.mem.Allocator {
|
|
switch (state.options.parse_allocator) {
|
|
.none, .none_with_fallback => return null,
|
|
.parse_arena => return state.aa,
|
|
.custom => |a| {
|
|
const alloc = switch (scope) {
|
|
.key => a.key_allocator,
|
|
.value => a.value_allocator,
|
|
};
|
|
switch (alloc) {
|
|
.none, .none_with_fallback => return null,
|
|
.parse_arena => return state.aa,
|
|
.allocator => |c| return c,
|
|
}
|
|
},
|
|
}
|
|
}
|
|
/// Logs a parse error to diagnostics. Note that the allocator provided should
|
|
/// *NOT* be an arena, as the message must outlive the parse results, which will
|
|
/// be otherwise cleaned up in the arena deinit
|
|
inline fn parseError(message: []const u8, state: *RecordIterator.State) ParseError!void {
|
|
log.debug("Parse error. Parse state {f}, message: {s}", .{ state, message });
|
|
if (state.options.diagnostics) |d| {
|
|
try d.addError(.{
|
|
.message = message,
|
|
.level = .err,
|
|
.line = state.line,
|
|
.column = state.column,
|
|
});
|
|
} else {
|
|
return ParseError.ParseFailed;
|
|
}
|
|
}
|
|
|
|
// Test-only types extracted to module level so that their pub methods
|
|
// (required by std.meta.hasMethod) do not appear in generated documentation.
|
|
const TestRecType = enum {
|
|
foo,
|
|
bar,
|
|
};
|
|
const TestCustomType = struct {
|
|
const Self = @This();
|
|
pub fn srfParse(val: []const u8) !CoercionResult(Self) {
|
|
if (std.mem.eql(u8, "hi", val)) return .init(.{});
|
|
return error.ValueNotEqualHi;
|
|
}
|
|
pub fn srfFormat(self: Self, comptime field_name: []const u8, writer: *std.Io.Writer) std.Io.Writer.Error!void {
|
|
_ = self;
|
|
try writer.writeAll(field_name);
|
|
try writer.writeAll("::hi");
|
|
}
|
|
};
|
|
|
|
test "long format single record, no eof" {
|
|
const data =
|
|
\\#!srfv1 # mandatory comment with format and version. Parser instructions start with #!
|
|
\\#!long # Mandatory to use multiline records, compact format is optional #!compact
|
|
\\# A comment
|
|
\\# empty lines ignored
|
|
\\
|
|
\\key::string value, with any data except a \n. an optional string length between the colons
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var it = try iterator(&reader, allocator, .{});
|
|
defer it.deinit();
|
|
const maybe_record = try it.next();
|
|
try std.testing.expect(maybe_record != null);
|
|
const rec = maybe_record.?;
|
|
const maybe_field = try rec.next();
|
|
try std.testing.expect(maybe_field != null);
|
|
const field = maybe_field.?;
|
|
try std.testing.expectEqualStrings("key", field.key);
|
|
try std.testing.expectEqualStrings(
|
|
"string value, with any data except a \\n. an optional string length between the colons",
|
|
field.value.?.string,
|
|
);
|
|
try std.testing.expect(try rec.next() == null);
|
|
try std.testing.expect(try it.next() == null);
|
|
}
|
|
test "long format from README - generic data structures, first record only" {
|
|
const data =
|
|
\\#!srfv1 # mandatory comment with format and version. Parser instructions start with #!
|
|
\\#!requireeof # Set this if you want parsing to fail when #!eof not present on last line
|
|
\\#!long # Mandatory to use multiline records, compact format is optional #!compact
|
|
\\# A comment
|
|
\\# empty lines ignored
|
|
\\
|
|
\\this is a number:num: 5
|
|
\\#!eof
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
const records = try iterator(&reader, allocator, .{});
|
|
defer records.deinit();
|
|
const first = (try records.next()).?;
|
|
try std.testing.expect((try first.next()) != null);
|
|
try std.testing.expect((try first.next()) == null);
|
|
}
|
|
|
|
test "long format from README - generic data structures" {
|
|
const data =
|
|
\\#!srfv1 # mandatory comment with format and version. Parser instructions start with #!
|
|
\\#!requireeof # Set this if you want parsing to fail when #!eof not present on last line
|
|
\\#!long # Mandatory to use multiline records, compact format is optional #!compact
|
|
\\# A comment
|
|
\\# empty lines ignored
|
|
\\
|
|
\\key::string value, with any data except a \n. an optional string length between the colons
|
|
\\this is a number:num: 5
|
|
\\null value:null:
|
|
\\array::array's don't exist. Use json or toml or something
|
|
\\data with newlines must have a length:7:foo
|
|
\\bar
|
|
\\boolean value:bool:false
|
|
\\ # Empty line separates records
|
|
\\
|
|
\\key::this is the second record
|
|
\\this is a number:num:42
|
|
\\null value:null:
|
|
\\array::array's still don't exist
|
|
\\data with newlines must have a length::single line
|
|
\\#!eof # eof marker, useful to make sure your file wasn't cut in half. Only considered if requireeof set at top
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
const records = try iterator(&reader, allocator, .{});
|
|
defer records.deinit();
|
|
var rec = (try records.next()).?;
|
|
var field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("key", field.key);
|
|
try std.testing.expectEqualStrings("string value, with any data except a \\n. an optional string length between the colons", field.value.?.string);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("this is a number", field.key);
|
|
try std.testing.expectEqual(@as(f64, 5), field.value.?.number);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("null value", field.key);
|
|
try std.testing.expect(field.value == null);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("array", field.key);
|
|
try std.testing.expectEqualStrings(
|
|
"array's don't exist. Use json or toml or something",
|
|
field.value.?.string,
|
|
);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("data with newlines must have a length", field.key);
|
|
try std.testing.expectEqualStrings("foo\nbar", field.value.?.string);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("boolean value", field.key);
|
|
try std.testing.expect(!field.value.?.boolean);
|
|
try std.testing.expect(try rec.next() == null);
|
|
|
|
const second = (try records.next()).?;
|
|
field = (try second.next()).?;
|
|
try std.testing.expectEqualStrings("key", field.key);
|
|
try std.testing.expectEqualStrings("this is the second record", field.value.?.string);
|
|
field = (try second.next()).?;
|
|
try std.testing.expectEqualStrings("this is a number", field.key);
|
|
try std.testing.expectEqual(@as(f64, 42), field.value.?.number);
|
|
field = (try second.next()).?;
|
|
try std.testing.expectEqualStrings("null value", field.key);
|
|
try std.testing.expect(field.value == null);
|
|
field = (try second.next()).?;
|
|
try std.testing.expectEqualStrings("array", field.key);
|
|
try std.testing.expectEqualStrings("array's still don't exist", field.value.?.string);
|
|
field = (try second.next()).?;
|
|
try std.testing.expectEqualStrings("data with newlines must have a length", field.key);
|
|
try std.testing.expectEqualStrings("single line", field.value.?.string);
|
|
try std.testing.expect(try second.next() == null);
|
|
}
|
|
|
|
test "compact format from README - generic data structures" {
|
|
const data =
|
|
\\#!srfv1 # mandatory comment with format and version. Parser instructions start with #!
|
|
\\key::string value must have a length between colons or end with a comma,this is a number:num:5 ,null value:null:,array::array's don't exist. Use json or toml or something,data with newlines must have a length:7:foo
|
|
\\bar,boolean value:bool:false
|
|
\\key::this is the second record
|
|
;
|
|
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
// We want "parse" and "parseLeaky" probably. Second parameter is a diagnostics
|
|
const records = try iterator(&reader, allocator, .{});
|
|
defer records.deinit();
|
|
var first = (try records.next()).?;
|
|
var field = (try first.next()).?;
|
|
try std.testing.expectEqualStrings("key", field.key);
|
|
try std.testing.expectEqualStrings("string value must have a length between colons or end with a comma", field.value.?.string);
|
|
field = (try first.next()).?;
|
|
try std.testing.expectEqualStrings("this is a number", field.key);
|
|
try std.testing.expectEqual(@as(f64, 5), field.value.?.number);
|
|
field = (try first.next()).?;
|
|
try std.testing.expectEqualStrings("null value", field.key);
|
|
try std.testing.expect(field.value == null);
|
|
field = (try first.next()).?;
|
|
try std.testing.expectEqualStrings("array", field.key);
|
|
try std.testing.expectEqualStrings("array's don't exist. Use json or toml or something", field.value.?.string);
|
|
field = (try first.next()).?;
|
|
try std.testing.expectEqualStrings("data with newlines must have a length", field.key);
|
|
try std.testing.expectEqualStrings("foo\nbar", field.value.?.string);
|
|
field = (try first.next()).?;
|
|
try std.testing.expectEqualStrings("boolean value", field.key);
|
|
try std.testing.expect(!field.value.?.boolean);
|
|
try std.testing.expect(try first.next() == null);
|
|
|
|
const second = (try records.next()).?;
|
|
field = (try second.next()).?;
|
|
try std.testing.expectEqualStrings("key", field.key);
|
|
try std.testing.expectEqualStrings("this is the second record", field.value.?.string);
|
|
try std.testing.expect(try second.next() == null);
|
|
try std.testing.expect(try records.next() == null);
|
|
}
|
|
test "serialize/deserialize" {
|
|
const Data = struct {
|
|
foo: []const u8,
|
|
bar: u8,
|
|
qux: ?TestRecType = .foo,
|
|
b: bool = false,
|
|
f: f32 = 4.2,
|
|
custom: ?TestCustomType = null,
|
|
};
|
|
|
|
const compact =
|
|
\\#!srfv1
|
|
\\foo::bar,foo:null:,foo:binary:YmFy,foo:num:42,bar:num:42
|
|
\\foo::bar,foo:null:,foo:binary:YmFy,foo:num:42,bar:num:42
|
|
\\foo::bar,foo:null:,foo:binary:YmFy,foo:num:42,bar:num:42,qux::bar
|
|
\\foo::bar,foo:null:,foo:binary:YmFy,foo:num:42,bar:num:42,qux::bar,b:bool:true,f:num:6.9,custom:string:hi
|
|
\\
|
|
;
|
|
// Now we'll do it with the iterator version
|
|
var it_reader = std.Io.Reader.fixed(compact);
|
|
const ri = try iterator(&it_reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
const rec1_it = try (try ri.next()).?.to(Data, .{});
|
|
try std.testing.expectEqualStrings("bar", rec1_it.foo);
|
|
try std.testing.expectEqual(@as(u8, 42), rec1_it.bar);
|
|
try std.testing.expectEqual(@as(TestRecType, .foo), rec1_it.qux);
|
|
_ = try ri.next();
|
|
_ = try ri.next();
|
|
const rec4_it = try (try ri.next()).?.to(Data, .{});
|
|
try std.testing.expectEqualStrings("bar", rec4_it.foo);
|
|
try std.testing.expectEqual(@as(u8, 42), rec4_it.bar);
|
|
try std.testing.expectEqual(@as(TestRecType, .bar), rec4_it.qux.?);
|
|
try std.testing.expectEqual(true, rec4_it.b);
|
|
try std.testing.expectEqual(@as(f32, 6.9), rec4_it.f);
|
|
|
|
const all_data: []const Data = &.{
|
|
.{ .foo = "hi", .bar = 42, .qux = .bar, .b = true, .f = 6.0, .custom = .{} },
|
|
.{ .foo = "bar", .bar = 69 },
|
|
};
|
|
var buf: [4096]u8 = undefined;
|
|
const compact_from = try std.fmt.bufPrint(
|
|
&buf,
|
|
"{f}",
|
|
.{fmt(Data, all_data, .{})},
|
|
);
|
|
|
|
const expect =
|
|
\\#!srfv1
|
|
\\foo::hi,bar:num:42,qux::bar,b:bool:true,f:num:6,custom::hi
|
|
\\foo::bar,bar:num:69
|
|
\\
|
|
;
|
|
try std.testing.expectEqualStrings(expect, compact_from);
|
|
}
|
|
test "serialize/deserialize allows overflow lifetime semantics" {
|
|
const Data = struct {
|
|
foo: []const u8,
|
|
bar: u8,
|
|
qux: ?TestRecType = .foo,
|
|
b: bool = false,
|
|
f: f32 = 4.2,
|
|
custom: ?TestCustomType = null,
|
|
};
|
|
|
|
const compact =
|
|
\\#!srfv1
|
|
\\foo:binary:YmFy,bar:num:42
|
|
\\foo:binary:YmFy,bar:num:42
|
|
\\foo:binary:YmFy,bar:num:42,qux::bar
|
|
\\foo:binary:YmFy,bar:num:42,qux::bar,b:bool:true,f:num:6.9,custom:string:hi
|
|
\\
|
|
;
|
|
// Round trip and make sure we get equivalent objects back
|
|
var compact_reader = std.Io.Reader.fixed(compact);
|
|
var it = try iterator(
|
|
&compact_reader,
|
|
std.testing.allocator,
|
|
.{ .parse_allocator = .none_with_fallback },
|
|
);
|
|
|
|
const rec1 = try (try it.next()).?.to(Data, .{});
|
|
try std.testing.expect(it.state.fallback_arena != null);
|
|
const fallback = it.toOwnedFallback();
|
|
defer fallback.deinit();
|
|
// This would not be possible otherwise
|
|
try std.testing.expectEqualStrings("bar", rec1.foo);
|
|
try std.testing.expectEqual(@as(u8, 42), rec1.bar);
|
|
try std.testing.expectEqual(@as(TestRecType, .foo), rec1.qux);
|
|
|
|
var another_reader = std.Io.Reader.fixed(compact);
|
|
var another_it = try iterator(
|
|
&another_reader,
|
|
std.testing.allocator,
|
|
.{ .parse_allocator = .none },
|
|
);
|
|
defer another_it.deinit();
|
|
var fi = (try another_it.next()).?;
|
|
try std.testing.expectError(error.AllocationRequired, fi.next());
|
|
}
|
|
test "conversion from string true/false to proper type" {
|
|
const Data = struct {
|
|
foo: []const u8,
|
|
bar: u8,
|
|
qux: ?TestRecType = .foo,
|
|
b: bool = false,
|
|
f: f32 = 4.2,
|
|
custom: ?TestCustomType = null,
|
|
};
|
|
const compact =
|
|
\\#!srfv1
|
|
\\foo::bar,foo:null:,foo:binary:YmFy,foo:num:42,bar:num:42,qux::bar,b::true,f:num:6.9,custom:string:hi
|
|
\\foo::bar,foo:null:,foo:binary:YmFy,foo:num:42,bar:num:42,qux::bar,b::false,f:num:6.9,custom:string:hi
|
|
\\foo::bar,foo:null:,foo:binary:YmFy,foo:num:42,bar:num:42,qux::bar,b::noneoftheabove,f:num:6.9,custom:string:hi
|
|
\\
|
|
;
|
|
// Round trip and make sure we get equivalent objects back
|
|
var compact_reader = std.Io.Reader.fixed(compact);
|
|
var it = try iterator(&compact_reader, std.testing.allocator, .{});
|
|
defer it.deinit();
|
|
|
|
const rec1 = try (try it.next()).?.to(Data, .{});
|
|
try std.testing.expect(rec1.b);
|
|
const rec2 = try (try it.next()).?.to(Data, .{});
|
|
try std.testing.expect(!rec2.b);
|
|
const rec3_raw = (try it.next()).?;
|
|
try std.testing.expectError(
|
|
error.StringValueOfBooleanMustBetrueOrfalse,
|
|
rec3_raw.to(Data, .{}),
|
|
);
|
|
}
|
|
test "iterator with blank" {
|
|
const Data = struct {
|
|
foo: []const u8,
|
|
bar: u8,
|
|
qux: ?TestRecType = .foo,
|
|
b: bool = false,
|
|
f: f32 = 4.2,
|
|
custom: ?TestCustomType = null,
|
|
};
|
|
const compact =
|
|
\\#!srfv1
|
|
\\foo::1,bar:num:42
|
|
\\
|
|
\\# second record
|
|
\\
|
|
\\foo::2,bar:num:24
|
|
\\
|
|
;
|
|
// Round trip and make sure we get equivalent objects back
|
|
var compact_reader = std.Io.Reader.fixed(compact);
|
|
const it = try iterator(&compact_reader, std.testing.allocator, .{});
|
|
defer it.deinit();
|
|
|
|
const rec1 = try (try it.next()).?.to(Data, .{});
|
|
try std.testing.expectEqual('1', rec1.foo[0]);
|
|
try std.testing.expectEqual(@as(u8, 42), rec1.bar);
|
|
const rec2 = try (try it.next()).?.to(Data, .{});
|
|
try std.testing.expectEqual('2', rec2.foo[0]);
|
|
try std.testing.expectEqual(@as(u8, 24), rec2.bar);
|
|
}
|
|
test "unions" {
|
|
const Foo = struct {
|
|
number: u8,
|
|
true_or_false: bool,
|
|
};
|
|
const Bar = struct {
|
|
sentence: []const u8,
|
|
decimal: f64,
|
|
};
|
|
const MixedData = union(enum) {
|
|
foo: Foo,
|
|
bar: Bar,
|
|
|
|
// pub const srf_tag_field = "foobar";
|
|
};
|
|
|
|
const data: []const MixedData = &.{
|
|
.{ .foo = .{ .number = 42, .true_or_false = true } },
|
|
.{ .bar = .{ .sentence = "foobar", .decimal = 6.9 } },
|
|
};
|
|
var buf: [4096]u8 = undefined;
|
|
const compact_from = try std.fmt.bufPrint(
|
|
&buf,
|
|
"{f}",
|
|
.{fmt(MixedData, data, .{})},
|
|
);
|
|
const expect =
|
|
\\#!srfv1
|
|
\\type::foo,number:num:42,true_or_false:bool:true
|
|
\\type::bar,sentence::foobar,decimal:num:6.9
|
|
\\
|
|
;
|
|
try std.testing.expectEqualStrings(expect, compact_from);
|
|
|
|
var compact_reader = std.Io.Reader.fixed(expect);
|
|
const it = try iterator(&compact_reader, std.testing.allocator, .{});
|
|
defer it.deinit();
|
|
|
|
const rec1 = try (try it.next()).?.to(MixedData, .{});
|
|
try std.testing.expectEqualDeep(data[0], rec1);
|
|
const rec2 = try (try it.next()).?.to(MixedData, .{});
|
|
try std.testing.expectEqualDeep(data[1], rec2);
|
|
}
|
|
test "union with custom-allocator initTo: dispatch tag must not leak" {
|
|
// Repro for: FieldIterator.to(T) on a tagged union dupes the
|
|
// dispatch tag's string value via the value-side allocator, then
|
|
// discards the slice without freeing it. With `.parse_arena`,
|
|
// the leak is masked by `it.deinit()`. With
|
|
// `.{ .custom = .initTo(my_alloc) }`, the slice lives in the
|
|
// caller's allocator and DebugAllocator catches the leak.
|
|
//
|
|
// Each parsed union record produces exactly one leaked
|
|
// allocation: the `type::<variant>` string.
|
|
const Foo = struct {
|
|
number: u8,
|
|
};
|
|
const Bar = struct {
|
|
decimal: f64,
|
|
};
|
|
const MixedData = union(enum) {
|
|
foo: Foo,
|
|
bar: Bar,
|
|
};
|
|
const compact =
|
|
\\#!srfv1
|
|
\\type::foo,number:num:42
|
|
\\type::bar,decimal:num:6.9
|
|
\\
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(compact);
|
|
var ri = try iterator(&reader, allocator, .{
|
|
.parse_allocator = .{ .custom = .initTo(allocator) },
|
|
});
|
|
defer ri.deinit();
|
|
// Each `to(MixedData, .{})` call leaks one string allocation
|
|
// (the dispatch tag) into `allocator`. Walk both records; on
|
|
// exit, DebugAllocator should report 2 leaks.
|
|
const rec1 = try (try ri.next()).?.to(MixedData, .{});
|
|
try std.testing.expectEqual(@as(u8, 42), rec1.foo.number);
|
|
const rec2 = try (try ri.next()).?.to(MixedData, .{});
|
|
try std.testing.expectEqual(@as(f64, 6.9), rec2.bar.decimal);
|
|
}
|
|
test "enums" {
|
|
const Types = enum {
|
|
foo,
|
|
bar,
|
|
|
|
// pub const srf_tag_field = "foobar";
|
|
};
|
|
|
|
const Data = struct {
|
|
data_type: ?Types = null,
|
|
yo: u8,
|
|
};
|
|
// The original test here was using the non-iterator version of the parser
|
|
// (now deleted). That allowed random access, so we could coerce to
|
|
// multiple data types if they had a similar shape. I'm not sure why you
|
|
// might want to do that, but since we are consuming the reader in a single
|
|
// pass this is no longer possible. This note is here to provide some
|
|
// context in case this functionality is actually ever needed, we can
|
|
// maybe figure this out and re-enable this portion of the test
|
|
// const Data2 = struct {
|
|
// data_type: Types = .bar,
|
|
// yo: u8,
|
|
// };
|
|
|
|
const data: []const Data = &.{
|
|
.{ .data_type = .foo, .yo = 42 },
|
|
.{ .data_type = null, .yo = 69 },
|
|
};
|
|
var buf: [4096]u8 = undefined;
|
|
const compact_from = try std.fmt.bufPrint(
|
|
&buf,
|
|
"{f}",
|
|
.{fmt(Data, data, .{})},
|
|
);
|
|
const expect =
|
|
\\#!srfv1
|
|
\\data_type::foo,yo:num:42
|
|
\\yo:num:69
|
|
\\
|
|
;
|
|
try std.testing.expectEqualStrings(expect, compact_from);
|
|
|
|
var compact_reader = std.Io.Reader.fixed(expect);
|
|
const it = try iterator(&compact_reader, std.testing.allocator, .{});
|
|
defer it.deinit();
|
|
|
|
const rec1 = try (try it.next()).?.to(Data, .{});
|
|
try std.testing.expectEqualDeep(data[0], rec1);
|
|
const rec2 = try (try it.next()).?.to(Data, .{});
|
|
try std.testing.expectEqualDeep(data[1], rec2);
|
|
|
|
const missing_tag =
|
|
\\#!srfv1
|
|
\\yo:num:69
|
|
\\
|
|
;
|
|
var mt_reader = std.Io.Reader.fixed(missing_tag);
|
|
const mt = try iterator(&mt_reader, std.testing.allocator, .{});
|
|
defer mt.deinit();
|
|
const mt_rec1 = (try mt.next()).?;
|
|
const mt_data = try mt_rec1.to(Data, .{});
|
|
try std.testing.expect(mt_data.data_type == null);
|
|
|
|
// const mt_dt2 = try mt_rec1.to(Data2, .{});
|
|
// try std.testing.expect(mt_dt2.data_type == .bar);
|
|
}
|
|
test "compact format length-prefixed string as last field" {
|
|
// When a length-prefixed value is the last field on the line,
|
|
// rest_of_data.len == size exactly. The check on line 216 uses
|
|
// strict > instead of >=, falling through to the multi-line path
|
|
// where size - rest_of_data.len - 1 underflows.
|
|
const data =
|
|
\\#!srfv1
|
|
\\name::alice,desc:5:world
|
|
;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
const it = try iterator(&reader, std.testing.allocator, .{});
|
|
defer it.deinit();
|
|
const rec = (try it.next()).?;
|
|
const field1 = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("name", field1.key);
|
|
try std.testing.expectEqualStrings("alice", field1.value.?.string);
|
|
const field2 = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("desc", field2.key);
|
|
try std.testing.expectEqualStrings("world", field2.value.?.string);
|
|
try std.testing.expectEqual(null, try rec.next());
|
|
try std.testing.expectEqual(null, try it.next());
|
|
}
|
|
test iterator {
|
|
// Example: streaming through records and fields using the iterator API.
|
|
// This is the preferred parsing approach -- no intermediate slices are
|
|
// allocated for fields or records.
|
|
const data =
|
|
\\#!srfv1
|
|
\\name::alice,desc:5:world
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, allocator, .{});
|
|
defer ri.deinit();
|
|
|
|
// Advance to the first (and only) record
|
|
const fi = (try ri.next()).?;
|
|
|
|
// Iterate fields within the record
|
|
const field1 = (try fi.next()).?;
|
|
try std.testing.expectEqualStrings("name", field1.key);
|
|
try std.testing.expectEqualStrings("alice", field1.value.?.string);
|
|
const field2 = (try fi.next()).?;
|
|
try std.testing.expectEqualStrings("desc", field2.key);
|
|
try std.testing.expectEqualStrings("world", field2.value.?.string);
|
|
|
|
// No more fields in this record
|
|
try std.testing.expect(try fi.next() == null);
|
|
// No more records
|
|
try std.testing.expect(try ri.next() == null);
|
|
}
|
|
test "iterator with custom allocator" {
|
|
// Example: streaming through records and fields using the iterator API.
|
|
// This is the preferred parsing approach -- no intermediate slices are
|
|
// allocated for fields or records.
|
|
const data =
|
|
\\#!srfv1
|
|
\\name::alice,desc:5:world
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(
|
|
&reader,
|
|
allocator,
|
|
.{
|
|
.parse_allocator = .{ .custom = .initIterator(std.testing.allocator) },
|
|
},
|
|
);
|
|
defer ri.deinit();
|
|
|
|
// Advance to the first (and only) record
|
|
const fi = (try ri.next()).?;
|
|
|
|
// Iterate fields within the record
|
|
const field1 = (try fi.next()).?;
|
|
defer allocator.free(field1.key);
|
|
defer allocator.free(field1.value.?.string);
|
|
try std.testing.expectEqualStrings("name", field1.key);
|
|
try std.testing.expectEqualStrings("alice", field1.value.?.string);
|
|
const field2 = (try fi.next()).?;
|
|
defer allocator.free(field2.key);
|
|
defer allocator.free(field2.value.?.string);
|
|
try std.testing.expectEqualStrings("desc", field2.key);
|
|
try std.testing.expectEqualStrings("world", field2.value.?.string);
|
|
|
|
// No more fields in this record
|
|
try std.testing.expect(try fi.next() == null);
|
|
// No more records
|
|
try std.testing.expect(try ri.next() == null);
|
|
}
|
|
test "iterator with custom allocator - to() pattern" {
|
|
// const ll = std.testing.log_level;
|
|
// std.testing.log_level = .debug;
|
|
// defer std.testing.log_level = ll;
|
|
// Example: streaming through records and fields using the iterator API.
|
|
// This is the preferred parsing approach -- no intermediate slices are
|
|
// allocated for fields or records.
|
|
const data =
|
|
\\#!srfv1
|
|
\\name::alice,desc:5:world
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(
|
|
&reader,
|
|
allocator,
|
|
.{
|
|
.parse_allocator = .{ .custom = .initTo(std.testing.allocator) },
|
|
},
|
|
);
|
|
defer ri.deinit();
|
|
|
|
// Advance to the first (and only) record
|
|
const fi = (try ri.next()).?;
|
|
const rec = try fi.to(struct { name: []const u8, desc: []const u8 }, .{});
|
|
defer allocator.free(rec.name);
|
|
defer allocator.free(rec.desc);
|
|
try std.testing.expectEqualStrings("alice", rec.name);
|
|
try std.testing.expectEqualStrings("world", rec.desc);
|
|
|
|
// No more fields in this record
|
|
try std.testing.expect(try fi.next() == null);
|
|
// No more records
|
|
try std.testing.expect(try ri.next() == null);
|
|
}
|
|
test "iterator with custom allocator - to() pattern, relaxed and custom coercion" {
|
|
// Example: streaming through records and fields using the iterator API.
|
|
// This is the preferred parsing approach -- no intermediate slices are
|
|
// allocated for fields or records.
|
|
// Should this also allow the lienient number parsing? probably so
|
|
const data =
|
|
\\#!srfv1
|
|
\\name::alice,desc:5:world,cost::$5
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(
|
|
&reader,
|
|
allocator,
|
|
.{
|
|
.parse_allocator = .{ .custom = .initTo(std.testing.allocator) },
|
|
},
|
|
);
|
|
defer ri.deinit();
|
|
|
|
// Advance to the first (and only) record
|
|
const fi = (try ri.next()).?;
|
|
const rec = try fi.to(struct { name: []const u8, desc: []const u8, cost: usize }, .{ .strings_to_numbers = true });
|
|
defer allocator.free(rec.name);
|
|
defer allocator.free(rec.desc);
|
|
try std.testing.expectEqualStrings("alice", rec.name);
|
|
try std.testing.expectEqualStrings("world", rec.desc);
|
|
|
|
// No more fields in this record
|
|
try std.testing.expect(try fi.next() == null);
|
|
// No more records
|
|
try std.testing.expect(try ri.next() == null);
|
|
}
|
|
test "iterator basic long form" {
|
|
// Example: batch parsing collects all records and fields into slices.
|
|
// Prefer `iterator` for streaming; use `parse` when random access to
|
|
// all records is needed.
|
|
const data =
|
|
\\#!srfv1
|
|
\\#!long
|
|
\\name::alice
|
|
\\age:num:30
|
|
\\
|
|
\\name::bob
|
|
\\age:num:25
|
|
\\#!eof
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
// Diagnostics are optional, but if you would like them, include
|
|
// these three lines and set the options field:
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
var it = try iterator(&reader, allocator, .{ .diagnostics = &diag });
|
|
defer it.deinit();
|
|
|
|
const first_field_first_record = (try ((try it.next()).?.next())).?;
|
|
try std.testing.expectEqualStrings("alice", first_field_first_record.value.?.string);
|
|
const first_field_second_record = (try ((try it.next()).?.next())).?;
|
|
try std.testing.expectEqualStrings("bob", first_field_second_record.value.?.string);
|
|
try std.testing.expectEqual(null, try it.next());
|
|
}
|
|
test "parse tolerates commas and currency in numbers" {
|
|
// Example: batch parsing collects all records and fields into slices.
|
|
// Prefer `iterator` for streaming; use `parse` when random access to
|
|
// all records is needed.
|
|
const data =
|
|
\\#!srfv1
|
|
\\#!long
|
|
\\name::bananas
|
|
\\cost:num:$30.00
|
|
\\
|
|
\\name::spaceship
|
|
\\cost:num:$1,000,000,000.42
|
|
\\
|
|
\\name::Omikase in Tokyo
|
|
\\cost:num:¥15,000
|
|
\\
|
|
\\name::Airbus A380
|
|
\\cost:num:€410,000,000
|
|
\\
|
|
\\name::Bread in London
|
|
\\cost:num:5 €
|
|
\\
|
|
\\name::The other way
|
|
\\cost:num:€ 5
|
|
\\#!eof
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var it = try iterator(&reader, allocator, .{ .strict_number_parsing = false });
|
|
defer it.deinit();
|
|
|
|
var rec = (try it.next()).?;
|
|
var field = (try rec.next()).?;
|
|
// Dollars are single byte currency
|
|
try std.testing.expectEqualStrings("bananas", field.value.?.string);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqual(@as(f64, 30), field.value.?.number);
|
|
|
|
// Add commas to the mix
|
|
rec = (try it.next()).?;
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("spaceship", field.value.?.string);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqual(@as(f64, 1_000_000_000.42), field.value.?.number);
|
|
|
|
// Yen symbol is two bytes long
|
|
rec = (try it.next()).?;
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("Omikase in Tokyo", field.value.?.string);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqual(@as(f64, 15_000), field.value.?.number);
|
|
|
|
rec = (try it.next()).?;
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("Airbus A380", field.value.?.string);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqual(@as(f64, 410_000_000), field.value.?.number);
|
|
|
|
rec = (try it.next()).?;
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("Bread in London", field.value.?.string);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqual(@as(f64, 5), field.value.?.number);
|
|
|
|
rec = (try it.next()).?;
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqualStrings("The other way", field.value.?.string);
|
|
field = (try rec.next()).?;
|
|
try std.testing.expectEqual(@as(f64, 5), field.value.?.number);
|
|
|
|
try std.testing.expectEqual(null, try it.next());
|
|
}
|
|
test fmt {
|
|
// Example: serialize typed Zig values directly to SRF format.
|
|
const Data = struct {
|
|
name: []const u8,
|
|
age: u8,
|
|
};
|
|
const values: []const Data = &.{
|
|
.{ .name = "alice", .age = 30 },
|
|
.{ .name = "bob", .age = 25 },
|
|
};
|
|
var buf: [4096]u8 = undefined;
|
|
const result = try std.fmt.bufPrint(
|
|
&buf,
|
|
"{f}",
|
|
.{fmt(Data, values, .{})},
|
|
);
|
|
try std.testing.expectEqualStrings(
|
|
\\#!srfv1
|
|
\\name::alice,age:num:30
|
|
\\name::bob,age:num:25
|
|
\\
|
|
, result);
|
|
}
|
|
test "fmt commas" {
|
|
// Example: serialize typed Zig values directly to SRF format.
|
|
const Data = struct {
|
|
name: []const u8 = "bob",
|
|
age: u8,
|
|
};
|
|
const values: []const Data = &.{
|
|
.{ .name = "alice, yo", .age = 30 },
|
|
};
|
|
var buf: [4096]u8 = undefined;
|
|
const result = try std.fmt.bufPrint(
|
|
&buf,
|
|
"{f}",
|
|
.{fmt(Data, values, .{})},
|
|
);
|
|
try std.testing.expectEqualStrings(
|
|
\\#!srfv1
|
|
\\name:9:alice, yo,age:num:30
|
|
\\
|
|
, result);
|
|
}
|
|
test "fmt outputs defaults with option" {
|
|
// Example: serialize typed Zig values directly to SRF format.
|
|
const Data = struct {
|
|
name: []const u8 = "bob",
|
|
age: u8,
|
|
};
|
|
const values: []const Data = &.{
|
|
.{ .age = 30 },
|
|
};
|
|
var buf: [4096]u8 = undefined;
|
|
const result = try std.fmt.bufPrint(
|
|
&buf,
|
|
"{f}",
|
|
.{fmt(Data, values, .{ .emit_default_values = true })},
|
|
);
|
|
try std.testing.expectEqualStrings(
|
|
\\#!srfv1
|
|
\\name::bob,age:num:30
|
|
\\
|
|
, result);
|
|
}
|
|
test "parse with diagnostics" {
|
|
// Example: batch parsing collects all records and fields into slices.
|
|
// Prefer `iterator` for streaming; use `parse` when random access to
|
|
// all records is needed.
|
|
const data =
|
|
\\#!srfv1
|
|
\\#!long
|
|
\\name:num:alice
|
|
\\age:num:30
|
|
\\
|
|
\\name::bob
|
|
\\age:num:25
|
|
\\#!eof
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
try std.testing.expect(
|
|
blk: {
|
|
var it = iterator(&reader, allocator, .{ .diagnostics = &diag }) catch break :blk false;
|
|
defer it.deinit();
|
|
while (true) {
|
|
_ = it.next() catch |err| break :blk err == error.ParseFailed;
|
|
}
|
|
break :blk false;
|
|
},
|
|
);
|
|
const errors = diags.errors();
|
|
try std.testing.expectEqual(@as(usize, 1), errors.len);
|
|
const first = errors[0];
|
|
try std.testing.expectEqual(@as(usize, 3), first.line);
|
|
// TODO: this is at the end of the line. I'm not sure what the actual column
|
|
// should be here, but it's probably not that. Maybe at the beginning of
|
|
// "alice", which would be column 10?
|
|
try std.testing.expectEqual(@as(usize, 15), first.column);
|
|
try std.testing.expectEqualStrings("error parsing numeric value", first.message);
|
|
// const second = errors[1];
|
|
// try std.testing.expectEqualStrings("yo", second.message);
|
|
}
|
|
|
|
// ── Tombstone ────────────────────────────────────────────────
|
|
|
|
test "Tombstone.parse: forever without reason" {
|
|
const t = try Tombstone.parse("forever");
|
|
try std.testing.expect(t.isForever());
|
|
try std.testing.expectEqual(@as(i64, std.math.maxInt(i64)), t.until);
|
|
try std.testing.expect(t.reason() == null);
|
|
}
|
|
|
|
test "Tombstone.parse: forever with multi-word reason" {
|
|
const t = try Tombstone.parse("forever not found in tiingo");
|
|
try std.testing.expect(t.isForever());
|
|
try std.testing.expectEqualStrings("not found in tiingo", t.reason().?);
|
|
}
|
|
|
|
test "Tombstone.parse: numeric until without reason" {
|
|
const t = try Tombstone.parse("1782852900");
|
|
try std.testing.expect(!t.isForever());
|
|
try std.testing.expectEqual(@as(i64, 1782852900), t.until);
|
|
try std.testing.expect(t.reason() == null);
|
|
}
|
|
|
|
test "Tombstone.parse: numeric until with reason" {
|
|
const t = try Tombstone.parse("1782852900 delisted");
|
|
try std.testing.expect(!t.isForever());
|
|
try std.testing.expectEqual(@as(i64, 1782852900), t.until);
|
|
try std.testing.expectEqualStrings("delisted", t.reason().?);
|
|
}
|
|
|
|
test "Tombstone.parse: invalid until is a parse error" {
|
|
try std.testing.expectError(error.ParseFailed, Tombstone.parse("notanumber"));
|
|
try std.testing.expectError(error.ParseFailed, Tombstone.parse("notanumber with reason"));
|
|
}
|
|
|
|
test "Tombstone.parse: 256-byte reason accepted, 257 rejected" {
|
|
var buf: [8 + 257]u8 = undefined;
|
|
@memcpy(buf[0..8], "forever ");
|
|
@memset(buf[8..], 'x');
|
|
|
|
const ok = try Tombstone.parse(buf[0 .. 8 + 256]);
|
|
try std.testing.expectEqual(@as(usize, 256), ok.reason().?.len);
|
|
|
|
try std.testing.expectError(error.ParseFailed, Tombstone.parse(buf[0 .. 8 + 257]));
|
|
}
|
|
|
|
test "Tombstone isFresh/dataAvailable: forever" {
|
|
const data = "#!srfv1\n#!tombstoned=forever\n";
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
try std.testing.expect(ri.tombstoned != null);
|
|
try std.testing.expect(ri.tombstoned.?.isForever());
|
|
try std.testing.expect(ri.isFresh(std.testing.io)); // never re-fetch
|
|
try std.testing.expect(!ri.dataAvailable()); // but data is absent
|
|
}
|
|
|
|
test "Tombstone isFresh: future recheck fresh, past recheck stale" {
|
|
{
|
|
// recheck ~year 3000 -> still fresh, do not re-fetch yet
|
|
const data = "#!srfv1\n#!tombstoned=32503680000\n";
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
try std.testing.expect(ri.isFresh(std.testing.io));
|
|
try std.testing.expect(!ri.dataAvailable());
|
|
}
|
|
{
|
|
// recheck at epoch+1 -> in the past -> stale, re-fetch due
|
|
const data = "#!srfv1\n#!tombstoned=1\n";
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
try std.testing.expect(!ri.isFresh(std.testing.io));
|
|
try std.testing.expect(!ri.dataAvailable());
|
|
}
|
|
}
|
|
|
|
test "Tombstone dataAvailable: true without a tombstone" {
|
|
const data = "#!srfv1\nkey::value\n";
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
try std.testing.expect(ri.tombstoned == null);
|
|
try std.testing.expect(ri.dataAvailable());
|
|
}
|
|
|
|
test "Tombstone round-trip: forever with reason" {
|
|
const Rec = struct { k: []const u8 };
|
|
const items: []const Rec = &.{};
|
|
const tomb = try Tombstone.parse("forever not found in tiingo");
|
|
var buf: [512]u8 = undefined;
|
|
const out = try std.fmt.bufPrint(&buf, "{f}", .{fmt(Rec, items, .{ .tombstoned = tomb })});
|
|
try std.testing.expect(std.mem.indexOf(u8, out, "#!tombstoned=forever not found in tiingo") != null);
|
|
|
|
var reader = std.Io.Reader.fixed(out);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
try std.testing.expect(ri.tombstoned.?.isForever());
|
|
try std.testing.expectEqualStrings("not found in tiingo", ri.tombstoned.?.reason().?);
|
|
try std.testing.expect(!ri.dataAvailable());
|
|
}
|
|
|
|
test "Tombstone round-trip: numeric until with reason" {
|
|
const Rec = struct { k: []const u8 };
|
|
const items: []const Rec = &.{};
|
|
const tomb = try Tombstone.parse("1782852900 delisted");
|
|
var buf: [512]u8 = undefined;
|
|
const out = try std.fmt.bufPrint(&buf, "{f}", .{fmt(Rec, items, .{ .tombstoned = tomb })});
|
|
try std.testing.expect(std.mem.indexOf(u8, out, "#!tombstoned=1782852900 delisted") != null);
|
|
|
|
var reader = std.Io.Reader.fixed(out);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
try std.testing.expect(!ri.tombstoned.?.isForever());
|
|
try std.testing.expectEqual(@as(i64, 1782852900), ri.tombstoned.?.until);
|
|
try std.testing.expectEqualStrings("delisted", ri.tombstoned.?.reason().?);
|
|
}
|
|
|
|
test "Tombstone round-trip: forever without reason" {
|
|
const Rec = struct { k: []const u8 };
|
|
const items: []const Rec = &.{};
|
|
const tomb = try Tombstone.parse("forever");
|
|
var buf: [512]u8 = undefined;
|
|
const out = try std.fmt.bufPrint(&buf, "{f}", .{fmt(Rec, items, .{ .tombstoned = tomb })});
|
|
try std.testing.expect(std.mem.indexOf(u8, out, "#!tombstoned=forever\n") != null);
|
|
|
|
var reader = std.Io.Reader.fixed(out);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
try std.testing.expect(ri.tombstoned.?.isForever());
|
|
try std.testing.expect(ri.tombstoned.?.reason() == null);
|
|
}
|
|
|
|
test "Tombstone reason truncates at an inline '#' (Directive comment stripping)" {
|
|
// Directive.parse strips inline comments (everything from a bare '#'
|
|
// onward) before Tombstone.parse sees the value, so a '#' embedded in a
|
|
// tombstone reason truncates it. This pins that known limitation:
|
|
// reasons should be plain text with no '#'. Note the truncation happens
|
|
// via the iterator/Directive path, NOT Tombstone.parse directly.
|
|
const data = "#!srfv1\n#!tombstoned=forever gone#not-part-of-reason\n";
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
try std.testing.expect(ri.tombstoned.?.isForever());
|
|
try std.testing.expectEqualStrings("gone", ri.tombstoned.?.reason().?);
|
|
}
|
|
test "coerce: unknown enum member is an error" {
|
|
const data =
|
|
\\#!srfv1
|
|
\\kind::gamma
|
|
;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
const fi = (try ri.next()).?;
|
|
try std.testing.expectError(
|
|
error.StringValueNotValidEnumMember,
|
|
fi.to(struct { kind: enum { alpha, beta } }, .{}),
|
|
);
|
|
}
|
|
|
|
test "coerce: non-string enum value is an error" {
|
|
const data =
|
|
\\#!srfv1
|
|
\\kind:num:1
|
|
;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
const fi = (try ri.next()).?;
|
|
try std.testing.expectError(
|
|
error.EnumValueNotString,
|
|
fi.to(struct { kind: enum { alpha, beta } }, .{}),
|
|
);
|
|
}
|
|
|
|
test "coerce: valid enum member still round-trips" {
|
|
const data =
|
|
\\#!srfv1
|
|
\\kind::beta
|
|
;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
const fi = (try ri.next()).?;
|
|
const rec = try fi.to(struct { kind: enum { alpha, beta } }, .{});
|
|
try std.testing.expectEqual(.beta, rec.kind);
|
|
}
|
|
|
|
test "coerce: string in a numeric field errors when strings_to_numbers is off" {
|
|
// The `close_price::200.00` shape that took down a downstream
|
|
// consumer: string separator on a numeric field.
|
|
const data =
|
|
\\#!srfv1
|
|
\\cost::200.00
|
|
;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
const fi = (try ri.next()).?;
|
|
try std.testing.expectError(
|
|
error.FloatNotNumberType,
|
|
fi.to(struct { cost: f64 }, .{}),
|
|
);
|
|
}
|
|
|
|
test "raw values returned iff include_raw is set to true" {
|
|
const data =
|
|
\\#!srfv1
|
|
\\cost:num:200.00
|
|
;
|
|
// raw option off
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var ri = try iterator(&reader, std.testing.allocator, .{});
|
|
defer ri.deinit();
|
|
const fi = (try ri.next()).?;
|
|
try std.testing.expect((try fi.next()).?.raw == null);
|
|
var raw_reader = std.Io.Reader.fixed(data);
|
|
var raw_ri = try iterator(&raw_reader, std.testing.allocator, .{ .include_raw = true });
|
|
defer raw_ri.deinit();
|
|
const raw_fi = (try raw_ri.next()).?;
|
|
try std.testing.expectEqualStrings("200.00", (try raw_fi.next()).?.raw.?);
|
|
}
|
|
|
|
test "BoundedDiagnostics.errors returns a view into the diagnostics, not a copy" {
|
|
var diags: BoundedDiagnostics(4) = .empty;
|
|
try diags.addError(.{
|
|
.message = "error parsing numeric value",
|
|
.level = .err,
|
|
.line = 3,
|
|
.column = 15,
|
|
});
|
|
|
|
// `errors()` hands out a slice of `self.buffer`, so it has to be *our*
|
|
// buffer. Taking `self` by value copies the whole struct, and the slice then
|
|
// points into a temporary that dies with the call: every read through it is
|
|
// a read of a dead stack frame.
|
|
try std.testing.expectEqual(
|
|
@intFromPtr(&diags.buffer),
|
|
@intFromPtr(diags.errors().ptr),
|
|
);
|
|
}
|
|
|
|
test "compact format: a length prefix shorter than the line is reported exactly once" {
|
|
// const lvl = std.testing.log_level;
|
|
// defer std.testing.log_level = lvl;
|
|
// std.testing.log_level = .debug;
|
|
|
|
const data =
|
|
\\#!srfv1
|
|
\\k:3:hello,foo::bar
|
|
\\
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
|
|
var it = try iterator(&reader, allocator, .{ .diagnostics = &diag });
|
|
defer it.deinit();
|
|
const record = try it.next();
|
|
try std.testing.expect(record != null);
|
|
const k = try record.?.next();
|
|
try std.testing.expect(k != null);
|
|
try std.testing.expect(k.?.value != null);
|
|
try std.testing.expect(k.?.value.? == .string);
|
|
try std.testing.expectEqualStrings("hel", k.?.value.?.string);
|
|
|
|
const errors = diags.errors();
|
|
try std.testing.expectEqual(@as(usize, 1), errors.len);
|
|
try std.testing.expectEqual(@as(usize, 2), errors[0].line);
|
|
|
|
// The parser should recover from this underflow on the length in this
|
|
// situation, but we should still see a ParseError at the end of the stream
|
|
const foo = try record.?.next();
|
|
try std.testing.expect(foo != null);
|
|
try std.testing.expect(foo.?.value != null);
|
|
try std.testing.expect(foo.?.value.? == .string);
|
|
try std.testing.expectEqualStrings("bar", foo.?.value.?.string);
|
|
|
|
// Any attempt to move to the next record should trigger parsefailed. But
|
|
// at least we've gotten to the point of consuming the full record
|
|
try std.testing.expectError(error.ParseFailed, it.next());
|
|
}
|
|
test "long format: a length prefix shorter than the line is reported" {
|
|
// In long format `next` takes the `field_delimiter == '\n'` branch and calls
|
|
// nextLine() unconditionally, discarding whatever was left of the line. So
|
|
// `k:3:hello` silently becomes "hel": no diagnostic, no error, no log. The
|
|
// compact branch right below it reports exactly this condition.
|
|
const data =
|
|
\\#!srfv1
|
|
\\#!long
|
|
\\k:3:hello
|
|
\\
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
|
|
try std.testing.expect(blk: {
|
|
var it = iterator(&reader, allocator, .{ .diagnostics = &diag }) catch break :blk false;
|
|
defer it.deinit();
|
|
while (true) {
|
|
const record = it.next() catch |err| break :blk err == error.ParseFailed;
|
|
if (record == null) break :blk false; // parsed cleanly: the surplus was dropped
|
|
}
|
|
});
|
|
|
|
const errors = diags.errors();
|
|
try std.testing.expectEqual(@as(usize, 1), errors.len);
|
|
try std.testing.expectEqual(@as(usize, 3), errors[0].line);
|
|
}
|
|
|
|
test "long format: no error with comment after field val" {
|
|
// In long format `next` takes the `field_delimiter == '\n'` branch and calls
|
|
// nextLine() unconditionally, discarding whatever was left of the line. So
|
|
// `k:3:hello` silently becomes "hel": no diagnostic, no error, no log. The
|
|
// compact branch right below it reports exactly this condition.
|
|
const data =
|
|
\\#!srfv1
|
|
\\#!long
|
|
\\k:5:hello # a comment
|
|
\\
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
|
|
try std.testing.expect(blk: {
|
|
var it = iterator(&reader, allocator, .{ .diagnostics = &diag }) catch break :blk false;
|
|
defer it.deinit();
|
|
while (true) {
|
|
const record = it.next() catch |err| break :blk err == error.ParseFailed;
|
|
if (record == null) break :blk true; // parsed cleanly: the surplus was dropped
|
|
}
|
|
});
|
|
|
|
const errors = diags.errors();
|
|
try std.testing.expectEqual(@as(usize, 0), errors.len);
|
|
}
|
|
test "long format: a length prefix shorter than the line is fatal (multiple lines)" {
|
|
// In long format `next` takes the `field_delimiter == '\n'` branch and calls
|
|
// nextLine() unconditionally, discarding whatever was left of the line. So
|
|
// `k:3:hello` silently becomes "hel": no diagnostic, no error, no log. The
|
|
// compact branch right below it reports exactly this condition.
|
|
const data =
|
|
\\#!srfv1
|
|
\\#!long
|
|
\\k:4:he
|
|
\\llo
|
|
\\
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
|
|
try std.testing.expect(blk: {
|
|
var it = iterator(&reader, allocator, .{ .diagnostics = &diag }) catch break :blk false;
|
|
defer it.deinit();
|
|
while (true) {
|
|
const record = it.next() catch |err| break :blk err == error.ParseFailed;
|
|
if (record == null) break :blk false; // parsed cleanly: the surplus was dropped
|
|
}
|
|
});
|
|
|
|
const errors = diags.errors();
|
|
try std.testing.expectEqual(@as(usize, 1), errors.len);
|
|
try std.testing.expectEqual(@as(usize, 4), errors[0].line);
|
|
}
|
|
|
|
test "long format: a length prefix that consumes the whole line still parses" {
|
|
const data =
|
|
\\#!srfv1
|
|
\\#!long
|
|
\\k:5:hello
|
|
\\
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
|
|
var it = try iterator(&reader, allocator, .{ .diagnostics = &diag });
|
|
defer it.deinit();
|
|
while (try it.next()) |_| {}
|
|
try std.testing.expectEqual(@as(usize, 0), diags.errors().len);
|
|
}
|
|
|
|
test "long format: a multi-line length-prefixed value still parses" {
|
|
const data =
|
|
\\#!srfv1
|
|
\\#!long
|
|
\\bio:7:foo
|
|
\\bar
|
|
\\
|
|
;
|
|
const allocator = std.testing.allocator;
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
|
|
var it = try iterator(&reader, allocator, .{ .diagnostics = &diag });
|
|
defer it.deinit();
|
|
while (try it.next()) |_| {}
|
|
try std.testing.expectEqual(@as(usize, 0), diags.errors().len);
|
|
}
|
|
/// Column of the first reported diagnostic, or an error if there wasn't one.
|
|
fn firstErrorColumn(data: []const u8) !usize {
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
|
|
var it = try iterator(&reader, std.testing.allocator, .{ .diagnostics = &diag });
|
|
defer it.deinit();
|
|
while (it.next() catch null) |_| {}
|
|
|
|
const errors = diags.errors();
|
|
if (errors.len == 0) return error.NoDiagnosticReported;
|
|
return errors[0].column;
|
|
}
|
|
|
|
test "column accounts for indentation that nextLine trimmed" {
|
|
// `nextLine` trims leading whitespace but resets column to 1, so an indented
|
|
// field and an unindented one report the same column even though the whole
|
|
// field moved right. Asserting the *difference* rather than an absolute
|
|
// column keeps this independent of where the column should point within the
|
|
// field, which is still an open question elsewhere in these tests.
|
|
const plain = "#!srfv1\n#!long\nname:num:abc\n";
|
|
const indented = "#!srfv1\n#!long\n name:num:abc\n";
|
|
|
|
const plain_col = try firstErrorColumn(plain);
|
|
const indented_col = try firstErrorColumn(indented);
|
|
try std.testing.expectEqual(plain_col + 4, indented_col);
|
|
}
|
|
|
|
test "a tab counts as one column" {
|
|
const plain = "#!srfv1\n#!long\nname:num:abc\n";
|
|
const tabbed = "#!srfv1\n#!long\n\tname:num:abc\n";
|
|
try std.testing.expectEqual(
|
|
(try firstErrorColumn(plain)) + 1,
|
|
try firstErrorColumn(tabbed),
|
|
);
|
|
}
|
|
fn drain(data: []const u8) !void {
|
|
var reader = std.Io.Reader.fixed(data);
|
|
var diags: BoundedDiagnostics(10) = .empty;
|
|
var diag: Diagnostics = diags.diagnostics();
|
|
var it = try iterator(&reader, std.testing.allocator, .{ .diagnostics = &diag });
|
|
defer it.deinit();
|
|
while (it.next() catch null) |fields| {
|
|
while (fields.next() catch null) |_| {}
|
|
}
|
|
}
|
|
|
|
test "compact: a short length prefix with no delimiter after it, then another line" {
|
|
try drain(
|
|
\\#!srfv1
|
|
\\k:2:abc
|
|
\\z::a
|
|
\\
|
|
);
|
|
}
|
|
|
|
test "compact: a recovered short length prefix, then another line" {
|
|
try drain("#!srfv1\nk:3:hello,foo::bar\nz::a,b\n");
|
|
}
|
|
|
|
test "compact: trailing whitespace after a short prefix, then another line" {
|
|
try drain("#!srfv1\nk:5:hello \nz::a\n");
|
|
}
|
|
|
|
test "compact: single-line documents are unaffected" {
|
|
try drain("#!srfv1\nk:2:abc\n");
|
|
try drain("#!srfv1\nk:3:hello,foo::bar\n");
|
|
}
|
|
|
|
test "compact: correct length prefixes are unaffected" {
|
|
try drain("#!srfv1\nz:5:a,b,c,next::x\nq::v\n");
|
|
}
|
|
test "compact underflow, follower is an indented field" {
|
|
try drain(
|
|
\\#!srfv1
|
|
\\bio:5:foo
|
|
\\ indented::v
|
|
\\
|
|
);
|
|
}
|
|
test "control: follower contains a comma" {
|
|
try drain("#!srfv1\nbio:5:foo\n# com,ment\n");
|
|
}
|
|
test "control: follower is short enough to fit" {
|
|
try drain("#!srfv1\nbio:5:foo\nab\n");
|
|
}
|
|
test "control: no underflow" {
|
|
try drain("#!srfv1\nbio:7:foo\nbar\n");
|
|
}
|
|
test "control: same shape in long format" {
|
|
try drain("#!srfv1\n#!long\nbio:5:foo\n# comment\n");
|
|
}
|
|
//
|
|
// Reference: the same documents with the length srf itself suggests. These
|
|
// establish what recovery should be aiming at, and they pass today.
|
|
//
|
|
|
|
test "reference: bio:8: lands on a delimiter" {
|
|
try expectFields("#!srfv1\nbio:8:foo\nz::a,y::b\nfinal::v\n", &.{
|
|
.{ .record = 1, .key = "bio", .value = "foo\nz::a" },
|
|
.{ .record = 1, .key = "y", .value = "b" },
|
|
.{ .record = 2, .key = "final", .value = "v" },
|
|
}, 0);
|
|
}
|
|
|
|
test "reference: bio:13: lands on a delimiter" {
|
|
try expectFields("#!srfv1\nbio:13:foo\nz:5:a,b,c,q::x\nfinal::v\n", &.{
|
|
.{ .record = 1, .key = "bio", .value = "foo\nz:5:a,b,c" },
|
|
.{ .record = 1, .key = "q", .value = "x" },
|
|
.{ .record = 2, .key = "final", .value = "v" },
|
|
}, 0);
|
|
}
|
|
|
|
const ExpectedField = struct { key: []const u8, value: []const u8, record: usize };
|
|
|
|
/// Collects every field srf yields, with the record it belongs to, plus the
|
|
/// diagnostics. Parse failures end collection rather than aborting, so a
|
|
/// recovering parser can be checked on what it managed to produce.
|
|
fn collect(
|
|
allocator: std.mem.Allocator,
|
|
data: []const u8,
|
|
out: *std.ArrayList(ExpectedField),
|
|
diag_count: *usize,
|
|
first_diag: *[]const u8,
|
|
) !void {
|
|
var bounded = BoundedDiagnostics(10).empty;
|
|
var d: Diagnostics = bounded.diagnostics();
|
|
var reader = std.Io.Reader.fixed(data);
|
|
|
|
var it = try iterator(&reader, allocator, .{ .diagnostics = &d });
|
|
defer it.deinit();
|
|
|
|
var record: usize = 0;
|
|
outer: while (true) {
|
|
const fields = it.next() catch break :outer orelse break :outer;
|
|
record += 1;
|
|
while (true) {
|
|
const f = fields.next() catch break :outer orelse break;
|
|
const v = if (f.value) |vv| switch (vv) {
|
|
.string => |s| s,
|
|
else => "<non-string>",
|
|
} else "<null>";
|
|
try out.append(allocator, .{
|
|
.key = try allocator.dupe(u8, f.key),
|
|
.value = try allocator.dupe(u8, v),
|
|
.record = record,
|
|
});
|
|
}
|
|
}
|
|
|
|
diag_count.* = bounded.errors().len;
|
|
if (bounded.errors().len > 0) {
|
|
first_diag.* = try allocator.dupe(u8, bounded.errors()[0].message);
|
|
}
|
|
}
|
|
|
|
fn expectFields(data: []const u8, expected: []const ExpectedField, want_diags: usize) !void {
|
|
const allocator = std.testing.allocator;
|
|
var got: std.ArrayList(ExpectedField) = .empty;
|
|
defer {
|
|
for (got.items) |f| {
|
|
allocator.free(f.key);
|
|
allocator.free(f.value);
|
|
}
|
|
got.deinit(allocator);
|
|
}
|
|
var diag_count: usize = 0;
|
|
var first: []const u8 = "";
|
|
try collect(allocator, data, &got, &diag_count, &first);
|
|
defer if (first.len > 0) allocator.free(first);
|
|
|
|
if (first.len > 0) log.debug("diagnostic: {s}", .{first});
|
|
for (got.items) |f| {
|
|
log.debug(" got record {d}: '{s}' = '{s}'", .{ f.record, f.key, f.value });
|
|
}
|
|
|
|
try std.testing.expectEqual(want_diags, diag_count);
|
|
try std.testing.expectEqual(expected.len, got.items.len);
|
|
for (expected, got.items) |e, g| {
|
|
try std.testing.expectEqual(e.record, g.record);
|
|
try std.testing.expectEqualStrings(e.key, g.key);
|
|
try std.testing.expectEqualStrings(e.value, g.value);
|
|
}
|
|
}
|
|
|
|
test "compact: a value ending one byte short of a delimiter" {
|
|
// Declares 7, so bio is "foo\nz::" and the surplus is the single byte "a".
|
|
// Stepping over it and the following comma gives the same tail as the
|
|
// bio:8: reference.
|
|
try expectFields("#!srfv1\nbio:7:foo\nz::a,y::b\nfinal::v\n", &.{
|
|
.{ .record = 1, .key = "bio", .value = "foo\nz::" },
|
|
.{ .record = 1, .key = "y", .value = "b" },
|
|
.{ .record = 2, .key = "final", .value = "v" },
|
|
}, 1);
|
|
}
|
|
|
|
test "compact: a value ending one byte short, deeper into the next line" {
|
|
// Declares 12, so bio is "foo\nz:5:a,b," and the surplus is "c".
|
|
try expectFields("#!srfv1\nbio:12:foo\nz:5:a,b,c,q::x\nfinal::v\n", &.{
|
|
.{ .record = 1, .key = "bio", .value = "foo\nz:5:a,b," },
|
|
.{ .record = 1, .key = "q", .value = "x" },
|
|
.{ .record = 2, .key = "final", .value = "v" },
|
|
}, 1);
|
|
}
|