srf-lsp/src/document.zig

//! Locates the SRF construct under a byte offset.
//!
//! srf's parser is a streaming iterator and `srf.Field` carries no positions, so
//! answering "what is under the cursor?" needs its own pass. This module does
//! only that: it finds a token and hands back the surrounding field's text.
//! Interpreting the value is `hover.zig`'s job, and it defers to srf for that.
//!
//! The scan mirrors the format's two mode-dependent rules, which is why it
//! cannot be a simple line splitter:
//!
//!   * the field delimiter is `,` in compact format and end-of-line after
//!     `#!long`, so what counts as one field depends on a directive above it;
//!   * a numeric type hint means "the value is exactly N bytes", which can carry
//!     a value across newlines.

const std = @import("std");

pub const Span = struct {
    start: usize,
    end: usize,

    fn contains(self: Span, offset: usize) bool {
        // `end` is inclusive here: hovering with the cursor just past the last
        // character of a word is still hovering that word, and editors routinely
        // report exactly that position.
        return offset >= self.start and offset <= self.end;
    }
};

/// The field a token belongs to, as raw slices of the document.
pub const Field = struct {
    key: []const u8,
    /// null for the `key::value` form, which carries no hint.
    hint: ?[]const u8,
    /// Empty when the field has no value (`missing:null:`).
    value: []const u8,
    /// Whether the field was found under `#!long`.
    long_mode: bool,
    /// Set when a numeric hint's byte count does not agree with the document.
    length_problem: ?LengthProblem = null,

    /// The hint parsed as a byte count, for the length-prefixed form.
    pub fn declaredLength(self: Field) ?usize {
        const hint = self.hint orelse return null;
        if (hint.len == 0) return null;
        for (hint) |c| if (c < '0' or c > '9') return null;
        return std.fmt.parseInt(usize, hint, 10) catch null;
    }

    /// Which `srf.Value` variant this field's hint produces.
    ///
    /// Coercion cares about the variant, not the hint text: an empty hint,
    /// `string`, and a byte count all yield `.string` and coerce identically, so
    /// treating them as different types would flag correct documents. srf's own
    /// README uses `key:7:...` and `key::...` for the same key.
    ///
    /// Null when the field carries no type evidence:
    ///
    ///   * a `null` hint, which is an absent value and mixes legitimately with
    ///     any other type (`coerce` handles it through `.optional`);
    ///   * an empty value, which says nothing about the intended type;
    ///   * an unrecognised hint, which srf already reports as "unrecognized
    ///     metadata for key", so treating it as a type would double-report.
    ///
    /// The order mirrors `Value.parse`'s dispatch in srf.zig.
    pub fn valueKind(self: Field) ?ValueKind {
        if (self.value.len == 0) return null;

        const hint = self.hint orelse return .string;
        if (hint.len == 0 or std.mem.eql(u8, hint, "string")) return .string;
        if (std.mem.eql(u8, hint, "binary")) return .bytes;
        if (std.mem.eql(u8, hint, "num")) return .number;
        if (std.mem.eql(u8, hint, "bool")) return .boolean;
        if (std.mem.eql(u8, hint, "null")) return null;
        // A byte count is a length-prefixed string; anything else is a bad hint.
        if (self.declaredLength() != null) return .string;
        return null;
    }
};

/// The kinds of value an SRF field can hold, as `srf.Value` sees them.
pub const ValueKind = enum {
    string,
    number,
    boolean,
    bytes,

    /// How to name this kind in a message aimed at someone editing the file.
    pub fn describe(self: ValueKind) []const u8 {
        return switch (self) {
            .string => "a string",
            .number => "`num`",
            .boolean => "`bool`",
            .bytes => "`binary`",
        };
    }
};

pub const Directive = struct {
    /// Directive name without the `#!`, e.g. `long` or `expires`.
    name: []const u8,
    /// Text after `=`, if any.
    argument: ?[]const u8,
};

/// A length prefix that does not agree with the document.
///
/// Worth reporting separately because srf handles these badly: in long format it
/// silently keeps the declared bytes and drops the rest of the line, and for a
/// declaration past the end of the file it fails with no location at all.
pub const LengthProblem = union(enum) {
    /// The document ran out before the declared byte count.
    truncated: struct { declared: usize, available: usize },
    /// The declared count does not land on a field terminator, so the value
    /// really continues past where the declaration says it stops. In long format
    /// srf discards the remainder without a word.
    overrun: struct {
        declared: usize,
        /// Where the surplus starts, which is the declared end of the value.
        surplus_start: usize,
        /// Bytes from there to the end of that line: the text that does not fit
        /// the declaration, and which srf drops in long format.
        surplus: usize,
    },
};

/// A field and the spans of its parts.
pub const Located = struct {
    field: Field,
    key_span: Span,
    /// null for the `key::value` form.
    hint_span: ?Span,
    value_span: Span,
};

pub const Item = union(enum) {
    /// The `#!srfv1` header.
    magic: Span,
    directive: struct { span: Span, value: Directive },
    field: Located,
};

pub const Token = struct {
    span: Span,
    what: What,
};

pub const What = union(enum) {
    magic,
    directive: Directive,
    key: Field,
    hint: Field,
    value: Field,
};

/// Walks every construct in the document in order. Comments and blank lines are
/// skipped: nothing in them is addressable.
pub fn items(text: []const u8) Iterator {
    return .{ .scanner = .{ .text = text } };
}

pub const Iterator = struct {
    scanner: Scanner,

    pub fn next(self: *Iterator) ?Item {
        return self.scanner.next();
    }
};

/// Returns the token covering `offset`, or null if the offset lands on a comment,
/// a blank line, punctuation or past the end.
pub fn tokenAt(text: []const u8, offset: usize) ?Token {
    var it = items(text);
    while (it.next()) |item| {
        switch (item) {
            .magic => |span| {
                if (span.contains(offset)) return .{ .span = span, .what = .magic };
            },
            .directive => |d| {
                if (d.span.contains(offset)) {
                    return .{ .span = d.span, .what = .{ .directive = d.value } };
                }
            },
            .field => |located| {
                if (located.key_span.contains(offset)) {
                    return .{ .span = located.key_span, .what = .{ .key = located.field } };
                }
                if (located.hint_span) |hs| {
                    if (hs.contains(offset)) {
                        return .{ .span = hs, .what = .{ .hint = located.field } };
                    }
                }
                // An empty value has nothing to point at.
                if (located.value_span.end > located.value_span.start and
                    located.value_span.contains(offset))
                {
                    return .{ .span = located.value_span, .what = .{ .value = located.field } };
                }
            },
        }
    }
    return null;
}

const magic_header = "#!srfv1";

/// One forward pass over the document, yielding one construct per call.
///
/// It cannot be a line splitter, because two of the format's rules are not local
/// to a line: the field delimiter depends on a `#!long` / `#!compact` directive
/// above, and a numeric type hint means "exactly N bytes", which can carry a
/// value across newlines.
const Scanner = struct {
    text: []const u8,
    /// Start of the next line to classify.
    pos: usize = 0,
    long_mode: bool = false,
    /// Set while part way through a field line. Compact format packs several
    /// fields onto one line, so the scan has to be able to resume mid-line.
    field_cursor: ?usize = null,
    /// End of the logical line being walked. A length-prefixed value can move it.
    line_end: usize = 0,

    fn next(self: *Scanner) ?Item {
        // Finish the line we are part way through before reading another.
        if (self.field_cursor) |cursor| return self.scanField(cursor);

        while (self.pos < self.text.len) {
            const line_start = self.pos;
            const end = self.lineEnd(line_start);

            // Leading whitespace is insignificant; srf trims it before deciding
            // what a line is.
            var body = line_start;
            while (body < end and isSpace(self.text[body])) body += 1;

            if (body == end) { // blank line
                self.pos = self.nextLine(end);
                continue;
            }

            if (self.text[body] == '#') {
                const item = self.classifyHash(body, end);
                self.pos = self.nextLine(end);
                if (item) |i| return i;
                continue; // a plain comment
            }

            self.line_end = end;
            return self.scanField(body);
        }
        return null;
    }

    /// `#!srfv1`, a `#!directive`, or null for a plain comment.
    fn classifyHash(self: *Scanner, body: usize, line_end: usize) ?Item {
        const line = self.text[body..line_end];
        if (!std.mem.startsWith(u8, line, "#!")) return null;

        if (std.mem.startsWith(u8, line, magic_header)) {
            return .{ .magic = .{ .start = body, .end = body + magic_header.len } };
        }

        // `#!name` or `#!name=argument`, stopping at an inline comment.
        var end = body + 2;
        while (end < line_end and !isSpace(self.text[end])) end += 1;

        const decl = self.text[body + 2 .. end];
        const eq = std.mem.indexOfScalar(u8, decl, '=');
        const name = if (eq) |i| decl[0..i] else decl;

        // Mode directives take effect for everything below them.
        if (std.mem.eql(u8, name, "long")) self.long_mode = true;
        if (std.mem.eql(u8, name, "compact")) self.long_mode = false;

        return .{ .directive = .{
            .span = .{ .start = body, .end = end },
            .value = .{ .name = name, .argument = if (eq) |i| decl[i + 1 ..] else null },
        } };
    }

    /// Reads one field starting at `field_start`, leaving `field_cursor` set if
    /// another field follows it on the same line.
    fn scanField(self: *Scanner, field_start: usize) ?Item {
        // Key runs to the first ':'.
        const colon = std.mem.indexOfScalarPos(u8, self.text[0..self.line_end], field_start, ':') orelse
            return self.abandonLine();
        const head = self.splitHead(colon) orelse return self.abandonLine();

        const extent = self.valueExtent(head.hint, head.value_start);
        // A length-prefixed value can swallow newlines, moving the end of the
        // logical line being walked.
        self.line_end = extent.line_end;

        if (extent.next_field) |n| {
            self.field_cursor = n;
        } else {
            self.field_cursor = null;
            self.pos = self.nextLine(self.line_end);
        }

        return .{ .field = .{
            .field = .{
                .key = self.text[field_start..colon],
                .hint = head.hint,
                .value = self.text[head.value_start..extent.end],
                .long_mode = self.long_mode,
                .length_problem = extent.length_problem,
            },
            .key_span = .{ .start = field_start, .end = colon },
            .hint_span = head.hint_span,
            .value_span = .{ .start = head.value_start, .end = extent.end },
        } };
    }

    /// Gives up on the current line: it has no colon, so it is not a field.
    fn abandonLine(self: *Scanner) ?Item {
        self.field_cursor = null;
        self.pos = self.nextLine(self.line_end);
        return self.next();
    }

    const Head = struct {
        hint: ?[]const u8,
        hint_span: ?Span,
        value_start: usize,
    };

    /// Splits what follows the key's colon into an optional type hint and the
    /// start of the value. Null when the line has no second colon, so it is not
    /// a field at all.
    fn splitHead(self: *const Scanner, colon: usize) ?Head {
        if (colon + 1 < self.line_end and self.text[colon + 1] == ':') {
            // `key::value` carries no hint.
            return .{ .hint = null, .hint_span = null, .value_start = colon + 2 };
        }

        // `key:hint:value`, where the hint may be padded with whitespace.
        const second = std.mem.indexOfScalarPos(u8, self.text[0..self.line_end], colon + 1, ':') orelse
            return null;
        var start = colon + 1;
        var end = second;
        while (start < end and isSpace(self.text[start])) start += 1;
        while (end > start and isSpace(self.text[end - 1])) end -= 1;
        return .{
            .hint = self.text[start..end],
            .hint_span = .{ .start = start, .end = end },
            .value_start = second + 1,
        };
    }

    const Extent = struct {
        /// One past the last byte of the value.
        end: usize,
        /// Where the next field on this line starts, if any.
        next_field: ?usize,
        /// End of the logical line, which a length-prefixed value can move.
        line_end: usize,
        length_problem: ?LengthProblem,
    };

    /// How far the value runs, which is the one part of the format that depends
    /// on both the mode and the type hint.
    fn valueExtent(self: *const Scanner, hint: ?[]const u8, value_start: usize) Extent {
        if (parseLength(hint)) |declared| {
            // Exactly N bytes, newlines included. Clamped so a wrong declaration
            // cannot run us off the end of the document.
            const end = @min(value_start + declared, self.text.len);
            const new_line_end = self.lineEnd(end);
            const problem = self.lengthProblem(declared, value_start);

            // Where the next field starts. When the count lands on a delimiter
            // this is simply past it. When it does not, srf recovers by skipping
            // the surplus bytes *and* the delimiter after them, so mirror that:
            // otherwise hover would find no field where the parser reports one.
            const next_field: ?usize = if (self.long_mode)
                // srf discards the remainder of the line in long format, so
                // nothing further on it is addressable.
                null
            else if (end < new_line_end and self.text[end] == ',')
                end + 1
            else if (problem != null)
                if (std.mem.indexOfScalarPos(u8, self.text[0..new_line_end], end, ',')) |comma|
                    comma + 1
                else
                    null
            else
                null;

            return .{
                .end = end,
                .next_field = next_field,
                .line_end = new_line_end,
                .length_problem = problem,
            };
        }

        // Long format: the value runs to end of line and commas are literal text.
        if (self.long_mode) {
            return .{
                .end = self.line_end,
                .next_field = null,
                .line_end = self.line_end,
                .length_problem = null,
            };
        }

        // Compact format: the next comma ends the value and starts a field.
        if (std.mem.indexOfScalarPos(u8, self.text[0..self.line_end], value_start, ',')) |comma| {
            return .{
                .end = comma,
                .next_field = comma + 1,
                .line_end = self.line_end,
                .length_problem = null,
            };
        }
        return .{
            .end = self.line_end,
            .next_field = null,
            .line_end = self.line_end,
            .length_problem = null,
        };
    }

    /// Checks a declared byte count against the document.
    ///
    /// The test is what *follows* the declared bytes, not how many bytes we took:
    /// a length-prefixed value must be followed immediately by a field
    /// terminator. Comparing the count against a value sliced using that same
    /// count would always agree, which is no test at all.
    fn lengthProblem(self: *const Scanner, declared: usize, value_start: usize) ?LengthProblem {
        const available = self.text.len - value_start;
        if (available < declared) {
            return .{ .truncated = .{ .declared = declared, .available = available } };
        }

        const after = value_start + declared;
        // End of file terminates a field just as well as a newline does.
        if (after >= self.text.len) return null;

        if (self.text[after] == '\n') return null;
        if (!self.long_mode) {
            if (self.text[after] == ',') return null;
        } else {
            // Long format tolerates whitespace, and a comment, after the value.
            // Mirrors srf's `checkShortPrefix`, so hover cannot flag something
            // the parser is happy with.
            var i = after;
            while (i < self.text.len and isSpace(self.text[i])) i += 1;
            if (i >= self.text.len or self.text[i] == '\n' or self.text[i] == '#') return null;
        }

        // Measured from the declared end, not from the start of the value: a
        // multi-line length-prefixed value can put the surplus on a later line,
        // and a range built from the first line's end would come out inverted.
        //
        // How far the surplus runs has to match how srf counts it, or hover ends
        // up suggesting a different corrected length than the diagnostic does. srf
        // stops at the next delimiter in compact format (`extra_bytes` in
        // `checkShortPrefix`) and runs to end of line in long format, where a
        // comma is ordinary text.
        const line_end = self.lineEnd(after);
        const surplus_end = if (self.long_mode)
            line_end
        else if (std.mem.indexOfScalarPos(u8, self.text[0..line_end], after, ',')) |comma|
            comma
        else
            line_end;

        return .{ .overrun = .{
            .declared = declared,
            .surplus_start = after,
            .surplus = surplus_end - after,
        } };
    }

    fn lineEnd(self: *const Scanner, from: usize) usize {
        return std.mem.indexOfScalarPos(u8, self.text, from, '\n') orelse self.text.len;
    }

    fn nextLine(self: *const Scanner, line_end: usize) usize {
        return if (line_end < self.text.len) line_end + 1 else self.text.len;
    }
};

fn isSpace(c: u8) bool {
    return c == ' ' or c == '\t' or c == '\r';
}

/// The hint as a byte count, or null when it is a keyword hint.
fn parseLength(hint: ?[]const u8) ?usize {
    const h = hint orelse return null;
    if (h.len == 0) return null;
    for (h) |c| if (c < '0' or c > '9') return null;
    const n = std.fmt.parseInt(usize, h, 10) catch return null;
    // A zero length is an empty value, not a span to walk.
    return if (n == 0) null else n;
}

const testing = std.testing;

/// Finds `needle` in `text` and returns the offset of its midpoint, which is a
/// stable way to say "hover here" without hand-counting byte offsets.
fn midpointOf(text: []const u8, needle: []const u8) usize {
    const at = std.mem.indexOf(u8, text, needle).?;
    return at + needle.len / 2;
}

test "hovering the magic header" {
    const text = "#!srfv1\nname::alice\n";
    const token = tokenAt(text, midpointOf(text, "#!srfv1")).?;
    try testing.expect(token.what == .magic);
    try testing.expectEqualStrings("#!srfv1", text[token.span.start..token.span.end]);
}

test "hovering a bare directive" {
    const text = "#!srfv1\n#!long\nname::alice\n";
    const token = tokenAt(text, midpointOf(text, "#!long")).?;
    try testing.expectEqualStrings("long", token.what.directive.name);
    try testing.expectEqual(@as(?[]const u8, null), token.what.directive.argument);
}

test "hovering a directive with an argument" {
    const text = "#!srfv1\n#!expires=1772589213\nk::v\n";
    const token = tokenAt(text, midpointOf(text, "expires")).?;
    try testing.expectEqualStrings("expires", token.what.directive.name);
    try testing.expectEqualStrings("1772589213", token.what.directive.argument.?);
}

test "a directive's inline comment is not part of its argument" {
    const text = "#!srfv1\n#!long # use long format\nk::v\n";
    const token = tokenAt(text, midpointOf(text, "#!long")).?;
    try testing.expectEqualStrings("long", token.what.directive.name);
    try testing.expectEqualStrings("#!long", text[token.span.start..token.span.end]);
}

test "a plain comment has nothing to hover" {
    const text = "#!srfv1\n# just a comment\nk::v\n";
    try testing.expectEqual(@as(?Token, null), tokenAt(text, midpointOf(text, "just")));
}

test "hovering an untyped key" {
    const text = "#!srfv1\nname::alice\n";
    const token = tokenAt(text, midpointOf(text, "name")).?;
    try testing.expectEqualStrings("name", token.what.key.key);
    try testing.expectEqual(@as(?[]const u8, null), token.what.key.hint);
    try testing.expectEqualStrings("alice", token.what.key.value);
}

test "hovering a typed key, its hint and its value" {
    const text = "#!srfv1\nage:num:30\n";

    const key = tokenAt(text, midpointOf(text, "age")).?;
    try testing.expectEqualStrings("age", key.what.key.key);
    try testing.expectEqualStrings("num", key.what.key.hint.?);

    const hint = tokenAt(text, midpointOf(text, "num")).?;
    try testing.expect(hint.what == .hint);
    try testing.expectEqualStrings("num", text[hint.span.start..hint.span.end]);

    const value = tokenAt(text, midpointOf(text, "30")).?;
    try testing.expect(value.what == .value);
    try testing.expectEqualStrings("30", value.what.value.value);
}

test "whitespace around a hint is not part of it" {
    const text = "#!srfv1\nage: num :30\n";
    const token = tokenAt(text, midpointOf(text, "num")).?;
    try testing.expectEqualStrings("num", text[token.span.start..token.span.end]);
}

test "compact format splits fields on commas" {
    const text = "#!srfv1\nname::alice,age:num:30\n";

    const first = tokenAt(text, midpointOf(text, "alice")).?;
    try testing.expectEqualStrings("alice", first.what.value.value);
    try testing.expectEqualStrings("name", first.what.value.key);

    const second = tokenAt(text, midpointOf(text, "age")).?;
    try testing.expectEqualStrings("age", second.what.key.key);
    try testing.expectEqualStrings("30", second.what.key.value);
}

test "long format keeps commas inside the value" {
    const text = "#!srfv1\n#!long\nname::alice, bob and carol\n";
    const token = tokenAt(text, midpointOf(text, "bob")).?;
    try testing.expectEqualStrings("alice, bob and carol", token.what.value.value);
    try testing.expect(token.what.value.long_mode);
}

test "a compact directive switches back from long" {
    const text = "#!srfv1\n#!long\n#!compact\nname::alice,age:num:30\n";
    const token = tokenAt(text, midpointOf(text, "alice")).?;
    try testing.expectEqualStrings("alice", token.what.value.value);
    try testing.expect(!token.what.value.long_mode);
}

test "a length-prefixed value spans newlines" {
    const text = "#!srfv1\n#!long\nbio:7:foo\nbar\nname::alice\n";
    const token = tokenAt(text, midpointOf(text, "foo")).?;
    try testing.expect(token.what == .value);
    try testing.expectEqualStrings("foo\nbar", token.what.value.value);
    try testing.expectEqual(@as(?usize, 7), token.what.value.declaredLength());
}

test "the field after a length-prefixed value is still found" {
    const text = "#!srfv1\n#!long\nbio:7:foo\nbar\nname::alice\n";
    const token = tokenAt(text, midpointOf(text, "alice")).?;
    try testing.expectEqualStrings("name", token.what.value.key);
    try testing.expectEqualStrings("alice", token.what.value.value);
}

test "a length-prefixed value in compact format is followed by a comma" {
    const text = "#!srfv1\nk:5:a,b,c,next::x\n";

    const value = tokenAt(text, midpointOf(text, "a,b,c")).?;
    try testing.expectEqualStrings("a,b,c", value.what.value.value);

    const next = tokenAt(text, midpointOf(text, "next")).?;
    try testing.expectEqualStrings("next", next.what.key.key);
    try testing.expectEqualStrings("x", next.what.key.value);
}

test "a length longer than the document does not run off the end" {
    const text = "#!srfv1\n#!long\nk:9999:short\n";
    const token = tokenAt(text, midpointOf(text, "short")).?;
    try testing.expectEqualStrings("short\n", token.what.value.value);
}

test "indented fields are found" {
    const text = "#!srfv1\n#!long\n    name::alice\n";
    const token = tokenAt(text, midpointOf(text, "name")).?;
    try testing.expectEqualStrings("name", token.what.key.key);
}

test "a key containing a comma is still one key" {
    const text = "#!srfv1\n#!long\nlast, first::alice\n";
    const token = tokenAt(text, midpointOf(text, "first")).?;
    try testing.expectEqualStrings("last, first", token.what.key.key);
}

test "an empty value yields no value token" {
    const text = "#!srfv1\n#!long\nmissing:null:\n";
    const key = tokenAt(text, midpointOf(text, "missing")).?;
    try testing.expectEqualStrings("", key.what.key.value);
    // The colon itself is not a value.
    try testing.expectEqual(@as(?Token, null), tokenAt(text, text.len - 1));
}

test "a line with no colon has nothing to hover" {
    const text = "#!srfv1\n#!long\ngarbage line\nk::v\n";
    try testing.expectEqual(@as(?Token, null), tokenAt(text, midpointOf(text, "garbage")));
}

test "a blank line has nothing to hover" {
    const text = "#!srfv1\n\nk::v\n";
    try testing.expectEqual(@as(?Token, null), tokenAt(text, 8));
}

test "an offset past the end has nothing to hover" {
    const text = "#!srfv1\nk::v\n";
    try testing.expectEqual(@as(?Token, null), tokenAt(text, 9999));
}

test "an empty document has nothing to hover" {
    try testing.expectEqual(@as(?Token, null), tokenAt("", 0));
}

test "multi-byte values are located by byte offset" {
    const text = "#!srfv1\n#!long\ncost:num:¥15,000\n";
    const token = tokenAt(text, midpointOf(text, "15,000")).?;
    try testing.expectEqualStrings("¥15,000", token.what.value.value);
}

test "declaredLength only accepts an all-digit hint" {
    const text = "#!srfv1\nk:num:1\n";
    const token = tokenAt(text, midpointOf(text, "num")).?;
    try testing.expectEqual(@as(?usize, null), token.what.hint.declaredLength());
}

test "records after a blank line are still scanned" {
    const text = "#!srfv1\n#!long\nname::alice\n\nname::bob\n";
    const token = tokenAt(text, midpointOf(text, "bob")).?;
    try testing.expectEqualStrings("bob", token.what.value.value);
}

test "the eof directive is a directive" {
    const text = "#!srfv1\nk::v\n#!eof\n";
    const token = tokenAt(text, midpointOf(text, "#!eof")).?;
    try testing.expectEqualStrings("eof", token.what.directive.name);
}

test "a length prefix longer than the document swallows the rest of it" {
    // Deliberate: the format says the value is N bytes, so if the file ends
    // early the trailing text really is part of that value. Diagnostics flag the
    // mismatch; the locator just reports what the declaration implies rather
    // than inventing a boundary.
    const text = "#!srfv1\n#!long\nbad:99:short\n\n";
    const on_blank = tokenAt(text, text.len - 1).?;
    try testing.expect(on_blank.what == .value);
    try testing.expectEqualStrings("bad", on_blank.what.value.key);
    try testing.expectEqual(@as(?usize, 99), on_blank.what.value.declaredLength());
    // Everything from after the second colon to the end of the document.
    try testing.expectEqualStrings("short\n\n", on_blank.what.value.value);
}

test "a blank line outside any value has nothing to hover" {
    const text = "#!srfv1\n#!long\nname::alice\n\nname::bob\n";
    const blank_at = std.mem.indexOf(u8, text, "alice\n\n").? + 6;
    try testing.expectEqual(@as(?Token, null), tokenAt(text, blank_at));
}

/// The length problem for the field containing `offset`, if any.
fn problemAt(text: []const u8, needle: []const u8) ?LengthProblem {
    const offset = midpointOf(text, needle);
    var it = items(text);
    while (it.next()) |item| switch (item) {
        .field => |located| {
            if (located.key_span.contains(offset) or located.value_span.contains(offset) or
                (located.hint_span != null and located.hint_span.?.contains(offset)))
            {
                return located.field.length_problem;
            }
        },
        else => {},
    };
    return null;
}

test "a correct length prefix has no problem" {
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\n#!long\nk:5:hello\n", "hello"),
    );
}

test "a length shorter than the line is an overrun, which srf silently ignores" {
    // srf keeps the first 3 bytes and drops "lo" without a word, so this is the
    // one that most needs reporting.
    const problem = problemAt("#!srfv1\n#!long\nk:3:hello\n", "k:3:").?;
    try testing.expectEqual(@as(usize, 3), problem.overrun.declared);
    // "lo" is the surplus srf throws away.
    try testing.expectEqual(@as(usize, 2), problem.overrun.surplus);
}

test "a length past the end of the document is truncation" {
    const problem = problemAt("#!srfv1\n#!long\nk:99:hello\n", "hello").?;
    try testing.expectEqual(@as(usize, 99), problem.truncated.declared);
    // "hello\n" is all that remains.
    try testing.expectEqual(@as(usize, 6), problem.truncated.available);
}

test "a length ending exactly at end of file is fine" {
    // No trailing newline, so EOF is the terminator.
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\n#!long\nk:5:hello", "hello"),
    );
}

test "in compact format a comma terminates a length-prefixed value" {
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\nk:5:a,b,c,next::x\n", "a,b,c"),
    );
}

test "in compact format a length landing mid-token is an overrun" {
    // After 2 bytes comes 'b', neither a comma nor a newline.
    const problem = problemAt("#!srfv1\nk:2:abc\n", "k:2:").?;
    try testing.expectEqual(@as(usize, 2), problem.overrun.declared);
}

test "in long format a trailing comma after the declared bytes is an overrun" {
    // Long format has no comma delimiter, so the comma is stray text.
    const problem = problemAt("#!srfv1\n#!long\nk:5:hello,\n", "hello").?;
    try testing.expectEqual(@as(usize, 5), problem.overrun.declared);
    // Just the stray comma.
    try testing.expectEqual(@as(usize, 1), problem.overrun.surplus);
}

test "in long format, trailing whitespace after the declared bytes is accepted" {
    // srf's checkShortPrefix trims whitespace before deciding, so flagging this
    // would contradict the parser.
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\n#!long\nk:5:hello \n", "hello"),
    );
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\n#!long\nk:5:hello\t\n", "hello"),
    );
}

test "in long format, a trailing comment after the declared bytes is accepted" {
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\n#!long\nk:5:hello # a note\n", "hello"),
    );
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\n#!long\nk:5:hello   # spaced\n", "hello"),
    );
}

test "in compact format, trailing whitespace is still an overrun" {
    // Compact format has no comment-after-value allowance: the next byte must be
    // a comma or the line must end.
    const problem = problemAt("#!srfv1\nk:5:hello \n", "hello").?;
    try testing.expectEqual(@as(usize, 5), problem.overrun.declared);
}

test "a multi-line length-prefixed value has no problem" {
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\n#!long\nbio:7:foo\nbar\nname::alice\n", "foo"),
    );
}

test "a keyword hint never reports a length problem" {
    try testing.expectEqual(
        @as(?LengthProblem, null),
        problemAt("#!srfv1\n#!long\nk:string:hello\n", "hello"),
    );
}

test "items yields every construct in document order" {
    const text = "#!srfv1\n#!long\n# comment\nname::alice\n\nage:num:30\n#!eof\n";
    var it = items(text);
    var kinds: std.ArrayList([]const u8) = .empty;
    defer kinds.deinit(testing.allocator);
    while (it.next()) |item| {
        try kinds.append(testing.allocator, switch (item) {
            .magic => "magic",
            .directive => "directive",
            .field => "field",
        });
    }
    try testing.expectEqualDeep(
        @as([]const []const u8, &.{ "magic", "directive", "field", "field", "directive" }),
        kinds.items,
    );
}

test "items yields each field of a compact line separately" {
    const text = "#!srfv1\nname::alice,age:num:30,ok:bool:true\n";
    var it = items(text);
    var keys: std.ArrayList([]const u8) = .empty;
    defer keys.deinit(testing.allocator);
    while (it.next()) |item| switch (item) {
        .field => |f| try keys.append(testing.allocator, f.field.key),
        else => {},
    };
    try testing.expectEqualDeep(
        @as([]const []const u8, &.{ "name", "age", "ok" }),
        keys.items,
    );
}

test "items terminates on a line with no colon" {
    const text = "#!srfv1\ngarbage\nk::v\n";
    var it = items(text);
    var fields: usize = 0;
    while (it.next()) |item| switch (item) {
        .field => fields += 1,
        else => {},
    };
    try testing.expectEqual(@as(usize, 1), fields);
}

test "the surplus of a multi-line overrun is measured on its own line" {
    // Declares 5 bytes ("foo\nb"), so the surplus is "ar" on the second line.
    // Measuring from the first line's end would produce an inverted range.
    const problem = problemAt("#!srfv1\n#!long\nbio:5:foo\nbar\n", "bio").?;
    try testing.expectEqual(@as(usize, 5), problem.overrun.declared);
    try testing.expectEqual(@as(usize, 2), problem.overrun.surplus);
    const text = "#!srfv1\n#!long\nbio:5:foo\nbar\n";
    try testing.expectEqualStrings(
        "ar",
        text[problem.overrun.surplus_start..][0..problem.overrun.surplus],
    );
}

/// The value kind of the field containing the midpoint of `needle`.
fn kindAt(text: []const u8, needle: []const u8) ??ValueKind {
    const offset = midpointOf(text, needle);
    var it = items(text);
    while (it.next()) |item| switch (item) {
        .field => |located| {
            if (located.key_span.contains(offset) or located.value_span.contains(offset) or
                (located.hint_span != null and located.hint_span.?.contains(offset)))
            {
                return located.field.valueKind();
            }
        },
        else => {},
    };
    return null;
}

test "an untyped value is a string" {
    try testing.expectEqual(ValueKind.string, (kindAt("#!srfv1\nk::32\n", "k::").?).?);
}

test "an explicit string hint is a string" {
    try testing.expectEqual(ValueKind.string, (kindAt("#!srfv1\nk:string:32\n", "string").?).?);
}

test "a length prefix is a string, the same as the other two forms" {
    // This is what keeps srf's own README quiet: it uses `key:7:...` and
    // `key::...` for the same key in one document.
    try testing.expectEqual(ValueKind.string, (kindAt("#!srfv1\n#!long\nk:2:32\n", "k:2:").?).?);
}

test "num, bool and binary map to their own kinds" {
    try testing.expectEqual(ValueKind.number, (kindAt("#!srfv1\nk:num:32\n", "num").?).?);
    try testing.expectEqual(ValueKind.boolean, (kindAt("#!srfv1\nk:bool:true\n", "bool").?).?);
    try testing.expectEqual(ValueKind.bytes, (kindAt("#!srfv1\nk:binary:aGk=\n", "binary").?).?);
}

test "a null hint carries no type evidence" {
    try testing.expectEqual(@as(?ValueKind, null), kindAt("#!srfv1\nk:null:x\n", "null").?);
}

test "an empty value carries no type evidence" {
    try testing.expectEqual(@as(?ValueKind, null), kindAt("#!srfv1\n#!long\nk:num:\n", "k:num:").?);
    try testing.expectEqual(@as(?ValueKind, null), kindAt("#!srfv1\n#!long\nk::\n", "k::").?);
}

test "an unrecognised hint carries no type evidence, since srf reports it" {
    try testing.expectEqual(@as(?ValueKind, null), kindAt("#!srfv1\nk:nmu:32\n", "nmu").?);
}

test "value kinds describe themselves for a message" {
    try testing.expectEqualStrings("a string", ValueKind.string.describe());
    try testing.expectEqualStrings("`num`", ValueKind.number.describe());
    try testing.expectEqualStrings("`bool`", ValueKind.boolean.describe());
    try testing.expectEqualStrings("`binary`", ValueKind.bytes.describe());
}

test "in compact format the surplus stops at the next delimiter, as srf counts it" {
    // srf's `checkShortPrefix` reports `indexOfScalar(past_val, ',')` bytes, so
    // measuring to end of line here would make hover suggest a different
    // corrected length than the diagnostic does.
    const problem = problemAt("#!srfv1\nim_worth:4:23,000,000,000,really:bool:false\n", "im_worth").?;
    try testing.expectEqual(@as(usize, 4), problem.overrun.declared);
    // "23,0" then "00" then a comma: two surplus bytes, so 4 + 2 = 6.
    try testing.expectEqual(@as(usize, 2), problem.overrun.surplus);
}

test "in compact format a surplus with no delimiter runs to end of line" {
    const problem = problemAt("#!srfv1\nk:2:abcdef\n", "k:2:").?;
    try testing.expectEqual(@as(usize, 4), problem.overrun.surplus);
}

test "in long format the surplus runs to end of line, commas included" {
    // Long format has no comma delimiter, so a comma is ordinary text and srf
    // counts the whole remainder.
    const problem = problemAt("#!srfv1\n#!long\nim_worth:4:23,000,000\n", "im_worth").?;
    // Value is "23,0"; the remainder "00,000" is six bytes, commas and all.
    try testing.expectEqual(@as(usize, 6), problem.overrun.surplus);
}