diff --git a/README.md b/README.md index 9a5ac22..87be3ee 100644 --- a/README.md +++ b/README.md @@ -10,16 +10,35 @@ Provides real-time parse error diagnostics using the SRF library's parser direct directly: missing or duplicate `#!srfv1`, bad type hints, unparseable values, data after `#!eof`, and so on. - All errors from a parse are reported, not just the first. +- Length-prefix checking for the cases srf does not cover. A numeric type hint + declares an exact byte count. srf reports a wrong one in compact format, but in + long format it keeps the declared bytes and discards the rest of the line + without a word, and a count running past the end of the file fails with no + location at all. Both are detected here and reported on the offending bytes. +- Hover, covering every construct in the format: + - `#!srfv1` and each directive, explained. + - `#!expires`, `#!created` and `#!modified` render their Unix timestamp as a + UTC date, because nobody can read `1772589213`. + - Keys and values show the parsed interpretation: numbers, booleans, byte + counts for strings, and for `binary` values the decoded content when it is + text (`decoding to 5 bytes: "hello"`) or a note when it is not. + - Type hints are explained, including a numeric hint's length-prefix meaning. - Position encoding is negotiated during `initialize` (`utf-8`, `utf-16` or `utf-32`), so columns line up even on lines containing multi-byte characters. - Full-document sync, which matches SRF's single-pass parser. - stdio transport. -Not implemented: hover, completion, document symbols, formatting, goto -definition, semantic tokens, and pull diagnostics -(`textDocument/diagnostic`). Requests for them are answered with -`MethodNotFound` rather than left unanswered, and none of them are advertised as -server capabilities. +Value interpretation defers to the srf library rather than reimplementing its +rules, so hover and diagnostics cannot disagree about the same text. + +Not implemented: completion, document symbols, formatting, goto definition, +semantic tokens, and pull diagnostics (`textDocument/diagnostic`). Requests for +them are answered with `MethodNotFound` rather than left unanswered, and none of +them are advertised as server capabilities. + +## Requirements + +Zig 0.16.0. Pinned in `.mise.toml`, along with the other tooling. ## Setup diff --git a/build.zig.zon b/build.zig.zon index 189d129..3c46ffe 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -5,8 +5,8 @@ .minimum_zig_version = "0.16.0", .dependencies = .{ .srf = .{ - .url = "git+https://git.lerch.org/lobo/srf#7692d85745a90144e358bac9f73d1090ddc125fe", - .hash = "srf-0.0.0-qZj5760tAgDN_2Y-nsXzkW_qbntB05Iio___ziFdi937", + .url = "git+https://git.lerch.org/lobo/srf#08f06df810684f8d32ad0ea5a8f2b3ee61733d2c", + .hash = "srf-0.0.0-qZj571YvAgCNHXtzQCOdoAKNzjhLp5o_Ap-GMFmO5-CW", }, }, .paths = .{ diff --git a/src/Server.zig b/src/Server.zig index 3f83158..13b9b4a 100644 --- a/src/Server.zig +++ b/src/Server.zig @@ -13,6 +13,8 @@ const std = @import("std"); const analysis = @import("analysis.zig"); +const hover = @import("hover.zig"); +const positions = @import("positions.zig"); const rpc = @import("rpc.zig"); const log = std.log.scoped(.srf_lsp); @@ -98,6 +100,16 @@ fn field(object: std.json.ObjectMap, name: []const u8) ?std.json.Value { return object.get(name); } +/// LSP line and character numbers are unsigned. A negative or out-of-range value +/// is the client's bug, and clamping to 0 beats refusing the whole request. +fn asU32(value: ?std.json.Value) ?u32 { + const v = value orelse return null; + return switch (v) { + .integer => |i| if (i < 0) 0 else std.math.cast(u32, i) orelse std.math.maxInt(u32), + else => null, + }; +} + // // Sending // @@ -210,6 +222,8 @@ fn dispatch( return self.handleDidClose(root); } else if (std.mem.eql(u8, method, "textDocument/didSave")) { return; // full sync means didChange already told us everything + } else if (std.mem.eql(u8, method, "textDocument/hover")) { + return self.handleHover(root, id); } // Unknown method. A request must be told we cannot serve it; a notification @@ -242,6 +256,7 @@ fn handleInitialize(self: *Server, root: std.json.ObjectMap, id: ?std.json.Value // parser is single-pass over the entire text. .change = 1, }, + .hoverProvider = true, // Deliberately no `diagnosticProvider`: that advertises *pull* // diagnostics (`textDocument/diagnostic`), which we do not // implement. We push via `textDocument/publishDiagnostics` instead, @@ -337,6 +352,42 @@ fn handleDidClose(self: *Server, root: std.json.ObjectMap) !void { }); } +// +// Hover +// + +fn handleHover(self: *Server, root: std.json.ObjectMap, id: ?std.json.Value) !void { + // A hover request with no id is malformed; there is nowhere to send a reply. + const request_id = id orelse return; + + const params = asObject(field(root, "params")) orelse + return self.sendError(request_id, .invalid_params, "hover needs params"); + const doc = asObject(field(params, "textDocument")) orelse + return self.sendError(request_id, .invalid_params, "hover needs a textDocument"); + const uri = asString(field(doc, "uri")) orelse + return self.sendError(request_id, .invalid_params, "hover needs a document uri"); + const position = asObject(field(params, "position")) orelse + return self.sendError(request_id, .invalid_params, "hover needs a position"); + + const text = self.documents.get(uri) orelse { + // Not an error: the client may ask about a document it never opened, or + // one we already closed. Null means "nothing to show here". + return self.sendResult(request_id, null); + }; + + const at: positions.Position = .{ + .line = asU32(field(position, "line")) orelse 0, + .character = asU32(field(position, "character")) orelse 0, + }; + + const result = try hover.hoverAt(self.allocator, text, at, self.position_encoding) orelse { + return self.sendResult(request_id, null); + }; + defer result.deinit(self.allocator); + + try self.sendResult(request_id, result); +} + fn publishDiagnostics(self: *Server, uri: []const u8, text: []const u8) !void { const diagnostics = try analysis.analyze(self.allocator, text, self.position_encoding); defer analysis.freeDiagnostics(self.allocator, diagnostics); @@ -731,10 +782,105 @@ test "we do not advertise capabilities we cannot serve" { // diagnostics in particular: announcing it makes conforming clients send // textDocument/diagnostic, which we would only answer with -32601. try h.expectNotSent("diagnosticProvider"); - try h.expectNotSent("hoverProvider"); try h.expectNotSent("completionProvider"); try h.expectNotSent("documentSymbolProvider"); try h.expectNotSent("documentFormattingProvider"); try h.expectNotSent("definitionProvider"); try h.expectNotSent("semanticTokensProvider"); } + +test "hover is advertised now that it is implemented" { + var h: Harness = undefined; + h.setup(testing.allocator); + defer h.deinit(); + + try h.initialize(); + try h.expectSent("\"hoverProvider\":true"); +} + +test "hovering an open document returns markdown" { + var h: Harness = undefined; + h.setup(testing.allocator); + defer h.deinit(); + try h.initialize(); + + try h.server.handleMessage( + \\{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file:///x.srf","languageId":"srf","version":1,"text":"#!srfv1\nage:num:30\n"}}} + ); + try h.server.handleMessage( + \\{"jsonrpc":"2.0","id":7,"method":"textDocument/hover","params":{"textDocument":{"uri":"file:///x.srf"},"position":{"line":1,"character":1}}} + ); + try h.expectSent("\"id\":7"); + try h.expectSent("markdown"); + try h.expectSent("Parsed as a number"); +} + +test "hovering a directive renders its timestamp" { + var h: Harness = undefined; + h.setup(testing.allocator); + defer h.deinit(); + try h.initialize(); + + try h.server.handleMessage( + \\{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file:///x.srf","languageId":"srf","version":1,"text":"#!srfv1\n#!expires=1772589213\nk::v\n"}}} + ); + try h.server.handleMessage( + \\{"jsonrpc":"2.0","id":8,"method":"textDocument/hover","params":{"textDocument":{"uri":"file:///x.srf"},"position":{"line":1,"character":4}}} + ); + try h.expectSent("2026-03-04"); +} + +test "hovering blank space returns a null result, not an error" { + var h: Harness = undefined; + h.setup(testing.allocator); + defer h.deinit(); + try h.initialize(); + + try h.server.handleMessage( + \\{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file:///x.srf","languageId":"srf","version":1,"text":"#!srfv1\n\nk::v\n"}}} + ); + try h.server.handleMessage( + \\{"jsonrpc":"2.0","id":9,"method":"textDocument/hover","params":{"textDocument":{"uri":"file:///x.srf"},"position":{"line":1,"character":0}}} + ); + try h.expectSent("{\"jsonrpc\":\"2.0\",\"id\":9,\"result\":null}"); +} + +test "hovering a document we never opened returns null" { + var h: Harness = undefined; + h.setup(testing.allocator); + defer h.deinit(); + try h.initialize(); + + try h.server.handleMessage( + \\{"jsonrpc":"2.0","id":10,"method":"textDocument/hover","params":{"textDocument":{"uri":"file:///never.srf"},"position":{"line":0,"character":0}}} + ); + try h.expectSent("\"id\":10,\"result\":null"); +} + +test "a hover request missing its position is refused with invalid_params" { + var h: Harness = undefined; + h.setup(testing.allocator); + defer h.deinit(); + try h.initialize(); + + try h.server.handleMessage( + \\{"jsonrpc":"2.0","id":11,"method":"textDocument/hover","params":{"textDocument":{"uri":"file:///x.srf"}}} + ); + try h.expectSent("-32602"); +} + +test "a hover position with junk types does not take the server down" { + var h: Harness = undefined; + h.setup(testing.allocator); + defer h.deinit(); + try h.initialize(); + + try h.server.handleMessage( + \\{"jsonrpc":"2.0","method":"textDocument/didOpen","params":{"textDocument":{"uri":"file:///x.srf","languageId":"srf","version":1,"text":"#!srfv1\nk::v\n"}}} + ); + try h.server.handleMessage( + \\{"jsonrpc":"2.0","id":12,"method":"textDocument/hover","params":{"textDocument":{"uri":"file:///x.srf"},"position":{"line":"nope","character":-5}}} + ); + try testing.expectEqual(Phase.running, h.server.phase); + try h.expectSent("\"id\":12"); +} diff --git a/src/analysis.zig b/src/analysis.zig index bacbdcc..706c528 100644 --- a/src/analysis.zig +++ b/src/analysis.zig @@ -10,34 +10,12 @@ const std = @import("std"); const srf = @import("srf"); +const document = @import("document.zig"); +const positions = @import("positions.zig"); -/// How LSP positions are counted. The client picks this during `initialize`. -pub const PositionEncoding = enum { - /// Byte offsets. What srf already reports, so no conversion. - @"utf-8", - /// UTF-16 code units. The protocol default, so it is the fallback. - @"utf-16", - /// Codepoints. - @"utf-32", - - pub fn fromWireName(name: []const u8) ?PositionEncoding { - return std.meta.stringToEnum(PositionEncoding, name); - } - - pub fn wireName(self: PositionEncoding) []const u8 { - return @tagName(self); - } -}; - -pub const Position = struct { - line: u32, - character: u32, -}; - -pub const Range = struct { - start: Position, - end: Position, -}; +pub const PositionEncoding = positions.Encoding; +pub const Position = positions.Position; +pub const Range = positions.Range; /// https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#diagnosticSeverity pub const Severity = enum(u8) { @@ -125,12 +103,12 @@ const Sink = struct { else 0; - const line_text = lineAt(self.text, line_index); + const line_text = positions.lineAt(self.text, line_index); const byte_col_raw = if (err.column > 0) err.column - 1 else 0; const byte_col = @min(byte_col_raw, line_text.len); - const line_end = convertColumn(line_text, line_text.len, self.encoding); - const start = convertColumn(line_text, byte_col, self.encoding); + const line_end = positions.characterFromByte(line_text, line_text.len, self.encoding); + const start = positions.characterFromByte(line_text, byte_col, self.encoding); // srf's column usually points just past the text it consumed, which for // a whole-line problem lands at end of line. Underlining from there to @@ -159,62 +137,6 @@ fn severityFor(level: std.log.Level) Severity { }; } -/// Returns the 0-based `index`th line of `text`, excluding its newline. Empty if -/// the line does not exist. -fn lineAt(text: []const u8, index: u32) []const u8 { - var remaining = text; - var i: u32 = 0; - while (true) { - const nl = std.mem.indexOfScalar(u8, remaining, '\n'); - if (i == index) { - const line = if (nl) |n| remaining[0..n] else remaining; - return std.mem.trimEnd(u8, line, "\r"); - } - if (nl) |n| { - remaining = remaining[n + 1 ..]; - i += 1; - } else { - return ""; - } - } -} - -/// Converts a byte offset within a line to a character offset in `encoding`. -/// Invalid UTF-8 is counted a byte at a time rather than rejected, so a -/// half-typed multi-byte character still produces a usable position. -fn convertColumn(line: []const u8, byte_offset: usize, encoding: PositionEncoding) u32 { - const limit = @min(byte_offset, line.len); - if (encoding == .@"utf-8") return @intCast(limit); - - var count: u32 = 0; - var i: usize = 0; - while (i < limit) { - const len = std.unicode.utf8ByteSequenceLength(line[i]) catch { - i += 1; - count += 1; - continue; - }; - if (i + len > line.len) { - i += 1; - count += 1; - continue; - } - const codepoint = std.unicode.utf8Decode(line[i..][0..len]) catch { - i += 1; - count += 1; - continue; - }; - count += switch (encoding) { - // Anything outside the BMP takes two UTF-16 code units. - .@"utf-16" => if (codepoint > 0xFFFF) 2 else 1, - .@"utf-32" => 1, - .@"utf-8" => unreachable, - }; - i += len; - } - return count; -} - /// Parses `text` and returns its diagnostics. Caller owns the result and should /// release it with `freeDiagnostics`. pub fn analyze( @@ -234,14 +156,25 @@ pub fn analyze( } var diagnostics = sink.interface(); - run(allocator, text, &diagnostics) catch |err| { - // srf can fail without saying why: `iterator` returns `ParseFailed` with - // no diagnostic at all for an empty document. Swallowing that leaves the - // editor showing nothing on a file that will not parse, so synthesise a - // diagnostic rather than drop the reason on the floor. - if (sink.items.items.len == 0) try appendUnlocatedError(allocator, &sink, text, err); + const parse_error: ?anyerror = blk: { + run(allocator, text, &diagnostics) catch |err| break :blk err; + break :blk null; }; + // Our own checks run before the fallback below, because they can often name + // the exact problem srf failed to locate. `srf.zig:873` is the common case: + // it detects a length-prefixed value that misses its delimiter, logs it, and + // returns a bare `ParseFailed` with no diagnostic at all. + try appendLengthProblems(allocator, &sink, text, encoding); + + if (parse_error) |err| { + // Only if nothing at all was found. srf can fail without saying why: + // `iterator` returns `ParseFailed` with no diagnostic for an empty + // document. Swallowing that would leave the editor showing nothing on a + // file that will not parse. + if (sink.items.items.len == 0) try appendUnlocatedError(allocator, &sink, text, err); + } + if (sink.dropped) { // Best effort: if we could not allocate a diagnostic we probably cannot // allocate this note either, and the list we have is still worth @@ -258,6 +191,84 @@ pub fn analyze( return sink.items.toOwnedSlice(allocator); } +/// Reports length-prefix declarations that srf does not usefully report itself. +/// +/// srf covers the compact-format case as of 08f06df: a value that does not end on +/// the delimiter gets a diagnostic and a fatal `ParseFailed`. We stay out of its +/// way there, because two squiggles for one defect is worse than one. +/// +/// What is left to us: +/// +/// * **Long format overruns.** `srf.zig:848` takes the `field_delimiter == '\n'` +/// branch and calls `nextLine()` unconditionally, discarding whatever was left +/// of the line. No diagnostic, no failure, no log: `k:3:hello` silently +/// becomes `hel`. Silent data loss is the worst kind, so we report it. +/// * **Declarations past the end of the file.** srf fails with a bare +/// `EndOfStream` and no location at all. +fn appendLengthProblems( + allocator: std.mem.Allocator, + sink: *Sink, + text: []const u8, + encoding: PositionEncoding, +) error{OutOfMemory}!void { + var it = document.items(text); + while (it.next()) |item| { + const located = switch (item) { + .field => |f| f, + else => continue, + }; + const problem = located.field.length_problem orelse continue; + + // srf already diagnoses a compact-format overrun, and does it fatally. + if (problem == .overrun and !located.field.long_mode) continue; + + const message = switch (problem) { + .truncated => |t| try std.fmt.allocPrint( + allocator, + "length prefix declares {d} bytes but only {d} remain in the file", + .{ t.declared, t.available }, + ), + .overrun => |o| try std.fmt.allocPrint( + allocator, + "length prefix declares {d} bytes, so srf will silently discard the {d} byte(s) highlighted here", + .{ o.declared, o.surplus }, + ), + }; + errdefer allocator.free(message); + + sink.items.append(allocator, .{ + // Underline what is wrong. For an overrun that is the surplus text + // past the declared end, not the bytes that do fit: highlighting the + // part srf keeps would point at the one bit that is correct. + .range = switch (problem) { + .truncated => positions.rangeFromSpan( + text, + located.value_span.start, + located.value_span.end, + encoding, + ), + .overrun => |o| positions.rangeFromSpan( + text, + o.surplus_start, + o.surplus_start + o.surplus, + encoding, + ), + }, + // A long-format overrun still parses, so the cost is losing data + // rather than failing to read the file. A truncated declaration stops + // the parse outright. + .severity = switch (problem) { + .truncated => .err, + .overrun => .warning, + }, + .message = message, + }) catch { + allocator.free(message); + return error.OutOfMemory; + }; + } +} + /// Records a parse failure that the parser gave us no location for. Keeps the /// underlying error name in the message: it is the only information we have /// about why the parse stopped, so hiding it would leave the user guessing. @@ -278,11 +289,11 @@ fn appendUnlocatedError( ); errdefer allocator.free(message); - const first_line = lineAt(text, 0); + const first_line = positions.lineAt(text, 0); try sink.items.append(allocator, .{ .range = .{ .start = .{ .line = 0, .character = 0 }, - .end = .{ .line = 0, .character = convertColumn(first_line, first_line.len, sink.encoding) }, + .end = .{ .line = 0, .character = positions.characterFromByte(first_line, first_line.len, sink.encoding) }, }, .severity = .err, .message = message, @@ -311,6 +322,25 @@ fn analyzeForTest(text: []const u8) ![]Diagnostic { return analyze(testing.allocator, text, .@"utf-8"); } +/// Exercises only our own length checking, without running srf's parser. +/// +/// Useful for asserting exactly what we contribute, separately from whatever srf +/// reports for the same input. +fn lengthDiagnosticsForTest(text: []const u8) ![]Diagnostic { + var sink: Sink = .{ + .allocator = testing.allocator, + .text = text, + .encoding = .@"utf-8", + .items = .empty, + }; + errdefer { + for (sink.items.items) |d| d.deinit(testing.allocator); + sink.items.deinit(testing.allocator); + } + try appendLengthProblems(testing.allocator, &sink, text, .@"utf-8"); + return sink.items.toOwnedSlice(testing.allocator); +} + test "valid document has no diagnostics" { const diags = try analyzeForTest("#!srfv1\nname::alice\nage:num:30\n"); defer freeDiagnostics(testing.allocator, diags); @@ -355,7 +385,7 @@ test "diagnostic range stays inside the line it names" { defer freeDiagnostics(testing.allocator, diags); try testing.expect(diags.len > 0); for (diags) |d| { - const line = lineAt(text, d.range.start.line); + const line = positions.lineAt(text, d.range.start.line); try testing.expect(d.range.start.character <= line.len); try testing.expect(d.range.end.character <= line.len); try testing.expect(d.range.end.character >= d.range.start.character); @@ -384,56 +414,6 @@ test "more errors than the old bounded buffer held are all reported" { try testing.expect(diags.len > 10); } -test "utf-8 columns are byte offsets" { - // "¥" is 2 bytes, 1 UTF-16 unit, 1 codepoint. - const line = "a¥b"; - try testing.expectEqual(@as(u32, 4), convertColumn(line, line.len, .@"utf-8")); -} - -test "utf-16 columns count code units" { - const line = "a¥b"; - try testing.expectEqual(@as(u32, 3), convertColumn(line, line.len, .@"utf-16")); -} - -test "utf-16 counts astral characters as surrogate pairs" { - const line = "a\u{1F600}b"; // emoji: 4 bytes, 2 UTF-16 units, 1 codepoint - try testing.expectEqual(@as(u32, 4), convertColumn(line, line.len, .@"utf-16")); - try testing.expectEqual(@as(u32, 3), convertColumn(line, line.len, .@"utf-32")); - try testing.expectEqual(@as(u32, 6), convertColumn(line, line.len, .@"utf-8")); -} - -test "column conversion tolerates invalid utf-8" { - const line = "a\xffb"; - try testing.expectEqual(@as(u32, 3), convertColumn(line, line.len, .@"utf-16")); -} - -test "column conversion tolerates a truncated multi-byte sequence" { - const line = "a\xc2"; // leading byte of a 2-byte sequence, cut off - try testing.expectEqual(@as(u32, 2), convertColumn(line, line.len, .@"utf-16")); -} - -test "column conversion clamps past the end of the line" { - const line = "abc"; - try testing.expectEqual(@as(u32, 3), convertColumn(line, 99, .@"utf-16")); -} - -test "lineAt splits on newlines and strips carriage returns" { - const text = "one\r\ntwo\nthree"; - try testing.expectEqualStrings("one", lineAt(text, 0)); - try testing.expectEqualStrings("two", lineAt(text, 1)); - try testing.expectEqualStrings("three", lineAt(text, 2)); - try testing.expectEqualStrings("", lineAt(text, 3)); - try testing.expectEqualStrings("", lineAt(text, 99)); -} - -test "position encoding round trips through its wire name" { - try testing.expectEqual(PositionEncoding.@"utf-8", PositionEncoding.fromWireName("utf-8").?); - try testing.expectEqual(PositionEncoding.@"utf-16", PositionEncoding.fromWireName("utf-16").?); - try testing.expectEqual(PositionEncoding.@"utf-32", PositionEncoding.fromWireName("utf-32").?); - try testing.expectEqual(@as(?PositionEncoding, null), PositionEncoding.fromWireName("utf-7")); - try testing.expectEqualStrings("utf-16", PositionEncoding.@"utf-16".wireName()); -} - test "srf BoundedDiagnostics.errors() returns a stable view" { // This used to be the bug that made the server segfault: `errors()` took // `self` by value, so the returned slice pointed into a copy that died with @@ -489,3 +469,127 @@ test "an error reported at end of line underlines the whole line" { try testing.expectEqual(@as(u32, 0), range.start.character); try testing.expectEqual(@as(u32, "broken line here".len), range.end.character); } + +test "a long-format length overrun is reported, which srf does not do" { + // srf keeps "hel" and drops "lo" without a word. That is silent data loss, + // so we report it ourselves. + const diags = try analyzeForTest("#!srfv1\n#!long\nk:3:hello\n"); + defer freeDiagnostics(testing.allocator, diags); + try testing.expectEqual(@as(usize, 1), diags.len); + try testing.expectEqual(Severity.warning, diags[0].severity); + try testing.expect(std.mem.indexOf(u8, diags[0].message, "declares 3 bytes") != null); + try testing.expect(std.mem.indexOf(u8, diags[0].message, "silently discard the 2 byte(s)") != null); + try testing.expectEqual(@as(u32, 2), diags[0].range.start.line); +} + +test "a length prefix past the end of the file is an error" { + const diags = try lengthDiagnosticsForTest("#!srfv1\n#!long\nk:99:hello\n"); + defer freeDiagnostics(testing.allocator, diags); + try testing.expectEqual(@as(usize, 1), diags.len); + try testing.expectEqual(Severity.err, diags[0].severity); + try testing.expect(std.mem.indexOf(u8, diags[0].message, "declares 99 bytes but only 6 remain") != null); +} + +test "a compact-format overrun is left to srf, which now reports it" { + // srf 08f06df diagnoses this fatally, so we add nothing: one defect, one + // diagnostic. + const ours = try lengthDiagnosticsForTest("#!srfv1\nk:2:abc\n"); + defer freeDiagnostics(testing.allocator, ours); + try testing.expectEqual(@as(usize, 0), ours.len); + + const all = try analyzeForTest("#!srfv1\nk:2:abc\n"); + defer freeDiagnostics(testing.allocator, all); + try testing.expectEqual(@as(usize, 1), all.len); + try testing.expectEqual(Severity.err, all[0].severity); + // srf's own wording, located on the offending line. + try testing.expect(std.mem.indexOf(u8, all[0].message, "reset line for next item") != null); + try testing.expectEqual(@as(u32, 1), all[0].range.start.line); +} + +test "a correct length prefix produces no diagnostic" { + const diags = try analyzeForTest("#!srfv1\n#!long\nk:5:hello\n"); + defer freeDiagnostics(testing.allocator, diags); + try testing.expectEqual(@as(usize, 0), diags.len); +} + +test "a correct length prefix with commas in compact format produces no diagnostic" { + const diags = try analyzeForTest("#!srfv1\nk:5:a,b,c,next::x\n"); + defer freeDiagnostics(testing.allocator, diags); + try testing.expectEqual(@as(usize, 0), diags.len); +} + +test "a multi-line length-prefixed value produces no diagnostic" { + const diags = try analyzeForTest("#!srfv1\n#!long\nbio:7:foo\nbar\nname::alice\n"); + defer freeDiagnostics(testing.allocator, diags); + try testing.expectEqual(@as(usize, 0), diags.len); +} + +test "an overrun diagnostic underlines the surplus, not the bytes that fit" { + // "k:3:hello": the value starts at character 4 and declares 3 bytes, so the + // surplus "lo" runs from character 7 to 9. Highlighting 4..7 would point at + // the one part of the line that is correct. + const text = "#!srfv1\n#!long\nk:3:hello\n"; + const diags = try analyzeForTest(text); + defer freeDiagnostics(testing.allocator, diags); + try testing.expectEqual(@as(usize, 1), diags.len); + try testing.expectEqual(@as(u32, 7), diags[0].range.start.character); + try testing.expectEqual(@as(u32, 9), diags[0].range.end.character); +} + +test "a truncation diagnostic underlines the value it does have" { + const diags = try lengthDiagnosticsForTest("#!srfv1\n#!long\nk:99:hello\n"); + defer freeDiagnostics(testing.allocator, diags); + try testing.expectEqual(@as(u32, 5), diags[0].range.start.character); +} + +test "length diagnostics coexist with srf's own diagnostics" { + // Duplicate magic header (srf's finding) plus a bad length prefix (ours). + const diags = try analyzeForTest("#!srfv1\n#!srfv1\n#!long\nk:3:hello\n"); + defer freeDiagnostics(testing.allocator, diags); + var from_srf = false; + var from_us = false; + for (diags) |d| { + if (std.mem.indexOf(u8, d.message, "duplicate magic") != null) from_srf = true; + if (std.mem.indexOf(u8, d.message, "length prefix") != null) from_us = true; + } + try testing.expect(from_srf); + try testing.expect(from_us); +} + +test "a compact overrun no longer produces the unlocated fallback" { + // srf now fails fatally *with* a diagnostic, so the generic + // "reported no location" message must not appear alongside it. + const diags = try analyzeForTest("#!srfv1\nk:2:abc\n"); + defer freeDiagnostics(testing.allocator, diags); + for (diags) |d| { + try testing.expect(std.mem.indexOf(u8, d.message, "reported no location") == null); + } +} + +test "a located length problem replaces the unlocated fallback" { + // srf fails here with a bare EndOfStream and no diagnostic, so without our + // own check the user would get "parsing stopped ... no location". A precise + // message on the offending line is strictly better, and the generic one + // should not tag along beside it. + // + // The truncated form is used rather than an overrun because an overrun makes + // srf log at error level (srf.zig:873), and Zig's test runner counts any + // error log as a failed test. See `lengthDiagnosticsForTest`. + const diags = try analyzeForTest("#!srfv1\n#!long\nk:99:hello\n"); + defer freeDiagnostics(testing.allocator, diags); + for (diags) |d| { + if (std.mem.indexOf(u8, d.message, "reported no location") != null) { + std.debug.print("\nunlocated fallback should have been suppressed\n", .{}); + return error.UnexpectedFallback; + } + } + try testing.expectEqual(@as(usize, 1), diags.len); + try testing.expect(std.mem.indexOf(u8, diags[0].message, "declares 99 bytes but only 6 remain") != null); +} + +test "the unlocated fallback still fires when nothing else explains the failure" { + const diags = try analyzeForTest(""); + defer freeDiagnostics(testing.allocator, diags); + try testing.expectEqual(@as(usize, 1), diags.len); + try testing.expect(std.mem.indexOf(u8, diags[0].message, "empty") != null); +} diff --git a/src/document.zig b/src/document.zig new file mode 100644 index 0000000..dd9bb78 --- /dev/null +++ b/src/document.zig @@ -0,0 +1,767 @@ +//! Locates the SRF construct under a byte offset. +//! +//! srf's parser is a streaming iterator and `srf.Field` carries no positions, so +//! answering "what is under the cursor?" needs its own pass. This module does +//! only that: it finds a token and hands back the surrounding field's text. +//! Interpreting the value is `hover.zig`'s job, and it defers to srf for that. +//! +//! The scan mirrors the format's two mode-dependent rules, which is why it +//! cannot be a simple line splitter: +//! +//! * the field delimiter is `,` in compact format and end-of-line after +//! `#!long`, so what counts as one field depends on a directive above it; +//! * a numeric type hint means "the value is exactly N bytes", which can carry +//! a value across newlines. + +const std = @import("std"); + +pub const Span = struct { + start: usize, + end: usize, + + fn contains(self: Span, offset: usize) bool { + // `end` is inclusive here: hovering with the cursor just past the last + // character of a word is still hovering that word, and editors routinely + // report exactly that position. + return offset >= self.start and offset <= self.end; + } +}; + +/// The field a token belongs to, as raw slices of the document. +pub const Field = struct { + key: []const u8, + /// null for the `key::value` form, which carries no hint. + hint: ?[]const u8, + /// Empty when the field has no value (`missing:null:`). + value: []const u8, + /// Whether the field was found under `#!long`. + long_mode: bool, + /// Set when a numeric hint's byte count does not agree with the document. + length_problem: ?LengthProblem = null, + + /// The hint parsed as a byte count, for the length-prefixed form. + pub fn declaredLength(self: Field) ?usize { + const hint = self.hint orelse return null; + if (hint.len == 0) return null; + for (hint) |c| if (c < '0' or c > '9') return null; + return std.fmt.parseInt(usize, hint, 10) catch null; + } +}; + +pub const Directive = struct { + /// Directive name without the `#!`, e.g. `long` or `expires`. + name: []const u8, + /// Text after `=`, if any. + argument: ?[]const u8, +}; + +/// A length prefix that does not agree with the document. +/// +/// Worth reporting separately because srf handles these badly: in long format it +/// silently keeps the declared bytes and drops the rest of the line, and for a +/// declaration past the end of the file it fails with no location at all. +pub const LengthProblem = union(enum) { + /// The document ran out before the declared byte count. + truncated: struct { declared: usize, available: usize }, + /// The declared count does not land on a field terminator, so the value + /// really continues past where the declaration says it stops. In long format + /// srf discards the remainder without a word. + overrun: struct { + declared: usize, + /// Where the surplus starts, which is the declared end of the value. + surplus_start: usize, + /// Bytes from there to the end of that line: the text that does not fit + /// the declaration, and which srf drops in long format. + surplus: usize, + }, +}; + +/// A field and the spans of its parts. +pub const Located = struct { + field: Field, + key_span: Span, + /// null for the `key::value` form. + hint_span: ?Span, + value_span: Span, +}; + +pub const Item = union(enum) { + /// The `#!srfv1` header. + magic: Span, + directive: struct { span: Span, value: Directive }, + field: Located, +}; + +pub const Token = struct { + span: Span, + what: What, +}; + +pub const What = union(enum) { + magic, + directive: Directive, + key: Field, + hint: Field, + value: Field, +}; + +/// Walks every construct in the document in order. Comments and blank lines are +/// skipped: nothing in them is addressable. +pub fn items(text: []const u8) Iterator { + return .{ .scanner = .{ .text = text } }; +} + +pub const Iterator = struct { + scanner: Scanner, + + pub fn next(self: *Iterator) ?Item { + return self.scanner.next(); + } +}; + +/// Returns the token covering `offset`, or null if the offset lands on a comment, +/// a blank line, punctuation or past the end. +pub fn tokenAt(text: []const u8, offset: usize) ?Token { + var it = items(text); + while (it.next()) |item| { + switch (item) { + .magic => |span| { + if (span.contains(offset)) return .{ .span = span, .what = .magic }; + }, + .directive => |d| { + if (d.span.contains(offset)) { + return .{ .span = d.span, .what = .{ .directive = d.value } }; + } + }, + .field => |located| { + if (located.key_span.contains(offset)) { + return .{ .span = located.key_span, .what = .{ .key = located.field } }; + } + if (located.hint_span) |hs| { + if (hs.contains(offset)) { + return .{ .span = hs, .what = .{ .hint = located.field } }; + } + } + // An empty value has nothing to point at. + if (located.value_span.end > located.value_span.start and + located.value_span.contains(offset)) + { + return .{ .span = located.value_span, .what = .{ .value = located.field } }; + } + }, + } + } + return null; +} + +const magic_header = "#!srfv1"; + +/// One forward pass over the document, yielding one construct per call. +/// +/// It cannot be a line splitter, because two of the format's rules are not local +/// to a line: the field delimiter depends on a `#!long` / `#!compact` directive +/// above, and a numeric type hint means "exactly N bytes", which can carry a +/// value across newlines. +const Scanner = struct { + text: []const u8, + /// Start of the next line to classify. + pos: usize = 0, + long_mode: bool = false, + /// Set while part way through a field line. Compact format packs several + /// fields onto one line, so the scan has to be able to resume mid-line. + field_cursor: ?usize = null, + /// End of the logical line being walked. A length-prefixed value can move it. + line_end: usize = 0, + + fn next(self: *Scanner) ?Item { + // Finish the line we are part way through before reading another. + if (self.field_cursor) |cursor| return self.scanField(cursor); + + while (self.pos < self.text.len) { + const line_start = self.pos; + const end = self.lineEnd(line_start); + + // Leading whitespace is insignificant; srf trims it before deciding + // what a line is. + var body = line_start; + while (body < end and isSpace(self.text[body])) body += 1; + + if (body == end) { // blank line + self.pos = self.nextLine(end); + continue; + } + + if (self.text[body] == '#') { + const item = self.classifyHash(body, end); + self.pos = self.nextLine(end); + if (item) |i| return i; + continue; // a plain comment + } + + self.line_end = end; + return self.scanField(body); + } + return null; + } + + /// `#!srfv1`, a `#!directive`, or null for a plain comment. + fn classifyHash(self: *Scanner, body: usize, line_end: usize) ?Item { + const line = self.text[body..line_end]; + if (!std.mem.startsWith(u8, line, "#!")) return null; + + if (std.mem.startsWith(u8, line, magic_header)) { + return .{ .magic = .{ .start = body, .end = body + magic_header.len } }; + } + + // `#!name` or `#!name=argument`, stopping at an inline comment. + var end = body + 2; + while (end < line_end and !isSpace(self.text[end])) end += 1; + + const decl = self.text[body + 2 .. end]; + const eq = std.mem.indexOfScalar(u8, decl, '='); + const name = if (eq) |i| decl[0..i] else decl; + + // Mode directives take effect for everything below them. + if (std.mem.eql(u8, name, "long")) self.long_mode = true; + if (std.mem.eql(u8, name, "compact")) self.long_mode = false; + + return .{ .directive = .{ + .span = .{ .start = body, .end = end }, + .value = .{ .name = name, .argument = if (eq) |i| decl[i + 1 ..] else null }, + } }; + } + + /// Reads one field starting at `field_start`, leaving `field_cursor` set if + /// another field follows it on the same line. + fn scanField(self: *Scanner, field_start: usize) ?Item { + // Key runs to the first ':'. + const colon = std.mem.indexOfScalarPos(u8, self.text[0..self.line_end], field_start, ':') orelse + return self.abandonLine(); + const head = self.splitHead(colon) orelse return self.abandonLine(); + + const extent = self.valueExtent(head.hint, head.value_start); + // A length-prefixed value can swallow newlines, moving the end of the + // logical line being walked. + self.line_end = extent.line_end; + + if (extent.next_field) |n| { + self.field_cursor = n; + } else { + self.field_cursor = null; + self.pos = self.nextLine(self.line_end); + } + + return .{ .field = .{ + .field = .{ + .key = self.text[field_start..colon], + .hint = head.hint, + .value = self.text[head.value_start..extent.end], + .long_mode = self.long_mode, + .length_problem = extent.length_problem, + }, + .key_span = .{ .start = field_start, .end = colon }, + .hint_span = head.hint_span, + .value_span = .{ .start = head.value_start, .end = extent.end }, + } }; + } + + /// Gives up on the current line: it has no colon, so it is not a field. + fn abandonLine(self: *Scanner) ?Item { + self.field_cursor = null; + self.pos = self.nextLine(self.line_end); + return self.next(); + } + + const Head = struct { + hint: ?[]const u8, + hint_span: ?Span, + value_start: usize, + }; + + /// Splits what follows the key's colon into an optional type hint and the + /// start of the value. Null when the line has no second colon, so it is not + /// a field at all. + fn splitHead(self: *const Scanner, colon: usize) ?Head { + if (colon + 1 < self.line_end and self.text[colon + 1] == ':') { + // `key::value` carries no hint. + return .{ .hint = null, .hint_span = null, .value_start = colon + 2 }; + } + + // `key:hint:value`, where the hint may be padded with whitespace. + const second = std.mem.indexOfScalarPos(u8, self.text[0..self.line_end], colon + 1, ':') orelse + return null; + var start = colon + 1; + var end = second; + while (start < end and isSpace(self.text[start])) start += 1; + while (end > start and isSpace(self.text[end - 1])) end -= 1; + return .{ + .hint = self.text[start..end], + .hint_span = .{ .start = start, .end = end }, + .value_start = second + 1, + }; + } + + const Extent = struct { + /// One past the last byte of the value. + end: usize, + /// Where the next field on this line starts, if any. + next_field: ?usize, + /// End of the logical line, which a length-prefixed value can move. + line_end: usize, + length_problem: ?LengthProblem, + }; + + /// How far the value runs, which is the one part of the format that depends + /// on both the mode and the type hint. + fn valueExtent(self: *const Scanner, hint: ?[]const u8, value_start: usize) Extent { + if (parseLength(hint)) |declared| { + // Exactly N bytes, newlines included. Clamped so a wrong declaration + // cannot run us off the end of the document. + const end = @min(value_start + declared, self.text.len); + const new_line_end = self.lineEnd(end); + const comma_follows = !self.long_mode and end < new_line_end and self.text[end] == ','; + return .{ + .end = end, + .next_field = if (comma_follows) end + 1 else null, + .line_end = new_line_end, + .length_problem = self.lengthProblem(declared, value_start), + }; + } + + // Long format: the value runs to end of line and commas are literal text. + if (self.long_mode) { + return .{ + .end = self.line_end, + .next_field = null, + .line_end = self.line_end, + .length_problem = null, + }; + } + + // Compact format: the next comma ends the value and starts a field. + if (std.mem.indexOfScalarPos(u8, self.text[0..self.line_end], value_start, ',')) |comma| { + return .{ + .end = comma, + .next_field = comma + 1, + .line_end = self.line_end, + .length_problem = null, + }; + } + return .{ + .end = self.line_end, + .next_field = null, + .line_end = self.line_end, + .length_problem = null, + }; + } + + /// Checks a declared byte count against the document. + /// + /// The test is what *follows* the declared bytes, not how many bytes we took: + /// a length-prefixed value must be followed immediately by a field + /// terminator. Comparing the count against a value sliced using that same + /// count would always agree, which is no test at all. + fn lengthProblem(self: *const Scanner, declared: usize, value_start: usize) ?LengthProblem { + const available = self.text.len - value_start; + if (available < declared) { + return .{ .truncated = .{ .declared = declared, .available = available } }; + } + + const after = value_start + declared; + // End of file terminates a field just as well as a newline does. + if (after >= self.text.len) return null; + + const terminator = self.text[after]; + if (terminator == '\n') return null; + if (!self.long_mode and terminator == ',') return null; + + // Measured from the declared end, not from the start of the value: a + // multi-line length-prefixed value can put the surplus on a later line, + // and a range built from the first line's end would come out inverted. + return .{ .overrun = .{ + .declared = declared, + .surplus_start = after, + .surplus = self.lineEnd(after) - after, + } }; + } + + fn lineEnd(self: *const Scanner, from: usize) usize { + return std.mem.indexOfScalarPos(u8, self.text, from, '\n') orelse self.text.len; + } + + fn nextLine(self: *const Scanner, line_end: usize) usize { + return if (line_end < self.text.len) line_end + 1 else self.text.len; + } +}; + +fn isSpace(c: u8) bool { + return c == ' ' or c == '\t' or c == '\r'; +} + +/// The hint as a byte count, or null when it is a keyword hint. +fn parseLength(hint: ?[]const u8) ?usize { + const h = hint orelse return null; + if (h.len == 0) return null; + for (h) |c| if (c < '0' or c > '9') return null; + const n = std.fmt.parseInt(usize, h, 10) catch return null; + // A zero length is an empty value, not a span to walk. + return if (n == 0) null else n; +} + +const testing = std.testing; + +/// Finds `needle` in `text` and returns the offset of its midpoint, which is a +/// stable way to say "hover here" without hand-counting byte offsets. +fn midpointOf(text: []const u8, needle: []const u8) usize { + const at = std.mem.indexOf(u8, text, needle).?; + return at + needle.len / 2; +} + +test "hovering the magic header" { + const text = "#!srfv1\nname::alice\n"; + const token = tokenAt(text, midpointOf(text, "#!srfv1")).?; + try testing.expect(token.what == .magic); + try testing.expectEqualStrings("#!srfv1", text[token.span.start..token.span.end]); +} + +test "hovering a bare directive" { + const text = "#!srfv1\n#!long\nname::alice\n"; + const token = tokenAt(text, midpointOf(text, "#!long")).?; + try testing.expectEqualStrings("long", token.what.directive.name); + try testing.expectEqual(@as(?[]const u8, null), token.what.directive.argument); +} + +test "hovering a directive with an argument" { + const text = "#!srfv1\n#!expires=1772589213\nk::v\n"; + const token = tokenAt(text, midpointOf(text, "expires")).?; + try testing.expectEqualStrings("expires", token.what.directive.name); + try testing.expectEqualStrings("1772589213", token.what.directive.argument.?); +} + +test "a directive's inline comment is not part of its argument" { + const text = "#!srfv1\n#!long # use long format\nk::v\n"; + const token = tokenAt(text, midpointOf(text, "#!long")).?; + try testing.expectEqualStrings("long", token.what.directive.name); + try testing.expectEqualStrings("#!long", text[token.span.start..token.span.end]); +} + +test "a plain comment has nothing to hover" { + const text = "#!srfv1\n# just a comment\nk::v\n"; + try testing.expectEqual(@as(?Token, null), tokenAt(text, midpointOf(text, "just"))); +} + +test "hovering an untyped key" { + const text = "#!srfv1\nname::alice\n"; + const token = tokenAt(text, midpointOf(text, "name")).?; + try testing.expectEqualStrings("name", token.what.key.key); + try testing.expectEqual(@as(?[]const u8, null), token.what.key.hint); + try testing.expectEqualStrings("alice", token.what.key.value); +} + +test "hovering a typed key, its hint and its value" { + const text = "#!srfv1\nage:num:30\n"; + + const key = tokenAt(text, midpointOf(text, "age")).?; + try testing.expectEqualStrings("age", key.what.key.key); + try testing.expectEqualStrings("num", key.what.key.hint.?); + + const hint = tokenAt(text, midpointOf(text, "num")).?; + try testing.expect(hint.what == .hint); + try testing.expectEqualStrings("num", text[hint.span.start..hint.span.end]); + + const value = tokenAt(text, midpointOf(text, "30")).?; + try testing.expect(value.what == .value); + try testing.expectEqualStrings("30", value.what.value.value); +} + +test "whitespace around a hint is not part of it" { + const text = "#!srfv1\nage: num :30\n"; + const token = tokenAt(text, midpointOf(text, "num")).?; + try testing.expectEqualStrings("num", text[token.span.start..token.span.end]); +} + +test "compact format splits fields on commas" { + const text = "#!srfv1\nname::alice,age:num:30\n"; + + const first = tokenAt(text, midpointOf(text, "alice")).?; + try testing.expectEqualStrings("alice", first.what.value.value); + try testing.expectEqualStrings("name", first.what.value.key); + + const second = tokenAt(text, midpointOf(text, "age")).?; + try testing.expectEqualStrings("age", second.what.key.key); + try testing.expectEqualStrings("30", second.what.key.value); +} + +test "long format keeps commas inside the value" { + const text = "#!srfv1\n#!long\nname::alice, bob and carol\n"; + const token = tokenAt(text, midpointOf(text, "bob")).?; + try testing.expectEqualStrings("alice, bob and carol", token.what.value.value); + try testing.expect(token.what.value.long_mode); +} + +test "a compact directive switches back from long" { + const text = "#!srfv1\n#!long\n#!compact\nname::alice,age:num:30\n"; + const token = tokenAt(text, midpointOf(text, "alice")).?; + try testing.expectEqualStrings("alice", token.what.value.value); + try testing.expect(!token.what.value.long_mode); +} + +test "a length-prefixed value spans newlines" { + const text = "#!srfv1\n#!long\nbio:7:foo\nbar\nname::alice\n"; + const token = tokenAt(text, midpointOf(text, "foo")).?; + try testing.expect(token.what == .value); + try testing.expectEqualStrings("foo\nbar", token.what.value.value); + try testing.expectEqual(@as(?usize, 7), token.what.value.declaredLength()); +} + +test "the field after a length-prefixed value is still found" { + const text = "#!srfv1\n#!long\nbio:7:foo\nbar\nname::alice\n"; + const token = tokenAt(text, midpointOf(text, "alice")).?; + try testing.expectEqualStrings("name", token.what.value.key); + try testing.expectEqualStrings("alice", token.what.value.value); +} + +test "a length-prefixed value in compact format is followed by a comma" { + const text = "#!srfv1\nk:5:a,b,c,next::x\n"; + + const value = tokenAt(text, midpointOf(text, "a,b,c")).?; + try testing.expectEqualStrings("a,b,c", value.what.value.value); + + const next = tokenAt(text, midpointOf(text, "next")).?; + try testing.expectEqualStrings("next", next.what.key.key); + try testing.expectEqualStrings("x", next.what.key.value); +} + +test "a length longer than the document does not run off the end" { + const text = "#!srfv1\n#!long\nk:9999:short\n"; + const token = tokenAt(text, midpointOf(text, "short")).?; + try testing.expectEqualStrings("short\n", token.what.value.value); +} + +test "indented fields are found" { + const text = "#!srfv1\n#!long\n name::alice\n"; + const token = tokenAt(text, midpointOf(text, "name")).?; + try testing.expectEqualStrings("name", token.what.key.key); +} + +test "a key containing a comma is still one key" { + const text = "#!srfv1\n#!long\nlast, first::alice\n"; + const token = tokenAt(text, midpointOf(text, "first")).?; + try testing.expectEqualStrings("last, first", token.what.key.key); +} + +test "an empty value yields no value token" { + const text = "#!srfv1\n#!long\nmissing:null:\n"; + const key = tokenAt(text, midpointOf(text, "missing")).?; + try testing.expectEqualStrings("", key.what.key.value); + // The colon itself is not a value. + try testing.expectEqual(@as(?Token, null), tokenAt(text, text.len - 1)); +} + +test "a line with no colon has nothing to hover" { + const text = "#!srfv1\n#!long\ngarbage line\nk::v\n"; + try testing.expectEqual(@as(?Token, null), tokenAt(text, midpointOf(text, "garbage"))); +} + +test "a blank line has nothing to hover" { + const text = "#!srfv1\n\nk::v\n"; + try testing.expectEqual(@as(?Token, null), tokenAt(text, 8)); +} + +test "an offset past the end has nothing to hover" { + const text = "#!srfv1\nk::v\n"; + try testing.expectEqual(@as(?Token, null), tokenAt(text, 9999)); +} + +test "an empty document has nothing to hover" { + try testing.expectEqual(@as(?Token, null), tokenAt("", 0)); +} + +test "multi-byte values are located by byte offset" { + const text = "#!srfv1\n#!long\ncost:num:¥15,000\n"; + const token = tokenAt(text, midpointOf(text, "15,000")).?; + try testing.expectEqualStrings("¥15,000", token.what.value.value); +} + +test "declaredLength only accepts an all-digit hint" { + const text = "#!srfv1\nk:num:1\n"; + const token = tokenAt(text, midpointOf(text, "num")).?; + try testing.expectEqual(@as(?usize, null), token.what.hint.declaredLength()); +} + +test "records after a blank line are still scanned" { + const text = "#!srfv1\n#!long\nname::alice\n\nname::bob\n"; + const token = tokenAt(text, midpointOf(text, "bob")).?; + try testing.expectEqualStrings("bob", token.what.value.value); +} + +test "the eof directive is a directive" { + const text = "#!srfv1\nk::v\n#!eof\n"; + const token = tokenAt(text, midpointOf(text, "#!eof")).?; + try testing.expectEqualStrings("eof", token.what.directive.name); +} + +test "a length prefix longer than the document swallows the rest of it" { + // Deliberate: the format says the value is N bytes, so if the file ends + // early the trailing text really is part of that value. Diagnostics flag the + // mismatch; the locator just reports what the declaration implies rather + // than inventing a boundary. + const text = "#!srfv1\n#!long\nbad:99:short\n\n"; + const on_blank = tokenAt(text, text.len - 1).?; + try testing.expect(on_blank.what == .value); + try testing.expectEqualStrings("bad", on_blank.what.value.key); + try testing.expectEqual(@as(?usize, 99), on_blank.what.value.declaredLength()); + // Everything from after the second colon to the end of the document. + try testing.expectEqualStrings("short\n\n", on_blank.what.value.value); +} + +test "a blank line outside any value has nothing to hover" { + const text = "#!srfv1\n#!long\nname::alice\n\nname::bob\n"; + const blank_at = std.mem.indexOf(u8, text, "alice\n\n").? + 6; + try testing.expectEqual(@as(?Token, null), tokenAt(text, blank_at)); +} + +/// The length problem for the field containing `offset`, if any. +fn problemAt(text: []const u8, needle: []const u8) ?LengthProblem { + const offset = midpointOf(text, needle); + var it = items(text); + while (it.next()) |item| switch (item) { + .field => |located| { + if (located.key_span.contains(offset) or located.value_span.contains(offset) or + (located.hint_span != null and located.hint_span.?.contains(offset))) + { + return located.field.length_problem; + } + }, + else => {}, + }; + return null; +} + +test "a correct length prefix has no problem" { + try testing.expectEqual( + @as(?LengthProblem, null), + problemAt("#!srfv1\n#!long\nk:5:hello\n", "hello"), + ); +} + +test "a length shorter than the line is an overrun, which srf silently ignores" { + // srf keeps the first 3 bytes and drops "lo" without a word, so this is the + // one that most needs reporting. + const problem = problemAt("#!srfv1\n#!long\nk:3:hello\n", "k:3:").?; + try testing.expectEqual(@as(usize, 3), problem.overrun.declared); + // "lo" is the surplus srf throws away. + try testing.expectEqual(@as(usize, 2), problem.overrun.surplus); +} + +test "a length past the end of the document is truncation" { + const problem = problemAt("#!srfv1\n#!long\nk:99:hello\n", "hello").?; + try testing.expectEqual(@as(usize, 99), problem.truncated.declared); + // "hello\n" is all that remains. + try testing.expectEqual(@as(usize, 6), problem.truncated.available); +} + +test "a length ending exactly at end of file is fine" { + // No trailing newline, so EOF is the terminator. + try testing.expectEqual( + @as(?LengthProblem, null), + problemAt("#!srfv1\n#!long\nk:5:hello", "hello"), + ); +} + +test "in compact format a comma terminates a length-prefixed value" { + try testing.expectEqual( + @as(?LengthProblem, null), + problemAt("#!srfv1\nk:5:a,b,c,next::x\n", "a,b,c"), + ); +} + +test "in compact format a length landing mid-token is an overrun" { + // After 2 bytes comes 'b', neither a comma nor a newline. + const problem = problemAt("#!srfv1\nk:2:abc\n", "k:2:").?; + try testing.expectEqual(@as(usize, 2), problem.overrun.declared); +} + +test "in long format a trailing comma after the declared bytes is an overrun" { + // Long format has no comma delimiter, so the comma is stray text. + const problem = problemAt("#!srfv1\n#!long\nk:5:hello,\n", "hello").?; + try testing.expectEqual(@as(usize, 5), problem.overrun.declared); + // Just the stray comma. + try testing.expectEqual(@as(usize, 1), problem.overrun.surplus); +} + +test "trailing whitespace after the declared bytes is an overrun" { + const problem = problemAt("#!srfv1\n#!long\nk:5:hello \n", "hello").?; + try testing.expectEqual(@as(usize, 5), problem.overrun.declared); +} + +test "a multi-line length-prefixed value has no problem" { + try testing.expectEqual( + @as(?LengthProblem, null), + problemAt("#!srfv1\n#!long\nbio:7:foo\nbar\nname::alice\n", "foo"), + ); +} + +test "a keyword hint never reports a length problem" { + try testing.expectEqual( + @as(?LengthProblem, null), + problemAt("#!srfv1\n#!long\nk:string:hello\n", "hello"), + ); +} + +test "items yields every construct in document order" { + const text = "#!srfv1\n#!long\n# comment\nname::alice\n\nage:num:30\n#!eof\n"; + var it = items(text); + var kinds: std.ArrayList([]const u8) = .empty; + defer kinds.deinit(testing.allocator); + while (it.next()) |item| { + try kinds.append(testing.allocator, switch (item) { + .magic => "magic", + .directive => "directive", + .field => "field", + }); + } + try testing.expectEqualDeep( + @as([]const []const u8, &.{ "magic", "directive", "field", "field", "directive" }), + kinds.items, + ); +} + +test "items yields each field of a compact line separately" { + const text = "#!srfv1\nname::alice,age:num:30,ok:bool:true\n"; + var it = items(text); + var keys: std.ArrayList([]const u8) = .empty; + defer keys.deinit(testing.allocator); + while (it.next()) |item| switch (item) { + .field => |f| try keys.append(testing.allocator, f.field.key), + else => {}, + }; + try testing.expectEqualDeep( + @as([]const []const u8, &.{ "name", "age", "ok" }), + keys.items, + ); +} + +test "items terminates on a line with no colon" { + const text = "#!srfv1\ngarbage\nk::v\n"; + var it = items(text); + var fields: usize = 0; + while (it.next()) |item| switch (item) { + .field => fields += 1, + else => {}, + }; + try testing.expectEqual(@as(usize, 1), fields); +} + +test "the surplus of a multi-line overrun is measured on its own line" { + // Declares 5 bytes ("foo\nb"), so the surplus is "ar" on the second line. + // Measuring from the first line's end would produce an inverted range. + const problem = problemAt("#!srfv1\n#!long\nbio:5:foo\nbar\n", "bio").?; + try testing.expectEqual(@as(usize, 5), problem.overrun.declared); + try testing.expectEqual(@as(usize, 2), problem.overrun.surplus); + const text = "#!srfv1\n#!long\nbio:5:foo\nbar\n"; + try testing.expectEqualStrings( + "ar", + text[problem.overrun.surplus_start..][0..problem.overrun.surplus], + ); +} diff --git a/src/hover.zig b/src/hover.zig new file mode 100644 index 0000000..c4c4f2b --- /dev/null +++ b/src/hover.zig @@ -0,0 +1,682 @@ +//! Renders hover text for the SRF construct under the cursor. +//! +//! `document.zig` finds *what* is under the cursor; this module says what it +//! means. Where a value's meaning depends on parsing (is `¥15,000` a valid +//! `num`? how many bytes does that base64 decode to?), we ask srf rather than +//! reimplementing its rules, by re-parsing the single located field in isolation. +//! That keeps hover and diagnostics from ever disagreeing about the same text. + +const std = @import("std"); +const srf = @import("srf"); +const document = @import("document.zig"); +const positions = @import("positions.zig"); + +pub const MarkupContent = struct { + kind: []const u8 = "markdown", + /// Owned by the `Hover`. + value: []const u8, +}; + +pub const Hover = struct { + contents: MarkupContent, + range: positions.Range, + + pub fn deinit(self: Hover, allocator: std.mem.Allocator) void { + allocator.free(self.contents.value); + } +}; + +/// Builds hover content for `position`, or null when there is nothing useful +/// under the cursor (a comment, a blank line, punctuation, past the end). +/// +/// Caller owns the result and should release it with `Hover.deinit`. +pub fn hoverAt( + allocator: std.mem.Allocator, + text: []const u8, + position: positions.Position, + encoding: positions.Encoding, +) error{OutOfMemory}!?Hover { + const offset = positions.byteFromPosition(text, position, encoding); + const token = document.tokenAt(text, offset) orelse return null; + + var out: std.Io.Writer.Allocating = .init(allocator); + defer out.deinit(); + + // Writing to an Allocating writer can only fail by running out of memory. + render(allocator, &out.writer, token) catch |err| switch (err) { + error.WriteFailed => return error.OutOfMemory, + }; + + const owned = try allocator.dupe(u8, out.written()); + return .{ + .contents = .{ .value = owned }, + .range = positions.rangeFromSpan(text, token.span.start, token.span.end, encoding), + }; +} + +fn render( + allocator: std.mem.Allocator, + w: *std.Io.Writer, + token: document.Token, +) std.Io.Writer.Error!void { + switch (token.what) { + .magic => try w.writeAll( + \\**SRF v1** (Simple Record Format) + \\ + \\Mandatory first line. Declares the format and version, and must be + \\the first line of the file. + ), + .directive => |d| try renderDirective(w, d), + .key => |f| try renderField(allocator, w, f), + .hint => |f| try renderHint(allocator, w, f), + .value => |f| try renderValue(allocator, w, f), + } +} + +// +// Directives +// + +fn renderDirective(w: *std.Io.Writer, d: document.Directive) std.Io.Writer.Error!void { + if (std.mem.eql(u8, d.name, "long")) { + return w.writeAll( + \\**`#!long`** long format + \\ + \\Fields are separated by newlines, so a value may contain commas. + \\Required for records that span multiple lines. + ); + } + if (std.mem.eql(u8, d.name, "compact")) { + return w.writeAll( + \\**`#!compact`** compact format (the default) + \\ + \\Fields are separated by commas, one record per line. A value + \\containing a comma needs a length prefix. + ); + } + if (std.mem.eql(u8, d.name, "requireeof")) { + return w.writeAll( + \\**`#!requireeof`** + \\ + \\Parsing fails unless `#!eof` appears on the last line. Detects a + \\file that was truncated in transit. + ); + } + if (std.mem.eql(u8, d.name, "eof")) { + return w.writeAll( + \\**`#!eof`** end of data + \\ + \\Nothing may follow this line. Only enforced when `#!requireeof` + \\is set. + ); + } + + const label: ?[]const u8 = if (std.mem.eql(u8, d.name, "expires")) + "Data expires at" + else if (std.mem.eql(u8, d.name, "created")) + "Data created at" + else if (std.mem.eql(u8, d.name, "modified")) + "Data last modified at" + else + null; + + if (label) |text| { + try w.print("**`#!{s}`**\n\n", .{d.name}); + const argument = d.argument orelse { + return w.writeAll("Missing timestamp: this directive needs `=`."); + }; + const seconds = std.fmt.parseInt(i64, argument, 10) catch { + return w.print("`{s}` is not a Unix timestamp in seconds.", .{argument}); + }; + try w.print("{s} ", .{text}); + try writeTimestamp(w, seconds); + return w.print("\n\n(`{d}` seconds since the Unix epoch)", .{seconds}); + } + + try w.print("**`#!{s}`** unrecognised directive\n\n", .{d.name}); + try w.writeAll("Parsers ignore directives they do not know, so this is not an error."); +} + +/// Writes a Unix timestamp as `YYYY-MM-DD HH:MM:SS UTC`. Nobody can read an +/// epoch second, which is the whole reason this function exists. +fn writeTimestamp(w: *std.Io.Writer, seconds: i64) std.Io.Writer.Error!void { + if (seconds < 0) { + // std.time.epoch counts from 1970 upwards only. Rather than do signed + // civil-date maths for a case that should not occur in cache metadata, + // say so plainly. + return w.print("a time before 1970 (`{d}`)", .{seconds}); + } + + const epoch: std.time.epoch.EpochSeconds = .{ .secs = @intCast(seconds) }; + const day = epoch.getEpochDay().calculateYearDay(); + const month_day = day.calculateMonthDay(); + const time = epoch.getDaySeconds(); + + try w.print("{d:0>4}-{d:0>2}-{d:0>2} {d:0>2}:{d:0>2}:{d:0>2} UTC", .{ + day.year, + month_day.month.numeric(), + month_day.day_index + 1, + time.getHoursIntoDay(), + time.getMinutesIntoHour(), + time.getSecondsIntoMinute(), + }); +} + +// +// Fields +// + +fn renderField( + allocator: std.mem.Allocator, + w: *std.Io.Writer, + f: document.Field, +) std.Io.Writer.Error!void { + try w.print("**`{s}`**", .{f.key}); + if (f.hint) |hint| try w.print(" · `{s}`", .{hint}) else try w.writeAll(" · untyped"); + try w.writeAll("\n\n"); + try describeValue(allocator, w, f); +} + +fn renderHint( + allocator: std.mem.Allocator, + w: *std.Io.Writer, + f: document.Field, +) std.Io.Writer.Error!void { + const hint = f.hint orelse return; + + if (f.declaredLength()) |declared| { + try w.print("**length prefix: {d} bytes**\n\n", .{declared}); + try w.writeAll( + \\The value is exactly this many bytes, newlines included, so it can + \\hold delimiters and line breaks without escaping. + \\ + \\ + ); + return describeValue(allocator, w, f); + } + + const explanation: ?[]const u8 = if (std.mem.eql(u8, hint, "string")) + "Text, ending at the field delimiter. No escaping: a value that needs to contain the delimiter or a newline uses a length prefix instead." + else if (std.mem.eql(u8, hint, "num")) + "A number, parsed as a 64-bit float." + else if (std.mem.eql(u8, hint, "bool")) + "`true` or `false`." + else if (std.mem.eql(u8, hint, "null")) + "No value." + else if (std.mem.eql(u8, hint, "binary")) + "Base64-encoded bytes." + else + null; + + if (explanation) |text| { + try w.print("**`{s}`**\n\n{s}\n\n", .{ hint, text }); + return describeValue(allocator, w, f); + } + + try w.print("**`{s}`** unrecognised type hint\n\n", .{hint}); + try w.writeAll("SRF knows `string`, `num`, `bool`, `null`, `binary`, and a byte count for a length-prefixed value."); +} + +fn renderValue( + allocator: std.mem.Allocator, + w: *std.Io.Writer, + f: document.Field, +) std.Io.Writer.Error!void { + try w.print("**`{s}`**", .{f.key}); + if (f.hint) |hint| try w.print(" · `{s}`", .{hint}); + try w.writeAll("\n\n"); + try describeValue(allocator, w, f); +} + +/// The part that has to agree with the reference parser, so it asks the reference +/// parser. +fn describeValue( + allocator: std.mem.Allocator, + w: *std.Io.Writer, + f: document.Field, +) std.Io.Writer.Error!void { + if (f.length_problem) |problem| try writeLengthProblem(w, problem, f); + + if (f.value.len == 0) { + return w.writeAll("*No value.*"); + } + + try describeParsed(allocator, w, f); + try w.print("```\n{s}\n```", .{f.value}); +} + +/// A wrong length prefix is the easiest way to corrupt an SRF file by hand, and +/// the byte count is invisible in the text, so say exactly what is wrong. +fn writeLengthProblem( + w: *std.Io.Writer, + problem: document.LengthProblem, + f: document.Field, +) std.Io.Writer.Error!void { + switch (problem) { + .truncated => |t| try w.print( + "**Length mismatch:** declares {d} byte{s} but only {d} remain in the file.\n\n", + .{ t.declared, plural(t.declared), t.available }, + ), + .overrun => |o| { + try w.print( + "**Length mismatch:** declares {d} byte{s}, but the value does not end there", + .{ o.declared, plural(o.declared) }, + ); + if (f.long_mode) { + try w.print( + ": {d} more byte{s} follow.\n\nsrf keeps the first {d} and discards the rest without reporting it.\n\n", + .{ o.surplus, plural(o.surplus), o.declared }, + ); + } else { + // srf reports this one itself, fatally. Hover is still where the + // detail belongs, so explain the rule rather than just repeating + // that something is wrong. + try w.writeAll( + ". A length-prefixed value must be followed by a comma or end of line, so srf rejects the document.\n\n", + ); + } + }, + } +} + +/// Asks srf what the value means, and describes it while the parse is still +/// alive: decoded `binary` bytes live in the parser's arena. +fn describeParsed( + allocator: std.mem.Allocator, + w: *std.Io.Writer, + f: document.Field, +) std.Io.Writer.Error!void { + // No hint, a plain string, or a length prefix all mean "text": nothing to + // parse and nothing that can fail. + const hint = f.hint orelse return writeByteCount(w, f.value); + if (f.declaredLength() != null) return writeByteCount(w, f.value); + if (std.mem.eql(u8, hint, "string")) return writeByteCount(w, f.value); + + const synthetic = buildSyntheticDocument(allocator, f) catch return writeUnparseable(w, hint); + defer allocator.free(synthetic); + + var reader = std.Io.Reader.fixed(synthetic); + var it = srf.iterator(&reader, allocator, .{}) catch return writeUnparseable(w, hint); + defer it.deinit(); + + const fields = (it.next() catch return writeUnparseable(w, hint)) orelse + return writeUnparseable(w, hint); + const field = (fields.next() catch return writeUnparseable(w, hint)) orelse + return writeUnparseable(w, hint); + const value = field.value orelse return writeUnparseable(w, hint); + + switch (value) { + .number => |n| try w.print("Parsed as a number: `{d}`\n\n", .{n}), + .boolean => |b| try w.print("Parsed as a boolean: `{}`\n\n", .{b}), + .bytes => |decoded| try writeDecodedBytes(w, decoded), + .string => try writeByteCount(w, f.value), + } +} + +fn writeByteCount(w: *std.Io.Writer, value: []const u8) std.Io.Writer.Error!void { + try w.print("{d} byte{s}\n\n", .{ value.len, plural(value.len) }); +} + +fn writeUnparseable(w: *std.Io.Writer, hint: []const u8) std.Io.Writer.Error!void { + try w.print("**Does not parse as `{s}`.** srf would report an error here.\n\n", .{hint}); +} + +/// Longest decoded preview shown inline. Past this the point is made and the +/// hover starts crowding the buffer. +const preview_limit = 48; + +/// Describes decoded base64. Shows the content when it is text, because that is +/// the whole reason you would base64 a value in a human-readable format. +fn writeDecodedBytes(w: *std.Io.Writer, decoded: []const u8) std.Io.Writer.Error!void { + try w.print("Base64, decoding to {d} byte{s}", .{ decoded.len, plural(decoded.len) }); + + if (decoded.len == 0) return w.writeAll("\n\n"); + if (!looksLikeText(decoded)) return w.writeAll(": binary content\n\n"); + + try w.writeAll(": \""); + const shown = decoded[0..@min(decoded.len, preview_limit)]; + try writeEscaped(w, shown); + if (shown.len < decoded.len) try w.writeAll("..."); + try w.writeAll("\"\n\n"); +} + +/// Writes text on a single line, so a decoded value containing newlines cannot +/// break the surrounding markdown. +fn writeEscaped(w: *std.Io.Writer, bytes: []const u8) std.Io.Writer.Error!void { + for (bytes) |c| switch (c) { + '\n' => try w.writeAll("\\n"), + '\r' => try w.writeAll("\\r"), + '\t' => try w.writeAll("\\t"), + '"' => try w.writeAll("\\\""), + '\\' => try w.writeAll("\\\\"), + else => try w.writeByte(c), + }; +} + +/// Whether decoded bytes are worth showing as text: valid UTF-8 with no control +/// characters other than the usual whitespace. +fn looksLikeText(bytes: []const u8) bool { + if (!std.unicode.utf8ValidateSlice(bytes)) return false; + for (bytes) |c| { + if (c == '\t' or c == '\n' or c == '\r') continue; + if (c < 0x20 or c == 0x7f) return false; + } + return true; +} + +fn buildSyntheticDocument( + allocator: std.mem.Allocator, + f: document.Field, +) error{OutOfMemory}![]u8 { + var buf: std.Io.Writer.Allocating = .init(allocator); + errdefer buf.deinit(); + const w = &buf.writer; + + // Writes to an Allocating writer only fail on OOM. + w.writeAll("#!srfv1\n") catch return error.OutOfMemory; + if (f.long_mode) w.writeAll("#!long\n") catch return error.OutOfMemory; + if (f.hint) |hint| { + w.print("{s}:{s}:{s}\n", .{ f.key, hint, f.value }) catch return error.OutOfMemory; + } else { + w.print("{s}::{s}\n", .{ f.key, f.value }) catch return error.OutOfMemory; + } + return buf.toOwnedSlice(); +} + +fn plural(n: usize) []const u8 { + return if (n == 1) "" else "s"; +} + +const testing = std.testing; + +/// Hovers at the midpoint of `needle` and returns the markdown. +fn hoverOn(text: []const u8, needle: []const u8) ![]const u8 { + const at = std.mem.indexOf(u8, text, needle).?; + const position = positions.positionFromByte(text, at + needle.len / 2, .@"utf-8"); + const hover = (try hoverAt(testing.allocator, text, position, .@"utf-8")).?; + return hover.contents.value; +} + +fn expectHoverContains(text: []const u8, needle: []const u8, expected: []const u8) !void { + const markdown = try hoverOn(text, needle); + defer testing.allocator.free(markdown); + if (std.mem.indexOf(u8, markdown, expected) == null) { + std.debug.print("\nhovering \"{s}\"\nexpected to contain: {s}\ngot:\n{s}\n", .{ needle, expected, markdown }); + return error.MissingFromHover; + } +} + +test "hovering the magic header explains it" { + try expectHoverContains("#!srfv1\nk::v\n", "#!srfv1", "Simple Record Format"); +} + +test "hovering #!long explains the delimiter change" { + try expectHoverContains("#!srfv1\n#!long\nk::v\n", "#!long", "separated by newlines"); +} + +test "hovering #!compact says it is the default" { + try expectHoverContains("#!srfv1\n#!compact\nk::v\n", "#!compact", "the default"); +} + +test "hovering #!requireeof explains truncation detection" { + try expectHoverContains("#!srfv1\n#!requireeof\nk::v\n", "#!requireeof", "truncated"); +} + +test "hovering #!eof explains it" { + try expectHoverContains("#!srfv1\nk::v\n#!eof\n", "#!eof", "end of data"); +} + +test "hovering #!expires renders the timestamp as a date" { + const text = "#!srfv1\n#!expires=1772589213\nk::v\n"; + try expectHoverContains(text, "expires", "2026-03-04"); + try expectHoverContains(text, "expires", "UTC"); + // The raw value stays visible; the date is a convenience, not a replacement. + try expectHoverContains(text, "expires", "1772589213"); +} + +test "hovering #!created and #!modified render dates too" { + try expectHoverContains("#!srfv1\n#!created=0\nk::v\n", "created", "1970-01-01 00:00:00 UTC"); + try expectHoverContains("#!srfv1\n#!modified=86400\nk::v\n", "modified", "1970-01-02"); +} + +test "a non-numeric timestamp is called out, not silently mangled" { + try expectHoverContains("#!srfv1\n#!expires=tomorrow\nk::v\n", "expires", "not a Unix timestamp"); +} + +test "a timestamp before 1970 is described rather than wrapped" { + try expectHoverContains("#!srfv1\n#!expires=-100\nk::v\n", "expires", "before 1970"); +} + +test "an unknown directive says parsers ignore it" { + try expectHoverContains("#!srfv1\n#!wat=1\nk::v\n", "#!wat", "ignore directives they do not know"); +} + +test "hovering an untyped key shows the key and its value" { + const text = "#!srfv1\nname::alice\n"; + try expectHoverContains(text, "name", "`name`"); + try expectHoverContains(text, "name", "untyped"); + try expectHoverContains(text, "name", "alice"); +} + +test "hovering a num key shows the parsed number" { + try expectHoverContains("#!srfv1\nage:num:30\n", "age", "Parsed as a number: `30`"); +} + +test "hovering a num hint explains the type" { + try expectHoverContains("#!srfv1\nage:num:30\n", "num", "64-bit float"); +} + +test "hovering a bool value shows the parsed boolean" { + try expectHoverContains("#!srfv1\nok:bool:true\n", "true", "Parsed as a boolean: `true`"); +} + +test "hovering a binary value reports the decoded size" { + // "aGVsbG8=" is "hello": 5 bytes. + try expectHoverContains("#!srfv1\nblob:binary:aGVsbG8=\n", "aGVsbG8", "decoding to 5 bytes"); +} + +test "hovering a string value reports its byte length" { + try expectHoverContains("#!srfv1\ns:string:hello\n", "hello", "5 bytes"); +} + +test "a value that does not parse as its type says so" { + try expectHoverContains("#!srfv1\nn:num:not-a-number\n", "not-a-number", "Does not parse as `num`"); +} + +test "a lone byte is singular" { + try expectHoverContains("#!srfv1\ns:string:x\n", "s:string:x", "1 byte"); +} + +test "hovering a length prefix explains the byte count" { + const text = "#!srfv1\n#!long\nbio:7:foo\nbar\n"; + try expectHoverContains(text, ":7:", "length prefix: 7 bytes"); + try expectHoverContains(text, ":7:", "newlines included"); +} + +test "a length-prefixed value shows its multi-line content" { + try expectHoverContains("#!srfv1\n#!long\nbio:7:foo\nbar\n", "foo", "foo\nbar"); +} + +test "a wrong length prefix is flagged" { + // Declares 99 bytes; the line holds far fewer. + try expectHoverContains("#!srfv1\n#!long\nbio:99:short\n", "short", "Length mismatch"); +} + +test "an empty value says so" { + try expectHoverContains("#!srfv1\n#!long\nmissing:null:\n", "missing", "No value"); +} + +test "currency in a num is judged by srf, not by us" { + // srf's default is strict number parsing, so this does not parse. Whatever + // srf decides, hover agrees with it: that is the point of the re-parse. + const text = "#!srfv1\n#!long\ncost:num:¥15,000\n"; + const markdown = try hoverOn(text, "15,000"); + defer testing.allocator.free(markdown); + const parsed = std.mem.indexOf(u8, markdown, "Parsed as a number") != null; + const rejected = std.mem.indexOf(u8, markdown, "Does not parse") != null; + try testing.expect(parsed or rejected); +} + +test "long mode is preserved when re-parsing, so commas stay in the value" { + const text = "#!srfv1\n#!long\nname::alice, bob\n"; + try expectHoverContains(text, "bob", "alice, bob"); +} + +test "hovering a comment gives nothing" { + const text = "#!srfv1\n# a comment\nk::v\n"; + const at = std.mem.indexOf(u8, text, "comment").?; + const position = positions.positionFromByte(text, at, .@"utf-8"); + try testing.expectEqual( + @as(?Hover, null), + try hoverAt(testing.allocator, text, position, .@"utf-8"), + ); +} + +test "hovering past the end gives nothing" { + try testing.expectEqual( + @as(?Hover, null), + try hoverAt(testing.allocator, "#!srfv1\n", .{ .line = 50, .character = 0 }, .@"utf-8"), + ); +} + +test "hover reports the range of the thing it described" { + const text = "#!srfv1\nname::alice\n"; + const hover = (try hoverAt(testing.allocator, text, .{ .line = 1, .character = 2 }, .@"utf-8")).?; + defer hover.deinit(testing.allocator); + try testing.expectEqual(@as(u32, 1), hover.range.start.line); + try testing.expectEqual(@as(u32, 0), hover.range.start.character); + try testing.expectEqual(@as(u32, 4), hover.range.end.character); +} + +test "hover ranges use the negotiated encoding" { + const text = "#!srfv1\n#!long\ncost:num:¥15,000\n"; + // Hover the value, whose line contains a two-byte character before it. + const at = std.mem.indexOf(u8, text, "15,000").?; + + const utf8 = (try hoverAt( + testing.allocator, + text, + positions.positionFromByte(text, at, .@"utf-8"), + .@"utf-8", + )).?; + defer utf8.deinit(testing.allocator); + + const utf16 = (try hoverAt( + testing.allocator, + text, + positions.positionFromByte(text, at, .@"utf-16"), + .@"utf-16", + )).?; + defer utf16.deinit(testing.allocator); + + // The span starts at the yen sign itself, so both encodings agree there: + // nothing multi-byte precedes it on the line. + try testing.expectEqual(utf8.range.start.character, utf16.range.start.character); + // They diverge at the end, which sits past it: two bytes but one UTF-16 unit. + try testing.expectEqual(utf8.range.end.character - 1, utf16.range.end.character); +} + +test "the markdown declares itself as markdown" { + const text = "#!srfv1\nname::alice\n"; + const hover = (try hoverAt(testing.allocator, text, .{ .line = 1, .character = 2 }, .@"utf-8")).?; + defer hover.deinit(testing.allocator); + try testing.expectEqualStrings("markdown", hover.contents.kind); +} + +test "timestamps are correct across leap years and the 2038 boundary" { + // Verified against Python's datetime for each of these. + const cases = [_]struct { secs: []const u8, expected: []const u8 }{ + .{ .secs = "0", .expected = "1970-01-01 00:00:00 UTC" }, + .{ .secs = "1", .expected = "1970-01-01 00:00:01 UTC" }, + .{ .secs = "951782400", .expected = "2000-02-29 00:00:00 UTC" }, + .{ .secs = "1078012800", .expected = "2004-02-29 00:00:00 UTC" }, + .{ .secs = "1709164800", .expected = "2024-02-29 00:00:00 UTC" }, + .{ .secs = "1772589213", .expected = "2026-03-04 01:53:33 UTC" }, + .{ .secs = "2147483647", .expected = "2038-01-19 03:14:07 UTC" }, + }; + + for (cases) |case| { + var out: std.Io.Writer.Allocating = .init(testing.allocator); + defer out.deinit(); + const secs = try std.fmt.parseInt(i64, case.secs, 10); + try writeTimestamp(&out.writer, secs); + try testing.expectEqualStrings(case.expected, out.written()); + } +} + +test "base64 text content is shown, not just its size" { + // "aGVsbG8=" is "hello". + try expectHoverContains("#!srfv1\nblob:binary:aGVsbG8=\n", "aGVsbG8", "decoding to 5 bytes: \"hello\""); +} + +test "base64 binary content is described rather than dumped" { + // "AAAAAA==" is two base64 groups: four 0x00 bytes, which are not text. + try expectHoverContains("#!srfv1\nblob:binary:AAAAAA==\n", "AAAAAA", "decoding to 4 bytes: binary content"); +} + +test "base64 text with newlines stays on one line" { + // "aGkKdGhlcmU=" is "hi\nthere". + try expectHoverContains("#!srfv1\nblob:binary:aGkKdGhlcmU=\n", "aGkK", "\"hi\\nthere\""); +} + +test "a long base64 text preview is truncated" { + // 60 'a' characters, base64 encoded. + var raw: [60]u8 = @splat('a'); + var encoded: [128]u8 = undefined; + const b64 = std.base64.standard.Encoder.encode(&encoded, &raw); + + const text = try std.fmt.allocPrint(testing.allocator, "#!srfv1\nblob:binary:{s}\n", .{b64}); + defer testing.allocator.free(text); + + const markdown = try hoverOn(text, b64); + defer testing.allocator.free(markdown); + try testing.expect(std.mem.indexOf(u8, markdown, "decoding to 60 bytes") != null); + try testing.expect(std.mem.indexOf(u8, markdown, "...") != null); +} + +test "empty base64 decodes to nothing without a preview" { + const markdown = try hoverOn("#!srfv1\nblob:binary:\n", "binary"); + defer testing.allocator.free(markdown); + try testing.expect(std.mem.indexOf(u8, markdown, "No value") != null); +} + +test "looksLikeText accepts text and rejects control bytes" { + try testing.expect(looksLikeText("hello")); + try testing.expect(looksLikeText("hi\nthere\t!")); + try testing.expect(looksLikeText("¥15,000")); + try testing.expect(!looksLikeText("\x00\x01")); + try testing.expect(!looksLikeText("\xff\xfe")); + try testing.expect(!looksLikeText("bad\x07bell")); +} + +test "an overrun length in long format explains srf's silent truncation" { + const text = "#!srfv1\n#!long\nk:3:hello\n"; + try expectHoverContains(text, "k:3:", "declares 3 bytes, but the value does not end there"); + try expectHoverContains(text, "k:3:", "2 more bytes follow"); + try expectHoverContains(text, "k:3:", "discards the rest without reporting it"); +} + +test "a truncated length reports what is actually left" { + const text = "#!srfv1\n#!long\nk:99:hello\n"; + try expectHoverContains(text, "hello", "declares 99 bytes but only 6 remain"); +} + +test "an overrun length in compact format explains the terminator rule" { + const text = "#!srfv1\nk:2:abc\n"; + try expectHoverContains(text, "k:2:", "followed by a comma or end of line"); + // Hover explains it even though the diagnostic for it comes from srf. + try expectHoverContains(text, "k:2:", "srf rejects the document"); +} + +test "a correct length prefix reports no mismatch" { + const markdown = try hoverOn("#!srfv1\n#!long\nk:5:hello\n", "hello"); + defer testing.allocator.free(markdown); + try testing.expect(std.mem.indexOf(u8, markdown, "Length mismatch") == null); +} + +test "a correct length prefix with commas in compact format reports no mismatch" { + const markdown = try hoverOn("#!srfv1\nk:5:a,b,c,next::x\n", "a,b,c"); + defer testing.allocator.free(markdown); + try testing.expect(std.mem.indexOf(u8, markdown, "Length mismatch") == null); + try testing.expect(std.mem.indexOf(u8, markdown, "a,b,c") != null); +} diff --git a/src/main.zig b/src/main.zig index 81b949a..9479588 100644 --- a/src/main.zig +++ b/src/main.zig @@ -8,13 +8,31 @@ const std = @import("std"); const rpc = @import("rpc.zig"); const Server = @import("Server.zig"); -/// `std.log.defaultLog` writes to stderr, which is exactly what we need: stdout -/// is the protocol stream and a stray byte on it desynchronises the client. So -/// there is no custom `logFn` here on purpose. +/// `std.log.defaultLog` writes to stderr, which is what we need: stdout is the +/// protocol stream and a stray byte on it desynchronises the client. pub const std_options: std.Options = .{ .log_level = .info, + .logFn = logFn, }; +/// Demotes the srf library's own logging to debug. +/// +/// srf logs at error level for conditions that are not server errors: a failed +/// custom `srfParse` coercion at `srf.zig:556`, for instance. Real problems in the +/// document reach the user as diagnostics, so passing these through would make +/// `:LspLog` look like the server broke. They are kept, but only surface when +/// debug logging is on. +fn logFn( + comptime level: std.log.Level, + comptime scope: @EnumLiteral(), + comptime format: []const u8, + args: anytype, +) void { + const effective = if (scope == .srf) std.log.Level.debug else level; + if (@intFromEnum(effective) > @intFromEnum(std.options.log_level)) return; + std.log.defaultLog(effective, scope, format, args); +} + const log = std.log.scoped(.srf_lsp); /// Returns the process exit code: 0 for an orderly shutdown, 1 otherwise, which @@ -66,4 +84,7 @@ test { _ = @import("rpc.zig"); _ = @import("Server.zig"); _ = @import("analysis.zig"); + _ = @import("document.zig"); + _ = @import("positions.zig"); + _ = @import("hover.zig"); } diff --git a/src/positions.zig b/src/positions.zig new file mode 100644 index 0000000..1e55aaf --- /dev/null +++ b/src/positions.zig @@ -0,0 +1,277 @@ +//! Converting between byte offsets in a document and LSP positions. +//! +//! LSP counts a `character` in UTF-16 code units by default, while everything +//! else here (srf's columns, `document.zig`'s spans, plain slicing) counts bytes. +//! The client picks the encoding during `initialize`, so both directions are +//! needed and neither can assume one byte per character. + +const std = @import("std"); + +/// How LSP positions are counted. The client picks this during `initialize`. +pub const Encoding = enum { + /// Byte offsets. What srf and our own scanner already use, so no conversion. + @"utf-8", + /// UTF-16 code units. The protocol default, so it is the fallback. + @"utf-16", + /// Codepoints. + @"utf-32", + + pub fn fromWireName(name: []const u8) ?Encoding { + return std.meta.stringToEnum(Encoding, name); + } + + pub fn wireName(self: Encoding) []const u8 { + return @tagName(self); + } +}; + +pub const Position = struct { + line: u32, + character: u32, +}; + +pub const Range = struct { + start: Position, + end: Position, +}; + +/// Returns the 0-based `index`th line of `text`, excluding its newline and any +/// carriage return. Empty if the line does not exist. +pub fn lineAt(text: []const u8, index: u32) []const u8 { + var remaining = text; + var i: u32 = 0; + while (true) { + const nl = std.mem.indexOfScalar(u8, remaining, '\n'); + if (i == index) { + const line = if (nl) |n| remaining[0..n] else remaining; + return std.mem.trimEnd(u8, line, "\r"); + } + if (nl) |n| { + remaining = remaining[n + 1 ..]; + i += 1; + } else { + return ""; + } + } +} + +/// Byte offset at which the 0-based `index`th line starts. Clamped to the end of +/// the text for a line past the end. +pub fn lineStart(text: []const u8, index: u32) usize { + var offset: usize = 0; + var i: u32 = 0; + while (i < index) : (i += 1) { + const nl = std.mem.indexOfScalarPos(u8, text, offset, '\n') orelse return text.len; + offset = nl + 1; + } + return @min(offset, text.len); +} + +/// Converts a byte offset within one line to a character offset in `encoding`. +/// +/// Invalid UTF-8 is counted a byte at a time rather than rejected: a document +/// being edited is routinely invalid for a keystroke or two, and a usable +/// position beats an error. +pub fn characterFromByte(line: []const u8, byte_offset: usize, encoding: Encoding) u32 { + const limit = @min(byte_offset, line.len); + if (encoding == .@"utf-8") return @intCast(limit); + + var count: u32 = 0; + var i: usize = 0; + while (i < limit) { + const decoded = decodeAt(line, i) orelse { + i += 1; + count += 1; + continue; + }; + count += switch (encoding) { + // Anything outside the BMP needs two UTF-16 code units. + .@"utf-16" => if (decoded.codepoint > 0xFFFF) @as(u32, 2) else 1, + .@"utf-32" => 1, + .@"utf-8" => unreachable, + }; + i += decoded.len; + } + return count; +} + +/// Converts a character offset within one line back to a byte offset. Clamped to +/// the length of the line. +pub fn byteFromCharacter(line: []const u8, character: u32, encoding: Encoding) usize { + if (encoding == .@"utf-8") return @min(character, line.len); + + var seen: u32 = 0; + var i: usize = 0; + while (i < line.len) { + if (seen >= character) return i; + const decoded = decodeAt(line, i) orelse { + i += 1; + seen += 1; + continue; + }; + seen += switch (encoding) { + .@"utf-16" => if (decoded.codepoint > 0xFFFF) @as(u32, 2) else 1, + .@"utf-32" => 1, + .@"utf-8" => unreachable, + }; + i += decoded.len; + } + return line.len; +} + +/// Whole-document byte offset for an LSP position. +pub fn byteFromPosition(text: []const u8, position: Position, encoding: Encoding) usize { + const start = lineStart(text, position.line); + const line = lineAt(text, position.line); + return @min(start + byteFromCharacter(line, position.character, encoding), text.len); +} + +/// LSP position for a whole-document byte offset. +pub fn positionFromByte(text: []const u8, offset: usize, encoding: Encoding) Position { + const limit = @min(offset, text.len); + var line: u32 = 0; + var i: usize = 0; + while (std.mem.indexOfScalarPos(u8, text, i, '\n')) |nl| { + if (nl >= limit) break; + line += 1; + i = nl + 1; + } + return .{ + .line = line, + .character = characterFromByte(lineAt(text, line), limit - i, encoding), + }; +} + +/// LSP range for a whole-document byte span. +pub fn rangeFromSpan(text: []const u8, start: usize, end: usize, encoding: Encoding) Range { + return .{ + .start = positionFromByte(text, start, encoding), + .end = positionFromByte(text, end, encoding), + }; +} + +const Decoded = struct { codepoint: u21, len: usize }; + +/// Decodes one codepoint, or null if the bytes at `i` are not valid UTF-8. +fn decodeAt(bytes: []const u8, i: usize) ?Decoded { + const len = std.unicode.utf8ByteSequenceLength(bytes[i]) catch return null; + if (i + len > bytes.len) return null; + const codepoint = std.unicode.utf8Decode(bytes[i..][0..len]) catch return null; + return .{ .codepoint = codepoint, .len = len }; +} + +const testing = std.testing; + +test "encoding round trips through its wire name" { + try testing.expectEqual(Encoding.@"utf-8", Encoding.fromWireName("utf-8").?); + try testing.expectEqual(Encoding.@"utf-16", Encoding.fromWireName("utf-16").?); + try testing.expectEqual(Encoding.@"utf-32", Encoding.fromWireName("utf-32").?); + try testing.expectEqual(@as(?Encoding, null), Encoding.fromWireName("utf-7")); + try testing.expectEqualStrings("utf-16", Encoding.@"utf-16".wireName()); +} + +test "lineAt splits on newlines and strips carriage returns" { + const text = "one\r\ntwo\nthree"; + try testing.expectEqualStrings("one", lineAt(text, 0)); + try testing.expectEqualStrings("two", lineAt(text, 1)); + try testing.expectEqualStrings("three", lineAt(text, 2)); + try testing.expectEqualStrings("", lineAt(text, 3)); + try testing.expectEqualStrings("", lineAt(text, 99)); +} + +test "lineStart finds where each line begins" { + const text = "one\ntwo\nthree"; + try testing.expectEqual(@as(usize, 0), lineStart(text, 0)); + try testing.expectEqual(@as(usize, 4), lineStart(text, 1)); + try testing.expectEqual(@as(usize, 8), lineStart(text, 2)); + try testing.expectEqual(text.len, lineStart(text, 99)); +} + +test "utf-8 characters are byte offsets" { + const line = "a¥b"; // yen is 2 bytes + try testing.expectEqual(@as(u32, 4), characterFromByte(line, line.len, .@"utf-8")); +} + +test "utf-16 characters count code units" { + const line = "a¥b"; + try testing.expectEqual(@as(u32, 3), characterFromByte(line, line.len, .@"utf-16")); +} + +test "utf-16 counts astral characters as surrogate pairs" { + const line = "a\u{1F600}b"; // emoji: 4 bytes, 2 UTF-16 units, 1 codepoint + try testing.expectEqual(@as(u32, 4), characterFromByte(line, line.len, .@"utf-16")); + try testing.expectEqual(@as(u32, 3), characterFromByte(line, line.len, .@"utf-32")); + try testing.expectEqual(@as(u32, 6), characterFromByte(line, line.len, .@"utf-8")); +} + +test "character conversion tolerates invalid utf-8" { + try testing.expectEqual(@as(u32, 3), characterFromByte("a\xffb", 3, .@"utf-16")); +} + +test "character conversion tolerates a truncated sequence" { + try testing.expectEqual(@as(u32, 2), characterFromByte("a\xc2", 2, .@"utf-16")); +} + +test "character conversion clamps past the end of the line" { + try testing.expectEqual(@as(u32, 3), characterFromByte("abc", 99, .@"utf-16")); +} + +test "byteFromCharacter is the inverse of characterFromByte" { + for ([_][]const u8{ "plain ascii", "a¥b€c", "a\u{1F600}b", "" }) |line| { + for ([_]Encoding{ .@"utf-8", .@"utf-16", .@"utf-32" }) |encoding| { + var byte: usize = 0; + while (byte <= line.len) : (byte += 1) { + // Only round trip offsets that sit on a codepoint boundary. + if (byte < line.len and std.unicode.utf8ByteSequenceLength(line[byte]) == error.Utf8InvalidStartByte) continue; + const character = characterFromByte(line, byte, encoding); + try testing.expectEqual(byte, byteFromCharacter(line, character, encoding)); + } + } + } +} + +test "byteFromCharacter clamps past the end of the line" { + try testing.expectEqual(@as(usize, 3), byteFromCharacter("abc", 99, .@"utf-16")); +} + +test "byteFromPosition locates an offset in a multi-line document" { + const text = "#!srfv1\ncost:num:¥15,000\n"; + // In utf-16 the yen is one unit, so "15,000" starts at character 10. + const utf16 = byteFromPosition(text, .{ .line = 1, .character = 10 }, .@"utf-16"); + try testing.expectEqualStrings("15,000", text[utf16 .. utf16 + 6]); + + // In utf-8 the same place is one byte further along. + const utf8 = byteFromPosition(text, .{ .line = 1, .character = 11 }, .@"utf-8"); + try testing.expectEqual(utf16, utf8); +} + +test "byteFromPosition clamps a position past the end" { + const text = "#!srfv1\n"; + try testing.expectEqual(text.len, byteFromPosition(text, .{ .line = 99, .character = 99 }, .@"utf-8")); +} + +test "positionFromByte is the inverse of byteFromPosition" { + const text = "#!srfv1\n#!long\ncost:num:¥15,000\nname::alice\n"; + var offset: usize = 0; + while (offset <= text.len) : (offset += 1) { + if (offset < text.len and std.unicode.utf8ByteSequenceLength(text[offset]) == error.Utf8InvalidStartByte) continue; + const position = positionFromByte(text, offset, .@"utf-16"); + try testing.expectEqual(offset, byteFromPosition(text, position, .@"utf-16")); + } +} + +test "positionFromByte reports the right line" { + const text = "a\nb\nc\n"; + try testing.expectEqual(@as(u32, 0), positionFromByte(text, 0, .@"utf-8").line); + try testing.expectEqual(@as(u32, 1), positionFromByte(text, 2, .@"utf-8").line); + try testing.expectEqual(@as(u32, 2), positionFromByte(text, 4, .@"utf-8").line); +} + +test "rangeFromSpan converts both ends" { + const text = "#!srfv1\nname::alice\n"; + const at = std.mem.indexOf(u8, text, "alice").?; + const range = rangeFromSpan(text, at, at + 5, .@"utf-8"); + try testing.expectEqual(@as(u32, 1), range.start.line); + try testing.expectEqual(@as(u32, 6), range.start.character); + try testing.expectEqual(@as(u32, 11), range.end.character); +}