handle full srf spec

.pre-commit-config.yaml

@@ -6,6 +6,9 @@ repos:
     hooks:
       - id: trailing-whitespace
       - id: end-of-file-fixer
+        # tree-sitter generate emits these two without a trailing newline, so
+        # the fixer and the generate hook below would undo each other forever.
+        exclude: ^src/(grammar|node-types)\.json$
       - id: check-yaml
       - id: check-added-large-files
   - repo: local

grammar.js

@@ -2,6 +2,11 @@
  * @file SRF (Simple Record Format) grammar for tree-sitter
  * @author Emil Lerch
  * @license MIT
+ *
+ * SRF's field delimiter depends on the format directive: ',' in compact format
+ * (the default) and '\n' after `#!long`. A numeric type hint additionally means
+ * "this value is exactly N bytes", newlines included. Neither is expressible
+ * here, so those tokens come from the external scanner in src/scanner.c.
  */
 
 /// <reference types="tree-sitter-cli/dsl" />
@@ -12,17 +17,34 @@ module.exports = grammar({
 
   extras: _ => [],
 
+  externals: $ => [
+    $._long_marker,
+    $._compact_marker,
+    $._length_hint,
+    $.value,
+    // Never produced by the scanner. Only valid while tree-sitter is in error
+    // recovery, which tells the scanner to keep its hands off.
+    $._error_sentinel,
+  ],
+
   rules: {
     document: $ => repeat($._line),
 
-    _line: $ => choice(
+    // Leading whitespace is insignificant on every kind of line: the reference
-      $.magic,
+    // parser trims the start of each line before deciding what it is.
-      $.directive,
+    _line: $ => seq(
-      $.comment,
+      optional($._ws),
-      $.field,
+      choice(
-      $.blank_line,
+        $.magic,
+        $.directive,
+        $.comment,
+        $._field_line,
+        $.blank_line,
+      ),
     ),
 
+    _ws: _ => /[ \t]+/,
+
     magic: $ => seq(
       '#!srfv1',
       optional($.inline_comment),
@@ -42,8 +64,11 @@ module.exports = grammar({
       $.directive_unknown,
     ),
 
-    directive_long: $ => seq('#!long', optional($.inline_comment), '\n'),
+    // The format directives are scanned externally so that the scanner learns
-    directive_compact: $ => seq('#!compact', optional($.inline_comment), '\n'),
+    // which delimiter applies to the rest of the document. If both appear, the
+    // last one wins, matching the reference parser.
+    directive_long: $ => seq($._long_marker, optional($.inline_comment), '\n'),
+    directive_compact: $ => seq($._compact_marker, optional($.inline_comment), '\n'),
     directive_requireeof: $ => seq('#!requireeof', optional($.inline_comment), '\n'),
     directive_eof: $ => seq('#!eof', optional($.inline_comment), '\n'),
     directive_expires: $ => seq(
@@ -77,6 +102,16 @@ module.exports = grammar({
 
     blank_line: _ => token(/[ \t]*\n/),
 
+    // A comma only ever separates fields in compact format: in long format the
+    // scanner has already swallowed it as part of the value, so the repetition
+    // below is unreachable there. Requiring a field after every comma is what
+    // makes a trailing separator (`key::value,`) an error, as the spec demands.
+    _field_line: $ => seq(
+      $.field,
+      repeat(seq(',', $.field)),
+      '\n',
+    ),
+
     field: $ => choice(
       $.typed_field,
       $.untyped_field,
@@ -85,30 +120,31 @@ module.exports = grammar({
     typed_field: $ => seq(
       $.key,
       ':',
+      optional($._ws),
       $.type_hint,
+      optional($._ws),
       ':',
       optional($.value),
-      /[,\n]/,
     ),
 
     untyped_field: $ => seq(
       $.key,
       '::',
       optional($.value),
-      /[,\n]/,
     ),
 
-    key: _ => /[^#\n:, \t][^:,]*/,
+    // Everything up to the first colon, so commas are fair game. A key may not
+    // start with one, which keeps a stray trailing comma in compact format from
+    // being absorbed into the next key.
+    key: _ => /[^#\n:, \t][^:\n]*/,
 
-    type_hint: _ => choice(
+    type_hint: $ => choice(
       'string',
       'num',
       'bool',
       'null',
       'binary',
-      /[0-9]+/,
+      $._length_hint,
     ),
-
-    value: _ => /[^\n,]+/,
   },
 });

queries/highlights.scm

@@ -22,3 +22,4 @@
 
 ":" @punctuation.delimiter
 "::" @punctuation.delimiter
+"," @punctuation.delimiter

src/grammar.json

@@ -10,30 +10,51 @@
       }
     },
     "_line": {
-      "type": "CHOICE",
+      "type": "SEQ",
       "members": [
         {
-          "type": "SYMBOL",
+          "type": "CHOICE",
-          "name": "magic"
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_ws"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
         },
         {
-          "type": "SYMBOL",
+          "type": "CHOICE",
-          "name": "directive"
+          "members": [
-        },
+            {
-        {
+              "type": "SYMBOL",
-          "type": "SYMBOL",
+              "name": "magic"
-          "name": "comment"
+            },
-        },
+            {
-        {
+              "type": "SYMBOL",
-          "type": "SYMBOL",
+              "name": "directive"
-          "name": "field"
+            },
-        },
+            {
-        {
+              "type": "SYMBOL",
-          "type": "SYMBOL",
+              "name": "comment"
-          "name": "blank_line"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "_field_line"
+            },
+            {
+              "type": "SYMBOL",
+              "name": "blank_line"
+            }
+          ]
         }
       ]
     },
+    "_ws": {
+      "type": "PATTERN",
+      "value": "[ \\t]+"
+    },
     "magic": {
       "type": "SEQ",
       "members": [
@@ -104,8 +125,8 @@
       "type": "SEQ",
       "members": [
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "#!long"
+          "name": "_long_marker"
         },
         {
           "type": "CHOICE",
@@ -129,8 +150,8 @@
       "type": "SEQ",
       "members": [
         {
-          "type": "STRING",
+          "type": "SYMBOL",
-          "value": "#!compact"
+          "name": "_compact_marker"
         },
         {
           "type": "CHOICE",
@@ -385,6 +406,35 @@
         "value": "[ \\t]*\\n"
       }
     },
+    "_field_line": {
+      "type": "SEQ",
+      "members": [
+        {
+          "type": "SYMBOL",
+          "name": "field"
+        },
+        {
+          "type": "REPEAT",
+          "content": {
+            "type": "SEQ",
+            "members": [
+              {
+                "type": "STRING",
+                "value": ","
+              },
+              {
+                "type": "SYMBOL",
+                "name": "field"
+              }
+            ]
+          }
+        },
+        {
+          "type": "STRING",
+          "value": "\n"
+        }
+      ]
+    },
     "field": {
       "type": "CHOICE",
       "members": [
@@ -409,10 +459,34 @@
           "type": "STRING",
           "value": ":"
         },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_ws"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
         {
           "type": "SYMBOL",
           "name": "type_hint"
         },
+        {
+          "type": "CHOICE",
+          "members": [
+            {
+              "type": "SYMBOL",
+              "name": "_ws"
+            },
+            {
+              "type": "BLANK"
+            }
+          ]
+        },
         {
           "type": "STRING",
           "value": ":"
@@ -428,10 +502,6 @@
               "type": "BLANK"
             }
           ]
-        },
-        {
-          "type": "PATTERN",
-          "value": "[,\\n]"
         }
       ]
     },
@@ -457,16 +527,12 @@
               "type": "BLANK"
             }
           ]
-        },
-        {
-          "type": "PATTERN",
-          "value": "[,\\n]"
         }
       ]
     },
     "key": {
       "type": "PATTERN",
-      "value": "[^#\\n:, \\t][^:,]*"
+      "value": "[^#\\n:, \\t][^:\\n]*"
     },
     "type_hint": {
       "type": "CHOICE",
@@ -492,20 +558,37 @@
           "value": "binary"
         },
         {
-          "type": "PATTERN",
+          "type": "SYMBOL",
-          "value": "[0-9]+"
+          "name": "_length_hint"
         }
       ]
-    },
-    "value": {
-      "type": "PATTERN",
-      "value": "[^\\n,]+"
     }
   },
   "extras": [],
   "conflicts": [],
   "precedences": [],
-  "externals": [],
+  "externals": [
+    {
+      "type": "SYMBOL",
+      "name": "_long_marker"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_compact_marker"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_length_hint"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "value"
+    },
+    {
+      "type": "SYMBOL",
+      "name": "_error_sentinel"
+    }
+  ],
   "inline": [],
   "supertypes": [],
   "reserved": {}

src/node-types.json

@@ -302,10 +302,6 @@
     "type": "\n",
     "named": false
   },
-  {
-    "type": "#!compact",
-    "named": false
-  },
   {
     "type": "#!created=",
     "named": false
@@ -318,10 +314,6 @@
     "type": "#!expires=",
     "named": false
   },
-  {
-    "type": "#!long",
-    "named": false
-  },
   {
     "type": "#!modified=",
     "named": false
@@ -334,6 +326,10 @@
     "type": "#!srfv1",
     "named": false
   },
+  {
+    "type": ",",
+    "named": false
+  },
   {
     "type": ":",
     "named": false

src/scanner.c

@@ -0,0 +1,255 @@
+/**
+ * @file External scanner for the SRF grammar
+ * @author Emil Lerch
+ * @license MIT
+ *
+ * SRF's field delimiter is not fixed by the syntax: it is ',' in compact
+ * format (the default) and '\n' after a `#!long` directive. On top of that, a
+ * numeric type hint (`key:7:foo`) means "the value is exactly 7 bytes",
+ * newlines included. Neither is expressible in a context-free grammar, so the
+ * mode-sensitive tokens live here and everything else stays in grammar.js.
+ *
+ * Tokens produced:
+ *   LONG_MARKER    `#!long`    - flips the scanner into long mode
+ *   COMPACT_MARKER `#!compact` - flips the scanner back into compact mode
+ *   LENGTH_HINT    digits in the type hint slot; records the byte count
+ *   VALUE          a field value, delimited by mode or by a pending length
+ *
+ * ERROR_SENTINEL is never returned. It is only valid during error recovery
+ * (when tree-sitter marks every token valid), which is our cue to bail out
+ * without touching scanner state.
+ */
+
+#include "tree_sitter/alloc.h"
+#include "tree_sitter/parser.h"
+
+enum TokenType {
+  LONG_MARKER,
+  COMPACT_MARKER,
+  LENGTH_HINT,
+  VALUE,
+  ERROR_SENTINEL,
+};
+
+/// Reject absurd length hints rather than letting a typo scan its way through
+/// the rest of the buffer. 16MiB is far past any sane single SRF value.
+#define MAX_VALUE_LENGTH (16u * 1024u * 1024u)
+
+typedef struct {
+  /// false means compact format, which is the SRF default.
+  bool long_mode;
+  /// Set by LENGTH_HINT, consumed by the next VALUE.
+  bool has_length;
+  uint32_t length;
+} Scanner;
+
+/// Number of UTF-8 bytes the given codepoint occupies. Length hints count
+/// bytes, but `lexer->lookahead` hands us decoded codepoints.
+static inline uint32_t utf8_length(int32_t codepoint) {
+  if (codepoint < 0x80) return 1;
+  if (codepoint < 0x800) return 2;
+  if (codepoint < 0x10000) return 3;
+  return 4;
+}
+
+static inline bool at_directive_boundary(TSLexer *lexer) {
+  return lexer->eof(lexer) || lexer->lookahead == '\n' ||
+         lexer->lookahead == ' ' || lexer->lookahead == '\t';
+}
+
+/// Consumes the remaining characters of a directive keyword, which must be
+/// followed by end of directive so that `#!longhand` is not mistaken for
+/// `#!long`. Advances the lexer as it goes, so a false return leaves the lexer
+/// mid-keyword; only ever call it on a path that returns false all the way out.
+static bool match_keyword_tail(TSLexer *lexer, const char *tail) {
+  for (const char *c = tail; *c != '\0'; c++) {
+    if (lexer->lookahead != (int32_t)*c) return false;
+    lexer->advance(lexer, false);
+  }
+  return at_directive_boundary(lexer);
+}
+
+/// `#!long` or `#!compact`. Both are usually valid at the same position and
+/// the scanner cannot rewind, so this matches the shared `#!` prefix once and
+/// then branches on the character that distinguishes them.
+static bool scan_format_marker(Scanner *scanner, TSLexer *lexer,
+                               const bool *valid_symbols) {
+  if (lexer->lookahead != '#') return false;
+  lexer->advance(lexer, false);
+  if (lexer->lookahead != '!') return false;
+  lexer->advance(lexer, false);
+
+  if (valid_symbols[LONG_MARKER] && lexer->lookahead == 'l') {
+    lexer->advance(lexer, false);
+    if (!match_keyword_tail(lexer, "ong")) return false;
+    scanner->long_mode = true;
+    lexer->result_symbol = LONG_MARKER;
+    return true;
+  }
+
+  if (valid_symbols[COMPACT_MARKER] && lexer->lookahead == 'c') {
+    lexer->advance(lexer, false);
+    if (!match_keyword_tail(lexer, "ompact")) return false;
+    scanner->long_mode = false;
+    lexer->result_symbol = COMPACT_MARKER;
+    return true;
+  }
+
+  return false;
+}
+
+/// A run of digits in the type hint slot, e.g. the `7` of `key:7:value`.
+/// The token ends at the last digit; the optional whitespace and the ':' that
+/// follow are matched by the grammar, so we only look past them to confirm we
+/// really are in a type hint slot.
+static bool scan_length_hint(Scanner *scanner, TSLexer *lexer) {
+  if (lexer->lookahead < '0' || lexer->lookahead > '9') return false;
+
+  uint32_t length = 0;
+  while (lexer->lookahead >= '0' && lexer->lookahead <= '9') {
+    length = length * 10 + (uint32_t)(lexer->lookahead - '0');
+    if (length > MAX_VALUE_LENGTH) return false;
+    lexer->advance(lexer, false);
+  }
+  lexer->mark_end(lexer);
+
+  while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
+    lexer->advance(lexer, false);
+  }
+  if (lexer->lookahead != ':') return false;
+
+  // `key:0:` is an empty value, which is the same thing as no value at all, so
+  // leave the pending length unset and let the delimited scan decline. This
+  // also keeps the invariant below intact: a pending length always produces a
+  // value token, so it always gets cleared.
+  if (length > 0) {
+    scanner->has_length = true;
+    scanner->length = length;
+  }
+  lexer->result_symbol = LENGTH_HINT;
+  return true;
+}
+
+/// Exactly `scanner->length` bytes, newlines and all. Stops rather than
+/// straddling a codepoint boundary when the length is wrong; the field
+/// terminator the grammar then demands will be missing, which surfaces the bad
+/// length as an error instead of silently eating the file.
+///
+/// Note this is stricter than the reference parser in long format, where a
+/// length shorter than the rest of the line leaves trailing bytes that the
+/// parser skips along with the rest of the line. Here those bytes have nowhere
+/// to go, so `key:5:hello world` is flagged. Compact format rejects it in the
+/// reference parser too, and either way the length is wrong.
+static bool scan_length_prefixed_value(Scanner *scanner, TSLexer *lexer) {
+  uint32_t remaining = scanner->length;
+  bool consumed = false;
+
+  while (remaining > 0 && !lexer->eof(lexer)) {
+    uint32_t width = utf8_length(lexer->lookahead);
+    if (width > remaining) break;
+    remaining -= width;
+    lexer->advance(lexer, false);
+    consumed = true;
+  }
+
+  // Only reachable at EOF, i.e. a truncated file. Nothing follows, so leaving
+  // the pending length set cannot mislead a later token.
+  if (!consumed) return false;
+
+  scanner->has_length = false;
+  scanner->length = 0;
+  lexer->result_symbol = VALUE;
+  return true;
+}
+
+/// Everything up to the delimiter for the current mode: end of line in long
+/// mode, end of line or a comma in compact mode.
+static bool scan_delimited_value(const Scanner *scanner, TSLexer *lexer) {
+  bool consumed = false;
+
+  while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
+    if (!scanner->long_mode && lexer->lookahead == ',') break;
+    lexer->advance(lexer, false);
+    consumed = true;
+  }
+
+  if (!consumed) return false;
+  lexer->result_symbol = VALUE;
+  return true;
+}
+
+void *tree_sitter_srf_external_scanner_create(void) {
+  return ts_calloc(1, sizeof(Scanner));
+}
+
+void tree_sitter_srf_external_scanner_destroy(void *payload) {
+  ts_free(payload);
+}
+
+unsigned tree_sitter_srf_external_scanner_serialize(void *payload, char *buffer) {
+  Scanner *scanner = (Scanner *)payload;
+  buffer[0] = (char)scanner->long_mode;
+  buffer[1] = (char)scanner->has_length;
+  buffer[2] = (char)(scanner->length & 0xff);
+  buffer[3] = (char)((scanner->length >> 8) & 0xff);
+  buffer[4] = (char)((scanner->length >> 16) & 0xff);
+  buffer[5] = (char)((scanner->length >> 24) & 0xff);
+  return 6;
+}
+
+void tree_sitter_srf_external_scanner_deserialize(void *payload, const char *buffer,
+                                                  unsigned length) {
+  Scanner *scanner = (Scanner *)payload;
+
+  // Called with length 0 to reset the scanner to its initial state, which for
+  // SRF is compact format with no pending length.
+  scanner->long_mode = false;
+  scanner->has_length = false;
+  scanner->length = 0;
+  if (length < 6) return;
+
+  scanner->long_mode = buffer[0] != 0;
+  scanner->has_length = buffer[1] != 0;
+  scanner->length = (uint32_t)(unsigned char)buffer[2] |
+                    ((uint32_t)(unsigned char)buffer[3] << 8) |
+                    ((uint32_t)(unsigned char)buffer[4] << 16) |
+                    ((uint32_t)(unsigned char)buffer[5] << 24);
+}
+
+bool tree_sitter_srf_external_scanner_scan(void *payload, TSLexer *lexer,
+                                           const bool *valid_symbols) {
+  Scanner *scanner = (Scanner *)payload;
+
+  // Error recovery marks every symbol valid. Producing tokens (and flipping
+  // format mode) off the back of that would be guesswork, so leave it to the
+  // internal lexer's recovery.
+  if (valid_symbols[ERROR_SENTINEL]) return false;
+
+  // Scanner state only survives when we return true, because tree-sitter
+  // serialises after a produced token and not after a declined one. So state is
+  // never cleared on a bail-out path; instead a pending length is guaranteed to
+  // be consumed by the very next token (see scan_length_hint).
+
+  // Whitespace is never skipped here: `_ws` handles indentation in the
+  // grammar, and leading whitespace inside a value is significant.
+  if (lexer->lookahead == '#' &&
+      (valid_symbols[LONG_MARKER] || valid_symbols[COMPACT_MARKER])) {
+    // A direct return rather than a fall-through: scan_format_marker advances
+    // past `#!` before it can fail, so the lexer is no longer at the token
+    // start and a value scan from here would produce garbage. Returning false
+    // resets the lexer to the token start, letting the internal lexer have a go
+    // at `#!created=`, a misspelled marker, a comment, and so on.
+    return scan_format_marker(scanner, lexer, valid_symbols);
+  }
+
+  if (valid_symbols[LENGTH_HINT] && scan_length_hint(scanner, lexer)) {
+    return true;
+  }
+
+  if (valid_symbols[VALUE]) {
+    if (scanner->has_length) return scan_length_prefixed_value(scanner, lexer);
+    return scan_delimited_value(scanner, lexer);
+  }
+
+  return false;
+}

test/corpus/basics.txt

@@ -1,7 +1,6 @@
 ==================
 Basic long format
 ==================
-
 #!srfv1
 #!long
 name::alice
@@ -10,7 +9,6 @@ age:num:30
 ---
 
 (document
-  (blank_line)
   (magic)
   (directive
     (directive_long))
@@ -27,14 +25,12 @@ age:num:30
 ==================
 Basic compact format
 ==================
-
 #!srfv1
 name::alice,age:num:30
 
 ---
 
 (document
-  (blank_line)
   (magic)
   (field
     (untyped_field
@@ -49,7 +45,6 @@ name::alice,age:num:30
 ==================
 Directives
 ==================
-
 #!srfv1
 #!requireeof
 #!long
@@ -63,7 +58,6 @@ name::alice
 ---
 
 (document
-  (blank_line)
   (magic)
   (directive
     (directive_requireeof))
@@ -89,7 +83,6 @@ name::alice
 ==================
 Comments
 ==================
-
 #!srfv1
 # This is a comment
 name::alice
@@ -97,7 +90,6 @@ name::alice
 ---
 
 (document
-  (blank_line)
   (magic)
   (comment)
   (field
@@ -108,7 +100,6 @@ name::alice
 ==================
 All type hints
 ==================
-
 #!srfv1
 name:string:alice
 age:num:30
@@ -120,7 +111,6 @@ bio:12:hello world!
 ---
 
 (document
-  (blank_line)
   (magic)
   (field
     (typed_field
@@ -155,7 +145,6 @@ bio:12:hello world!
 ==================
 Inline comments on directives
 ==================
-
 #!srfv1 # version header
 #!long # use long format
 name::alice
@@ -163,7 +152,6 @@ name::alice
 ---
 
 (document
-  (blank_line)
   (magic
     (inline_comment))
   (directive
@@ -177,7 +165,6 @@ name::alice
 ==================
 Multiple records long format
 ==================
-
 #!srfv1
 #!long
 name::alice
@@ -189,7 +176,6 @@ age:num:25
 ---
 
 (document
-  (blank_line)
   (magic)
   (directive
     (directive_long))

test/corpus/compact_format.txt

@@ -0,0 +1,201 @@
+==================
+Commas delimit fields by default
+==================
+#!srfv1
+name::alice,age:num:30
+name::bob,age:num:25
+
+---
+
+(document
+  (magic)
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Explicit compact directive
+==================
+#!srfv1
+#!compact
+name::alice,age:num:30
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_compact))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Last format directive wins
+==================
+#!srfv1
+#!long
+#!compact
+name::alice,age:num:30
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (directive
+    (directive_compact))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Directives sharing a prefix with the format markers
+==================
+#!srfv1
+#!created=1772500000
+#!longhand=1
+#!compaction=1
+name::alice,age:num:30
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_created
+      value: (timestamp)))
+  (directive
+    (directive_unknown))
+  (directive
+    (directive_unknown))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+A format marker inside an inline comment is not a directive
+==================
+#!srfv1
+#!long # compact format is optional #!compact
+name::alice, bob
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long
+      (inline_comment)))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+A format marker in a value is not a directive
+==================
+#!srfv1
+k::#!long,j::#!compact
+name::alice,age:num:30
+
+---
+
+(document
+  (magic)
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Compact format from the SRF README
+==================
+#!srfv1 # mandatory comment with format and version. Parser instructions start with #!
+key::string value must have a length between colons or end with a comma,this is a number:num:5 ,null value:null:,array::array's don't exist. Use json or toml or something,data with newlines must have a length:7:foo
+bar,boolean value:bool:false
+key::this is the second record
+
+---
+
+(document
+  (magic
+    (inline_comment))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value))))

test/corpus/errors.txt

@@ -0,0 +1,58 @@
+==================
+Line with no colon
+:error
+==================
+#!srfv1
+#!long
+garbage
+name::alice
+
+---
+
+==================
+Empty key
+:error
+==================
+#!srfv1
+#!long
+::value
+
+---
+
+==================
+Length hint beyond the sanity limit
+:error
+==================
+#!srfv1
+#!long
+k:99999999999:v
+
+---
+
+==================
+Length hint splitting a multi-byte character
+:error
+==================
+#!srfv1
+#!long
+yen:1:¥
+
+---
+
+==================
+Unterminated final line
+:error
+==================
+#!srfv1
+#!long
+name::alice
+---
+
+==================
+Trailing field separator in compact format
+:error
+==================
+#!srfv1
+name::alice,
+
+---

test/corpus/length_prefixed.txt

@@ -0,0 +1,155 @@
+==================
+Length-prefixed value spanning lines
+==================
+#!srfv1
+#!long
+data with newlines must have a length:7:foo
+bar
+name::alice
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+Length-prefixed value from the SRF type table
+==================
+#!srfv1
+#!long
+bio:12:hello
+world!
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Length-prefixed value containing commas in compact format
+==================
+#!srfv1
+k:5:a,b,c,next::x
+
+---
+
+(document
+  (magic)
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+Length counts bytes, not characters
+==================
+#!srfv1
+#!long
+yen:3:¥5
+next::ok
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+Length-prefixed value containing field syntax
+==================
+#!srfv1
+#!long
+raw:14:k::not a field
+name::alice
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+Zero length is an empty value
+==================
+#!srfv1
+#!long
+empty:0:
+name::alice
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+Whitespace around a length hint
+==================
+#!srfv1
+#!long
+padded: 7 :foo
+bar
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))

test/corpus/long_format.txt

@@ -0,0 +1,302 @@
+==================
+Value containing commas
+==================
+#!srfv1
+#!long
+name::alice, bob and carol
+cost:num:$1,000,000,000.42
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Key containing a comma
+==================
+#!srfv1
+#!long
+last, first::alice
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+Non-ASCII values
+==================
+#!srfv1
+#!long
+tokyo dinner:num:¥15,000
+airbus:num:€410,000,000
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Leading whitespace is insignificant
+==================
+#!srfv1
+  #!long
+  name::alice, bob
+	# indented comment
+age:num:30
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (comment)
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Whitespace around the type hint
+==================
+#!srfv1
+#!long
+n: num : 5
+m:num:6
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Empty values
+==================
+#!srfv1
+#!long
+missing:null:
+also missing::
+name::alice
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (typed_field
+      (key)
+      (type_hint)))
+  (field
+    (untyped_field
+      (key)))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+Values that look like other syntax
+==================
+#!srfv1
+#!long
+url::https://x.example/a?b=1,c=2
+note::alice # not an inline comment
+shrug::,
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value))))
+
+==================
+Records separated by blank lines
+==================
+#!srfv1
+#!long
+name::alice, of the alices
+age:num:30
+
+name::bob, of the bobs
+age:num:25
+
+---
+
+(document
+  (magic)
+  (directive
+    (directive_long))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (blank_line)
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value))))
+
+==================
+Long format from the SRF README
+==================
+#!srfv1 # mandatory comment with format and version. Parser instructions start with #!
+#!requireeof # Set this if you want parsing to fail when #!eof not present on last line
+#!long # Mandatory to use multiline records, compact format is optional #!compact
+# A comment
+# empty lines ignored
+
+key::string value, with any data except a \n. an optional string length between the colons
+this is a number:num: 5
+null value:null:
+array::array's don't exist. Use json or toml or something
+data with newlines must have a length:7:foo
+bar
+boolean value:bool:false
+
+  # Empty line separates records, but comments don't count as empty
+key::this is the second record
+this is a number:num:42
+null value:null:
+array::array's still don't exist
+data with newlines must have a length::single line
+#!eof # eof marker, useful to make sure your file wasn't cut in half. Only considered if requireeof set at top
+
+---
+
+(document
+  (magic
+    (inline_comment))
+  (directive
+    (directive_requireeof
+      (inline_comment)))
+  (directive
+    (directive_long
+      (inline_comment)))
+  (comment)
+  (comment)
+  (blank_line)
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (blank_line)
+  (comment)
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)
+      (value)))
+  (field
+    (typed_field
+      (key)
+      (type_hint)))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (field
+    (untyped_field
+      (key)
+      (value)))
+  (directive
+    (directive_eof
+      (inline_comment))))