handle full srf spec
This commit is contained in:
parent
eacd98c7ff
commit
469235afb3
GPG key ID:
A7B62D657EF764F8
12 changed files with 2303 additions and 1031 deletions
|
|
@ -6,6 +6,9 @@ repos:
|
|||
hooks:
|
||||
- id: trailing-whitespace
|
||||
- id: end-of-file-fixer
|
||||
# tree-sitter generate emits these two without a trailing newline, so
|
||||
# the fixer and the generate hook below would undo each other forever.
|
||||
exclude: ^src/(grammar|node-types)\.json$
|
||||
- id: check-yaml
|
||||
- id: check-added-large-files
|
||||
- repo: local
|
||||
|
|
|
|||
66
grammar.js
66
grammar.js
|
|
@ -2,6 +2,11 @@
|
|||
* @file SRF (Simple Record Format) grammar for tree-sitter
|
||||
* @author Emil Lerch
|
||||
* @license MIT
|
||||
*
|
||||
* SRF's field delimiter depends on the format directive: ',' in compact format
|
||||
* (the default) and '\n' after `#!long`. A numeric type hint additionally means
|
||||
* "this value is exactly N bytes", newlines included. Neither is expressible
|
||||
* here, so those tokens come from the external scanner in src/scanner.c.
|
||||
*/
|
||||
|
||||
/// <reference types="tree-sitter-cli/dsl" />
|
||||
|
|
@ -12,17 +17,34 @@ module.exports = grammar({
|
|||
|
||||
extras: _ => [],
|
||||
|
||||
externals: $ => [
|
||||
$._long_marker,
|
||||
$._compact_marker,
|
||||
$._length_hint,
|
||||
$.value,
|
||||
// Never produced by the scanner. Only valid while tree-sitter is in error
|
||||
// recovery, which tells the scanner to keep its hands off.
|
||||
$._error_sentinel,
|
||||
],
|
||||
|
||||
rules: {
|
||||
document: $ => repeat($._line),
|
||||
|
||||
_line: $ => choice(
|
||||
$.magic,
|
||||
$.directive,
|
||||
$.comment,
|
||||
$.field,
|
||||
$.blank_line,
|
||||
// Leading whitespace is insignificant on every kind of line: the reference
|
||||
// parser trims the start of each line before deciding what it is.
|
||||
_line: $ => seq(
|
||||
optional($._ws),
|
||||
choice(
|
||||
$.magic,
|
||||
$.directive,
|
||||
$.comment,
|
||||
$._field_line,
|
||||
$.blank_line,
|
||||
),
|
||||
),
|
||||
|
||||
_ws: _ => /[ \t]+/,
|
||||
|
||||
magic: $ => seq(
|
||||
'#!srfv1',
|
||||
optional($.inline_comment),
|
||||
|
|
@ -42,8 +64,11 @@ module.exports = grammar({
|
|||
$.directive_unknown,
|
||||
),
|
||||
|
||||
directive_long: $ => seq('#!long', optional($.inline_comment), '\n'),
|
||||
directive_compact: $ => seq('#!compact', optional($.inline_comment), '\n'),
|
||||
// The format directives are scanned externally so that the scanner learns
|
||||
// which delimiter applies to the rest of the document. If both appear, the
|
||||
// last one wins, matching the reference parser.
|
||||
directive_long: $ => seq($._long_marker, optional($.inline_comment), '\n'),
|
||||
directive_compact: $ => seq($._compact_marker, optional($.inline_comment), '\n'),
|
||||
directive_requireeof: $ => seq('#!requireeof', optional($.inline_comment), '\n'),
|
||||
directive_eof: $ => seq('#!eof', optional($.inline_comment), '\n'),
|
||||
directive_expires: $ => seq(
|
||||
|
|
@ -77,6 +102,16 @@ module.exports = grammar({
|
|||
|
||||
blank_line: _ => token(/[ \t]*\n/),
|
||||
|
||||
// A comma only ever separates fields in compact format: in long format the
|
||||
// scanner has already swallowed it as part of the value, so the repetition
|
||||
// below is unreachable there. Requiring a field after every comma is what
|
||||
// makes a trailing separator (`key::value,`) an error, as the spec demands.
|
||||
_field_line: $ => seq(
|
||||
$.field,
|
||||
repeat(seq(',', $.field)),
|
||||
'\n',
|
||||
),
|
||||
|
||||
field: $ => choice(
|
||||
$.typed_field,
|
||||
$.untyped_field,
|
||||
|
|
@ -85,30 +120,31 @@ module.exports = grammar({
|
|||
typed_field: $ => seq(
|
||||
$.key,
|
||||
':',
|
||||
optional($._ws),
|
||||
$.type_hint,
|
||||
optional($._ws),
|
||||
':',
|
||||
optional($.value),
|
||||
/[,\n]/,
|
||||
),
|
||||
|
||||
untyped_field: $ => seq(
|
||||
$.key,
|
||||
'::',
|
||||
optional($.value),
|
||||
/[,\n]/,
|
||||
),
|
||||
|
||||
key: _ => /[^#\n:, \t][^:,]*/,
|
||||
// Everything up to the first colon, so commas are fair game. A key may not
|
||||
// start with one, which keeps a stray trailing comma in compact format from
|
||||
// being absorbed into the next key.
|
||||
key: _ => /[^#\n:, \t][^:\n]*/,
|
||||
|
||||
type_hint: _ => choice(
|
||||
type_hint: $ => choice(
|
||||
'string',
|
||||
'num',
|
||||
'bool',
|
||||
'null',
|
||||
'binary',
|
||||
/[0-9]+/,
|
||||
$._length_hint,
|
||||
),
|
||||
|
||||
value: _ => /[^\n,]+/,
|
||||
},
|
||||
});
|
||||
|
|
|
|||
|
|
@ -22,3 +22,4 @@
|
|||
|
||||
":" @punctuation.delimiter
|
||||
"::" @punctuation.delimiter
|
||||
"," @punctuation.delimiter
|
||||
|
|
|
|||
157
src/grammar.json
157
src/grammar.json
|
|
@ -10,30 +10,51 @@
|
|||
}
|
||||
},
|
||||
"_line": {
|
||||
"type": "CHOICE",
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "magic"
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_ws"
|
||||
},
|
||||
{
|
||||
"type": "BLANK"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "directive"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "comment"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "field"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "blank_line"
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "magic"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "directive"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "comment"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_field_line"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "blank_line"
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
},
|
||||
"_ws": {
|
||||
"type": "PATTERN",
|
||||
"value": "[ \\t]+"
|
||||
},
|
||||
"magic": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
|
|
@ -104,8 +125,8 @@
|
|||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "#!long"
|
||||
"type": "SYMBOL",
|
||||
"name": "_long_marker"
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
|
|
@ -129,8 +150,8 @@
|
|||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "#!compact"
|
||||
"type": "SYMBOL",
|
||||
"name": "_compact_marker"
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
|
|
@ -385,6 +406,35 @@
|
|||
"value": "[ \\t]*\\n"
|
||||
}
|
||||
},
|
||||
"_field_line": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "field"
|
||||
},
|
||||
{
|
||||
"type": "REPEAT",
|
||||
"content": {
|
||||
"type": "SEQ",
|
||||
"members": [
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": ","
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "field"
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": "\n"
|
||||
}
|
||||
]
|
||||
},
|
||||
"field": {
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
|
|
@ -409,10 +459,34 @@
|
|||
"type": "STRING",
|
||||
"value": ":"
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_ws"
|
||||
},
|
||||
{
|
||||
"type": "BLANK"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "type_hint"
|
||||
},
|
||||
{
|
||||
"type": "CHOICE",
|
||||
"members": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_ws"
|
||||
},
|
||||
{
|
||||
"type": "BLANK"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "STRING",
|
||||
"value": ":"
|
||||
|
|
@ -428,10 +502,6 @@
|
|||
"type": "BLANK"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[,\\n]"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
|
@ -457,16 +527,12 @@
|
|||
"type": "BLANK"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[,\\n]"
|
||||
}
|
||||
]
|
||||
},
|
||||
"key": {
|
||||
"type": "PATTERN",
|
||||
"value": "[^#\\n:, \\t][^:,]*"
|
||||
"value": "[^#\\n:, \\t][^:\\n]*"
|
||||
},
|
||||
"type_hint": {
|
||||
"type": "CHOICE",
|
||||
|
|
@ -492,20 +558,37 @@
|
|||
"value": "binary"
|
||||
},
|
||||
{
|
||||
"type": "PATTERN",
|
||||
"value": "[0-9]+"
|
||||
"type": "SYMBOL",
|
||||
"name": "_length_hint"
|
||||
}
|
||||
]
|
||||
},
|
||||
"value": {
|
||||
"type": "PATTERN",
|
||||
"value": "[^\\n,]+"
|
||||
}
|
||||
},
|
||||
"extras": [],
|
||||
"conflicts": [],
|
||||
"precedences": [],
|
||||
"externals": [],
|
||||
"externals": [
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_long_marker"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_compact_marker"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_length_hint"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "value"
|
||||
},
|
||||
{
|
||||
"type": "SYMBOL",
|
||||
"name": "_error_sentinel"
|
||||
}
|
||||
],
|
||||
"inline": [],
|
||||
"supertypes": [],
|
||||
"reserved": {}
|
||||
|
|
|
|||
|
|
@ -302,10 +302,6 @@
|
|||
"type": "\n",
|
||||
"named": false
|
||||
},
|
||||
{
|
||||
"type": "#!compact",
|
||||
"named": false
|
||||
},
|
||||
{
|
||||
"type": "#!created=",
|
||||
"named": false
|
||||
|
|
@ -318,10 +314,6 @@
|
|||
"type": "#!expires=",
|
||||
"named": false
|
||||
},
|
||||
{
|
||||
"type": "#!long",
|
||||
"named": false
|
||||
},
|
||||
{
|
||||
"type": "#!modified=",
|
||||
"named": false
|
||||
|
|
@ -334,6 +326,10 @@
|
|||
"type": "#!srfv1",
|
||||
"named": false
|
||||
},
|
||||
{
|
||||
"type": ",",
|
||||
"named": false
|
||||
},
|
||||
{
|
||||
"type": ":",
|
||||
"named": false
|
||||
|
|
|
|||
2110
src/parser.c
2110
src/parser.c
File diff suppressed because it is too large
Load diff
255
src/scanner.c
Normal file
255
src/scanner.c
Normal file
|
|
@ -0,0 +1,255 @@
|
|||
/**
|
||||
* @file External scanner for the SRF grammar
|
||||
* @author Emil Lerch
|
||||
* @license MIT
|
||||
*
|
||||
* SRF's field delimiter is not fixed by the syntax: it is ',' in compact
|
||||
* format (the default) and '\n' after a `#!long` directive. On top of that, a
|
||||
* numeric type hint (`key:7:foo`) means "the value is exactly 7 bytes",
|
||||
* newlines included. Neither is expressible in a context-free grammar, so the
|
||||
* mode-sensitive tokens live here and everything else stays in grammar.js.
|
||||
*
|
||||
* Tokens produced:
|
||||
* LONG_MARKER `#!long` - flips the scanner into long mode
|
||||
* COMPACT_MARKER `#!compact` - flips the scanner back into compact mode
|
||||
* LENGTH_HINT digits in the type hint slot; records the byte count
|
||||
* VALUE a field value, delimited by mode or by a pending length
|
||||
*
|
||||
* ERROR_SENTINEL is never returned. It is only valid during error recovery
|
||||
* (when tree-sitter marks every token valid), which is our cue to bail out
|
||||
* without touching scanner state.
|
||||
*/
|
||||
|
||||
#include "tree_sitter/alloc.h"
|
||||
#include "tree_sitter/parser.h"
|
||||
|
||||
enum TokenType {
|
||||
LONG_MARKER,
|
||||
COMPACT_MARKER,
|
||||
LENGTH_HINT,
|
||||
VALUE,
|
||||
ERROR_SENTINEL,
|
||||
};
|
||||
|
||||
/// Reject absurd length hints rather than letting a typo scan its way through
|
||||
/// the rest of the buffer. 16MiB is far past any sane single SRF value.
|
||||
#define MAX_VALUE_LENGTH (16u * 1024u * 1024u)
|
||||
|
||||
typedef struct {
|
||||
/// false means compact format, which is the SRF default.
|
||||
bool long_mode;
|
||||
/// Set by LENGTH_HINT, consumed by the next VALUE.
|
||||
bool has_length;
|
||||
uint32_t length;
|
||||
} Scanner;
|
||||
|
||||
/// Number of UTF-8 bytes the given codepoint occupies. Length hints count
|
||||
/// bytes, but `lexer->lookahead` hands us decoded codepoints.
|
||||
static inline uint32_t utf8_length(int32_t codepoint) {
|
||||
if (codepoint < 0x80) return 1;
|
||||
if (codepoint < 0x800) return 2;
|
||||
if (codepoint < 0x10000) return 3;
|
||||
return 4;
|
||||
}
|
||||
|
||||
static inline bool at_directive_boundary(TSLexer *lexer) {
|
||||
return lexer->eof(lexer) || lexer->lookahead == '\n' ||
|
||||
lexer->lookahead == ' ' || lexer->lookahead == '\t';
|
||||
}
|
||||
|
||||
/// Consumes the remaining characters of a directive keyword, which must be
|
||||
/// followed by end of directive so that `#!longhand` is not mistaken for
|
||||
/// `#!long`. Advances the lexer as it goes, so a false return leaves the lexer
|
||||
/// mid-keyword; only ever call it on a path that returns false all the way out.
|
||||
static bool match_keyword_tail(TSLexer *lexer, const char *tail) {
|
||||
for (const char *c = tail; *c != '\0'; c++) {
|
||||
if (lexer->lookahead != (int32_t)*c) return false;
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
return at_directive_boundary(lexer);
|
||||
}
|
||||
|
||||
/// `#!long` or `#!compact`. Both are usually valid at the same position and
|
||||
/// the scanner cannot rewind, so this matches the shared `#!` prefix once and
|
||||
/// then branches on the character that distinguishes them.
|
||||
static bool scan_format_marker(Scanner *scanner, TSLexer *lexer,
|
||||
const bool *valid_symbols) {
|
||||
if (lexer->lookahead != '#') return false;
|
||||
lexer->advance(lexer, false);
|
||||
if (lexer->lookahead != '!') return false;
|
||||
lexer->advance(lexer, false);
|
||||
|
||||
if (valid_symbols[LONG_MARKER] && lexer->lookahead == 'l') {
|
||||
lexer->advance(lexer, false);
|
||||
if (!match_keyword_tail(lexer, "ong")) return false;
|
||||
scanner->long_mode = true;
|
||||
lexer->result_symbol = LONG_MARKER;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (valid_symbols[COMPACT_MARKER] && lexer->lookahead == 'c') {
|
||||
lexer->advance(lexer, false);
|
||||
if (!match_keyword_tail(lexer, "ompact")) return false;
|
||||
scanner->long_mode = false;
|
||||
lexer->result_symbol = COMPACT_MARKER;
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// A run of digits in the type hint slot, e.g. the `7` of `key:7:value`.
|
||||
/// The token ends at the last digit; the optional whitespace and the ':' that
|
||||
/// follow are matched by the grammar, so we only look past them to confirm we
|
||||
/// really are in a type hint slot.
|
||||
static bool scan_length_hint(Scanner *scanner, TSLexer *lexer) {
|
||||
if (lexer->lookahead < '0' || lexer->lookahead > '9') return false;
|
||||
|
||||
uint32_t length = 0;
|
||||
while (lexer->lookahead >= '0' && lexer->lookahead <= '9') {
|
||||
length = length * 10 + (uint32_t)(lexer->lookahead - '0');
|
||||
if (length > MAX_VALUE_LENGTH) return false;
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
lexer->mark_end(lexer);
|
||||
|
||||
while (lexer->lookahead == ' ' || lexer->lookahead == '\t') {
|
||||
lexer->advance(lexer, false);
|
||||
}
|
||||
if (lexer->lookahead != ':') return false;
|
||||
|
||||
// `key:0:` is an empty value, which is the same thing as no value at all, so
|
||||
// leave the pending length unset and let the delimited scan decline. This
|
||||
// also keeps the invariant below intact: a pending length always produces a
|
||||
// value token, so it always gets cleared.
|
||||
if (length > 0) {
|
||||
scanner->has_length = true;
|
||||
scanner->length = length;
|
||||
}
|
||||
lexer->result_symbol = LENGTH_HINT;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Exactly `scanner->length` bytes, newlines and all. Stops rather than
|
||||
/// straddling a codepoint boundary when the length is wrong; the field
|
||||
/// terminator the grammar then demands will be missing, which surfaces the bad
|
||||
/// length as an error instead of silently eating the file.
|
||||
///
|
||||
/// Note this is stricter than the reference parser in long format, where a
|
||||
/// length shorter than the rest of the line leaves trailing bytes that the
|
||||
/// parser skips along with the rest of the line. Here those bytes have nowhere
|
||||
/// to go, so `key:5:hello world` is flagged. Compact format rejects it in the
|
||||
/// reference parser too, and either way the length is wrong.
|
||||
static bool scan_length_prefixed_value(Scanner *scanner, TSLexer *lexer) {
|
||||
uint32_t remaining = scanner->length;
|
||||
bool consumed = false;
|
||||
|
||||
while (remaining > 0 && !lexer->eof(lexer)) {
|
||||
uint32_t width = utf8_length(lexer->lookahead);
|
||||
if (width > remaining) break;
|
||||
remaining -= width;
|
||||
lexer->advance(lexer, false);
|
||||
consumed = true;
|
||||
}
|
||||
|
||||
// Only reachable at EOF, i.e. a truncated file. Nothing follows, so leaving
|
||||
// the pending length set cannot mislead a later token.
|
||||
if (!consumed) return false;
|
||||
|
||||
scanner->has_length = false;
|
||||
scanner->length = 0;
|
||||
lexer->result_symbol = VALUE;
|
||||
return true;
|
||||
}
|
||||
|
||||
/// Everything up to the delimiter for the current mode: end of line in long
|
||||
/// mode, end of line or a comma in compact mode.
|
||||
static bool scan_delimited_value(const Scanner *scanner, TSLexer *lexer) {
|
||||
bool consumed = false;
|
||||
|
||||
while (!lexer->eof(lexer) && lexer->lookahead != '\n') {
|
||||
if (!scanner->long_mode && lexer->lookahead == ',') break;
|
||||
lexer->advance(lexer, false);
|
||||
consumed = true;
|
||||
}
|
||||
|
||||
if (!consumed) return false;
|
||||
lexer->result_symbol = VALUE;
|
||||
return true;
|
||||
}
|
||||
|
||||
void *tree_sitter_srf_external_scanner_create(void) {
|
||||
return ts_calloc(1, sizeof(Scanner));
|
||||
}
|
||||
|
||||
void tree_sitter_srf_external_scanner_destroy(void *payload) {
|
||||
ts_free(payload);
|
||||
}
|
||||
|
||||
unsigned tree_sitter_srf_external_scanner_serialize(void *payload, char *buffer) {
|
||||
Scanner *scanner = (Scanner *)payload;
|
||||
buffer[0] = (char)scanner->long_mode;
|
||||
buffer[1] = (char)scanner->has_length;
|
||||
buffer[2] = (char)(scanner->length & 0xff);
|
||||
buffer[3] = (char)((scanner->length >> 8) & 0xff);
|
||||
buffer[4] = (char)((scanner->length >> 16) & 0xff);
|
||||
buffer[5] = (char)((scanner->length >> 24) & 0xff);
|
||||
return 6;
|
||||
}
|
||||
|
||||
void tree_sitter_srf_external_scanner_deserialize(void *payload, const char *buffer,
|
||||
unsigned length) {
|
||||
Scanner *scanner = (Scanner *)payload;
|
||||
|
||||
// Called with length 0 to reset the scanner to its initial state, which for
|
||||
// SRF is compact format with no pending length.
|
||||
scanner->long_mode = false;
|
||||
scanner->has_length = false;
|
||||
scanner->length = 0;
|
||||
if (length < 6) return;
|
||||
|
||||
scanner->long_mode = buffer[0] != 0;
|
||||
scanner->has_length = buffer[1] != 0;
|
||||
scanner->length = (uint32_t)(unsigned char)buffer[2] |
|
||||
((uint32_t)(unsigned char)buffer[3] << 8) |
|
||||
((uint32_t)(unsigned char)buffer[4] << 16) |
|
||||
((uint32_t)(unsigned char)buffer[5] << 24);
|
||||
}
|
||||
|
||||
bool tree_sitter_srf_external_scanner_scan(void *payload, TSLexer *lexer,
|
||||
const bool *valid_symbols) {
|
||||
Scanner *scanner = (Scanner *)payload;
|
||||
|
||||
// Error recovery marks every symbol valid. Producing tokens (and flipping
|
||||
// format mode) off the back of that would be guesswork, so leave it to the
|
||||
// internal lexer's recovery.
|
||||
if (valid_symbols[ERROR_SENTINEL]) return false;
|
||||
|
||||
// Scanner state only survives when we return true, because tree-sitter
|
||||
// serialises after a produced token and not after a declined one. So state is
|
||||
// never cleared on a bail-out path; instead a pending length is guaranteed to
|
||||
// be consumed by the very next token (see scan_length_hint).
|
||||
|
||||
// Whitespace is never skipped here: `_ws` handles indentation in the
|
||||
// grammar, and leading whitespace inside a value is significant.
|
||||
if (lexer->lookahead == '#' &&
|
||||
(valid_symbols[LONG_MARKER] || valid_symbols[COMPACT_MARKER])) {
|
||||
// A direct return rather than a fall-through: scan_format_marker advances
|
||||
// past `#!` before it can fail, so the lexer is no longer at the token
|
||||
// start and a value scan from here would produce garbage. Returning false
|
||||
// resets the lexer to the token start, letting the internal lexer have a go
|
||||
// at `#!created=`, a misspelled marker, a comment, and so on.
|
||||
return scan_format_marker(scanner, lexer, valid_symbols);
|
||||
}
|
||||
|
||||
if (valid_symbols[LENGTH_HINT] && scan_length_hint(scanner, lexer)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (valid_symbols[VALUE]) {
|
||||
if (scanner->has_length) return scan_length_prefixed_value(scanner, lexer);
|
||||
return scan_delimited_value(scanner, lexer);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
|
@ -1,7 +1,6 @@
|
|||
==================
|
||||
Basic long format
|
||||
==================
|
||||
|
||||
#!srfv1
|
||||
#!long
|
||||
name::alice
|
||||
|
|
@ -10,7 +9,6 @@ age:num:30
|
|||
---
|
||||
|
||||
(document
|
||||
(blank_line)
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
|
|
@ -27,14 +25,12 @@ age:num:30
|
|||
==================
|
||||
Basic compact format
|
||||
==================
|
||||
|
||||
#!srfv1
|
||||
name::alice,age:num:30
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(blank_line)
|
||||
(magic)
|
||||
(field
|
||||
(untyped_field
|
||||
|
|
@ -49,7 +45,6 @@ name::alice,age:num:30
|
|||
==================
|
||||
Directives
|
||||
==================
|
||||
|
||||
#!srfv1
|
||||
#!requireeof
|
||||
#!long
|
||||
|
|
@ -63,7 +58,6 @@ name::alice
|
|||
---
|
||||
|
||||
(document
|
||||
(blank_line)
|
||||
(magic)
|
||||
(directive
|
||||
(directive_requireeof))
|
||||
|
|
@ -89,7 +83,6 @@ name::alice
|
|||
==================
|
||||
Comments
|
||||
==================
|
||||
|
||||
#!srfv1
|
||||
# This is a comment
|
||||
name::alice
|
||||
|
|
@ -97,7 +90,6 @@ name::alice
|
|||
---
|
||||
|
||||
(document
|
||||
(blank_line)
|
||||
(magic)
|
||||
(comment)
|
||||
(field
|
||||
|
|
@ -108,7 +100,6 @@ name::alice
|
|||
==================
|
||||
All type hints
|
||||
==================
|
||||
|
||||
#!srfv1
|
||||
name:string:alice
|
||||
age:num:30
|
||||
|
|
@ -120,7 +111,6 @@ bio:12:hello world!
|
|||
---
|
||||
|
||||
(document
|
||||
(blank_line)
|
||||
(magic)
|
||||
(field
|
||||
(typed_field
|
||||
|
|
@ -155,7 +145,6 @@ bio:12:hello world!
|
|||
==================
|
||||
Inline comments on directives
|
||||
==================
|
||||
|
||||
#!srfv1 # version header
|
||||
#!long # use long format
|
||||
name::alice
|
||||
|
|
@ -163,7 +152,6 @@ name::alice
|
|||
---
|
||||
|
||||
(document
|
||||
(blank_line)
|
||||
(magic
|
||||
(inline_comment))
|
||||
(directive
|
||||
|
|
@ -177,7 +165,6 @@ name::alice
|
|||
==================
|
||||
Multiple records long format
|
||||
==================
|
||||
|
||||
#!srfv1
|
||||
#!long
|
||||
name::alice
|
||||
|
|
@ -189,7 +176,6 @@ age:num:25
|
|||
---
|
||||
|
||||
(document
|
||||
(blank_line)
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
|
|
|
|||
201
test/corpus/compact_format.txt
Normal file
201
test/corpus/compact_format.txt
Normal file
|
|
@ -0,0 +1,201 @@
|
|||
==================
|
||||
Commas delimit fields by default
|
||||
==================
|
||||
#!srfv1
|
||||
name::alice,age:num:30
|
||||
name::bob,age:num:25
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Explicit compact directive
|
||||
==================
|
||||
#!srfv1
|
||||
#!compact
|
||||
name::alice,age:num:30
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_compact))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Last format directive wins
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
#!compact
|
||||
name::alice,age:num:30
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(directive
|
||||
(directive_compact))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Directives sharing a prefix with the format markers
|
||||
==================
|
||||
#!srfv1
|
||||
#!created=1772500000
|
||||
#!longhand=1
|
||||
#!compaction=1
|
||||
name::alice,age:num:30
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_created
|
||||
value: (timestamp)))
|
||||
(directive
|
||||
(directive_unknown))
|
||||
(directive
|
||||
(directive_unknown))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
A format marker inside an inline comment is not a directive
|
||||
==================
|
||||
#!srfv1
|
||||
#!long # compact format is optional #!compact
|
||||
name::alice, bob
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long
|
||||
(inline_comment)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
A format marker in a value is not a directive
|
||||
==================
|
||||
#!srfv1
|
||||
k::#!long,j::#!compact
|
||||
name::alice,age:num:30
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Compact format from the SRF README
|
||||
==================
|
||||
#!srfv1 # mandatory comment with format and version. Parser instructions start with #!
|
||||
key::string value must have a length between colons or end with a comma,this is a number:num:5 ,null value:null:,array::array's don't exist. Use json or toml or something,data with newlines must have a length:7:foo
|
||||
bar,boolean value:bool:false
|
||||
key::this is the second record
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic
|
||||
(inline_comment))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
58
test/corpus/errors.txt
Normal file
58
test/corpus/errors.txt
Normal file
|
|
@ -0,0 +1,58 @@
|
|||
==================
|
||||
Line with no colon
|
||||
:error
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
garbage
|
||||
name::alice
|
||||
|
||||
---
|
||||
|
||||
==================
|
||||
Empty key
|
||||
:error
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
::value
|
||||
|
||||
---
|
||||
|
||||
==================
|
||||
Length hint beyond the sanity limit
|
||||
:error
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
k:99999999999:v
|
||||
|
||||
---
|
||||
|
||||
==================
|
||||
Length hint splitting a multi-byte character
|
||||
:error
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
yen:1:¥
|
||||
|
||||
---
|
||||
|
||||
==================
|
||||
Unterminated final line
|
||||
:error
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
name::alice
|
||||
---
|
||||
|
||||
==================
|
||||
Trailing field separator in compact format
|
||||
:error
|
||||
==================
|
||||
#!srfv1
|
||||
name::alice,
|
||||
|
||||
---
|
||||
155
test/corpus/length_prefixed.txt
Normal file
155
test/corpus/length_prefixed.txt
Normal file
|
|
@ -0,0 +1,155 @@
|
|||
==================
|
||||
Length-prefixed value spanning lines
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
data with newlines must have a length:7:foo
|
||||
bar
|
||||
name::alice
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Length-prefixed value from the SRF type table
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
bio:12:hello
|
||||
world!
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Length-prefixed value containing commas in compact format
|
||||
==================
|
||||
#!srfv1
|
||||
k:5:a,b,c,next::x
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Length counts bytes, not characters
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
yen:3:¥5
|
||||
next::ok
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Length-prefixed value containing field syntax
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
raw:14:k::not a field
|
||||
name::alice
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Zero length is an empty value
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
empty:0:
|
||||
name::alice
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Whitespace around a length hint
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
padded: 7 :foo
|
||||
bar
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
302
test/corpus/long_format.txt
Normal file
302
test/corpus/long_format.txt
Normal file
|
|
@ -0,0 +1,302 @@
|
|||
==================
|
||||
Value containing commas
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
name::alice, bob and carol
|
||||
cost:num:$1,000,000,000.42
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Key containing a comma
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
last, first::alice
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Non-ASCII values
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
tokyo dinner:num:¥15,000
|
||||
airbus:num:€410,000,000
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Leading whitespace is insignificant
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
name::alice, bob
|
||||
# indented comment
|
||||
age:num:30
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(comment)
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Whitespace around the type hint
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
n: num : 5
|
||||
m:num:6
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Empty values
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
missing:null:
|
||||
also missing::
|
||||
name::alice
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Values that look like other syntax
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
url::https://x.example/a?b=1,c=2
|
||||
note::alice # not an inline comment
|
||||
shrug::,
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Records separated by blank lines
|
||||
==================
|
||||
#!srfv1
|
||||
#!long
|
||||
name::alice, of the alices
|
||||
age:num:30
|
||||
|
||||
name::bob, of the bobs
|
||||
age:num:25
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic)
|
||||
(directive
|
||||
(directive_long))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(blank_line)
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value))))
|
||||
|
||||
==================
|
||||
Long format from the SRF README
|
||||
==================
|
||||
#!srfv1 # mandatory comment with format and version. Parser instructions start with #!
|
||||
#!requireeof # Set this if you want parsing to fail when #!eof not present on last line
|
||||
#!long # Mandatory to use multiline records, compact format is optional #!compact
|
||||
# A comment
|
||||
# empty lines ignored
|
||||
|
||||
key::string value, with any data except a \n. an optional string length between the colons
|
||||
this is a number:num: 5
|
||||
null value:null:
|
||||
array::array's don't exist. Use json or toml or something
|
||||
data with newlines must have a length:7:foo
|
||||
bar
|
||||
boolean value:bool:false
|
||||
|
||||
# Empty line separates records, but comments don't count as empty
|
||||
key::this is the second record
|
||||
this is a number:num:42
|
||||
null value:null:
|
||||
array::array's still don't exist
|
||||
data with newlines must have a length::single line
|
||||
#!eof # eof marker, useful to make sure your file wasn't cut in half. Only considered if requireeof set at top
|
||||
|
||||
---
|
||||
|
||||
(document
|
||||
(magic
|
||||
(inline_comment))
|
||||
(directive
|
||||
(directive_requireeof
|
||||
(inline_comment)))
|
||||
(directive
|
||||
(directive_long
|
||||
(inline_comment)))
|
||||
(comment)
|
||||
(comment)
|
||||
(blank_line)
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(blank_line)
|
||||
(comment)
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)
|
||||
(value)))
|
||||
(field
|
||||
(typed_field
|
||||
(key)
|
||||
(type_hint)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(field
|
||||
(untyped_field
|
||||
(key)
|
||||
(value)))
|
||||
(directive
|
||||
(directive_eof
|
||||
(inline_comment))))
|
||||
Loading…
Add table
Reference in a new issue