sys

README (2300B)

---

 This module provides a general purpose lexer machine.
 
 The user add actions callbacks to the lexer. The longest pattern matched
 prefix wins. In case of ties, the pattern with the highest precedence
 wins.
 
 The user prepare a backend to use with a lexer. A backend take a list of
 action to compile its core.
 
 	let actions: []lex::action = [];
 	defer free(actions);
 
 	append(actions, lex::action {
 		expr = `"([^\\"]|(\\.))*"`,
 		cb = &literal,
 		name = "LIT_STR",
 		...
 	})!;
 
 	const backend = lex::def_backend()!(actions)!; // use default backend (DFA without environment variable)
 	defer lex::destroy(backend);
 
 	const lexer = lex::init(backend, in);
 	defer lex::finish(&lexer);
 
 An action callback is associated with an regular expression to
 match the tokens. The action callbacks are free to initialize tokens as
 they please, but the [[scanner]] object provide convenient functions.
 
 	fn literal(
 		scan: *lex::scanner,
 		lexeme: const str,
 		user: nullable *opaque,
 	) (str | *lex::token | lex::error) = {
 		return lex::scan_token(scan, void, lexeme);
 	};
 
 This action callback would return a token of the added action type
 (ex: "LIT_STR"), with a void value, and lexing the full lexeme pattern
 matched string (ex: "foo").
 
 When the callback return a string, it represents the lexeme to swallow.
 
 	append(actions, lex::action {
 		expr = "( |\t|\n|\r)+",
 		cb = &skip,
 		...
 	})!;
 
 	fn skip(
 		scan: *lex::scanner,
 		lexeme: const str,
 		user: nullable *opaque,
 	) (str | *lex::token | lex::error) = {
 		return lexeme;
 	};
 
 Action callbacks can be used to match hatch symbols, and then to lex the
 scanned input manually.
 
 	append(actions, lex::action {
 		expr = `\<`,
 		cb = &html,
 		name = "ID"
 		...
 	})!;
 
 	fn html(
 		scan: *lex::scanner,
 		lexeme: const str,
 		user: nullable *opaque,
 	) (str | *lex::token | lex::error) = {
 		let buf: []u8 = [];
 		defer free(buf);
 
 		append(buf, strings::toutf8(lexeme)...)!;
 
 		let brk = 1z;
 		const start = scan.start;
 
 		for (let byte .. strings::toutf8(scan.in)) {
 			append(buf, byte)?;
 			if (byte == '<') {
 				brk += 1;
 			} else if (byte == '>') {
 				brk -= 1;
 			};
 			if (brk == 0) {
 				const lexeme = strings::fromutf8(buf)!;
 				return lex::scan_token(scan, void, lexeme);
 			};
 		};
 
 		return lex::syntaxf(start, "unclosed HTML literal");
 	};