sys

lex.ha (5693B)

---

 use regex;
 use strconv;
 use fmt;
 use strings;
 use encoding::utf8;
 
 // The default token constructor.
 export fn default_token(
 	scan: *scanner,
 	name: const str,
 	value: value,
 	morphene: const str,
 	lexeme: const str,
 ) (*token | error) = {
 	const end = scan.start;
 	const last = end;
 	const decoder = utf8::decode(strings::toutf8(morphene));
 	for (let r => utf8::next(&decoder)!) {
 		last = end;
 		forwardr(&end, [r as rune]);
 	};
 	return alloc(token {
 		name = name,
 		value = value,
 		morphene = morphene,
 		lexeme = lexeme,
 		start = scan.start,
 		end = last,
 		tostrfn = &tokstr,
 		freefn = &tokfree,
 	})?;
 };
 
 fn tokstr(tok: *token) str = {
 	match (tok.value) {
 	case void => return tok.morphene;
 	case let val: f64 => return strconv::f64tos(val);
 	case let val: i64 => return strconv::i64tos(val);
 	case let val: size => return strconv::ztos(val);
 	case let val: u64 => return strconv::u64tos(val);
 	case let val: str => return val;
 	case let val: rune => return strings::fromutf8_unsafe(utf8::encoderune(val));
 	};
 };
 
 fn tokfree(tok: *token) void = {
 	free(tok);
 };
 
 // Format a token as a string.
 export fn strtoken(tok: *token) str = tok.tostrfn(tok);
 
 // Initialize a new [[lexer]] to lex the input bytes. The caller must free
 // associated resources with [[finish]].
 export fn init(
 	be: *backend,
 	in: const str = "",
 	tokfn: nullable *tokenfn = null,
 ) lexer = {
 	const loc = location {
 		off = 0,
 		line = 1,
 		col = 1,
 	};
 	return lexer {
 		be = be,
 		in = in,
 		token = if (tokfn is *tokenfn) tokfn as *tokenfn else &default_token,
 		un = null,
 		loc = loc,
 		prevunlocs = [(loc, loc)...],
 		...
 	};
 };
 
 // Convenient function to reuse an existing lexer with a new input string.
 export fn reuse(lex: *lexer, in: const str) void = {
 	const loc = location {
 		off = 0,
 		line = 1,
 		col = 1,
 	};
 	lex.in = in;
 	lex.un = null;
 	lex.loc = loc;
 	lex.prevunlocs = [(loc, loc)...];
 	for (let tok .. lex.tokens)
 		tok.freefn(tok);
 	delete(lex.tokens[..]);
 	match (lex.reuse) {
 	case null => void;
 	case let reuse: *reusecb => reuse(lex);
 	};
 };
 
 // Free resources associated with a [[lexer]].
 export fn finish(lex: *lexer) void = {
 	for (let tok .. lex.tokens)
 		tok.freefn(tok);
 	free(lex.tokens);
 };
 
 // Format a [[syntax]] error as a formatable.
 export fn syntaxf(loc: location, fmt: const str, args: fmt::field...) syntax = {
 	static let buf: [2048]u8 = [0...];
 	const msg = fmt::bsprintf(buf, fmt, args...)!;
 	return (loc, msg);
 };
 
 // Gives the current location of the lexer.
 export fn mkloc(lex: *lexer) location = {
 	match (lex.un) {
 	case null => return lex.loc;
 	case let tok: *token => return lex.prevunlocs[1].1;
 	};
 };
 
 // Gives the previous location of the lexer.
 export fn prevloc(lex: *lexer) location = {
 	match (lex.un) {
 	case null => return lex.prevrloc;
 	case let tok: *token => return lex.prevunlocs[1].0;
 	};
 };
 
 // Initialize a token based on the scan context. When the lexeme is not present,
 // this considers the morphene as both. When the name is not present, the name
 // comes from the action will be use.
 export fn scan_token(
 	scan: *scanner,
 	value: value,
 	first: const str,
 	second: const str = "",
 	name: const str = ""
 ) (*token | error) = {
 	const (morphene, lexeme) = if (second == "") {
 		yield (first, first);
 	} else {
 		yield (first, second);
 	};
 	return scan.lex.token(
 		scan,
 		if (name == "") scan.name else name,
 		value,
 		morphene,
 		lexeme
 	);
 };
 
 // Return the lexer associated to a scanner.
 export fn scan_lexer(scan: *scanner) *lexer = scan.lex;
 
 // Return the token name associated to a scanner.
 export fn scan_name(scan: *scanner) str = scan.name;
 
 // Return a string representing the error.
 export fn strerror(err: error) str = {
 	static let buf: [2048]u8 = [0...];
 	match (err) {
 	case let s: syntax =>
 		return fmt::bsprintf(buf, "{}:{}: syntax error: {}",
 			s.0.line, s.0.col, s.1)!;
 	case let com: compile => return com;
 	case nomem => return "nomem";
 	};
 };
 
 // Give the next token from the lexer.
 export fn next(lex: *lexer) (*token | error) = {
 	if (lex.un is *token) {
 		const prev = lex.un as *token;
 		lex.un = null;
 		return prev;
 	};
 
 	defer {
 		lex.prevunlocs[1] = lex.prevunlocs[0];
 		lex.prevunlocs[0] = (prevloc(lex), mkloc(lex));
 	};
 
 
 	let scan = scanner {
 		lex = lex,
 		start = lex.loc,
 		...
 	};
 
 	for (true) {
 		scan.in = strings::sub(lex.in, lex.loc.off, strings::end);
 		if (len(scan.in) == 0) {
 			const tok = scan.lex.token(&scan, EOF, void, "", "")?;
 			append(lex.tokens, tok)?;
 			return tok;
 		};
 
 		const (action, lexeme) = match (perform(lex.be, scan.in)) {
 		case void => return syntaxf(mkloc(lex), "no matching token");
 		case let this: (*action, str) => yield this;
 		};
 
 		scan.name = action.name;
 
 		match (action.cb(&scan, lexeme, action.user)?) {
 		case let lexeme: str =>
 			forwardlex(lex, lexeme);
 			scan.start = lex.loc;
 		case let tok: *token =>
 			forwardlex(lex, tok.lexeme);
 			append(lex.tokens, tok)?;
 			return tok;
 		};
 	};
 };
 
 // Unlex a token, so that it get lexed back with [[next]].
 export fn unlex(lex: *lexer, value: *token) void = {
 	assert(lex.un is null);
 	lex.un = value;
 };
 
 fn forwardlex(lex: *lexer, in: str) void = {
 	const decoder = utf8::decode(strings::toutf8(in));
 	for (let r => utf8::next(&decoder)!) {
 		lex.prevrloc = lex.loc;
 		forwardr(&lex.loc, [r as rune]);
 	};
 };
 
 // Moves a location based on bytes.
 export fn forward(loc: *location, in: str) void = {
 	const decoder = utf8::decode(strings::toutf8(in));
 	for (let r => utf8::next(&decoder)!) {
 		forwardr(loc, [r as rune]);
 	};
 };
 
 fn forwardr(loc: *location, r: []rune) void = {
 	for (let r .. r) {
 		loc.off += 1;
 		loc.col += 1;
 		if (r == '\n') {
 			loc.col = 1;
 			loc.line += 1;
 		};
 	};
 };