tokens.ha (2285B)
1 use bufio; 2 use io; 3 use fmt; 4 use ascii; 5 use strings; 6 use memio; 7 8 // The different token types the lexer can return 9 export type types = enum { 10 OBRACE, 11 CBRACE, 12 NAME, 13 NUM, 14 EOF 15 }; 16 17 // Converts a type to a relvant string 18 export fn strtypes(ty: types) str = switch (ty) { 19 case types::OBRACE => yield "'('"; 20 case types::CBRACE => yield "')'"; 21 case types::NAME => yield "NAME"; 22 case types::NUM => yield "NUM"; 23 case types::EOF => yield "EOF"; 24 case => yield "FUCK"; 25 }; 26 27 28 fn consumeRune(l: *lexer, ty: types, r: rune) (token | invalid) = { 29 match (bufio::read_rune(l.in)) { 30 case let read: rune => 31 return if (read == r) token{ty = ty, data = ""} 32 else invalid; 33 case io::EOF => return invalid; 34 case => fmt::fatal("Fucked up input"); 35 }; 36 }; 37 38 fn consumeBlock(l: *lexer, ty: types, allow: *fn(_: rune) bool) (token | invalid) = { 39 let runes: []rune = []; 40 for (true) { 41 let r = match (bufio::read_rune(l.in)) { 42 case let r: rune => yield r; 43 case io::EOF => return invalid; 44 case => fmt::fatal("Fucked up input"); 45 }; 46 47 if (allow(r)) 48 append(runes, r)! 49 else { 50 io::seek(l.in, -1, io::whence::CUR)!; 51 break; 52 }; 53 }; 54 55 if (len(runes) == 0) return invalid; 56 return token{ty = ty, data = strings::fromrunes(runes)!}; 57 }; 58 59 fn consumeEOF(l: *lexer) (token | invalid) = { 60 match (bufio::read_rune(l.in)) { 61 case io::EOF => return token{ty = types::EOF, data = ""}; 62 case rune => return invalid; 63 case => fmt::fatal("Fucked up input"); 64 }; 65 }; 66 67 fn consumeObrace(l: *lexer) (token | invalid) = consumeRune(l, types::OBRACE, '('); 68 fn consumeCbrace(l: *lexer) (token | invalid) = consumeRune(l, types::CBRACE, ')'); 69 fn consumeName(l: *lexer) (token | invalid) = consumeBlock(l, types::NAME, &ascii::isalpha); 70 fn consumeNum(l: *lexer) (token | invalid) = consumeBlock(l, types::NUM, &ascii::isdigit); 71 72 const items = [ 73 lexItem{ty = types::OBRACE, f = &consumeObrace, nexts = [types::NAME]}, 74 lexItem{ty = types::CBRACE, f = &consumeCbrace, nexts = [types::NUM, types::CBRACE, types::OBRACE, types::EOF]}, 75 lexItem{ty = types::NAME, f = &consumeName, nexts = [types::NAME, types::NUM, types::OBRACE, types::CBRACE]}, 76 lexItem{ty = types::NUM, f = &consumeNum, nexts = [types::NAME, types::NUM, types::OBRACE, types::CBRACE]}, 77 lexItem{ty = types::EOF, f = &consumeEOF, nexts = []}, 78 ];