type.ha (1893B)
1 use regex; 2 3 // A location. 4 export type location = struct { 5 line: uint, 6 col: uint, 7 off: uint, 8 }; 9 10 // A token value. 11 export type value = (f64 | i64 | size | u64 | str | rune | void); 12 13 // A lexed token. 14 export type token = struct { 15 name: const str, 16 value: value, 17 morphene: const str, // meaningfull part "foo" 18 lexeme: const str, // all swallowed bytes "foos" 19 start: location, 20 end: location, 21 tostrfn: *tokstrfn, 22 freefn: *tokfreefn, 23 }; 24 25 export def EOF = "EOF"; 26 27 // Function to format a token as a string. 28 export type tokstrfn = fn(tok: *token) str; 29 // Function to free resources associated with a token. 30 export type tokfreefn = fn(tok: *token) void; 31 32 // A syntax error. 33 export type syntax = !(location, str); 34 35 // A backend compile error. 36 export type compile = !str; 37 38 // All possible errors for this module. 39 export type error = !(syntax | compile | nomem); 40 41 // A lexer. 42 export type lexer = struct { 43 be: *backend, 44 in: const str, // the full bytes 45 loc: location, // the cursor location 46 prevrloc: location, // last rune location 47 prevunlocs: [2](location, location), 48 un: nullable *token, 49 tokens: []*token, // to free them 50 token: *tokenfn, // the function to build tokens with [[scan_token]] 51 reuse: nullable *reusecb, // the callback when reusing with [[reuse]] 52 }; 53 54 // The toolkit given to the [[actioncb]] callback to help the user to 55 // initialize the tokens. 56 export type scanner = struct { 57 lex: *lexer, 58 in: const str, // the remaining bytes 59 name: const str, // the scanned token name 60 start: location, // the lexeme start location 61 }; 62 63 // A function to initialize tokens. 64 export type tokenfn = fn( 65 scan: *scanner, 66 name: const str, // token type name 67 value: value, // token value 68 morphene: const str, // meaningfull part 69 lexeme: const str, // full bytes to swallow 70 ) (*token | error); 71 72 // A callback to reuse a lexer. 73 export type reusecb = fn(lex: *lexer) void;