vis

a vi-like editor based on Plan 9's structural regular expressions
factor.lua

(2869B)
      1 -- Copyright 2013 Michael T. Richter. See LICENSE.
      2 -- Copyright 2024 John Benediktsson <mrjbq7@gmail.com>
      3 -- Factor lexer (http://factorcode.org).
      4 
      5 -- At this time the lexer is usable, but not perfect.  Problems include:
      6 --  * identifiers like (foo) get treated and coloured like stack declarations
      7 --  * other as-yet unknown display bugs  :-)
      8 
      9 local lexer = lexer
     10 local P, R, S = lpeg.P, lpeg.R, lpeg.S
     11 
     12 local lex = lexer.new(...)
     13 
     14 -- General building blocks.
     15 local pre = lexer.upper^1
     16 local post = pre
     17 local opt_pre = pre^-1
     18 local opt_post = opt_pre
     19 
     20 -- Comments.
     21 lex:add_rule('comment', lex:tag(lexer.COMMENT, P('#')^-1 * lexer.to_eol('!')))
     22 
     23 -- Strings.
     24 local dq1_str = opt_pre * lexer.range('"', true)
     25 lex:add_rule('string', lex:tag(lexer.STRING, dq1_str))
     26 
     27 -- Numbers.
     28 -- Note that complex literals like C{ 1/3 27.3 } are not covered by this lexer.
     29 -- The C{ ... } notation is treated as an operator--to be specific a
     30 -- "constructor" (for want of a better term).
     31 -- Also note that we cannot use lexer.number because numbers do not support the '+' prefix or
     32 -- case-insensitive base-changing prefixes.
     33 local hex_digits = lexer.xdigit^1
     34 local binary = P('-')^-1 * '0b' * S('01')^1
     35 local octal = P('-')^-1 * '0o' * R('07')^1
     36 local decimal = P('-')^-1 * lexer.digit^1
     37 local hexadecimal = P('-')^-1 * '0x' * hex_digits
     38 local integer = binary + octal + hexadecimal + decimal
     39 local ratio = decimal * '/' * decimal
     40 local dfloat_component = decimal * '.' * decimal^-1
     41 local hfloat_component = hexadecimal * ('.' * hex_digits^-1)^-1
     42 local float = (dfloat_component * (S('eE') * decimal)^-1) + (hfloat_component * S('pP') * decimal) +
     43 	(ratio * '.') + (P('-')^-1 * '1/0.') + ('0/0')
     44 lex:add_rule('number', lex:tag(lexer.NUMBER, (float + ratio + integer)))
     45 
     46 -- Keywords.
     47 -- Things like NAN:, USE:, USING:, POSTPONE:, etc. are considered keywords, as are similar
     48 -- words that end in #.
     49 -- Patterns like <<WORD ... WORD>> are similarly considered to be "keywords" (for want of a
     50 -- better term).
     51 local colon_words = pre * S(':#') + S(':;')^1
     52 local angle_words = (P('<')^1 * post) + (pre * P('>')^1)
     53 lex:add_rule('keyword', lex:tag(lexer.KEYWORD, (colon_words + angle_words)))
     54 
     55 -- Operators.
     56 -- The usual suspects like braces, brackets, angle brackets, parens, etc. are considered to be
     57 -- operators. They may, however, have prefixes like C{ ... }.
     58 local constructor_words = opt_pre * S('{[<') + S('}]>') + pre * '(' + ')'
     59 local stack_declaration = lexer.range('(', ')')
     60 local other_operators = S('+-*/<>')
     61 lex:add_rule('operator',
     62 	lex:tag(lexer.OPERATOR, (stack_declaration + constructor_words + other_operators)))
     63 
     64 -- Identifiers.
     65 -- Identifiers can be practically anything but whitespace.
     66 local symbols = S('`~!@#$%^&*()_-+={[<>]}:;X,?/')
     67 lex:add_rule('identifier', lex:tag(lexer.IDENTIFIER, (lexer.alnum + symbols)^1))
     68 
     69 lexer.property['factor.comment'] = '!'
     70 
     71 return lex