from __future__ import annotations __copyright__ = """ Copyright (C) 2009-2013 Andreas Kloeckner Copyright (C) 2013- University of Illinois Board of Trustees """ __license__ = """ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ import re from collections.abc import Mapping, Sequence from dataclasses import dataclass from typing import Literal, NoReturn, TypeAlias, cast from typing_extensions import Self, final, overload, override @final class RuleError(RuntimeError): def __init__(self, rule: LexRule) -> None: RuntimeError.__init__(self) self.Rule = rule @override def __str__(self) -> str: return repr(self.Rule) @final class InvalidTokenError(RuntimeError): def __init__(self, s: str, str_index: int) -> None: RuntimeError.__init__(self) self.string = s self.index = str_index @override def __str__(self) -> str: return "at index {}: ...{}...".format( self.index, self.string[self.index:self.index+20]) @final class ParseError(RuntimeError): def __init__(self, msg: str, s: str, token: Lexed | None) -> None: RuntimeError.__init__(self) self.message = msg self.string = s self.Token = token @override def __str__(self) -> str: if self.Token is None: return f"{self.message} at end of input" return "{} at index {}: ...{}...".format( self.message, self.Token[2], self.string[self.Token[2]:self.Token[2]+20]) @final class RE: def __init__(self, s: str, flags: int = 0) -> None: self.Content = s self.RE = re.compile(s, flags) @override def __repr__(self) -> str: return f"RE({self.Content})" LexRule: TypeAlias = tuple[str, RE | tuple[str | RE, ...]] LexTable: TypeAlias = Sequence[LexRule] BasicLexed: TypeAlias = tuple[str, str, int] LexedWithMatch: TypeAlias = tuple[str, str, int, re.Match[str]] Lexed: TypeAlias = BasicLexed | LexedWithMatch def _matches_rule( rule: RE | str | tuple[str | RE, ...], s: str, start: int, rule_dict: Mapping[str, RE | tuple[str | RE, ...]], debug: bool = False ) -> tuple[int, re.Match[str] | None]: if debug: print("Trying", rule, "on", s[start:]) if isinstance(rule, tuple): if rule[0] == "|": for subrule in rule[1:]: length, match_obj = _matches_rule( subrule, s, start, rule_dict, debug) if not length: continue return length, match_obj else: my_match_length = 0 for subrule in rule: length, _ = _matches_rule( subrule, s, start, rule_dict, debug) if not length: break my_match_length += length start += length else: return my_match_length, None return 0, None if isinstance(rule, str): return _matches_rule(rule_dict[rule], s, start, rule_dict, debug) if isinstance(rule, RE): match_obj = rule.RE.match(s, start) if match_obj: return match_obj.end()-start, match_obj return 0, None raise RuleError(rule) @overload def lex( lex_table: LexTable, s: str, *, debug: bool = ..., match_objects: Literal[False] = False, ) -> Sequence[BasicLexed]: ... @overload def lex( lex_table: LexTable, s: str, *, debug: bool = ..., match_objects: Literal[True], ) -> Sequence[LexedWithMatch]: ... def lex( lex_table: LexTable, s: str, *, debug: bool = False, match_objects: bool = False ) -> Sequence[Lexed]: rule_dict = dict(lex_table) result: list[Lexed] = [] i = 0 while i < len(s): for name, rule in lex_table: length, match_obj = _matches_rule(rule, s, i, rule_dict, debug) if length: if match_objects: assert match_obj result.append((name, s[i:i+length], i, match_obj)) else: result.append((name, s[i:i+length], i)) i += length break else: raise InvalidTokenError(s, i) return result @dataclass class LexIterator: lexed: Sequence[tuple[str, str, int] | tuple[str, str, int, re.Match[str]]] raw_string: str index: int = 0 def copy(self) -> Self: return type(self)(self.lexed, self.raw_string, self.index) def assign(self, rhs: LexIterator) -> None: assert self.lexed is rhs.lexed assert self.raw_string is rhs.raw_string self.index = rhs.index def next_tag(self, i: int = 0) -> str: return self.lexed[self.index + i][0] def next_str(self, i: int = 0) -> str: return self.lexed[self.index + i][1] def next_match_obj(self) -> re.Match[str]: _tok, _s, _i, match = cast("LexedWithMatch", self.lexed[self.index]) return match def next_str_and_advance(self) -> str: result = self.next_str() self.advance() return result def advance(self) -> None: self.index += 1 def is_at_end(self, i: int = 0) -> bool: return self.index + i >= len(self.lexed) def is_next(self, tag: str, i: int = 0) -> bool: return ( self.index + i < len(self.lexed) and self.next_tag(i) is tag) def raise_parse_error(self, msg: str) -> NoReturn: if self.is_at_end(): raise ParseError(msg, self.raw_string, None) raise ParseError(msg, self.raw_string, self.lexed[self.index]) def expected(self, what_expected: str) -> NoReturn: if self.is_at_end(): self.raise_parse_error( f"{what_expected} expected, end of input found instead") else: self.raise_parse_error( f"{what_expected} expected, {self.next_tag()} found instead") def expect_not_end(self) -> None: if self.is_at_end(): self.raise_parse_error("unexpected end of input") def expect(self, tag: str) -> None: self.expect_not_end() if not self.is_next(tag): self.expected(str(tag))