mirror of
https://github.com/seL4/seL4.git
synced 2026-03-27 10:29:57 +00:00
79da079239
This commit also converts our own copyright headers to directly use SPDX, but leaves all other copyright header intact, only adding the SPDX ident. As far as possible this commit also merges multiple Data61 copyright statements/headers into one for consistency.
1112 lines
40 KiB
Python
1112 lines
40 KiB
Python
# SPDX-License-Identifier: BSD-3-Clause
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# ply: lex.py
|
|
#
|
|
# Copyright (C) 2001-2009,
|
|
# David M. Beazley (Dabeaz LLC)
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are
|
|
# met:
|
|
#
|
|
# * Redistributions of source code must retain the above copyright notice,
|
|
# this list of conditions and the following disclaimer.
|
|
# * Redistributions in binary form must reproduce the above copyright notice,
|
|
# this list of conditions and the following disclaimer in the documentation
|
|
# and/or other materials provided with the distribution.
|
|
# * Neither the name of the David Beazley or Dabeaz LLC may be used to
|
|
# endorse or promote products derived from this software without
|
|
# specific prior written permission.
|
|
#
|
|
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
|
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
|
|
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
|
|
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
|
|
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
|
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
|
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
|
|
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
# -----------------------------------------------------------------------------
|
|
|
|
__version__ = "3.2"
|
|
__tabversion__ = "3.2" # Version of table file used
|
|
|
|
from past.builtins import cmp
|
|
import re
|
|
import sys
|
|
import types
|
|
import copy
|
|
import os
|
|
|
|
# Python3 doesn't have a build-in cmp function.
|
|
# We need to import it here, even though it isn't called in this file
|
|
# when interpreted by python3, to prevent pylint from treating it as
|
|
# an error.
|
|
|
|
# This tuple contains known string types
|
|
try:
|
|
# Python 2.6
|
|
StringTypes = (types.StringType, types.UnicodeType)
|
|
except AttributeError:
|
|
# Python 3.0
|
|
StringTypes = (str, bytes)
|
|
|
|
# Extract the code attribute of a function. Different implementations
|
|
# are for Python 2/3 compatibility.
|
|
|
|
if sys.version_info[0] < 3:
|
|
def func_code(f):
|
|
return f.func_code
|
|
else:
|
|
def func_code(f):
|
|
return f.__code__
|
|
|
|
# This regular expression is used to match valid token names
|
|
_is_identifier = re.compile(r'^[a-zA-Z0-9_]+$')
|
|
|
|
# Exception thrown when invalid token encountered and no default error
|
|
# handler is defined.
|
|
|
|
|
|
class LexError(Exception):
|
|
def __init__(self, message, s):
|
|
self.args = (message,)
|
|
self.text = s
|
|
|
|
# Token class. This class is used to represent the tokens produced.
|
|
|
|
|
|
class LexToken(object):
|
|
def __str__(self):
|
|
return "LexToken(%s,%r,%d,%d)" % (self.type, self.value, self.lineno, self.lexpos)
|
|
|
|
def __repr__(self):
|
|
return str(self)
|
|
|
|
# This object is a stand-in for a logging object created by the
|
|
# logging module.
|
|
|
|
|
|
class PlyLogger(object):
|
|
def __init__(self, f):
|
|
self.f = f
|
|
|
|
def critical(self, msg, *args, **kwargs):
|
|
self.f.write((msg % args) + "\n")
|
|
|
|
def warning(self, msg, *args, **kwargs):
|
|
self.f.write("WARNING: " + (msg % args) + "\n")
|
|
|
|
def error(self, msg, *args, **kwargs):
|
|
self.f.write("ERROR: " + (msg % args) + "\n")
|
|
|
|
info = critical
|
|
debug = critical
|
|
|
|
# Null logger is used when no output is generated. Does nothing.
|
|
|
|
|
|
class NullLogger(object):
|
|
def __getattribute__(self, name):
|
|
return self
|
|
|
|
def __call__(self, *args, **kwargs):
|
|
return self
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# === Lexing Engine ===
|
|
#
|
|
# The following Lexer class implements the lexer runtime. There are only
|
|
# a few public methods and attributes:
|
|
#
|
|
# input() - Store a new string in the lexer
|
|
# token() - Get the next token
|
|
# clone() - Clone the lexer
|
|
#
|
|
# lineno - Current line number
|
|
# lexpos - Current position in the input string
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
class Lexer:
|
|
def __init__(self):
|
|
self.lexre = None # Master regular expression. This is a list of
|
|
# tuples (re,findex) where re is a compiled
|
|
# regular expression and findex is a list
|
|
# mapping regex group numbers to rules
|
|
self.lexretext = None # Current regular expression strings
|
|
self.lexstatere = {} # Dictionary mapping lexer states to master regexs
|
|
self.lexstateretext = {} # Dictionary mapping lexer states to regex strings
|
|
self.lexstaterenames = {} # Dictionary mapping lexer states to symbol names
|
|
self.lexstate = "INITIAL" # Current lexer state
|
|
self.lexstatestack = [] # Stack of lexer states
|
|
self.lexstateinfo = None # State information
|
|
self.lexstateignore = {} # Dictionary of ignored characters for each state
|
|
self.lexstateerrorf = {} # Dictionary of error functions for each state
|
|
self.lexreflags = 0 # Optional re compile flags
|
|
self.lexdata = None # Actual input data (as a string)
|
|
self.lexpos = 0 # Current position in input text
|
|
self.lexlen = 0 # Length of the input text
|
|
self.lexerrorf = None # Error rule (if any)
|
|
self.lextokens = None # List of valid tokens
|
|
self.lexignore = "" # Ignored characters
|
|
self.lexliterals = "" # Literal characters that can be passed through
|
|
self.lexmodule = None # Module
|
|
self.lineno = 1 # Current line number
|
|
self.lexoptimize = 0 # Optimized mode
|
|
|
|
def clone(self, object=None):
|
|
c = copy.copy(self)
|
|
|
|
# If the object parameter has been supplied, it means we are attaching the
|
|
# lexer to a new object. In this case, we have to rebind all methods in
|
|
# the lexstatere and lexstateerrorf tables.
|
|
|
|
if object:
|
|
newtab = {}
|
|
for key, ritem in self.lexstatere.items():
|
|
newre = []
|
|
for cre, findex in ritem:
|
|
newfindex = []
|
|
for f in findex:
|
|
if not f or not f[0]:
|
|
newfindex.append(f)
|
|
continue
|
|
newfindex.append((getattr(object, f[0].__name__), f[1]))
|
|
newre.append((cre, newfindex))
|
|
newtab[key] = newre
|
|
c.lexstatere = newtab
|
|
c.lexstateerrorf = {}
|
|
for key, ef in self.lexstateerrorf.items():
|
|
c.lexstateerrorf[key] = getattr(object, ef.__name__)
|
|
c.lexmodule = object
|
|
return c
|
|
|
|
# ------------------------------------------------------------
|
|
# writetab() - Write lexer information to a table file
|
|
# ------------------------------------------------------------
|
|
def writetab(self, tabfile, outputdir=""):
|
|
if isinstance(tabfile, types.ModuleType):
|
|
return
|
|
basetabfilename = tabfile.split(".")[-1]
|
|
filename = os.path.join(outputdir, basetabfilename)+".py"
|
|
tf = open(filename, "w")
|
|
tf.write("# %s.py. This file automatically created by PLY (version %s). Don't edit!\n" %
|
|
(tabfile, __version__))
|
|
tf.write("_tabversion = %s\n" % repr(__version__))
|
|
tf.write("_lextokens = %s\n" % repr(self.lextokens))
|
|
tf.write("_lexreflags = %s\n" % repr(self.lexreflags))
|
|
tf.write("_lexliterals = %s\n" % repr(self.lexliterals))
|
|
tf.write("_lexstateinfo = %s\n" % repr(self.lexstateinfo))
|
|
|
|
tabre = {}
|
|
# Collect all functions in the initial state
|
|
initial = self.lexstatere["INITIAL"]
|
|
initialfuncs = []
|
|
for part in initial:
|
|
for f in part[1]:
|
|
if f and f[0]:
|
|
initialfuncs.append(f)
|
|
|
|
for key, lre in self.lexstatere.items():
|
|
titem = []
|
|
for i in range(len(lre)):
|
|
titem.append((self.lexstateretext[key][i], _funcs_to_names(
|
|
lre[i][1], self.lexstaterenames[key][i])))
|
|
tabre[key] = titem
|
|
|
|
tf.write("_lexstatere = %s\n" % repr(tabre))
|
|
tf.write("_lexstateignore = %s\n" % repr(self.lexstateignore))
|
|
|
|
taberr = {}
|
|
for key, ef in self.lexstateerrorf.items():
|
|
if ef:
|
|
taberr[key] = ef.__name__
|
|
else:
|
|
taberr[key] = None
|
|
tf.write("_lexstateerrorf = %s\n" % repr(taberr))
|
|
tf.close()
|
|
|
|
# ------------------------------------------------------------
|
|
# readtab() - Read lexer information from a tab file
|
|
# ------------------------------------------------------------
|
|
def readtab(self, tabfile, fdict):
|
|
if isinstance(tabfile, types.ModuleType):
|
|
lextab = tabfile
|
|
else:
|
|
if sys.version_info[0] < 3:
|
|
exec("import %s as lextab" % tabfile)
|
|
else:
|
|
env = {}
|
|
exec("import %s as lextab" % tabfile, env, env)
|
|
lextab = env['lextab']
|
|
|
|
if getattr(lextab, "_tabversion", "0.0") != __version__:
|
|
raise ImportError("Inconsistent PLY version")
|
|
|
|
self.lextokens = lextab._lextokens
|
|
self.lexreflags = lextab._lexreflags
|
|
self.lexliterals = lextab._lexliterals
|
|
self.lexstateinfo = lextab._lexstateinfo
|
|
self.lexstateignore = lextab._lexstateignore
|
|
self.lexstatere = {}
|
|
self.lexstateretext = {}
|
|
for key, lre in lextab._lexstatere.items():
|
|
titem = []
|
|
txtitem = []
|
|
for i in range(len(lre)):
|
|
titem.append((re.compile(lre[i][0], lextab._lexreflags),
|
|
_names_to_funcs(lre[i][1], fdict)))
|
|
txtitem.append(lre[i][0])
|
|
self.lexstatere[key] = titem
|
|
self.lexstateretext[key] = txtitem
|
|
self.lexstateerrorf = {}
|
|
for key, ef in lextab._lexstateerrorf.items():
|
|
self.lexstateerrorf[key] = fdict[ef]
|
|
self.begin('INITIAL')
|
|
|
|
# ------------------------------------------------------------
|
|
# input() - Push a new string into the lexer
|
|
# ------------------------------------------------------------
|
|
def input(self, s):
|
|
# Pull off the first character to see if s looks like a string
|
|
c = s[:1]
|
|
if not isinstance(c, StringTypes):
|
|
raise ValueError("Expected a string")
|
|
self.lexdata = s
|
|
self.lexpos = 0
|
|
self.lexlen = len(s)
|
|
|
|
# ------------------------------------------------------------
|
|
# begin() - Changes the lexing state
|
|
# ------------------------------------------------------------
|
|
def begin(self, state):
|
|
if not state in self.lexstatere:
|
|
raise ValueError("Undefined state")
|
|
self.lexre = self.lexstatere[state]
|
|
self.lexretext = self.lexstateretext[state]
|
|
self.lexignore = self.lexstateignore.get(state, "")
|
|
self.lexerrorf = self.lexstateerrorf.get(state, None)
|
|
self.lexstate = state
|
|
|
|
# ------------------------------------------------------------
|
|
# push_state() - Changes the lexing state and saves old on stack
|
|
# ------------------------------------------------------------
|
|
def push_state(self, state):
|
|
self.lexstatestack.append(self.lexstate)
|
|
self.begin(state)
|
|
|
|
# ------------------------------------------------------------
|
|
# pop_state() - Restores the previous state
|
|
# ------------------------------------------------------------
|
|
def pop_state(self):
|
|
self.begin(self.lexstatestack.pop())
|
|
|
|
# ------------------------------------------------------------
|
|
# current_state() - Returns the current lexing state
|
|
# ------------------------------------------------------------
|
|
def current_state(self):
|
|
return self.lexstate
|
|
|
|
# ------------------------------------------------------------
|
|
# skip() - Skip ahead n characters
|
|
# ------------------------------------------------------------
|
|
def skip(self, n):
|
|
self.lexpos += n
|
|
|
|
# ------------------------------------------------------------
|
|
# opttoken() - Return the next token from the Lexer
|
|
#
|
|
# Note: This function has been carefully implemented to be as fast
|
|
# as possible. Don't make changes unless you really know what
|
|
# you are doing
|
|
# ------------------------------------------------------------
|
|
def token(self):
|
|
# Make local copies of frequently referenced attributes
|
|
lexpos = self.lexpos
|
|
lexlen = self.lexlen
|
|
lexignore = self.lexignore
|
|
lexdata = self.lexdata
|
|
|
|
while lexpos < lexlen:
|
|
# This code provides some short-circuit code for whitespace, tabs, and other ignored characters
|
|
if lexdata[lexpos] in lexignore:
|
|
lexpos += 1
|
|
continue
|
|
|
|
# Look for a regular expression match
|
|
for lexre, lexindexfunc in self.lexre:
|
|
m = lexre.match(lexdata, lexpos)
|
|
if not m:
|
|
continue
|
|
|
|
# Create a token for return
|
|
tok = LexToken()
|
|
tok.value = m.group()
|
|
tok.lineno = self.lineno
|
|
tok.lexpos = lexpos
|
|
|
|
i = m.lastindex
|
|
func, tok.type = lexindexfunc[i]
|
|
|
|
if not func:
|
|
# If no token type was set, it's an ignored token
|
|
if tok.type:
|
|
self.lexpos = m.end()
|
|
return tok
|
|
else:
|
|
lexpos = m.end()
|
|
break
|
|
|
|
lexpos = m.end()
|
|
|
|
# If token is processed by a function, call it
|
|
|
|
tok.lexer = self # Set additional attributes useful in token rules
|
|
self.lexmatch = m
|
|
self.lexpos = lexpos
|
|
|
|
newtok = func(tok)
|
|
|
|
# Every function must return a token, if nothing, we just move to next token
|
|
if not newtok:
|
|
lexpos = self.lexpos # This is here in case user has updated lexpos.
|
|
lexignore = self.lexignore # This is here in case there was a state change
|
|
break
|
|
|
|
# Verify type of the token. If not in the token map, raise an error
|
|
if not self.lexoptimize:
|
|
if not newtok.type in self.lextokens:
|
|
raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % (
|
|
func_code(func).co_filename, func_code(func).co_firstlineno,
|
|
func.__name__, newtok.type), lexdata[lexpos:])
|
|
|
|
return newtok
|
|
else:
|
|
# No match, see if in literals
|
|
if lexdata[lexpos] in self.lexliterals:
|
|
tok = LexToken()
|
|
tok.value = lexdata[lexpos]
|
|
tok.lineno = self.lineno
|
|
tok.type = tok.value
|
|
tok.lexpos = lexpos
|
|
self.lexpos = lexpos + 1
|
|
return tok
|
|
|
|
# No match. Call t_error() if defined.
|
|
if self.lexerrorf:
|
|
tok = LexToken()
|
|
tok.value = self.lexdata[lexpos:]
|
|
tok.lineno = self.lineno
|
|
tok.type = "error"
|
|
tok.lexer = self
|
|
tok.lexpos = lexpos
|
|
self.lexpos = lexpos
|
|
newtok = self.lexerrorf(tok)
|
|
if lexpos == self.lexpos:
|
|
# Error method didn't change text position at all. This is an error.
|
|
raise LexError("Scanning error. Illegal character '%s'" %
|
|
(lexdata[lexpos]), lexdata[lexpos:])
|
|
lexpos = self.lexpos
|
|
if not newtok:
|
|
continue
|
|
return newtok
|
|
|
|
self.lexpos = lexpos
|
|
raise LexError("Illegal character '%s' at index %d" %
|
|
(lexdata[lexpos], lexpos), lexdata[lexpos:])
|
|
|
|
self.lexpos = lexpos + 1
|
|
if self.lexdata is None:
|
|
raise RuntimeError("No input string given with input()")
|
|
return None
|
|
|
|
# Iterator interface
|
|
def __iter__(self):
|
|
return self
|
|
|
|
def next(self):
|
|
t = self.token()
|
|
if t is None:
|
|
raise StopIteration
|
|
return t
|
|
|
|
__next__ = next
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# ==== Lex Builder ===
|
|
#
|
|
# The functions and classes below are used to collect lexing information
|
|
# and build a Lexer object from it.
|
|
# -----------------------------------------------------------------------------
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# get_caller_module_dict()
|
|
#
|
|
# This function returns a dictionary containing all of the symbols defined within
|
|
# a caller further down the call stack. This is used to get the environment
|
|
# associated with the yacc() call if none was provided.
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def get_caller_module_dict(levels):
|
|
try:
|
|
raise RuntimeError
|
|
except RuntimeError:
|
|
e, b, t = sys.exc_info()
|
|
f = t.tb_frame
|
|
while levels > 0:
|
|
f = f.f_back
|
|
levels -= 1
|
|
ldict = f.f_globals.copy()
|
|
if f.f_globals != f.f_locals:
|
|
ldict.update(f.f_locals)
|
|
|
|
return ldict
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# _funcs_to_names()
|
|
#
|
|
# Given a list of regular expression functions, this converts it to a list
|
|
# suitable for output to a table file
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def _funcs_to_names(funclist, namelist):
|
|
result = []
|
|
for f, name in zip(funclist, namelist):
|
|
if f and f[0]:
|
|
result.append((name, f[1]))
|
|
else:
|
|
result.append(f)
|
|
return result
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# _names_to_funcs()
|
|
#
|
|
# Given a list of regular expression function names, this converts it back to
|
|
# functions.
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def _names_to_funcs(namelist, fdict):
|
|
result = []
|
|
for n in namelist:
|
|
if n and n[0]:
|
|
result.append((fdict[n[0]], n[1]))
|
|
else:
|
|
result.append(n)
|
|
return result
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# _form_master_re()
|
|
#
|
|
# This function takes a list of all of the regex components and attempts to
|
|
# form the master regular expression. Given limitations in the Python re
|
|
# module, it may be necessary to break the master regex into separate expressions.
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def _form_master_re(relist, reflags, ldict, toknames):
|
|
if not relist:
|
|
return []
|
|
regex = "|".join(relist)
|
|
try:
|
|
lexre = re.compile(regex, re.VERBOSE | reflags)
|
|
|
|
# Build the index to function map for the matching engine
|
|
lexindexfunc = [None] * (max(lexre.groupindex.values())+1)
|
|
lexindexnames = lexindexfunc[:]
|
|
|
|
for f, i in lexre.groupindex.items():
|
|
handle = ldict.get(f, None)
|
|
if type(handle) in (types.FunctionType, types.MethodType):
|
|
lexindexfunc[i] = (handle, toknames[f])
|
|
lexindexnames[i] = f
|
|
elif handle is not None:
|
|
lexindexnames[i] = f
|
|
if f.find("ignore_") > 0:
|
|
lexindexfunc[i] = (None, None)
|
|
else:
|
|
lexindexfunc[i] = (None, toknames[f])
|
|
|
|
return [(lexre, lexindexfunc)], [regex], [lexindexnames]
|
|
except Exception:
|
|
m = int(len(relist)/2)
|
|
if m == 0:
|
|
m = 1
|
|
llist, lre, lnames = _form_master_re(relist[:m], reflags, ldict, toknames)
|
|
rlist, rre, rnames = _form_master_re(relist[m:], reflags, ldict, toknames)
|
|
return llist+rlist, lre+rre, lnames+rnames
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# def _statetoken(s,names)
|
|
#
|
|
# Given a declaration name s of the form "t_" and a dictionary whose keys are
|
|
# state names, this function returns a tuple (states,tokenname) where states
|
|
# is a tuple of state names and tokenname is the name of the token. For example,
|
|
# calling this with s = "t_foo_bar_SPAM" might return (('foo','bar'),'SPAM')
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def _statetoken(s, names):
|
|
nonstate = 1
|
|
parts = s.split("_")
|
|
for i in range(1, len(parts)):
|
|
if not parts[i] in names and parts[i] != 'ANY':
|
|
break
|
|
if i > 1:
|
|
states = tuple(parts[1:i])
|
|
else:
|
|
states = ('INITIAL',)
|
|
|
|
if 'ANY' in states:
|
|
states = tuple(names)
|
|
|
|
tokenname = "_".join(parts[i:])
|
|
return (states, tokenname)
|
|
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# LexerReflect()
|
|
#
|
|
# This class represents information needed to build a lexer as extracted from a
|
|
# user's input file.
|
|
# -----------------------------------------------------------------------------
|
|
class LexerReflect(object):
|
|
def __init__(self, ldict, log=None, reflags=0):
|
|
self.ldict = ldict
|
|
self.error_func = None
|
|
self.tokens = []
|
|
self.reflags = reflags
|
|
self.stateinfo = {'INITIAL': 'inclusive'}
|
|
self.files = {}
|
|
self.error = 0
|
|
|
|
if log is None:
|
|
self.log = PlyLogger(sys.stderr)
|
|
else:
|
|
self.log = log
|
|
|
|
# Get all of the basic information
|
|
def get_all(self):
|
|
self.get_tokens()
|
|
self.get_literals()
|
|
self.get_states()
|
|
self.get_rules()
|
|
|
|
# Validate all of the information
|
|
def validate_all(self):
|
|
self.validate_tokens()
|
|
self.validate_literals()
|
|
self.validate_rules()
|
|
return self.error
|
|
|
|
# Get the tokens map
|
|
def get_tokens(self):
|
|
tokens = self.ldict.get("tokens", None)
|
|
if not tokens:
|
|
self.log.error("No token list is defined")
|
|
self.error = 1
|
|
return
|
|
|
|
if not isinstance(tokens, (list, tuple)):
|
|
self.log.error("tokens must be a list or tuple")
|
|
self.error = 1
|
|
return
|
|
|
|
if not tokens:
|
|
self.log.error("tokens is empty")
|
|
self.error = 1
|
|
return
|
|
|
|
self.tokens = tokens
|
|
|
|
# Validate the tokens
|
|
def validate_tokens(self):
|
|
terminals = {}
|
|
for n in self.tokens:
|
|
if not _is_identifier.match(n):
|
|
self.log.error("Bad token name '%s'", n)
|
|
self.error = 1
|
|
if n in terminals:
|
|
self.log.warning("Token '%s' multiply defined", n)
|
|
terminals[n] = 1
|
|
|
|
# Get the literals specifier
|
|
def get_literals(self):
|
|
self.literals = self.ldict.get("literals", "")
|
|
|
|
# Validate literals
|
|
def validate_literals(self):
|
|
try:
|
|
for c in self.literals:
|
|
if not isinstance(c, StringTypes) or len(c) > 1:
|
|
self.log.error("Invalid literal %s. Must be a single character", repr(c))
|
|
self.error = 1
|
|
continue
|
|
|
|
except TypeError:
|
|
self.log.error(
|
|
"Invalid literals specification. literals must be a sequence of characters")
|
|
self.error = 1
|
|
|
|
def get_states(self):
|
|
self.states = self.ldict.get("states", None)
|
|
# Build statemap
|
|
if self.states:
|
|
if not isinstance(self.states, (tuple, list)):
|
|
self.log.error("states must be defined as a tuple or list")
|
|
self.error = 1
|
|
else:
|
|
for s in self.states:
|
|
if not isinstance(s, tuple) or len(s) != 2:
|
|
self.log.error(
|
|
"Invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')", repr(s))
|
|
self.error = 1
|
|
continue
|
|
name, statetype = s
|
|
if not isinstance(name, StringTypes):
|
|
self.log.error("State name %s must be a string", repr(name))
|
|
self.error = 1
|
|
continue
|
|
if not (statetype == 'inclusive' or statetype == 'exclusive'):
|
|
self.log.error(
|
|
"State type for state %s must be 'inclusive' or 'exclusive'", name)
|
|
self.error = 1
|
|
continue
|
|
if name in self.stateinfo:
|
|
self.log.error("State '%s' already defined", name)
|
|
self.error = 1
|
|
continue
|
|
self.stateinfo[name] = statetype
|
|
|
|
# Get all of the symbols with a t_ prefix and sort them into various
|
|
# categories (functions, strings, error functions, and ignore characters)
|
|
|
|
def get_rules(self):
|
|
tsymbols = [f for f in self.ldict if f[:2] == 't_']
|
|
|
|
# Now build up a list of functions and a list of strings
|
|
|
|
self.toknames = {} # Mapping of symbols to token names
|
|
self.funcsym = {} # Symbols defined as functions
|
|
self.strsym = {} # Symbols defined as strings
|
|
self.ignore = {} # Ignore strings by state
|
|
self.errorf = {} # Error functions by state
|
|
|
|
for s in self.stateinfo:
|
|
self.funcsym[s] = []
|
|
self.strsym[s] = []
|
|
|
|
if len(tsymbols) == 0:
|
|
self.log.error("No rules of the form t_rulename are defined")
|
|
self.error = 1
|
|
return
|
|
|
|
for f in tsymbols:
|
|
t = self.ldict[f]
|
|
states, tokname = _statetoken(f, self.stateinfo)
|
|
self.toknames[f] = tokname
|
|
|
|
if hasattr(t, "__call__"):
|
|
if tokname == 'error':
|
|
for s in states:
|
|
self.errorf[s] = t
|
|
elif tokname == 'ignore':
|
|
line = func_code(t).co_firstlineno
|
|
file = func_code(t).co_filename
|
|
self.log.error("%s:%d: Rule '%s' must be defined as a string",
|
|
file, line, t.__name__)
|
|
self.error = 1
|
|
else:
|
|
for s in states:
|
|
self.funcsym[s].append((f, t))
|
|
elif isinstance(t, StringTypes):
|
|
if tokname == 'ignore':
|
|
for s in states:
|
|
self.ignore[s] = t
|
|
if "\\" in t:
|
|
self.log.warning("%s contains a literal backslash '\\'", f)
|
|
|
|
elif tokname == 'error':
|
|
self.log.error("Rule '%s' must be defined as a function", f)
|
|
self.error = 1
|
|
else:
|
|
for s in states:
|
|
self.strsym[s].append((f, t))
|
|
else:
|
|
self.log.error("%s not defined as a function or string", f)
|
|
self.error = 1
|
|
|
|
# Sort the functions by line number
|
|
for f in self.funcsym.values():
|
|
if sys.version_info[0] < 3:
|
|
f.sort(lambda x, y: cmp(
|
|
func_code(x[1]).co_firstlineno, func_code(y[1]).co_firstlineno))
|
|
else:
|
|
# Python 3.0
|
|
f.sort(key=lambda x: func_code(x[1]).co_firstlineno)
|
|
|
|
# Sort the strings by regular expression length
|
|
for s in self.strsym.values():
|
|
if sys.version_info[0] < 3:
|
|
s.sort(lambda x, y: (len(x[1]) < len(y[1])) - (len(x[1]) > len(y[1])))
|
|
else:
|
|
# Python 3.0
|
|
s.sort(key=lambda x: len(x[1]), reverse=True)
|
|
|
|
# Validate all of the t_rules collected
|
|
def validate_rules(self):
|
|
for state in self.stateinfo:
|
|
# Validate all rules defined by functions
|
|
|
|
for fname, f in self.funcsym[state]:
|
|
line = func_code(f).co_firstlineno
|
|
file = func_code(f).co_filename
|
|
self.files[file] = 1
|
|
|
|
tokname = self.toknames[fname]
|
|
if isinstance(f, types.MethodType):
|
|
reqargs = 2
|
|
else:
|
|
reqargs = 1
|
|
nargs = func_code(f).co_argcount
|
|
if nargs > reqargs:
|
|
self.log.error("%s:%d: Rule '%s' has too many arguments",
|
|
file, line, f.__name__)
|
|
self.error = 1
|
|
continue
|
|
|
|
if nargs < reqargs:
|
|
self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__)
|
|
self.error = 1
|
|
continue
|
|
|
|
if not f.__doc__:
|
|
self.log.error("%s:%d: No regular expression defined for rule '%s'",
|
|
file, line, f.__name__)
|
|
self.error = 1
|
|
continue
|
|
|
|
try:
|
|
c = re.compile("(?P<%s>%s)" % (fname, f.__doc__), re.VERBOSE | self.reflags)
|
|
if c.match(""):
|
|
self.log.error(
|
|
"%s:%d: Regular expression for rule '%s' matches empty string", file, line, f.__name__)
|
|
self.error = 1
|
|
except re.error:
|
|
_etype, e, _etrace = sys.exc_info()
|
|
self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s",
|
|
file, line, f.__name__, e)
|
|
if '#' in f.__doc__:
|
|
self.log.error(
|
|
"%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'", file, line, f.__name__)
|
|
self.error = 1
|
|
|
|
# Validate all rules defined by strings
|
|
for name, r in self.strsym[state]:
|
|
tokname = self.toknames[name]
|
|
if tokname == 'error':
|
|
self.log.error("Rule '%s' must be defined as a function", name)
|
|
self.error = 1
|
|
continue
|
|
|
|
if not tokname in self.tokens and tokname.find("ignore_") < 0:
|
|
self.log.error("Rule '%s' defined for an unspecified token %s", name, tokname)
|
|
self.error = 1
|
|
continue
|
|
|
|
try:
|
|
c = re.compile("(?P<%s>%s)" % (name, r), re.VERBOSE | self.reflags)
|
|
if (c.match("")):
|
|
self.log.error(
|
|
"Regular expression for rule '%s' matches empty string", name)
|
|
self.error = 1
|
|
except re.error:
|
|
_etype, e, _etrace = sys.exc_info()
|
|
self.log.error("Invalid regular expression for rule '%s'. %s", name, e)
|
|
if '#' in r:
|
|
self.log.error("Make sure '#' in rule '%s' is escaped with '\\#'", name)
|
|
self.error = 1
|
|
|
|
if not self.funcsym[state] and not self.strsym[state]:
|
|
self.log.error("No rules defined for state '%s'", state)
|
|
self.error = 1
|
|
|
|
# Validate the error function
|
|
efunc = self.errorf.get(state, None)
|
|
if efunc:
|
|
f = efunc
|
|
line = func_code(f).co_firstlineno
|
|
file = func_code(f).co_filename
|
|
self.files[file] = 1
|
|
|
|
if isinstance(f, types.MethodType):
|
|
reqargs = 2
|
|
else:
|
|
reqargs = 1
|
|
nargs = func_code(f).co_argcount
|
|
if nargs > reqargs:
|
|
self.log.error("%s:%d: Rule '%s' has too many arguments",
|
|
file, line, f.__name__)
|
|
self.error = 1
|
|
|
|
if nargs < reqargs:
|
|
self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__)
|
|
self.error = 1
|
|
|
|
for f in self.files:
|
|
self.validate_file(f)
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# validate_file()
|
|
#
|
|
# This checks to see if there are duplicated t_rulename() functions or strings
|
|
# in the parser input file. This is done using a simple regular expression
|
|
# match on each line in the given file.
|
|
# -----------------------------------------------------------------------------
|
|
|
|
def validate_file(self, filename):
|
|
import os.path
|
|
base, ext = os.path.splitext(filename)
|
|
if ext != '.py':
|
|
return # No idea what the file is. Return OK
|
|
|
|
try:
|
|
f = open(filename)
|
|
lines = f.readlines()
|
|
f.close()
|
|
except IOError:
|
|
return # Couldn't find the file. Don't worry about it
|
|
|
|
fre = re.compile(r'\s*def\s+(t_[a-zA-Z_0-9]*)\(')
|
|
sre = re.compile(r'\s*(t_[a-zA-Z_0-9]*)\s*=')
|
|
|
|
counthash = {}
|
|
linen = 1
|
|
for l in lines:
|
|
m = fre.match(l)
|
|
if not m:
|
|
m = sre.match(l)
|
|
if m:
|
|
name = m.group(1)
|
|
prev = counthash.get(name)
|
|
if not prev:
|
|
counthash[name] = linen
|
|
else:
|
|
self.log.error(
|
|
"%s:%d: Rule %s redefined. Previously defined on line %d", filename, linen, name, prev)
|
|
self.error = 1
|
|
linen += 1
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# lex(module)
|
|
#
|
|
# Build all of the regular expression rules from definitions in the supplied module
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def lex(module=None, object=None, debug=0, optimize=0, lextab="lextab", reflags=0, nowarn=0, outputdir="", debuglog=None, errorlog=None):
|
|
global lexer
|
|
ldict = None
|
|
stateinfo = {'INITIAL': 'inclusive'}
|
|
lexobj = Lexer()
|
|
lexobj.lexoptimize = optimize
|
|
global token, input
|
|
|
|
if errorlog is None:
|
|
errorlog = PlyLogger(sys.stderr)
|
|
|
|
if debug:
|
|
if debuglog is None:
|
|
debuglog = PlyLogger(sys.stderr)
|
|
|
|
# Get the module dictionary used for the lexer
|
|
if object:
|
|
module = object
|
|
|
|
if module:
|
|
_items = [(k, getattr(module, k)) for k in dir(module)]
|
|
ldict = dict(_items)
|
|
else:
|
|
ldict = get_caller_module_dict(2)
|
|
|
|
# Collect parser information from the dictionary
|
|
linfo = LexerReflect(ldict, log=errorlog, reflags=reflags)
|
|
linfo.get_all()
|
|
if not optimize:
|
|
if linfo.validate_all():
|
|
raise SyntaxError("Can't build lexer")
|
|
|
|
if optimize and lextab:
|
|
try:
|
|
lexobj.readtab(lextab, ldict)
|
|
token = lexobj.token
|
|
input = lexobj.input
|
|
lexer = lexobj
|
|
return lexobj
|
|
|
|
except ImportError:
|
|
pass
|
|
|
|
# Dump some basic debugging information
|
|
if debug:
|
|
debuglog.info("lex: tokens = %r", linfo.tokens)
|
|
debuglog.info("lex: literals = %r", linfo.literals)
|
|
debuglog.info("lex: states = %r", linfo.stateinfo)
|
|
|
|
# Build a dictionary of valid token names
|
|
lexobj.lextokens = {}
|
|
for n in linfo.tokens:
|
|
lexobj.lextokens[n] = 1
|
|
|
|
# Get literals specification
|
|
if isinstance(linfo.literals, (list, tuple)):
|
|
lexobj.lexliterals = type(linfo.literals[0])().join(linfo.literals)
|
|
else:
|
|
lexobj.lexliterals = linfo.literals
|
|
|
|
# Get the stateinfo dictionary
|
|
stateinfo = linfo.stateinfo
|
|
|
|
regexs = {}
|
|
# Build the master regular expressions
|
|
for state in stateinfo:
|
|
regex_list = []
|
|
|
|
# Add rules defined by functions first
|
|
for fname, f in linfo.funcsym[state]:
|
|
line = func_code(f).co_firstlineno
|
|
file = func_code(f).co_filename
|
|
regex_list.append("(?P<%s>%s)" % (fname, f.__doc__))
|
|
if debug:
|
|
debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", fname, f.__doc__, state)
|
|
|
|
# Now add all of the simple rules
|
|
for name, r in linfo.strsym[state]:
|
|
regex_list.append("(?P<%s>%s)" % (name, r))
|
|
if debug:
|
|
debuglog.info("lex: Adding rule %s -> '%s' (state '%s')", name, r, state)
|
|
|
|
regexs[state] = regex_list
|
|
|
|
# Build the master regular expressions
|
|
|
|
if debug:
|
|
debuglog.info("lex: ==== MASTER REGEXS FOLLOW ====")
|
|
|
|
for state in regexs:
|
|
lexre, re_text, re_names = _form_master_re(regexs[state], reflags, ldict, linfo.toknames)
|
|
lexobj.lexstatere[state] = lexre
|
|
lexobj.lexstateretext[state] = re_text
|
|
lexobj.lexstaterenames[state] = re_names
|
|
if debug:
|
|
for i in range(len(re_text)):
|
|
debuglog.info("lex: state '%s' : regex[%d] = '%s'", state, i, re_text[i])
|
|
|
|
# For inclusive states, we need to add the regular expressions from the INITIAL state
|
|
for state, stype in stateinfo.items():
|
|
if state != "INITIAL" and stype == 'inclusive':
|
|
lexobj.lexstatere[state].extend(lexobj.lexstatere['INITIAL'])
|
|
lexobj.lexstateretext[state].extend(lexobj.lexstateretext['INITIAL'])
|
|
lexobj.lexstaterenames[state].extend(lexobj.lexstaterenames['INITIAL'])
|
|
|
|
lexobj.lexstateinfo = stateinfo
|
|
lexobj.lexre = lexobj.lexstatere["INITIAL"]
|
|
lexobj.lexretext = lexobj.lexstateretext["INITIAL"]
|
|
|
|
# Set up ignore variables
|
|
lexobj.lexstateignore = linfo.ignore
|
|
lexobj.lexignore = lexobj.lexstateignore.get("INITIAL", "")
|
|
|
|
# Set up error functions
|
|
lexobj.lexstateerrorf = linfo.errorf
|
|
lexobj.lexerrorf = linfo.errorf.get("INITIAL", None)
|
|
if not lexobj.lexerrorf:
|
|
errorlog.warning("No t_error rule is defined")
|
|
|
|
# Check state information for ignore and error rules
|
|
for s, stype in stateinfo.items():
|
|
if stype == 'exclusive':
|
|
if not s in linfo.errorf:
|
|
errorlog.warning("No error rule is defined for exclusive state '%s'", s)
|
|
if not s in linfo.ignore and lexobj.lexignore:
|
|
errorlog.warning("No ignore rule is defined for exclusive state '%s'", s)
|
|
elif stype == 'inclusive':
|
|
if not s in linfo.errorf:
|
|
linfo.errorf[s] = linfo.errorf.get("INITIAL", None)
|
|
if not s in linfo.ignore:
|
|
linfo.ignore[s] = linfo.ignore.get("INITIAL", "")
|
|
|
|
# Create global versions of the token() and input() functions
|
|
token = lexobj.token
|
|
input = lexobj.input
|
|
lexer = lexobj
|
|
|
|
# If in optimize mode, we write the lextab
|
|
if lextab and optimize:
|
|
lexobj.writetab(lextab, outputdir)
|
|
|
|
return lexobj
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# runmain()
|
|
#
|
|
# This runs the lexer as a main program
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def runmain(lexer=None, data=None):
|
|
if not data:
|
|
try:
|
|
filename = sys.argv[1]
|
|
f = open(filename)
|
|
data = f.read()
|
|
f.close()
|
|
except IndexError:
|
|
sys.stdout.write("Reading from standard input (type EOF to end):\n")
|
|
data = sys.stdin.read()
|
|
|
|
if lexer:
|
|
_input = lexer.input
|
|
else:
|
|
_input = input
|
|
_input(data)
|
|
if lexer:
|
|
_token = lexer.token
|
|
else:
|
|
_token = token
|
|
|
|
while 1:
|
|
tok = _token()
|
|
if not tok:
|
|
break
|
|
sys.stdout.write("(%s,%r,%d,%d)\n" % (tok.type, tok.value, tok.lineno, tok.lexpos))
|
|
|
|
# -----------------------------------------------------------------------------
|
|
# @TOKEN(regex)
|
|
#
|
|
# This decorator function can be used to set the regex expression on a function
|
|
# when its docstring might need to be set in an alternative way
|
|
# -----------------------------------------------------------------------------
|
|
|
|
|
|
def TOKEN(r):
|
|
def set_doc(f):
|
|
if hasattr(r, "__call__"):
|
|
f.__doc__ = r.__doc__
|
|
else:
|
|
f.__doc__ = r
|
|
return f
|
|
return set_doc
|
|
|
|
|
|
# Alternative spelling of the TOKEN decorator
|
|
Token = TOKEN
|