Coverage for python/lsst/daf/butler/exprParser/ply/lex.py : 6%

# ----------------------------------------------------------------------------- 

# ply: lex.py 

# 

# Copyright (C) 2001-2018 

# David M. Beazley (Dabeaz LLC) 

# All rights reserved. 

# 

# Redistribution and use in source and binary forms, with or without 

# modification, are permitted provided that the following conditions are 

# met: 

# 

# * Redistributions of source code must retain the above copyright notice, 

# this list of conditions and the following disclaimer. 

# * Redistributions in binary form must reproduce the above copyright notice, 

# this list of conditions and the following disclaimer in the documentation 

# and/or other materials provided with the distribution. 

# * Neither the name of the David Beazley or Dabeaz LLC may be used to 

# endorse or promote products derived from this software without 

# specific prior written permission. 

# 

# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 

# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 

# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 

# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 

# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 

# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 

# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 

# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 

# ----------------------------------------------------------------------------- 

__version__ = '3.11' 

__tabversion__ = '3.10' 

import re 

import sys 

import types 

import copy 

import os 

import inspect 

# This tuple contains known string types 

try: 

# Python 2.6 

StringTypes = (types.StringType, types.UnicodeType) 

except AttributeError: 

# Python 3.0 

StringTypes = (str, bytes) 

# This regular expression is used to match valid token names 

_is_identifier = re.compile(r'^[a-zA-Z0-9_]+$') 

# Exception thrown when invalid token encountered and no default error 

# handler is defined. 

class LexError(Exception): 

def __init__(self, message, s): 

self.args = (message,) 

self.text = s 

# Token class. This class is used to represent the tokens produced. 

class LexToken(object): 

def __str__(self): 

return 'LexToken(%s,%r,%d,%d)' % (self.type, self.value, self.lineno, self.lexpos) 

def __repr__(self): 

return str(self) 

# This object is a stand-in for a logging object created by the 

# logging module. 

class PlyLogger(object): 

def __init__(self, f): 

self.f = f 

def critical(self, msg, *args, **kwargs): 

self.f.write((msg % args) + '\n') 

def warning(self, msg, *args, **kwargs): 

self.f.write('WARNING: ' + (msg % args) + '\n') 

def error(self, msg, *args, **kwargs): 

self.f.write('ERROR: ' + (msg % args) + '\n') 

info = critical 

debug = critical 

# Null logger is used when no output is generated. Does nothing. 

class NullLogger(object): 

def __getattribute__(self, name): 

return self 

def __call__(self, *args, **kwargs): 

return self 

# ----------------------------------------------------------------------------- 

# === Lexing Engine === 

# 

# The following Lexer class implements the lexer runtime. There are only 

# a few public methods and attributes: 

# 

# input() - Store a new string in the lexer 

# token() - Get the next token 

# clone() - Clone the lexer 

# 

# lineno - Current line number 

# lexpos - Current position in the input string 

# ----------------------------------------------------------------------------- 

class Lexer: 

def __init__(self): 

self.lexre = None # Master regular expression. This is a list of 

# tuples (re, findex) where re is a compiled 

# regular expression and findex is a list 

# mapping regex group numbers to rules 

self.lexretext = None # Current regular expression strings 

self.lexstatere = {} # Dictionary mapping lexer states to master regexs 

self.lexstateretext = {} # Dictionary mapping lexer states to regex strings 

self.lexstaterenames = {} # Dictionary mapping lexer states to symbol names 

self.lexstate = 'INITIAL' # Current lexer state 

self.lexstatestack = [] # Stack of lexer states 

self.lexstateinfo = None # State information 

self.lexstateignore = {} # Dictionary of ignored characters for each state 

self.lexstateerrorf = {} # Dictionary of error functions for each state 

self.lexstateeoff = {} # Dictionary of eof functions for each state 

self.lexreflags = 0 # Optional re compile flags 

self.lexdata = None # Actual input data (as a string) 

self.lexpos = 0 # Current position in input text 

self.lexlen = 0 # Length of the input text 

self.lexerrorf = None # Error rule (if any) 

self.lexeoff = None # EOF rule (if any) 

self.lextokens = None # List of valid tokens 

self.lexignore = '' # Ignored characters 

self.lexliterals = '' # Literal characters that can be passed through 

self.lexmodule = None # Module 

self.lineno = 1 # Current line number 

self.lexoptimize = False # Optimized mode 

def clone(self, object=None): 

c = copy.copy(self) 

# If the object parameter has been supplied, it means we are attaching the 

# lexer to a new object. In this case, we have to rebind all methods in 

# the lexstatere and lexstateerrorf tables. 

if object: 

newtab = {} 

for key, ritem in self.lexstatere.items(): 

newre = [] 

for cre, findex in ritem: 

newfindex = [] 

for f in findex: 

if not f or not f[0]: 

newfindex.append(f) 

continue 

newfindex.append((getattr(object, f[0].__name__), f[1])) 

newre.append((cre, newfindex)) 

newtab[key] = newre 

c.lexstatere = newtab 

c.lexstateerrorf = {} 

for key, ef in self.lexstateerrorf.items(): 

c.lexstateerrorf[key] = getattr(object, ef.__name__) 

c.lexmodule = object 

return c 

# ------------------------------------------------------------ 

# writetab() - Write lexer information to a table file 

# ------------------------------------------------------------ 

def writetab(self, lextab, outputdir=''): 

if isinstance(lextab, types.ModuleType): 

raise IOError("Won't overwrite existing lextab module") 

basetabmodule = lextab.split('.')[-1] 

filename = os.path.join(outputdir, basetabmodule) + '.py' 

with open(filename, 'w') as tf: 

tf.write('# %s.py. This file automatically created by PLY (version %s). Don\'t edit!\n' % (basetabmodule, __version__)) 

tf.write('_tabversion = %s\n' % repr(__tabversion__)) 

tf.write('_lextokens = set(%s)\n' % repr(tuple(sorted(self.lextokens)))) 

tf.write('_lexreflags = %s\n' % repr(int(self.lexreflags))) 

tf.write('_lexliterals = %s\n' % repr(self.lexliterals)) 

tf.write('_lexstateinfo = %s\n' % repr(self.lexstateinfo)) 

# Rewrite the lexstatere table, replacing function objects with function names 

tabre = {} 

for statename, lre in self.lexstatere.items(): 

titem = [] 

for (pat, func), retext, renames in zip(lre, self.lexstateretext[statename], self.lexstaterenames[statename]): 

titem.append((retext, _funcs_to_names(func, renames))) 

tabre[statename] = titem 

tf.write('_lexstatere = %s\n' % repr(tabre)) 

tf.write('_lexstateignore = %s\n' % repr(self.lexstateignore)) 

taberr = {} 

for statename, ef in self.lexstateerrorf.items(): 

taberr[statename] = ef.__name__ if ef else None 

tf.write('_lexstateerrorf = %s\n' % repr(taberr)) 

tabeof = {} 

for statename, ef in self.lexstateeoff.items(): 

tabeof[statename] = ef.__name__ if ef else None 

tf.write('_lexstateeoff = %s\n' % repr(tabeof)) 

# ------------------------------------------------------------ 

# readtab() - Read lexer information from a tab file 

# ------------------------------------------------------------ 

def readtab(self, tabfile, fdict): 

if isinstance(tabfile, types.ModuleType): 

lextab = tabfile 

else: 

exec('import %s' % tabfile) 

lextab = sys.modules[tabfile] 

if getattr(lextab, '_tabversion', '0.0') != __tabversion__: 

raise ImportError('Inconsistent PLY version') 

self.lextokens = lextab._lextokens 

self.lexreflags = lextab._lexreflags 

self.lexliterals = lextab._lexliterals 

self.lextokens_all = self.lextokens | set(self.lexliterals) 

self.lexstateinfo = lextab._lexstateinfo 

self.lexstateignore = lextab._lexstateignore 

self.lexstatere = {} 

self.lexstateretext = {} 

for statename, lre in lextab._lexstatere.items(): 

titem = [] 

txtitem = [] 

for pat, func_name in lre: 

titem.append((re.compile(pat, lextab._lexreflags), _names_to_funcs(func_name, fdict))) 

self.lexstatere[statename] = titem 

self.lexstateretext[statename] = txtitem 

self.lexstateerrorf = {} 

for statename, ef in lextab._lexstateerrorf.items(): 

self.lexstateerrorf[statename] = fdict[ef] 

self.lexstateeoff = {} 

for statename, ef in lextab._lexstateeoff.items(): 

self.lexstateeoff[statename] = fdict[ef] 

self.begin('INITIAL') 

# ------------------------------------------------------------ 

# input() - Push a new string into the lexer 

# ------------------------------------------------------------ 

def input(self, s): 

# Pull off the first character to see if s looks like a string 

c = s[:1] 

if not isinstance(c, StringTypes): 

raise ValueError('Expected a string') 

self.lexdata = s 

self.lexpos = 0 

self.lexlen = len(s) 

# ------------------------------------------------------------ 

# begin() - Changes the lexing state 

# ------------------------------------------------------------ 

def begin(self, state): 

if state not in self.lexstatere: 

raise ValueError('Undefined state') 

self.lexre = self.lexstatere[state] 

self.lexretext = self.lexstateretext[state] 

self.lexignore = self.lexstateignore.get(state, '') 

self.lexerrorf = self.lexstateerrorf.get(state, None) 

self.lexeoff = self.lexstateeoff.get(state, None) 

self.lexstate = state 

# ------------------------------------------------------------ 

# push_state() - Changes the lexing state and saves old on stack 

# ------------------------------------------------------------ 

def push_state(self, state): 

self.lexstatestack.append(self.lexstate) 

self.begin(state) 

# ------------------------------------------------------------ 

# pop_state() - Restores the previous state 

# ------------------------------------------------------------ 

def pop_state(self): 

self.begin(self.lexstatestack.pop()) 

# ------------------------------------------------------------ 

# current_state() - Returns the current lexing state 

# ------------------------------------------------------------ 

def current_state(self): 

return self.lexstate 

# ------------------------------------------------------------ 

# skip() - Skip ahead n characters 

# ------------------------------------------------------------ 

def skip(self, n): 

self.lexpos += n 

# ------------------------------------------------------------ 

# opttoken() - Return the next token from the Lexer 

# 

# Note: This function has been carefully implemented to be as fast 

# as possible. Don't make changes unless you really know what 

# you are doing 

# ------------------------------------------------------------ 

def token(self): 

# Make local copies of frequently referenced attributes 

lexpos = self.lexpos 

lexlen = self.lexlen 

lexignore = self.lexignore 

lexdata = self.lexdata 

while lexpos < lexlen: 

# This code provides some short-circuit code for whitespace, tabs, and other ignored characters 

if lexdata[lexpos] in lexignore: 

lexpos += 1 

continue 

# Look for a regular expression match 

for lexre, lexindexfunc in self.lexre: 

m = lexre.match(lexdata, lexpos) 

if not m: 

continue 

# Create a token for return 

tok = LexToken() 

tok.value = m.group() 

tok.lineno = self.lineno 

tok.lexpos = lexpos 

i = m.lastindex 

func, tok.type = lexindexfunc[i] 

if not func: 

# If no token type was set, it's an ignored token 

if tok.type: 

self.lexpos = m.end() 

return tok 

else: 

lexpos = m.end() 

break 

lexpos = m.end() 

# If token is processed by a function, call it 

tok.lexer = self # Set additional attributes useful in token rules 

self.lexmatch = m 

self.lexpos = lexpos 

newtok = func(tok) 

# Every function must return a token, if nothing, we just move to next token 

if not newtok: 

lexpos = self.lexpos # This is here in case user has updated lexpos. 

lexignore = self.lexignore # This is here in case there was a state change 

break 

# Verify type of the token. If not in the token map, raise an error 

if not self.lexoptimize: 

if newtok.type not in self.lextokens_all: 

raise LexError("%s:%d: Rule '%s' returned an unknown token type '%s'" % ( 

func.__code__.co_filename, func.__code__.co_firstlineno, 

func.__name__, newtok.type), lexdata[lexpos:]) 

return newtok 

else: 

# No match, see if in literals 

if lexdata[lexpos] in self.lexliterals: 

tok = LexToken() 

tok.value = lexdata[lexpos] 

tok.lineno = self.lineno 

tok.type = tok.value 

tok.lexpos = lexpos 

self.lexpos = lexpos + 1 

return tok 

# No match. Call t_error() if defined. 

if self.lexerrorf: 

tok = LexToken() 

tok.value = self.lexdata[lexpos:] 

tok.lineno = self.lineno 

tok.type = 'error' 

tok.lexer = self 

tok.lexpos = lexpos 

self.lexpos = lexpos 

newtok = self.lexerrorf(tok) 

if lexpos == self.lexpos: 

# Error method didn't change text position at all. This is an error. 

raise LexError("Scanning error. Illegal character '%s'" % (lexdata[lexpos]), lexdata[lexpos:]) 

lexpos = self.lexpos 

if not newtok: 

continue 

return newtok 

self.lexpos = lexpos 

raise LexError("Illegal character '%s' at index %d" % (lexdata[lexpos], lexpos), lexdata[lexpos:]) 

if self.lexeoff: 

tok = LexToken() 

tok.type = 'eof' 

tok.value = '' 

tok.lineno = self.lineno 

tok.lexpos = lexpos 

tok.lexer = self 

self.lexpos = lexpos 

newtok = self.lexeoff(tok) 

return newtok 

self.lexpos = lexpos + 1 

if self.lexdata is None: 

raise RuntimeError('No input string given with input()') 

return None 

# Iterator interface 

def __iter__(self): 

return self 

def next(self): 

t = self.token() 

if t is None: 

raise StopIteration 

return t 

__next__ = next 

# ----------------------------------------------------------------------------- 

# ==== Lex Builder === 

# 

# The functions and classes below are used to collect lexing information 

# and build a Lexer object from it. 

# ----------------------------------------------------------------------------- 

# ----------------------------------------------------------------------------- 

# _get_regex(func) 

# 

# Returns the regular expression assigned to a function either as a doc string 

# or as a .regex attribute attached by the @TOKEN decorator. 

# ----------------------------------------------------------------------------- 

def _get_regex(func): 

return getattr(func, 'regex', func.__doc__) 

# ----------------------------------------------------------------------------- 

# get_caller_module_dict() 

# 

# This function returns a dictionary containing all of the symbols defined within 

# a caller further down the call stack. This is used to get the environment 

# associated with the yacc() call if none was provided. 

# ----------------------------------------------------------------------------- 

def get_caller_module_dict(levels): 

f = sys._getframe(levels) 

ldict = f.f_globals.copy() 

if f.f_globals != f.f_locals: 

ldict.update(f.f_locals) 

return ldict 

# ----------------------------------------------------------------------------- 

# _funcs_to_names() 

# 

# Given a list of regular expression functions, this converts it to a list 

# suitable for output to a table file 

# ----------------------------------------------------------------------------- 

def _funcs_to_names(funclist, namelist): 

result = [] 

for f, name in zip(funclist, namelist): 

if f and f[0]: 

result.append((name, f[1])) 

else: 

result.append(f) 

return result 

# ----------------------------------------------------------------------------- 

# _names_to_funcs() 

# 

# Given a list of regular expression function names, this converts it back to 

# functions. 

# ----------------------------------------------------------------------------- 

def _names_to_funcs(namelist, fdict): 

result = [] 

for n in namelist: 

if n and n[0]: 

result.append((fdict[n[0]], n[1])) 

else: 

result.append(n) 

return result 

# ----------------------------------------------------------------------------- 

# _form_master_re() 

# 

# This function takes a list of all of the regex components and attempts to 

# form the master regular expression. Given limitations in the Python re 

# module, it may be necessary to break the master regex into separate expressions. 

# ----------------------------------------------------------------------------- 

def _form_master_re(relist, reflags, ldict, toknames): 

if not relist: 

return [] 

regex = '|'.join(relist) 

try: 

lexre = re.compile(regex, reflags) 

# Build the index to function map for the matching engine 

lexindexfunc = [None] * (max(lexre.groupindex.values()) + 1) 

lexindexnames = lexindexfunc[:] 

for f, i in lexre.groupindex.items(): 

handle = ldict.get(f, None) 

if type(handle) in (types.FunctionType, types.MethodType): 

lexindexfunc[i] = (handle, toknames[f]) 

lexindexnames[i] = f 

elif handle is not None: 

lexindexnames[i] = f 

if f.find('ignore_') > 0: 

lexindexfunc[i] = (None, None) 

else: 

lexindexfunc[i] = (None, toknames[f]) 

return [(lexre, lexindexfunc)], [regex], [lexindexnames] 

except Exception: 

m = int(len(relist)/2) 

if m == 0: 

m = 1 

llist, lre, lnames = _form_master_re(relist[:m], reflags, ldict, toknames) 

rlist, rre, rnames = _form_master_re(relist[m:], reflags, ldict, toknames) 

return (llist+rlist), (lre+rre), (lnames+rnames) 

# ----------------------------------------------------------------------------- 

# def _statetoken(s,names) 

# 

# Given a declaration name s of the form "t_" and a dictionary whose keys are 

# state names, this function returns a tuple (states,tokenname) where states 

# is a tuple of state names and tokenname is the name of the token. For example, 

# calling this with s = "t_foo_bar_SPAM" might return (('foo','bar'),'SPAM') 

# ----------------------------------------------------------------------------- 

def _statetoken(s, names): 

parts = s.split('_') 

for i, part in enumerate(parts[1:], 1): 

if part not in names and part != 'ANY': 

break 

if i > 1: 

states = tuple(parts[1:i]) 

else: 

states = ('INITIAL',) 

if 'ANY' in states: 

states = tuple(names) 

tokenname = '_'.join(parts[i:]) 

return (states, tokenname) 

# ----------------------------------------------------------------------------- 

# LexerReflect() 

# 

# This class represents information needed to build a lexer as extracted from a 

# user's input file. 

# ----------------------------------------------------------------------------- 

class LexerReflect(object): 

def __init__(self, ldict, log=None, reflags=0): 

self.ldict = ldict 

self.error_func = None 

self.tokens = [] 

self.reflags = reflags 

self.stateinfo = {'INITIAL': 'inclusive'} 

self.modules = set() 

self.error = False 

self.log = PlyLogger(sys.stderr) if log is None else log 

# Get all of the basic information 

def get_all(self): 

self.get_tokens() 

self.get_literals() 

self.get_states() 

self.get_rules() 

# Validate all of the information 

def validate_all(self): 

self.validate_tokens() 

self.validate_literals() 

self.validate_rules() 

return self.error 

# Get the tokens map 

def get_tokens(self): 

tokens = self.ldict.get('tokens', None) 

if not tokens: 

self.log.error('No token list is defined') 

self.error = True 

return 

if not isinstance(tokens, (list, tuple)): 

self.log.error('tokens must be a list or tuple') 

self.error = True 

return 

if not tokens: 

self.log.error('tokens is empty') 

self.error = True 

return 

self.tokens = tokens 

# Validate the tokens 

def validate_tokens(self): 

terminals = {} 

for n in self.tokens: 

if not _is_identifier.match(n): 

self.log.error("Bad token name '%s'", n) 

self.error = True 

if n in terminals: 

self.log.warning("Token '%s' multiply defined", n) 

terminals[n] = 1 

# Get the literals specifier 

def get_literals(self): 

self.literals = self.ldict.get('literals', '') 

if not self.literals: 

self.literals = '' 

# Validate literals 

def validate_literals(self): 

try: 

for c in self.literals: 

if not isinstance(c, StringTypes) or len(c) > 1: 

self.log.error('Invalid literal %s. Must be a single character', repr(c)) 

self.error = True 

except TypeError: 

self.log.error('Invalid literals specification. literals must be a sequence of characters') 

self.error = True 

def get_states(self): 

self.states = self.ldict.get('states', None) 

# Build statemap 

if self.states: 

if not isinstance(self.states, (tuple, list)): 

self.log.error('states must be defined as a tuple or list') 

self.error = True 

else: 

for s in self.states: 

if not isinstance(s, tuple) or len(s) != 2: 

self.log.error("Invalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')", repr(s)) 

self.error = True 

continue 

name, statetype = s 

if not isinstance(name, StringTypes): 

self.log.error('State name %s must be a string', repr(name)) 

self.error = True 

continue 

if not (statetype == 'inclusive' or statetype == 'exclusive'): 

self.log.error("State type for state %s must be 'inclusive' or 'exclusive'", name) 

self.error = True 

continue 

if name in self.stateinfo: 

self.log.error("State '%s' already defined", name) 

self.error = True 

continue 

self.stateinfo[name] = statetype 

# Get all of the symbols with a t_ prefix and sort them into various 

# categories (functions, strings, error functions, and ignore characters) 

def get_rules(self): 

tsymbols = [f for f in self.ldict if f[:2] == 't_'] 

# Now build up a list of functions and a list of strings 

self.toknames = {} # Mapping of symbols to token names 

self.funcsym = {} # Symbols defined as functions 

self.strsym = {} # Symbols defined as strings 

self.ignore = {} # Ignore strings by state 

self.errorf = {} # Error functions by state 

self.eoff = {} # EOF functions by state 

for s in self.stateinfo: 

self.funcsym[s] = [] 

self.strsym[s] = [] 

if len(tsymbols) == 0: 

self.log.error('No rules of the form t_rulename are defined') 

self.error = True 

return 

for f in tsymbols: 

t = self.ldict[f] 

states, tokname = _statetoken(f, self.stateinfo) 

self.toknames[f] = tokname 

if hasattr(t, '__call__'): 

if tokname == 'error': 

for s in states: 

self.errorf[s] = t 

elif tokname == 'eof': 

for s in states: 

self.eoff[s] = t 

elif tokname == 'ignore': 

line = t.__code__.co_firstlineno 

file = t.__code__.co_filename 

self.log.error("%s:%d: Rule '%s' must be defined as a string", file, line, t.__name__) 

self.error = True 

else: 

for s in states: 

self.funcsym[s].append((f, t)) 

elif isinstance(t, StringTypes): 

if tokname == 'ignore': 

for s in states: 

self.ignore[s] = t 

if '\\' in t: 

self.log.warning("%s contains a literal backslash '\\'", f) 

elif tokname == 'error': 

self.log.error("Rule '%s' must be defined as a function", f) 

self.error = True 

else: 

for s in states: 

self.strsym[s].append((f, t)) 

else: 

self.log.error('%s not defined as a function or string', f) 

self.error = True 

# Sort the functions by line number 

for f in self.funcsym.values(): 

f.sort(key=lambda x: x[1].__code__.co_firstlineno) 

# Sort the strings by regular expression length 

for s in self.strsym.values(): 

s.sort(key=lambda x: len(x[1]), reverse=True) 

# Validate all of the t_rules collected 

def validate_rules(self): 

for state in self.stateinfo: 

# Validate all rules defined by functions 

for fname, f in self.funcsym[state]: 

line = f.__code__.co_firstlineno 

file = f.__code__.co_filename 

module = inspect.getmodule(f) 

self.modules.add(module) 

tokname = self.toknames[fname] 

if isinstance(f, types.MethodType): 

reqargs = 2 

else: 

reqargs = 1 

nargs = f.__code__.co_argcount 

if nargs > reqargs: 

self.log.error("%s:%d: Rule '%s' has too many arguments", file, line, f.__name__) 

self.error = True 

continue 

if nargs < reqargs: 

self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__) 

self.error = True 

continue 

if not _get_regex(f): 

self.log.error("%s:%d: No regular expression defined for rule '%s'", file, line, f.__name__) 

self.error = True 

continue 

try: 

c = re.compile('(?P<%s>%s)' % (fname, _get_regex(f)), self.reflags) 

if c.match(''): 

self.log.error("%s:%d: Regular expression for rule '%s' matches empty string", file, line, f.__name__) 

self.error = True 

except re.error as e: 

self.log.error("%s:%d: Invalid regular expression for rule '%s'. %s", file, line, f.__name__, e) 

if '#' in _get_regex(f): 

self.log.error("%s:%d. Make sure '#' in rule '%s' is escaped with '\\#'", file, line, f.__name__) 

self.error = True 

# Validate all rules defined by strings 

for name, r in self.strsym[state]: 

tokname = self.toknames[name] 

if tokname == 'error': 

self.log.error("Rule '%s' must be defined as a function", name) 

self.error = True 

continue 

if tokname not in self.tokens and tokname.find('ignore_') < 0: 

self.log.error("Rule '%s' defined for an unspecified token %s", name, tokname) 

self.error = True 

continue 

try: 

c = re.compile('(?P<%s>%s)' % (name, r), self.reflags) 

if (c.match('')): 

self.log.error("Regular expression for rule '%s' matches empty string", name) 

self.error = True 

except re.error as e: 

self.log.error("Invalid regular expression for rule '%s'. %s", name, e) 

if '#' in r: 

self.log.error("Make sure '#' in rule '%s' is escaped with '\\#'", name) 

self.error = True 

if not self.funcsym[state] and not self.strsym[state]: 

self.log.error("No rules defined for state '%s'", state) 

self.error = True 

# Validate the error function 

efunc = self.errorf.get(state, None) 

if efunc: 

f = efunc 

line = f.__code__.co_firstlineno 

file = f.__code__.co_filename 

module = inspect.getmodule(f) 

self.modules.add(module) 

if isinstance(f, types.MethodType): 

reqargs = 2 

else: 

reqargs = 1 

nargs = f.__code__.co_argcount 

if nargs > reqargs: 

self.log.error("%s:%d: Rule '%s' has too many arguments", file, line, f.__name__) 

self.error = True 

if nargs < reqargs: 

self.log.error("%s:%d: Rule '%s' requires an argument", file, line, f.__name__)