U
��V�� �
@ s" d Z d ZddlZddlZddlZddlZddlZddlZzejej fZ
W n e k
rd e e
fZ
Y nX e�d�ZG dd� de�ZG dd� de�ZG dd � d e�ZG d
d � d e�ZG d d
� d
�Zdd� Zdd� Zdd� Zdd� Zdd� Zdd� Z G dd� de�Z d$d d �Z d%d d!�Z d"d#� Z e Z!dS )&z3.8� Nz^[a-zA-Z0-9_]+$c @ s e Zd Zdd� ZdS )�LexErrorc C s |f| _ || _d S �N)�args�text)�self�message�s� r �/lex.py�__init__: s zLexError.__init__N)�__name__�
__module__� __qualname__r r r r r
r 9 s r c @ s e Zd Zdd� Zdd� ZdS )�LexTokenc C s d| j | j| j| jf S )NzLexToken(%s,%r,%d,%d))�type�value�lineno�lexpos�r r r r
�__str__A s zLexToken.__str__c C s t | �S r )�strr r r r
�__repr__D s zLexToken.__repr__N)r r
r r r r r r r
r @ s r c @ s4 e Zd Zdd� Zdd� Zdd� Zdd� ZeZeZd S )
� PlyLoggerc C s
|| _ d S r ��f)r r r r r
r L s zPlyLogger.__init__c O s | j �|| d � d S )N�
�r �write�r �msgr �kwargsr r r
�criticalO s zPlyLogger.criticalc O s | j �d|| d � d S )Nz WARNING: r r r r r r
�warningR s zPlyLogger.warningc O s | j �d|| d � d S )NzERROR: r r r r r r
�errorU s zPlyLogger.errorN) r r
r r r! r" r# �info�debugr r r r
r K s r c @ s e Zd Zdd� Zdd� ZdS )�
NullLoggerc C s | S r r )r �namer r r
�__getattribute__^ s zNullLogger.__getattribute__c O s | S r r )r r r r r r
�__call__a s zNullLogger.__call__N)r r
r r( r) r r r r
r& ] s r& c @ s| e Zd Zdd� Zd dd�Zd dd�Zd d
� Zd d � Zd
d� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dd� Zdd � ZeZdS ) �Lexerc C s� d | _ d | _i | _i | _i | _d| _g | _d | _i | _i | _ i | _
d| _ d | _ d| _
d| _d | _d | _d | _d| _d| _d | _d| _d| _d S )N�INITIALr � � F)�lexre� lexretext�
lexstatere�lexstateretext�lexstaterenames�lexstate�
lexstatestack� lexstateinfo�lexstateignore�lexstateerrorf� lexstateeoff�
lexreflags�lexdatar �lexlen� lexerrorf�lexeoff� lextokens� lexignore� lexliterals� lexmoduler � lexoptimizer r r r
r t s. zLexer.__init__Nc C s� t � | �}|r�i }| j�� D ]t\}}g }|D ]L\}}g } |D ]:}
|
rL|
d sX| �|
� q<| �t||
d j�|
d f� q<q,|�|| f� |||< q ||_i |_| j�� D ]\}} t|| j�|j|< q�||_|S �Nr r- )�copyr0 �items�append�getattrr r7 rA ) r �object�cZnewtab�keyZritemZnewreZcreZfindexZ newfindexr �efr r r
�clone� s(
"
z Lexer.cloner, c C s� t |tj�rtd��|�d�d }tj�||�d }t|d����}|� d|t
f � |� dt t � � |� dt | j
� � |� d t | j� � |� d
t | j� � |� d t | j� � i }| j�� D ]P\}}g } t|| j| | j| �D ]"\\}
} } }
| �| t| |
�f� q�| ||< q�|� d t |� � |� d
t | j� � i }| j�� D ] \}}|�r^|jnd ||< �qJ|� dt |� � i }| j�� D ] \}}|�r�|jnd ||< �q�|� dt |� � W 5 Q R X d S )Nz&Won't overwrite existing lextab module�.���z.py�wzJ# %s.py. This file automatically created by PLY (version %s). Don't edit!
z_tabversion = %s
z_lextokens = %s
z_lexreflags = %s
z_lexliterals = %s
z_lexstateinfo = %s
z_lexstatere = %s
z_lexstateignore = %s
z_lexstateerrorf = %s
z_lexstateeoff = %s
) �
isinstance�types�
ModuleType�IOError�split�os�path�join�openr � __version__�repr�__tabversion__r> r9 r@ r5 r0 rE �zipr1 r2 rF �_funcs_to_namesr6 r7 r r8 )r �lextab� outputdirZ
basetabmodule�filenameZtfZtabre� statename�lre�titem�pat�funcZretext�renamesZtaberrrK Ztabeofr r r
�writetab� s6 &
zLexer.writetabc C sH t |tj�r|}ntd| � tj| }t|dd�tkr@td��|j | _
|j | _ |j
| _| j
t| j�B | _|j| _|j| _i | _i | _|j�� D ]V\}}g }g }|D ],\}} |�t�||j tj B �t | |�f� q�|| j|< || j|< q�i | _ |j �� D ]\}}
||
| j |< q�i | _ |j!�� D ]\}}
||
| j |< �q | �"d� d S )N� import %sZ _tabversionz0.0zInconsistent PLY versionr+ )#rP rQ rR �exec�sys�modulesrG r[ � ImportErrorZ
_lextokensr> Z _lexreflagsr9 Z _lexliteralsr@ �set�
lextokens_allZ
_lexstateinfor5 Z_lexstateignorer6 r0 r1 Z _lexstatererE rF �re�compile�VERBOSE�_names_to_funcsr7 Z_lexstateerrorfr8 Z
_lexstateeoff�begin) r Ztabfile�fdictr^ ra rb rc Ztxtitemrd Z func_namerK r r r
�readtab� s8
&
z
Lexer.readtabc C s8 |d d� }t |t�s td��|| _d| _t|�| _d S )Nr- zExpected a stringr )rP � StringTypes�
ValueErrorr: r �lenr; )r r rI r r r
�input� s
z Lexer.inputc C sd || j krtd��| j | | _| j| | _| j�|d�| _| j�|d �| _ | j
�|d �| _ || _ d S )NzUndefined stater, )
r0 rw r. r1 r/ r6 �getr? r7 r< r8 r= r3 �r �stater r r
rs s
z Lexer.beginc C s | j �| j� | �|� d S r )r4 rF r3 rs r{ r r r
�
push_state s zLexer.push_statec C s | � | j�� � d S r )rs r4 �popr r r r
� pop_state s zLexer.pop_statec C s | j S r )r3 r r r r
�
current_state! s zLexer.current_statec C s | j |7 _ d S r )r )r �nr r r
�skip' s z
Lexer.skipc C s~ | j }| j}| j}| j}||k �r|| |kr8|d7 }q| jD ]�\}}|�||�}|sXq>t� }|�� |_| j |_ ||_ |j
} || \}
|_ |
s�|j r�|� � | _ | S |� � } q|� � }| |_
|| _|| _ |
|�} | s�| j }| j} q| j�s(| j | jk�r(td|
jj|
jj|
j| j f ||d � ��| S || | jk�rtt� }|| |_| j |_ |j|_ ||_ |d | _ |S | j�r�t� }| j|d � |_| j |_ d|_ | |_
||_ || _ | �|�} || j k�r�td|| ||d � ��| j }| �s�q| S || _ td|| |f ||d � ��q| j�r\t� }d|_ d|_| j |_ ||_ | |_
|| _ | �|�} | S |d | _ | jd k�rztd��d S ) Nr- z4%s:%d: Rule '%s' returned an unknown token type '%s'r# z&Scanning error. Illegal character '%s'z"Illegal character '%s' at index %d�eofr, z"No input string given with input())r r; r? r: r. �matchr �groupr r � lastindexr �end�lexerZlexmatchrB rn r �__code__� co_filename�co_firstlinenor r@ r< r= � RuntimeError) r r r; r? r: r. � lexindexfunc�m�tok�ire Znewtokr r r
�token1 s�
�
�
z Lexer.tokenc C s | S r r r r r r
�__iter__� s zLexer.__iter__c C s | � � }|d krt�|S r )r� �
StopIteration)r �tr r r
�next� s z
Lexer.next)N)r, )r r
r r rL rg ru ry rs r} r r� r� r� r� r� �__next__r r r r
r* s s
%(
nr* c C s t | d| j�S )N�regex)rG �__doc__)re r r r
�
_get_regex� s r� c C s0 t �| �}|j�� }|j|jkr,|�|j� |S r )rj � _getframe� f_globalsrD �f_locals�update)Zlevelsr �ldictr r r
�get_caller_module_dict� s
r� c C sF g }t | |�D ]2\}}|r6|d r6|�||d f� q|�|� q|S rC )r\ rF )Zfunclist�namelist�resultr r' r r r
r] � s r] c C sD g }| D ]6}|r4|d r4|� ||d |d f� q|� |� q|S rC )rF )r� rt r� r� r r r
rr � s rr c C sl | sg S d� | �}z�t�|tj|B �}d gt|j�� �d }|d d � }|j�� D ]z\}} |�|d �}
t |
�t
j t
j fkr�|
|| f|| < ||| < qT|
d k rT||| < |�
d�dkr�d|| < qTd || f|| < qT||fg|g|gfW S tk
�rf tt| �d �} | dk�rd} t| d | � |||�\} }
}t| | d � |||�\}}}| | |
| || f Y S X d S )N�|r- �ignore_r )NN� )rW ro rp rq �max�
groupindex�valuesrE rz r rQ � FunctionType�
MethodType�find� Exception�intrx �_form_master_re)Zrelist�reflagsr� �toknamesr� r. r� Z
lexindexnamesr r� Zhandler� Zllistrb ZlnamesZrlistZrreZrnamesr r r
r� � s2
r� c C s� d}| � d�}t|dd � d�D ] \}}||kr |dkr q>q |dkrXt|d|� �}nd}d|krlt|�}d�||d � �}||fS )Nr- �_�ANY)r+ )rT � enumerate�tuplerW )r �namesZnonstate�partsr� �part�statesZ tokennamer r r
� _statetoken s
r� c @ sf e Zd Zddd�Zdd� Zdd� Zd d
� Zd d � Zd
d� Zdd� Z dd� Z
dd� Z dd� Z dd� Z
dS )� LexerReflectNr c C sL || _ d | _g | _|| _ddi| _t� | _d| _|d krBtt j
�n|| _ d S )Nr+ � inclusiveF) r� Z
error_func�tokensr� � stateinform rk r# r rj �stderr�log)r r� r� r� r r r
r / s
zLexerReflect.__init__c C s$ | � � | �� | �� | �� d S r )�
get_tokens� get_literals�
get_states� get_rulesr r r r
�get_all: s zLexerReflect.get_allc C s | � � | �� | �� | jS r )�validate_tokens�validate_literals�validate_rulesr# r r r r
� validate_allA s zLexerReflect.validate_allc C sp | j �dd �}|s(| j�d� d| _d S t|ttf�sL| j�d� d| _d S |sf| j�d� d| _d S || _d S )Nr� zNo token list is definedTz tokens must be a list or tupleztokens is empty)r� rz r� r# rP �listr� r� )r r� r r r
r� H s zLexerReflect.get_tokensc C sP i }| j D ]@}t�|�s,| j�d|� d| _||krB| j�d|� d||< q
d S )NzBad token name '%s'TzToken '%s' multiply definedr- )r� �_is_identifierr� r� r# r" )r Z terminalsr� r r r
r� \ s
z LexerReflect.validate_tokensc C s | j �dd�| _| js d| _d S )N�literalsr, )r� rz r� r r r r
r� g s zLexerReflect.get_literalsc C sj z>| j D ]2}t|t�r"t|�dkr| j�dt|�� d| _qW n& tk
rd | j�d� d| _Y nX d S )Nr- z.Invalid literal %s. Must be a single characterTzIInvalid literals specification. literals must be a sequence of characters)r� rP rv rx r� r# rZ � TypeError)r rI r r r
r� m s
z LexerReflect.validate_literalsc C s� | j �dd �| _| jr�t| jttf�s:| j�d� d| _n�| jD ]�}t|t�rZt|�dkrt| j�dt |�� d| _q@|\}}t|t
�s�| j�dt |�� d| _q@|dks�|dks�| j�d |� d| _q@|| j kr�| j�d
|� d| _q@|| j |< q@d S ) Nr� z)states must be defined as a tuple or listTr� zMInvalid state specifier %s. Must be a tuple (statename,'exclusive|inclusive')z State name %s must be a stringr� � exclusivez:State type for state %s must be 'inclusive' or 'exclusive'zState '%s' already defined) r� rz r� rP r� r� r� r# rx rZ rv r� )r r r' Z statetyper r r
r� x s0
zLexerReflect.get_statesc C s, dd� | j D �}i | _i | _i | _i | _i | _i | _| jD ]}g | j|< g | j|< q:t|�dkrv| j �
d� d| _
d S |D �]d}| j | }t || j�\}}|| j|< t |d��rD|dkr�|D ]}|| j|< q�nt|dkr�|D ]}|| j|< q�nV|d k�r"|j
j}|j
j}| j �
d
|||j� d| _
n |D ]}| j| �||f� �q&qzt|t��r�|d k�r�|D ]}|| j|< �q^d |k�r�| j �d |� n@|dk�r�| j �
d
|� d| _
n |D ]}| j| �||f� �q�qz| j �
d|� d| _
qz| j�� D ]}|jdd� d� �q�| j�� D ]}|jdd� dd� �qd S )Nc S s g | ]}|d d� dkr|�qS )Nr� Zt_r )�.0r r r r
�
<listcomp>� s z*LexerReflect.get_rules.<locals>.<listcomp>r z+No rules of the form t_rulename are definedTr) r# r� �ignorez,%s:%d: Rule '%s' must be defined as a string�\z#%s contains a literal backslash '\'�'Rule '%s' must be defined as a functionz&%s not defined as a function or stringc S s | d j jS �Nr- )r� r� ��xr r r
�<lambda>� � z(LexerReflect.get_rules.<locals>.<lambda>)rJ c S s t | d �S r� )rx r� r r r
r� � r� )rJ �reverse)r� r� �funcsym�strsymr� �errorf�eoffr� rx r� r# r� �hasattrr� r� r� r rF rP rv r" r� �sort) r Ztsymbolsr r r� r� �tokname�line�filer r r
r� � sb
zLexerReflect.get_rulesc C s� | j D �]b}| j| D �]r\}}|jj}|jj}t�|�}| j�|� | j | }t
|t j �rbd}nd}|jj
} | |kr�| j�d|||j� d| _q| |k r�| j�d|||j� d| _qt|�s�| j�d|||j� d| _qzJt�d|t|�f tj| jB �}
|
�d��r"| j�d |||j� d| _W q tjk
�r� } zB| j�d
|||j| � d t|�k�rr| j�d |||j� d| _W 5 d } ~ X Y qX q| j| D �]\} }
| j | }|d
k�r�| j�d| � d| _�q�|| jk�r|�d�dk �r| j�d| |� d| _�q�z@t�d| |
f tj| jB �}
|
�d��r@| j�d| � d| _W nT tjk
�r� } z2| j�d| | � d |
k�r�| j�d| � d| _W 5 d } ~ X Y nX �q�| j| �s�| j| �s�| j�d|� d| _| j�|d �}|r|}|jj}|jj}t�|�}| j�|� t
|t j ��rd}nd}|jj
} | |k�rH| j�d|||j� d| _| |k r| j�d|||j� d| _q| jD ]}| � |� �qrd S )Nr� r- z'%s:%d: Rule '%s' has too many argumentsTz%%s:%d: Rule '%s' requires an argumentz2%s:%d: No regular expression defined for rule '%s'�
(?P<%s>%s)r, z<%s:%d: Regular expression for rule '%s' matches empty stringz3%s:%d: Invalid regular expression for rule '%s'. %s�#z6%s:%d. Make sure '#' in rule '%s' is escaped with '\#'r# r� r� r z-Rule '%s' defined for an unspecified token %sz5Regular expression for rule '%s' matches empty stringz,Invalid regular expression for rule '%s'. %sz/Make sure '#' in rule '%s' is escaped with '\#'z No rules defined for state '%s') r� r� r� r� r� �inspectZ getmodulerk �addr� rP rQ r� � co_argcountr� r# r r� ro rp rq r� r� r� r� r� r� rz �validate_module)r r| �fnamer r� r� �moduler� Zreqargs�nargsrI �er' �rZefuncr r r
r� � s�
zLexerReflect.validate_rulesc C s� t �|�\}}t�d�}t�d�}i }|d7 }|D ]n}|�|�}|sN|�|�}|r�|�d�} |�| �}
|
st||| < n$t �|�} | j� d| || |
� d| _ |d7 }q2d S )Nz \s*def\s+(t_[a-zA-Z_0-9]*)\(z\s*(t_[a-zA-Z_0-9]*)\s*=r- z7%s:%d: Rule %s redefined. Previously defined on line %dT)
r� Zgetsourcelinesro rp r� r� rz Z
getsourcefiler� r# ) r r� �linesZlinenZfreZsreZ counthashr� r� r' �prevr` r r r
r� @ s$
z LexerReflect.validate_module)Nr )r r
r r r� r� r� r� r� r� r� r� r� r� r r r r
r� . s
Bgr� Fr^ c
%
sr |d kr d}d }
ddi} t � } || _| d kr6ttj�} |rL|d krLttj�}|rT|� � r�� fdd�t� �D �}
t|
�}
d|
kr�tj|
d j|
d< nt d�}
|
�
d �}|r�t |t �r�d
|kr�|d
| }t
|
| |d �}|�� |s�|�� r�td ��|�r8|�r8z"| �||
� | ja| ja| a| W S tk
�r6 Y nX |�rh|�d
|j� |�d|j� |�d|j� t� | _|jD ]}| j� |� �qvt |jt t f��r�t |jd �� � |j�| _!n|j| _!| jt| j!�B | _"|j} i }| D ]�}g }|j#| D ]J\}}|j$j%}|j$j&}|�'d|t(|�f � |�r�|�d|t(|�|� �q�|j)| D ]2\}}|�'d||f � |�rH|�d|||� �qH|||< �q�|�r�|�d� |D ]h}t*|| ||
|j+�\}}} || j,|< || j-|< | | j.|< |�r�t/|�D ]\} } |�d|| | � �q�q�| �0� D ]`\}} |dk�r| dk�r| j,| �1| j,d � | j-| �1| j-d � | j.| �1| j.d � �q| | _2| j,d | _3| j-d | _4|| _5|j6| _7| j7�
dd�| _8|j9| _:|j9�
dd �| _;| j;�s�| �<d� |j=| _>|j=�
dd �| _?| �0� D ]�\} } | dk�rB| |j9k�r | �<d| � | |j6k�r�| j8�r�| �<d| � nJ| dk�r�| |j9k�rl|j9�
dd �|j9| < | |j6k�r�|j6�
dd�|j6| < �q�| ja| ja| a|�rn|�rn|d k�r&t |t@jA��r�|j}!nNd
|k�r�|
d }!n:|�Bd
�}"d
� |"d d� �}#tCd|# � tDtj|# dd�}!tEjF�G|!�}z| �H||� W n6 tIk
�rl }$ z| �<d ||$f � W 5 d }$~$X Y nX | S ) Nr^ r+ r� c s g | ]}|t � |�f�qS r )rG )r� �k�r� r r
r� w s zlex.<locals>.<listcomp>�__file__r
r� � __package__rM )r� r� zCan't build lexerzlex: tokens = %rzlex: literals = %rzlex: states = %rr r� z(lex: Adding rule %s -> '%s' (state '%s')z#lex: ==== MASTER REGEXS FOLLOW ====z"lex: state '%s' : regex[%d] = '%s'r, zNo t_error rule is definedr� z1No error rule is defined for exclusive state '%s'z2No ignore rule is defined for exclusive state '%s'rN rh z#Couldn't write lextab module %r. %s)Jr* rB r rj r� �dir�dictrk r� r� rz rP r r� r� r� � SyntaxErrorru r� ry r� rl r$ r� r� r� rm r> r� r� r� r rW r@ rn r� r� r� r� rF r� r� r� r� r0 r1 r2 r� rE �extendr5 r. r/ r9 r� r6 r? r� r7 r< r" r� r8 r= rQ rR rT ri rG rU rV �dirnamerg rS )%r� rH r% �optimizer^ r� Znowarnr_ ZdebuglogZerrorlogr� r� ZlexobjZ_itemsZpkgZlinfor� Zregexsr| Z
regex_listr� r r� r� r' r� r. Zre_textZre_namesr� r �styper Zsrcfiler� Zpkgnamer� r r� r
�lex\ s�
$r� c C s� |sVz&t jd }t|�}|�� }|�� W n* tk
rT t j�d� t j�� }Y nX | rb| j }nt }||� | rz| j
}nt
}|� }|s�q�t j�d|j |j |j
|jf � q~d S )Nr- z/Reading from standard input (type EOF to end):
z(%s,%r,%d,%d)
)rj �argvrX �read�close�
IndexError�stdoutr �stdinry r� r r r r )r� �datar` r Z_inputZ_tokenr� r r r
�runmain s(
r c s � fdd�}|S )Nc s t � d�rt� �| _n� | _| S )Nr) )r� r� r� r �r� r r
� set_regex? s
zTOKEN.<locals>.set_regexr )r� r r r r
�TOKEN> s r )
NNFFr^ r FNNN)NN)"rY r[ ro rj rQ rD rU r� Z
StringTypeZ UnicodeTyperv �AttributeErrorr �bytesrp r� r� r rH r r r& r* r� r� r] rr r� r� r� r� r r �Tokenr r r r
�<module>" sN
F
( 0 �
A
"