3
\NX = @ s d Z dZddlZddlZddlmZmZ ddlT ddlm Z dd e
e D d
d d g Z [ ye W n e
k
rz eZ Y nX d
d Zdd Zdd ZdZdZeede ee ZdZdZdZdZeddZeeeeeZd Zed d ee Z d e Z ee e Z ed e d! Z ee e eZ d"Z!d#Z"d$Z#d%Z$d&Z%ee%d' e%d( Z&ee%d) e%d* Z'ed+d,d-d.d/d0d1d2d3 Z(d4Z)ed5d6Z*ee(e)e*Z+ee e+e'eZ,ee, Z-ee%d7 ed8d e%d9 ed:d Z.edee&Z/eee/e e+e.e Z0e1e2ej3e-e0e#e$f\Z4Z5Z6Z7ej3e!ej3e"e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7e6e7ddddddddd;4Z8i Z9xdD ]Z:e:e9e:< qW i Z;xdD ]Z:e:e;e:< qW dZG dd de=Z?dd Z@e@fdd
ZAdd ZBG dd dZCej3dejDZEej3dejDZFdd ZGdd ZHdd ZIdd ZJeKdkrddlLZLeMeLjNdkrxeAeOeLjNd jP n eAeLjQjP dS )zKa-Ping Yee z@GvR, ESR, Tim Peters, Thomas Wouters, Fred Drake, Skip Montanaro N)BOM_UTF8lookup)* )tokenc C s g | ]}|d dkr|qS )r _ ).0xr r ./usr/lib64/python3.6/lib2to3/pgen2/tokenize.py
% s r tokenizegenerate_tokens
untokenizec G s ddj | d S )N(|))join)choicesr r r group0 s r c G s t | d S )Nr )r )r r r r any1 s r c G s t | d S )N?)r )r r r r maybe2 s r z[ \f\t]*z #[^\r\n]*z\\\r?\nz [a-zA-Z_]\w*z0[bB]_?[01]+(?:_[01]+)*z(0[xX]_?[\da-fA-F]+(?:_[\da-fA-F]+)*[lL]?z 0[oO]?_?[0-7]+(?:_[0-7]+)*[lL]?z[1-9]\d*(?:_\d+)*[lL]?z0[lL]?z[eE][-+]?\d+(?:_\d+)*z \d+(?:_\d+)*\.(?:\d+(?:_\d+)*)?z\.\d+(?:_\d+)*z \d+(?:_\d+)*z\d+(?:_\d+)*[jJ]z[jJ]z[^'\\]*(?:\\.[^'\\]*)*'z[^"\\]*(?:\\.[^"\\]*)*"z%[^'\\]*(?:(?:\\.|'(?!''))[^'\\]*)*'''z%[^"\\]*(?:(?:\\.|"(?!""))[^"\\]*)*"""z#(?:[uUrRbBfF]|[rR][bB]|[bBuU][rR])?'''"""z '[^\n'\\]*(?:\\.[^\n'\\]*)*'z "[^\n"\\]*(?:\\.[^\n"\\]*)*"z\*\*=?z>>=?z<<=?z<>z!=z//=?z->z[+\-*/%&@|^=<>]=?~z[][(){}]z\r?\nz[:;.,`@]z'[^\n'\\]*(?:\\.[^\n'\\]*)*'z"[^\n"\\]*(?:\\.[^\n"\\]*)*")4r r z'''z"""zr'''zr"""zu'''zu"""zb'''zb"""zf'''zf"""zur'''zur"""zbr'''zbr"""zrb'''zrb"""zR'''zR"""zU'''zU"""zB'''zB"""zF'''zF"""zuR'''zuR"""zUr'''zUr"""zUR'''zUR"""zbR'''zbR"""zBr'''zBr"""zBR'''zBR"""zrB'''zrB"""zRb'''zRb"""zRB'''zRB"""rRuUfFbBr'''r"""R'''R"""u'''u"""U'''U"""b'''b"""B'''B"""f'''f"""F'''F"""ur'''ur"""Ur'''Ur"""uR'''uR"""UR'''UR"""br'''br"""Br'''Br"""bR'''bR"""BR'''BR"""rb'''rb"""Rb'''Rb"""rB'''rB"""RB'''RB"""r'r"R'R"u'u"U'U"b'b"B'B"f'f"F'F"ur'ur"Ur'Ur"uR'uR"UR'UR"br'br"Br'Br"bR'bR"BR'BR"rb'rb"Rb'Rb"rB'rB"RB'RB" c @ s e Zd ZdS )
TokenErrorN)__name__
__module__ __qualname__r r r r rw s rw c @ s e Zd ZdS )StopTokenizingN)rx ry rz r r r r r{ s r{ c C s4 |\}}|\}}t d||||t| t|f d S )Nz%d,%d-%d,%d: %s %s)printtok_namerepr) typer Zxxx_todo_changemeZxxx_todo_changeme1lineZsrowZscolZerowZecolr r r
printtoken s r c C s( yt | | W n tk
r" Y nX d S )N)
tokenize_loopr{ )readline
tokeneaterr r r r
s
c C s xt | D ] }|| q
W d S )N)r )r r Z
token_infor r r r s r c @ s, e Zd Zdd Zdd Zdd Zdd Zd S )
Untokenizerc C s g | _ d| _d| _d S )Nr r )tokensprev_rowprev_col)selfr r r __init__ s zUntokenizer.__init__c C s* |\}}|| j }|r&| jjd| d S )N )r r append)r startrowcol
col_offsetr r r add_whitespace s
zUntokenizer.add_whitespacec C s xv|D ]n}t |dkr$| j|| P |\}}}}}| j| | jj| |\| _| _|ttfkr| jd7 _d| _qW dj | jS )N r r )
lencompatr r r r r NEWLINENLr )r iterablettok_typer r endr r r r r s
zUntokenizer.untokenizec C s d}g }| j j}|\}}|ttfkr,|d7 }|ttfkr|t kr|j
qBn*|ttfkrd}n|r|r||d d}|| qBW d S )NFr Tr r ) r r NAMENUMBERr r ASYNCAWAITINDENTDEDENTpop) r r r startlineindents toks_appendtoknumtokvaltokr r r r s0
zUntokenizer.compatN)rx ry rz r r r r r r r r r s r z&^[ \t\f]*#.*?coding[:=][ \t]*([-\w.]+)s ^[ \t\f]*(?:[#\r\n]|$)c C sH | d d j jdd}|dks*|jdr.dS |d ks@|jd
rDdS | S )N r -zutf-8zutf-8-latin-1
iso-8859-1 iso-latin-1latin-1- iso-8859-1- iso-latin-1-)r r r )r r r )lowerreplace
startswith)orig_encencr r r _get_normal_name s
r c s d d }d}fdd} fdd}| }|j trHd |dd }d }|sT|g fS ||}|rj||gfS tj|s~||gfS | }|s||gfS ||}|r|||gfS |||gfS )
NFzutf-8c s" y S t k
r t S X d S )N)
StopIterationbytesr )r r r read_or_stop s z%detect_encoding..read_or_stopc s y| j d}W n tk
r" d S X tj|}|s6d S t|jd}y t|}W n tk
rp td| Y nX r|j dkrtd|d7 }|S )Nasciir zunknown encoding: zutf-8zencoding problem: utf-8z-sig)
decodeUnicodeDecodeError cookie_rematchr r r LookupError SyntaxErrorname)r line_stringr encodingcodec) bom_foundr r find_cookie& s"
z$detect_encoding..find_cookieT z utf-8-sig)r r blank_rer )r r defaultr r firstsecondr )r r r detect_encoding s0
r c C s t }|j| S )N)r r )r utr r r r T s c ! c s d } }}t jd d }}d \}}d }dg} d }
d} d} d}
xy
| }W n tk
rd d}Y nX |d }dt| }}|rF|std||j|}|r|jd }}t||d | |||f|| fV d \}}d }nd|r0|d!d d kr0|d"d d kr0t|| ||t|f|fV d}d }qBn|| }|| }qBnF|dkrt| rt|s`P d}xf||k r|| d kr|d }n6|| d
kr|t d t }n|| dkrd}nP |d }qfW ||krP |
r|
V d }
|| dkr|| dkrh||d j
d}|t| }t |||f||t| f|fV t ||d ||f|t|f|fV qBt t f|| dk ||d ||f|t|f|fV qB|| d# kr| j
| t|d | |df||f|fV xt|| d$ k rJ|| krtdd|||f| d d% } | r.| | d& kr.d} d}
d} td||f||f|fV qW | r|
r| | d' krd} d}
d} n|std|dfd}x||k r8tj||}|r |jd\}}||f||f| }}}||| || } } | |ks| dkr| dkrt| |||fV q4| dkrft} |dkr8t } n
| rBd}
|
rR|
V d }
| | |||fV q4| dkr|
r|
V d }
t | |||fV q4| tkrt| }|j||}|r|jd}||| } |
r|
V d }
t| |||f|fV n||f}||d }|}P q4| tks@| d d tks@| d d
tkr| d( dkr||f}t| pxt| d pxt| d }||d d }}|}P n |
r|
V d }
t| |||fV q4| |kr| d)kr| r| dkrtnt| |||fV qt| |||f} | dkr |
r | }
q| dkrx|
rx|
d tkrx|
d dkrxd} | d* } t|
d |
d |
d
|
d fV d }
|
r|
V d }
| V nz| d kr|
r|
V d }
t | |||f|fV d}nF| d kr|d }n| d kr|d }|
r|
V d }
t| |||fV n(t|| ||f||d f|fV |d }qW qBW |
rN|
V d }
x.| dd D ] } td|df|dfdfV q\W t d|df|dfdfV d S )+Nr r
0123456789r Fr zEOF in multi-line stringr z\
r z\
r z#
#z
z3unindent does not match any outer indentation levelz
zEOF in multi-line statement.T
asyncawaitdef \z([{z)]})r r )r r r r r r r r )r r r ) stringZ
ascii_lettersr r rw r r STRING
ERRORTOKENtabsizerstripCOMMENTr r r IndentationErrorr
pseudoprogspanr r
triple_quotedendprogs
single_quotedr r r OP ENDMARKER)!r lnumparenlev continuedZ namecharsnumcharscontstrneedcontcontliner stashed async_defasync_def_indent async_def_nlr posmaxstrstartendprogendmatchr column
comment_tokennl_pos pseudomatchr sposeposr initialnewliner indentr r r r i sp
*
__main__)*r r r& r' r( r) r* r+ r, r- r. r/ r0 r1 r2 r3 r4 r5 r6 r7 r8 r9 r: r; r< r= r> r? r@ rA rB rC rD rE rF rG rH rI rJ rK rL rM )*r r rN rO rP rQ rR rS rT rU rV rW rX rY rZ r[ r\ r] r^ r_ r` ra rb rc rd re rf rg rh ri rj rk rl rm rn ro rp rq rr rs rt ru )R
__author__ __credits__r recodecsr r Zlib2to3.pgen2.tokenr r dir__all__r NameErrorstrr r r
WhitespaceCommentIgnoreName Binnumber Hexnumber Octnumber Decnumber IntnumberExponent
PointfloatExpfloat Floatnumber
ImagnumberNumberSingleDoubleSingle3Double3Z
_litprefixTripleStringOperatorBracketSpecialFunny
PlainTokenTokenContStr PseudoExtras PseudoTokenlistmapcompileZ tokenprogr Z single3progZ double3progr r r r r Exceptionrw r{ r r
r r ASCIIr r r r r r rx sysr argvopenr stdinr r r r s
8 I c