Ë
    ›¾j(€  ã                   ó�   — d dl Z d dlmZ d dlmZ d dlmZ d dlmZmZ  e«       Z	 e«       Z
 G d„ de«      Z G d„ d	e«      Zdd	gZy)
é    N)ÚPerluniprops)ÚNonbreakingPrefixes)Úis_cjk)ÚVIRAMASÚNUKTASc            
       óè  ‡ — e Zd ZdZ edj                  ej                  d«      «      «      Z edj                  ej                  d«      «      dj                  e	«      z   dj                  e
«      z   «      Z edj                  ej                  d«      «      «      Z edj                  ej                  d«      «      «      Z edj                  ej                  d«      «      dj                  e	«      z   dj                  e
«      z   «      Z edj                  ej                  d«      «      «      Z ej"                  d	«      d
fZ ej"                  d«      dfZdZdZdZ ej"                  dj/                  e«      «      dfZ ej"                  dj/                  e¬«      «      dfZ ej"                  d«      dfZ ej"                  d«      dfZ ej"                  d«      dfZ ej"                  dj/                  e«      «      dfZ ej"                  dj/                  e«      «      dfZ ej"                  dj/                  e«      «      dfZ ej"                  d«      d fZ  ej"                  d!«      d fZ! ej"                  d"«      d#fZ" ej"                  d$«      d%fZ# ej"                  d&«      d'fZ$ ej"                  d(«      d'fZ% ej"                  d)«      d*fZ& ej"                  d+«      d,fZ' ej"                  d-«      d.fZ( ej"                  d/«      d-fZ) ej"                  d0j/                  e¬1«      «      d2fZ* ej"                  d3j/                  e¬1«      «      d2fZ+ ej"                  d4j/                  e¬1«      «      d2fZ, ej"                  d5j/                  ee«      «      dfZ-d6j/                  e¬«      d7fZ. ej"                  d8«      d9fZ/ ej"                  d:«      dfZ0 ej"                  d;«      dfZ1 ej"                  d<«      d=fZ2 ej"                  d>«      d?fZ3 ej"                  d@«      dAfZ4 ej"                  dB«      dCfZ5 ej"                  dD«      dEfZ6 ej"                  dF«      dGfZ7 ej"                  dH«      dIfZ8 ej"                  dJ«      d
fZ9 ej"                  dK«      d
fZ: ej"                  dL«      dMfZ; ej"                  dN«      dOfZ< ej"                  dP«      dOfZ= ej"                  dQ«      dRfZ> ej"                  dS«      dTfZ? ej"                  dU«      dVfZ@ ej"                  dW«      dXfZA ej"                  dY«      dZfZB ej"                  d[«      d\fZC ej"                  d]«      d^fZD ej"                  d_«      d`fZE ej"                  da«      dbfZF ej"                  dc«      ddfZG ej"                  de«      dffZH ej"                  dg«      dhfZI ej"                  di«      djfZJ ej"                  dk«      dlfZK ej"                  dm«      dnfZL ej"                  do«      dpfZM ej"                  dq«      drfZN ej"                  ds«      dtfZO ej"                  du«      dvfZP ej"                  dw«      d
fZQ ej"                  dx«      dfZR ej"                  dy«      dfZS ej"                  dz«      d{fZT ej"                  d|«      d}fZU ej"                  d~«      dfZV ej"                  d€«      d�fZW ej"                  d‚«      dƒfZX ej"                  d„«      d…fZY ej"                  d@«      d†fZZ ej"                  d‡«      dˆfZ[ ej"                  d‰j/                  e¬Š«      «      d‹fZ\ ej"                  dŒj/                  ee¬�«      «      d‹fZ] ej"                  dŽj/                  e¬Š«      «      d‹fZ^ ej"                  d�j/                  e¬Š«      «      d�fZ_ ej"                  d‘j/                  e¬’«      «      d�fZ`e\e]e^e_e`gZa ej"                  d‰j/                  e¬Š«      «      d‹fZb ej"                  d“j/                  e¬Š«      «      d‹fZc ej"                  dŽj/                  e¬Š«      «      d‹fZd ej"                  d�j/                  e¬Š«      «      d”fZeebecedeegZf ej"                  d‚«      d•fZg ej"                  d–«      d—fZhd˜Zid™ZjdšZkd›ZldœZmg e‘e‘e ‘e!‘e"‘e#‘e$‘e%‘e&‘e'‘e(‘e*‘e+‘e,‘e-‘e.‘e/‘e0‘e1‘e2‘e3‘e4‘e5‘e6‘e7‘e8‘e9‘e:‘e;‘e<‘e=‘e>‘e?‘e@‘eA‘eB‘eC‘eD‘eE‘eF‘eG‘eH‘eI‘eJ‘eK‘eL‘eM‘eN‘eO‘eP‘Zne)eQeReSeTeUeVeWeXeYg
ZoeTeUeVeWeXeYeZe[gZpeiejekelemgZqg d�¢Zrd¨ˆ fdž„	ZsdŸ„ Ztd „ Zud¡„ Zvd¢„ Zwd£„ Zxd¤„ Zyd¥„ Zzd©d¦„Z{	 	 	 	 dªd§„Z|ˆ xZ}S )«ÚMosesTokenizerz–
    This is a Python port of the Moses Tokenizer from
    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl
    Ú ÚIsNÚIsAlnumÚIsScÚIsSoÚIsAlphaÚIsLowerz\s+ú z[\000-\037])z +r   )z^ r
   )z $r
   ú([^{}\s\.'\`\,\-])ú \1 ú ([{alphanum}])\-(?=[{alphanum}])©Úalphanumú\1 @-@ ú	\.([\.]+)z
 DOTMULTIúDOTMULTI\.([^\.])zDOTDOTMULTI ú
DOTMULTI\.ÚDOTDOTMULTIz
([^{}])[,]z\1 , z
[,]([^{}])z , \1z
([{}])[,]$z^``z`` z^"z^`([^`])z` \1z^'z`  z
([ ([{<])"z\1 `` z([ ([{<])``z([ ([{<])`([^`])z\1 ` \2z
([ ([{<])'z\1 ` z\.\.\.z _ELLIPSIS_ Ú
_ELLIPSIS_z([^{numbers}])[,]([^{numbers}]))Únumbersz\1 , \2z([{numbers}])[,]([^{numbers}])z([^{numbers}])[,]([{numbers}])z([;:@#\$%&{}{}])ú([{alphanum}])\/([{alphanum}])ú$1 \@\/\@ $2z([^.])([.])([\]\)}>"']*) ?$z\1 \2\3z([?!])z([\]\[\(\){}<>])z\(z-LRB-z\)z-RRB-z\[z-LSB-z\]z-RSB-z\{z-LCB-z\}z-RCB-z--z -- ú^ú$ú"z '' z([^'])' z\1 ' z([^'])'z'([sSmMdD]) z '\1 z'll z 'll z're z 're z've z 've zn't z n't z'LL z 'LL z'RE z 'RE z'VE z 'VE zN'T z N'T z ([Cc])annot z
 \1an not z ([Dd])'ye z \1' ye z ([Gg])imme z	 \1im me z ([Gg])onna z	 \1on na z ([Gg])otta z	 \1ot ta z ([Ll])emme z	 \1em me z ([Mm])ore'n z
 \1ore 'n z '([Tt])is z '\1 is z '([Tt])was z	 '\1 was z ([Ww])anna z	 \1an na z  *z^ *z *$ú&ú&amp;z\|ú&#124;ú<ú&lt;ú>ú&gt;z\'ú&apos;z\"ú&quot;ú&#91;ú]ú&#93;z([^{alpha}])[']([^{alpha}]))Úalphaz\1 ' \2z([^{alpha}{isn}])[']([{alpha}]))r/   Úisnz([{alpha}])[']([^{alpha}])z([{alpha}])[']([{alpha}])z\1 '\2z([{isn}])[']([s]))r0   z([^{alpha}])[']([{alpha}])z\1' \2z ' z\.' ?$z . ' z<\/?\S+\/?>z#<\S+( [a-zA-Z0-9]+\="?[^"]")+ ?\/?>z#<\S+( [a-zA-Z0-9]+\='?[^']')+ ?\/?>ú'[\w\-\_\.]+\@([\w\-\_]+\.)+[a-zA-Z]{2,}z/(http[s]?|ftp):\/\/[^:\/\s]+(\/\w+)*\/[\w\-\.]+)z"((https?|ftp|rsync)://|www\.)[^ ]*r1   z@[a-zA-Z0-9_]+z#[a-zA-Z0-9_]+c                 ó.  •— t         t        | �  «        || _        t        j                  |«      D �cg c]  }|j                  «       ‘Œ c}| _        |rsg | _        t        |d«      5 }|D ]Q  }|j                  «       }|sŒ|j                  d«      rŒ(|| j                  vsŒ7| j                  j                  |«       ŒS 	 d d d «       | j                  D �cg c]'  }| j                  |«      r|j                  d«      d   ‘Œ) c}| _        | j                  dv �rèd}| j                  dv r0|t        dj                  t         j#                  d«      «      «      z  }| j                  d	v r0|t        dj                  t         j#                  d
«      «      «      z  }| j                  dv r�|t        dj                  t         j#                  d«      «      «      z  }|t        dj                  t         j#                  d«      «      «      z  }|t        dj                  t         j#                  d
«      «      «      z  }| xj$                  |z  c_        | xj&                  |z  c_        t)        j*                  dj-                  | j&                  «      «      df| _        t)        j*                  dj-                  | j&                  ¬«      «      df| _        t)        j*                  dj-                  | j&                  ¬«      «      df| _        y y c c}w # 1 sw Y   �ŒIxY wc c}w )NÚrú#r   r   )ÚzhÚjaÚkoÚcjkr
   )r7   r8   ÚHangul)r5   r8   ÚHan)r6   r8   ÚHiraganaÚKatakanar   r   r   r   r   r   r   )Úsuperr	   Ú__init__ÚlangÚnonbreaking_prefixesÚwordsÚstripÚNONBREAKING_PREFIXESÚopenÚ
startswithÚappendÚhas_numeric_onlyÚ
rpartitionÚNUMERIC_ONLY_PREFIXESÚstrÚjoinÚperlunipropsÚcharsr   r   ÚreÚcompileÚformatÚPAD_NOT_ISALNUMÚAGGRESSIVE_HYPHEN_SPLITÚINTRATOKEN_SLASHES)	Úselfr?   Ú custom_nonbreaking_prefixes_fileÚ_nbpÚfinÚlineÚwÚ	cjk_charsÚ	__class__s	           €ú]/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/sacremoses/tokenize.pyr>   zMosesTokenizer.__init__(  s’  ø€ äŒn˜dÑ,Ô.ØˆŒ	ô &:×%?Ñ%?ÀÓ%Eö%
Ø!ˆD�J‰J�Lò%
ˆÔ!ñ
 ,Ø(*ˆDÔ%ÜÐ6¸Ó<ð CÀØò C�DØŸ:™:›<�DÚ D§O¡O°CÕ$8Ø t×'@Ñ'@Ò@Ø ×5Ñ5×<Ñ<¸TÕBñ	C÷Cð ×.Ñ.ö&
àØ×$Ñ$ QÔ'ð �L‰L˜Ó˜aÓ ò&
ˆÔ"ð �9‰9Ð1Ò1ØˆIØ�y‰y˜MÑ)ØœS §¡¬×);Ñ);¸HÓ)EÓ!FÓGÑG�	Ø�y‰y˜MÑ)ØœS §¡¬×);Ñ);¸EÓ)BÓ!CÓDÑD�	Ø�y‰y˜MÑ)ØœS §¡¬×);Ñ);¸JÓ)GÓ!HÓIÑI�	ØœS §¡¬×);Ñ);¸JÓ)GÓ!HÓIÑI�	ØœS §¡¬×);Ñ);¸EÓ)BÓ!CÓDÑD�	Ø�LŠL˜IÑ%�LØ�LŠL˜IÑ%�Lä#%§:¡:Ð.C×.JÑ.JÈ4Ï<É<Ó.XÓ#YÐ[bÐ#bˆDÔ ä—
‘
Ð>×EÑEÈtÏ|É|ÐEÓ\Ó]Øð,ˆDÔ(ô
 —
‘
Ð<×CÑCÈTÏ\É\ÐCÓZÓ[Øð'ˆDÕ#ð% 2ùò)%
÷Cñ Cüò&
s)   ²L Á&LÁ?LÂLÂ LÃ,LÌLc                 óö   — t        j                  dd|«      }t        j                  d«      }|j                  |«      r;t        j                  dd|«      }|j                  d|«      }|j                  |«      rŒ;|S )Nr   z DOTMULTI\1r   r   zDOTDOTMULTI \1r   )rN   ÚsubrO   Úsearch©rT   ÚtextÚdotmultis      r\   Úreplace_multidotsz MosesTokenizer.replace_multidotsY  sg   € Ü�v‰v�l N°DÓ9ˆÜ—:‘:˜mÓ,ˆØ�o‰o˜dÔ#Ü—6‘6Ð.Ð0AÀ4ÓHˆDØ—<‘< ¨tÓ4ˆDð �o‰o˜dÕ#ð ˆó    c                 óÄ   — t        j                  d«      }|j                  |«      r$|j                  d|«      }|j                  |«      rŒ$t        j                  dd|«      S )Nr   z	DOTMULTI.ÚDOTMULTIú.)rN   rO   r_   r^   r`   s      r\   Úrestore_multidotsz MosesTokenizer.restore_multidotsa  sM   € Ü—:‘:˜nÓ-ˆØ�o‰o˜dÔ#Ø—<‘< ¨dÓ3ˆDð �o‰o˜dÕ#ä�v‰v�k 4¨Ó.Ð.rd   c                 ó^   — t        |«      j                  t        | j                  «      «       S ©N)ÚsetÚ
differencer   ©rT   ra   s     r\   ÚislowerzMosesTokenizer.islowerg  s#   € Ü�t“9×'Ñ'¬¨D¯L©LÓ(9Ó:Ð:Ð:rd   c                 ón   — t        t        |«      j                  t        | j                  «      «      «      S rj   )Úanyrk   Úintersectionr   rm   s     r\   Ú
isanyalphazMosesTokenizer.isanyalphaj  s%   € Ü”3�t“9×)Ñ)¬#¨d¯l©lÓ*;Ó<Ó=Ð=rd   c                 ó@   — t        t        j                  d|«      «      S )Nz[\s]+(\#NUMERIC_ONLY\#))ÚboolrN   r_   rm   s     r\   rG   zMosesTokenizer.has_numeric_onlym  s   € Ü”B—I‘IÐ8¸$Ó?Ó@Ð@rd   c                 ó  — |j                  «       }t        |«      }t        |«      D ]Æ  \  }}t        j                  d|«      }|sŒ|j                  d«      }d|v r| j                  |«      sF|| j                  v r|| j                  vs*||dz
  k7  r#||dz      r| j                  ||dz      d   «      rŒŒ|| j                  v r%|dz   |k  rt        j                  d||dz      «      rŒ¿|dz   ||<   ŒÈ dj                  |«      S )Nz	^(\S+)\.$é   rg   r   z^[0-9]+z .r   )ÚsplitÚlenÚ	enumeraterN   r_   Úgrouprr   rC   rI   rn   rK   )rT   ra   ÚtokensÚ
num_tokensÚiÚtokenÚtoken_ends_with_periodÚprefixs           r\   Úhandles_nonbreaking_prefixesz+MosesTokenizer.handles_nonbreaking_prefixesp  s  € à—‘“ˆÜ˜“[ˆ
Ü! &Ó)ò "	.‰HˆAˆuä%'§Y¡Y¨|¸UÓ%CÐ"Ú%Ø/×5Ñ5°aÓ8�ð ˜F‘] t§¡°vÔ'>à $×";Ñ";Ñ;Ø"¨$×*DÑ*DÑDð ˜Z¨!™^Ò+Ø" 1 q¡5šMØ ŸL™L¨°°A±©°qÑ)9Ô:ð ð ˜d×8Ñ8Ñ8Ø˜Q™ *Ò,ÜŸ	™	 *¨f°Q¸±U©mÔ<àà &¨¡�F˜1’IðE"	.ðF �x‰x˜ÓÐrd   c                 óR   — | j                   D ]  \  }}|j                  ||«      }Œ |S rj   )ÚMOSES_ESCAPE_XML_REGEXESr^   ©rT   ra   ÚregexpÚsubstitutions       r\   Ú
escape_xmlzMosesTokenizer.escape_xml™  s1   € Ø$(×$AÑ$Aò 	2Ñ ˆF�LØ—:‘:˜l¨DÓ1‰Dð	2àˆrd   c                 óú   — t        |«      }| j                  D ]  \  }}|j                  ||«      }Œ | j                  |«      }| j                  D ]  \  }}|j                  ||«      }Œ |r|S |j                  «       S )z‚
        This is a Python port of the Penn treebank tokenizer adapted by the Moses
        machine translation community.
        )rJ   ÚMOSES_PENN_REGEXES_1r^   r�   ÚMOSES_PENN_REGEXES_2rw   )rT   ra   Ú
return_strr…   r†   s        r\   Úpenn_tokenizezMosesTokenizer.penn_tokenizež  s†   € ô �4‹yˆà$(×$=Ñ$=ò 	2Ñ ˆF�LØ—:‘:˜l¨DÓ1‰Dð	2ð ×0Ñ0°Ó6ˆà$(×$=Ñ$=ò 	2Ñ ˆF�LØ—:‘:˜l¨DÓ1‰Dð	2á!ˆtÐ3 t§z¡z£|Ð3rd   c                 ó8  — t        |«      }| j                  | j                  fD ]  \  }}|j                  ||«      }Œ |rÅ|D �cg c]&  }t	        j
                  |t        j                  «      ‘Œ( }}|D �	�
cg c](  }	|	j                  |«      D ]  }
|
j                  «       ‘Œ Œ* }}	}
t        |«      dk  sJ ‚t        t        |«      d„ d¬«      D ]4  \  }}dt        |«      j                  d«      z   }|j                  ||«      }Œ6 |j                  «       }	 | j                  \  }}|j                  ||«      }|r!| j                   \  }}|j                  ||«      }| j#                  |«      }| j$                  | j&                  | j(                  fD ]  \  }}|j                  ||«      }Œ | j*                  dk(  r'| j,                  D ]  \  }}|j                  ||«      }Œ nV| j*                  dv r'| j.                  D ]  \  }}|j                  ||«      }Œ n!| j0                  \  }}|j                  ||«      }| j3                  |«      }| j                  \  }}|j                  ||«      j                  «       }| j4                  \  }}|j                  ||«      }|rBt        «      D ]4  \  }}dt        |«      j                  d«      z   }|j                  ||«      }Œ6 | j7                  |«      }|r| j9                  |«      }|r|S |j;                  «       S c c}w c c}
}	w )	a  
        Python port of the Moses tokenizer.

            :param tokens: A single string, i.e. sentence text.
            :type tokens: str
            :param aggressive_dash_splits: Option to trigger dash split rules .
            :type aggressive_dash_splits: bool
        iè  c                 ó   — t        | d   «      S )Nrv   )rx   )Úpairs    r\   ú<lambda>z)MosesTokenizer.tokenize.<locals>.<lambda>Ð  s   € ÔPSÐTXÐYZÑT[ÓP\€ rd   T)ÚkeyÚreverseÚTHISISPROTECTEDé   Úen)ÚfrÚit)rJ   ÚDEDUPLICATE_SPACEÚ
ASCII_JUNKr^   rN   rO   Ú
IGNORECASEÚfinditerrz   rx   Úsortedry   ÚzfillÚreplacerB   rQ   rR   rc   ÚCOMMA_SEPARATE_1ÚCOMMA_SEPARATE_2ÚCOMMA_SEPARATE_3r?   ÚENGLISH_SPECIFIC_APOSTROPHEÚFR_IT_SPECIFIC_APOSTROPHEÚNON_SPECIFIC_APOSTROPHEr�   ÚTRAILING_DOT_APOSTROPHErh   r‡   rw   )rT   ra   Úaggressive_dash_splitsr‹   ÚescapeÚprotected_patternsr…   r†   ÚpÚprotected_patternÚmatchÚprotected_tokensr}   r~   Úsubstituitions                  r\   ÚtokenizezMosesTokenizer.tokenize¯  s/  € ô" �4‹yˆà%)×%;Ñ%;¸T¿_¹_Ð$Mò 	2Ñ ˆF�LØ—:‘:˜l¨DÓ1‰Dð	2ñ ØHZÖ![À1¤"§*¡*¨Q´·±Õ">Ð![ÐÐ![ð *<÷ à%Ø.×7Ñ7¸Ó=ò ð ð —‘•ð Øð Ðñ  ô
 Ð'Ó(¨DÒ0Ð0Ð0ô #¤9Ð-=Ó#>ÑD\ÐfjÔkò :‘��5Ø 1´C¸³F·L±LÀ³OÑ C�Ø—|‘| E¨=Ó9‘ð:ð
 �z‰z‹|ˆð	ð  $×3Ñ3Ñˆ�Ø�z‰z˜,¨Ó-ˆá!Ø#'×#?Ñ#?Ñ ˆF�LØ—:‘:˜l¨DÓ1ˆDð ×%Ñ% dÓ+ˆð ×!Ñ!Ø×!Ñ!Ø×!Ñ!ð%
ò 	2Ñ ˆF�Lð
 —:‘:˜l¨DÓ1‰Dð	2ð �9‰9˜ÒØ(,×(HÑ(Hò 6Ñ$�˜Ø—z‘z ,°Ó5‘ñ6à�Y‰Y˜,Ñ&Ø(,×(FÑ(Fò 6Ñ$�˜Ø—z‘z ,°Ó5‘ñ6ð $(×#?Ñ#?Ñ ˆF�LØ—:‘:˜l¨DÓ1ˆDð ×0Ñ0°Ó6ˆà#×5Ñ5Ñˆ�Ø�z‰z˜,¨Ó-×3Ñ3Ó5ˆà $× <Ñ <Ñˆ�Ø�z‰z˜-¨Ó.ˆñ Ü%Ð&6Ó7ò :‘��5Ø 1´C¸³F·L±LÀ³OÑ C�Ø—|‘| M°5Ó9‘ð:ð
 ×%Ñ% dÓ+ˆÙà—?‘? 4Ó(ˆDá!ˆtÐ3 t§z¡z£|Ð3ùòm "\ùó s   Á+LÁ6-L)r•   N)F)FFTN)~Ú__name__Ú
__module__Ú__qualname__Ú__doc__rJ   rK   rL   rM   r   r   r   r   r   r   r   r   rN   rO   r˜   r™   Ú	MID_STRIPÚ
LEFT_STRIPÚRIGHT_STRIPrP   rQ   rR   Ú REPLACE_DOT_WITH_LITERALSTRING_1Ú REPLACE_DOT_WITH_LITERALSTRING_2Ú REPLACE_DOT_WITH_LITERALSTRING_3rŸ   r    r¡   ÚDIRECTIONAL_QUOTE_1ÚDIRECTIONAL_QUOTE_2ÚDIRECTIONAL_QUOTE_3ÚDIRECTIONAL_QUOTE_4ÚDIRECTIONAL_QUOTE_5ÚDIRECTIONAL_QUOTE_6ÚDIRECTIONAL_QUOTE_7ÚDIRECTIONAL_QUOTE_8ÚREPLACE_ELLIPSISÚRESTORE_ELLIPSISÚCOMMA_1ÚCOMMA_2ÚCOMMA_3ÚSYMBOLSrS   ÚFINAL_PERIODÚPAD_QUESTION_EXCLAMATION_MARKÚPAD_PARENTHESISÚCONVERT_PARENTHESIS_1ÚCONVERT_PARENTHESIS_2ÚCONVERT_PARENTHESIS_3ÚCONVERT_PARENTHESIS_4ÚCONVERT_PARENTHESIS_5ÚCONVERT_PARENTHESIS_6ÚPAD_DOUBLE_DASHESÚPAD_START_OF_STRÚPAD_END_OF_STRÚCONVERT_DOUBLE_TO_SINGLE_QUOTESÚHANDLES_SINGLE_QUOTESÚ
APOSTROPHEÚCONTRACTION_1ÚCONTRACTION_2ÚCONTRACTION_3ÚCONTRACTION_4ÚCONTRACTION_5ÚCONTRACTION_6ÚCONTRACTION_7ÚCONTRACTION_8ÚCONTRACTION_9ÚCONTRACTION_10ÚCONTRACTION_11ÚCONTRACTION_12ÚCONTRACTION_13ÚCONTRACTION_14ÚCONTRACTION_15ÚCONTRACTION_16ÚCONTRACTION_17ÚCONTRACTION_18ÚCONTRACTION_19ÚCLEAN_EXTRA_SPACE_1ÚCLEAN_EXTRA_SPACE_2ÚCLEAN_EXTRA_SPACE_3ÚESCAPE_AMPERSANDÚESCAPE_PIPEÚESCAPE_LEFT_ANGLE_BRACKETÚESCAPE_RIGHT_ANGLE_BRACKETÚESCAPE_SINGLE_QUOTEÚESCAPE_DOUBLE_QUOTEÚESCAPE_LEFT_SQUARE_BRACKETÚESCAPE_RIGHT_SQUARE_BRACKETÚEN_SPECIFIC_1ÚEN_SPECIFIC_2ÚEN_SPECIFIC_3ÚEN_SPECIFIC_4ÚEN_SPECIFIC_5r¢   ÚFR_IT_SPECIFIC_1ÚFR_IT_SPECIFIC_2ÚFR_IT_SPECIFIC_3ÚFR_IT_SPECIFIC_4r£   r¤   r¥   ÚBASIC_PROTECTED_PATTERN_1ÚBASIC_PROTECTED_PATTERN_2ÚBASIC_PROTECTED_PATTERN_3ÚBASIC_PROTECTED_PATTERN_4ÚBASIC_PROTECTED_PATTERN_5r‰   rŠ   rƒ   ÚBASIC_PROTECTED_PATTERNSÚWEB_PROTECTED_PATTERNSr>   rc   rh   rn   rr   rG   r�   r‡   rŒ   r®   Ú__classcell__©r[   s   @r\   r	   r	      sI  ø„ ññ ˆb�g‰g�l×(Ñ(¨Ó/Ó0Ó
1€CÙØ
�‰�×"Ñ" 9Ó-Ó.°·±¸Ó1AÑAÀBÇGÁGÈFÃOÑSó€Gñ ˆr�w‰w�|×)Ñ)¨&Ó1Ó2Ó3€DÙˆr�w‰w�|×)Ñ)¨&Ó1Ó2Ó3€DÙØ
�‰�×"Ñ" 9Ó-Ó.°·±¸Ó1AÑAÀBÇGÁGÈFÃOÑSó€Gñ �"—'‘'˜,×,Ñ,¨YÓ7Ó8Ó9€Gð #˜Ÿ
™
 6Ó*¨DÐ0ÐØ�—‘˜NÓ+¨SÐ0€Jð €IØ€JØ€Kð !�b—j‘jÐ!6×!=Ñ!=¸gÓ!FÓGÈÐP€Oð
 	ˆ�
‰
Ð6×=Ñ=ÀwÐ=ÓOÓPØðÐð (2 r§z¡z°,Ó'?ÀÐ'NÐ$Ø'1 r§z¡zÐ2FÓ'GÐIYÐ'YÐ$Ø'1 r§z¡z°-Ó'@À-Ð'OÐ$ð "�r—z‘z -×"6Ñ"6°sÓ";Ó<¸hÐFÐØ!�r—z‘z -×"6Ñ"6°sÓ";Ó<¸hÐFÐØ!�r—z‘z -×"6Ñ"6°sÓ";Ó<¸hÐFÐð %˜"Ÿ*™* VÓ,¨fÐ4ÐØ$˜"Ÿ*™* UÓ+¨VÐ3ÐØ$˜"Ÿ*™* [Ó1°7Ð:ÐØ$˜"Ÿ*™* UÓ+¨VÐ3ÐØ$˜"Ÿ*™* ]Ó3°YÐ>ÐØ$˜"Ÿ*™* ^Ó4°iÐ?ÐØ$˜"Ÿ*™*Ð%8Ó9¸:ÐEÐØ$˜"Ÿ*™* ]Ó3°XÐ=Ðð "�r—z‘z )Ó,¨oÐ=Ðà!�r—z‘z -Ó0°)Ð;Ðð ˆb�j‰jÐ;×BÑBÈ3ÐBÓOÓPÐR\Ð\€GØˆb�j‰jÐ:×AÑAÈ#ÐAÓNÓOÐQ[Ð[€GØˆb�j‰jÐ:×AÑAÈ#ÐAÓNÓOÐQ[Ð[€Gð ˆb�j‰jÐ,×3Ñ3°D¸$Ó?Ó@À'ÐI€Gð 	*×0Ñ0¸'Ð0ÓBØðÐð �2—:‘:Ð@ÓAÀ:ÐM€Là$. B§J¡J¨yÓ$9¸7Ð$BÐ!ð !�b—j‘jÐ!4Ó5°wÐ>€OØ&˜BŸJ™J uÓ-¨wÐ6ÐØ&˜BŸJ™J uÓ-¨wÐ6ÐØ&˜BŸJ™J uÓ-¨wÐ6ÐØ&˜BŸJ™J uÓ-¨wÐ6ÐØ&˜BŸJ™J uÓ-¨wÐ6ÐØ&˜BŸJ™J uÓ-¨wÐ6Ðð #˜Ÿ
™
 5Ó)¨6Ð1Ðð "�r—z‘z $Ó'¨Ð,ÐØ�R—Z‘Z Ó% sÐ*€Nð '1 b§j¡j°Ó&6¸Ð&>Ð#à&˜BŸJ™J {Ó3°XÐ=Ðð �—‘˜JÓ'¨Ð1€Jð �B—J‘J˜Ó/°Ð9€MØ�B—J‘J˜wÓ'¨Ð1€MØ�B—J‘J˜wÓ'¨Ð1€MØ�B—J‘J˜wÓ'¨Ð1€MØ�B—J‘J˜wÓ'¨Ð1€MØ�B—J‘J˜wÓ'¨Ð1€MØ�B—J‘J˜wÓ'¨Ð1€MØ�B—J‘J˜wÓ'¨Ð1€MØ�B—J‘J˜wÓ'¨Ð1€Mð  �R—Z‘ZÐ 0Ó1°=Ð@€NØ�R—Z‘Z Ó/°Ð<€NØ�R—Z‘Z Ó0°,Ð>€NØ�R—Z‘Z Ó0°,Ð>€NØ�R—Z‘Z Ó0°,Ð>€NØ�R—Z‘Z Ó0°,Ð>€NØ�R—Z‘ZÐ 0Ó1°=Ð@€NØ�R—Z‘Z Ó/°Ð<€NØ�R—Z‘Z Ó0°,Ð>€NØ�R—Z‘Z Ó0°,Ð>€Nð %˜"Ÿ*™* VÓ,¨dÐ2ÐØ$˜"Ÿ*™* VÓ,¨cÐ1ÐØ$˜"Ÿ*™* VÓ,¨cÐ1Ðð "�r—z‘z $Ó'¨Ð1ÐØ�"—*‘*˜UÓ# YÐ.€KØ * §
¡
¨4Ó 0°'Ð 9ÐØ!+ §¡¨DÓ!1°7Ð!:ÐØ$˜"Ÿ*™* UÓ+¨YÐ6ÐØ$˜"Ÿ*™* UÓ+¨YÐ6ÐØ!+ §¡¨EÓ!2°HÐ!<ÐØ", "§*¡*¨TÓ"2°HÐ"<Ðà�B—J‘JÐ=×DÑDÈ7ÐDÓSÓTÐV`Ð`€Màˆ�
‰
Ð5×<Ñ<À7ÐPSÐ<ÓTÓUØð€Mð �B—J‘JÐ<×CÑCÈ'ÐCÓRÓSÐU_Ð_€MØ�B—J‘JÐ;×BÑBÈÐBÓQÓRÐT]Ð]€MØ�B—J‘JÐ3×:Ñ:¸sÐ:ÓCÓDÀiÐO€Mð 	ØØØØð#Ðð "�r—z‘zÐ"@×"GÑ"GÈgÐ"GÓ"VÓWÐYcÐcÐØ!�r—z‘zÐ"?×"FÑ"FÈWÐ"FÓ"UÓVÐXbÐbÐØ!�r—z‘zÐ"?×"FÑ"FÈWÐ"FÓ"UÓVÐXbÐbÐØ!�r—z‘zÐ">×"EÑ"EÈGÐ"EÓ"TÓUÐW`Ð`Ðð 	ØØØð	!Ðð )˜bŸj™j¨Ó/°Ð6Ðà(˜bŸj™j¨Ó3°WÐ<Ðà .ÐØ FÐØ FÐØ JÐØ RÐð3Øð3àð3ð 	ð3ð 	ð	3ð
 	ð3ð 	ð3ð 	ð3ð 	ð3ð 	ð3ð 	ð3ð 	ð3ð 	ð3ð 	ð3ð 	ð3ð 	ð3ð  	ð!3ð" 	ð#3ð$ 	&ð%3ð& 	ð'3ð( 	ð)3ð* 	ð+3ð, 	ð-3ð. 	ð/3ð0 	ð13ð2 	ð33ð4 	ð53ð6 	ð73ð8 	ð93ð: 	(ð;3ð< 	ð=3ð> 	ð?3ð@ 	ðA3ðB 	ðC3ðD 	ðE3ðF 	ðG3ðH 	ðI3ðJ 	ðK3ðL 	ðM3ðN 	ðO3ðP 	ðQ3ðR 	ðS3ðT 	ðU3ðV 	ðW3ðX 	ðY3ðZ 	ð[3ð\ 	ð]3ð^ 	ð_3ð` 	ða3ðb 	ðc3ðd 	ðe3Ððl 	ØØØØØØ!Ø"ØØðÐð 	ØØ!Ø"ØØØ"Ø#ð	 Ðð 	"Ø!Ø!Ø!Ø!ð ÐòÐõ/òbò/ò;ò>òAò' òRó
4ð(  %ØØØ÷m4rd   r	   c                   óH  ‡ — e Zd ZdZ edj                  ej                  d«      «      «      Z edj                  ej                  d«      «      «      Z	 edj                  ej                  d«      «      «      Z
 ej                  d«      dfZ ej                  d«      d	fZ ej                  d
«      dfZ ej                  d«      dfZ ej                  d«      dfZ ej                  d«      dfZ ej                  d«      dfZ ej                  d«      dfZ ej                  d«      dfZ ej                  d«      dfZ ej                  d«      dfZ ej                  d«      dfZ ej                  d«      dfZeeeeeeeeeeegZg d¢Zg d¢Zg d¢Z ej                  d j=                  dj                  e«      dj                  e«      dj                  e«      «      «      Z ej                  d!j=                  e
«      «      Z  ej                  d"j=                  e	«      «      Z! ej                  d#j=                  e	«      «      Z" ej                  d$j=                  e	«      «      Z# ej                  d%«      Z$ ej                  d&«      Z%d+ˆ fd'„	Z&d(„ Z'd,d)„Z(d,d*„Z)ˆ xZ*S )-ÚMosesDetokenizerz›
    This is a Python port of the Moses Detokenizer from
    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/detokenizer.perl

    r
   r   r   r   z \@\-\@ ú-z {2,}r   r%   ú|r'   r&   r)   r(   r+   r"   r*   ú'r,   ú[r.   r-   r$   r#   z&bar;z&bra;z&ket;)6ÚNÚnÚAÚaõ   Ã„õ   Ã¤ÚssaÚSsau   ssÃ¤u   SsÃ¤Ústau   stÃ¤ÚStau   StÃ¤ÚhunÚHunÚhynÚHynÚhanr:   õ   hÃ¤nu   HÃ¤nu   hÃ¶nu   HÃ¶nÚunÚUnÚynÚYnÚanÚAnu   Ã¤nu   Ã„nu   Ã¶nu   Ã–nÚseenÚSeenÚllaÚLlau   llÃ¤u   LlÃ¤ÚltaÚLtau   ltÃ¤u   LtÃ¤ÚlleÚLleÚksiÚKsiÚkseÚKseÚttaÚTtaÚineÚIne)ÚniÚsiÚmmeÚnneÚnsa)	r7   u   kÃ¶r  r  Úpau   pÃ¤Úkaanu   kÃ¤Ã¤nÚkinz^({})({})?({})$u   ^[{}\(\[\{{\Â¿\Â¡]+$z^['][{}]z[{}][']$z^[{}]z^[\,\.\?\!\:\;\\\%\}\]\)]+$u   ^[\'\"â€žâ€œ`]+$c                 ó8   •— t         t        | �  «        || _        y rj   )r=   r  r>   r?   )rT   r?   r[   s     €r\   r>   zMosesDetokenizer.__init__¥  s   ø€ ÜÔ Ñ.Ô0Øˆ�	rd   c                 óR   — | j                   D ]  \  }}|j                  ||«      }Œ |S rj   )ÚMOSES_UNESCAPE_XML_REGEXESr^   r„   s       r\   Úunescape_xmlzMosesDetokenizer.unescape_xml©  s1   € Ø$(×$CÑ$Cò 	2Ñ ˆF�LØ—:‘:˜l¨DÓ1‰Dð	2àˆrd   c                 ó	  — dj                  dj                  |«      «      }t        |«      }| j                  \  }}|j	                  ||«      }|r| j                  |«      }ddddddœ}d}d}	|j                  «       }t        t        |«      «      D �]¶  \  }
}t        |d   «      r9| j                  dk7  r*|
dkD  rt        ||
dz
     d   «      r|	|z  }	n|	||z   z  }	d}ŒN| j                  j                  |«      r|	||z   z  }	d}Œt| j                  j                  |«      r2| j                  d	k(  rt        j                  d
|«      r|	dz  }	|	|z  }	d}ŒÁ| j                  dk(  r(|
dkD  r#| j                  j                  |«      r|	|z  }	d}Œø| j                  dk(  rV|
dkD  rQt        j                  d|d   «      r8t        j                  d|d   «      rt        j                  d|«      r	|	|z  }	d}�Œ]| j                  dv rY|
t!        |«      dz
  k  rH| j"                  j                  |«      r-| j$                  j                  ||
dz      «      r|	||z   z  }	d}�ŒÄ| j                  dk(  r”|
t!        |«      dz
  k  rƒ| j"                  j                  |«      rht        j                  d||
dz      «      rLt        j                  d||
dz      t        j&                  «      r!|	||z   ||
dz      z   z  }	t)        |d«       d}�Œg| j*                  j                  |«      rÙ|}t        j                  d|«      rd}|j-                  |d«      ||<   | j                  dk(  r
|dk(  rd||<   | j                  dk(  r
|dk(  rd||<   ||   dz  dk(  rW| j                  dk(  r/|dk(  r*|
dkD  r%t        j                  d||
dz
     «      r	|	|z  }	d}�Œ,|	||z   z  }	d}||xx   dz  cc<   �ŒE|	|z  }	d}||xx   dz  cc<   �Œ[| j                  dk(  rCt        j                  d||
dz
     «      r'| j.                  j                  |«      r|	||z   z  }	d}�Œ­|	||z   z  }	d}�Œ¹ | j0                  \  }}|j	                  ||	«      }	|	j3                  «       }	|r|	S |	j                  «       S )z«
        Python port of the Moses detokenizer.
        :param tokens: A list of strings, i.e. tokenized text.
        :type tokens: list(str)
        :return: str
        z {} r   r   )r
  r"   z``ú`z''r
   r7   rv   éÿÿÿÿr–   z^[\?\!\:\;\\\%]$r•   Úcsz^[0-9]+$éþÿÿÿz^[.,]$)r–   r—   Úgaé   r”   u   ^[-â€“]$z^li$|^mail.*Nu   ^[â€žâ€œâ€�]+$r"   u   â€žu   â€œr
  z[s]$Úfiz:$)rP   rK   rJ   rR   r^   r=  rw   ry   Úiterr   r?   ÚIS_CURRENCY_SYMBOLr_   ÚIS_PUNCTrN   ÚIS_ENGLISH_CONTRACTIONrx   ÚIS_FRENCH_CONRTACTIONÚSTARTS_WITH_ALPHArš   ÚnextÚIS_OPEN_QUOTEÚgetÚFINNISH_REGEXÚ	ONE_SPACErB   )rT   r{   r‹   Úunescapera   r…   r†   Úquote_countsÚprepend_spaceÚdetokenized_textr}   r~   Únormalized_quos                r\   r®   zMosesDetokenizer.tokenize®  s¿  € ð �~‰~˜cŸh™h vÓ.Ó/ˆä�4‹yˆà#×;Ñ;Ñˆ�Ø�z‰z˜,¨Ó-ˆÙà×$Ñ$ TÓ*ˆDà Q¨a°a¸qÑAˆð ˆØÐØ—‘“ˆä!¤$ v£,Ó/ó s	$‰HˆAˆuä�e˜A‘hÔ D§I¡I°Ò$5à�q’5œV F¨1¨q©5¡M°"Ñ$5Ô6Ø$¨Ñ-Ñ$ð %¨¸Ñ(=Ñ=Ð$Ø #‘à×(Ñ(×/Ñ/°Ô6à  M°EÑ$9Ñ9Ð Ø "‘à—‘×%Ñ% eÔ,à—9‘9 Ò$¬¯©Ð3FÈÔ)NØ$¨Ñ+Ð$à  EÑ)Ð Ø #‘ð —	‘	˜TÒ!Ø˜’EØ×/Ñ/×6Ñ6°uÔ=ð ! EÑ)Ð Ø #‘ð —	‘	˜TÒ!Ø˜’EÜ—I‘IØ ¨¡ôô —I‘I˜i¨°©Ô4Ü—I‘I˜k¨5Ô1ð ! EÑ)Ð Ø #’ð —	‘	Ð/Ñ/Øœ˜V› q™Ò(Ø×.Ñ.×5Ñ5°eÔ<Ø×*Ñ*×1Ñ1°&¸¸Q¹±-Ô@ð ! M°EÑ$9Ñ9Ð Ø "’ð —	‘	˜TÒ!Øœ˜V› q™Ò(Ø×.Ñ.×5Ñ5°eÔ<Ü—I‘I˜k¨6°!°a±%©=Ô9Ü—I‘I˜o¨v°a¸!±e©}¼b¿m¹mÔLð ! M°EÑ$9¸FÀ1ÀqÁ5¹MÑ$IÑIÐ Ü�V˜TÔ"Ø "’ð ×#Ñ#×*Ñ*¨5Ô1Ø!&�Ü—9‘9Ð.°Ô6Ø%(�NØ/;×/?Ñ/?ÀÐPQÓ/R�˜^Ñ,à—9‘9 Ò$¨°%ªØ34�L Ñ0Ø—9‘9 Ò$¨°%ªØ34�L Ñ0à Ñ/°!Ñ3°qÒ8àŸ	™	 TÒ)Ø! SšLØ šEÜŸI™I g¨v°a¸!±e©}Ô=ð )¨EÑ1Ð(Ø(+šð )¨M¸EÑ,AÑAÐ(Ø(*˜Ø$ ^Ó4¸Ñ9Õ4ð %¨Ñ-Ð$Ø$'�MØ  Ó0°AÑ5Õ0ð —	‘	˜TÒ!Ü—I‘I˜e V¨A°©E¡]Ô3Ø×&Ñ&×-Ñ-¨eÔ4ð ! M°EÑ$9Ñ9Ð Ø #’ð ! M°EÑ$9Ñ9Ð Ø #’ðgs	$ðl  $Ÿ~™~Ñˆ�Ø!Ÿ:™: lÐ4DÓEÐà+×1Ñ1Ó3Ðá#-ÐÐKÐ3C×3IÑ3IÓ3KÐKrd   c                 ó(   — | j                  |||«      S )z&Duck-typing the abstract *tokenize()*.)r®   )rT   r{   r‹   rQ  s       r\   Ú
detokenizezMosesDetokenizer.detokenizeG  s   € à�}‰}˜V Z°Ó:Ð:rd   )r•   )TT)+r¯   r°   r±   r²   rJ   rK   rL   rM   r   r   r   rN   rO   rR   rP  ÚUNESCAPE_FACTOR_SEPARATORÚUNESCAPE_LEFT_ANGLE_BRACKETÚUNESCAPE_RIGHT_ANGLE_BRACKETÚUNESCAPE_DOUBLE_QUOTEÚUNESCAPE_SINGLE_QUOTEÚ UNESCAPE_SYNTAX_NONTERMINAL_LEFTÚ!UNESCAPE_SYNTAX_NONTERMINAL_RIGHTÚUNESCAPE_AMPERSANDÚ UNESCAPE_FACTOR_SEPARATOR_LEGACYÚ'UNESCAPE_SYNTAX_NONTERMINAL_LEFT_LEGACYÚ(UNESCAPE_SYNTAX_NONTERMINAL_RIGHT_LEGACYr<  ÚFINNISH_MORPHSET_1ÚFINNISH_MORPHSET_2ÚFINNISH_MORPHSET_3rP   rO  rG  rI  rJ  rK  rH  rM  r>   r=  r®   rW  r  r  s   @r\   r  r    sº  ø„ ññ �"—'‘'˜,×,Ñ,¨YÓ7Ó8Ó9€GÙ�"—'‘'˜,×,Ñ,¨YÓ7Ó8Ó9€GÙˆr�w‰w�|×)Ñ)¨&Ó1Ó2Ó3€Dà(˜bŸj™j¨Ó5°tÐ;Ðð �—
‘
˜8Ó$ cÐ)€Ið !+ §
¡
¨9Ó 5°tÐ ;ÐØ", "§*¡*¨WÓ"5°tÐ";ÐØ#- 2§:¡:¨gÓ#6¸Ð#<Ð Ø&˜BŸJ™J yÓ1°4Ð7ÐØ&˜BŸJ™J yÓ1°4Ð7ÐØ'1 r§z¡z°(Ó';¸TÐ'AÐ$Ø(2¨¯
©
°8Ó(<¸dÐ(BÐ%Ø#˜Ÿ™ HÓ-¨tÐ3Ðà'1 r§z¡z°(Ó';¸TÐ'AÐ$Ø.8¨b¯j©j¸Ó.BÀDÐ.HÐ+Ø/9¨r¯z©z¸(Ó/CÀTÐ/IÐ,ð 	)Ø!Ø#Ø$Ø/Ø0ØØØ(Ø)Øð"Ðò7Ðòr ;Ðò
Ðð �B—J‘JÐ1×8Ñ8Ø�‰Ð#Ó$Ø�‰Ð#Ó$Ø�‰Ð#Ó$ó ó €Mð $˜Ÿ™Ð$;×$BÑ$BÀ4Ó$HÓIÐà'˜RŸZ™Z¨×(:Ñ(:¸7Ó(CÓDÐà&˜BŸJ™J {×'9Ñ'9¸'Ó'BÓCÐà"˜Ÿ
™
 8§?¡?°7Ó#;Ó<Ðàˆr�z‰zÐ8Ó9€Hà�B—J‘JÐ6Ó7€Mõòó
WL÷r;rd   r  )rN   Úsacremoses.corpusr   r   Úsacremoses.utilr   Úsacremoses.indicr   r   rL   r@   Úobjectr	   r  Ú__all__© rd   r\   ú<module>rl     sQ   ðó 
å *Ý 1Ý "ß ,á‹~€Ù*Ó,Ð ôM4�Vô M4ô`j;�vô j;ðZ	 Ð/Ð
0�rd   