Ë
    ›¾j™"  ã                   óÔ  — d dl Z d dlmZ d dlmZ d dlmZ d dlZd dlmZm	Z	 d dl
mZmZ d dlmZ d dlmZ  ed	d
g¬«      Z ej&                  de¬«       ej(                  dddd¬«       ej(                  dddd¬«       ej(                  dddd¬«       ej(                  ddddd¬«       ej*                  «       d „ «       «       «       «       «       «       Z eej0                  j3                  d!«      d    «      d"k  rej4                  nej6                  Z e«       d#„ «       Zd$„ Zd%„ Zej?                  d&«       ej(                  d'd(ddd)¬*«       ej(                  d+d,ddd-¬*«       ej(                  d.d/d0¬1«       ej(                  d2d3d4¬1«      ed5„ «       «       «       «       «       «       Z ej?                  d6«       ej(                  d7d,ddd8¬*«      ed9„ «       «       «       Z!ej?                  d:«       ej(                  d;dddd<¬*«       ej(                  d=d>ddd?¬*«       ej(                  d@d/dddA¬*«       ej(                  dBd3dddC¬*«      edD„ «       «       «       «       «       «       Z"ej?                  dE«       ej(                  dFdGddH¬I«       ej(                  dJd(dddK¬*«       ej(                  dLd/dddM¬*«      edN„ «       «       «       «       «       Z#ej?                  dO«       ej(                  dFdGddP¬I«       ej(                  dJd(dddK¬*«       ej(                  dLd/dddQ¬*«      edR„ «       «       «       «       «       Z$ej?                  dS«       ej(                  dTd(dddU¬*«      edV„ «       «       «       Z%y)Wé    N)Údeepcopy©Úpartial©Úupdate_wrapper)ÚMosesTokenizerÚMosesDetokenizer)ÚMosesTruecaserÚMosesDetruecaser)ÚMosesPunctNormalizer©Úparallelize_preprocessz-hz--help)Úhelp_option_namesT)ÚchainÚcontext_settingsz
--languagez-lÚenz+Use language specific rules when tokenizing)ÚdefaultÚhelpz--processesz-jé   zNo. of processes.z
--encodingz-eÚutf8zSpecify encoding of file.z--quietz-qFzDisable progress bar.)Úis_flagr   r   c                  ó   — y ©N© )ÚlanguageÚencodingÚ	processesÚquiets       úX/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/sacremoses/cli.pyÚclir       s   € ð 	ó    ú.é   c                 óÎ   — t        j                  d|¬«      5 }|}| D ]  } |t        |«      fi |¤Ž}Œ |r|D ]  }t        j                  |«       Œ d d d «       y # 1 sw Y   y xY w)NÚstdin©r   )ÚclickÚget_text_streamÚlistÚecho)Ú
processorsr   ÚkwargsÚfinÚiteratorÚprocÚitems          r   Úprocess_pipeliner1   $   sn   € ä	×	Ñ	˜w°Ô	:ð !¸cØˆØò 	6ˆDÙœD ›NÑ5¨fÑ5‰Hð	6áØ ò !�Ü—
‘
˜4Õ ð!÷!÷ !ñ !ús   ˜:AÁA$c                 ó(   ‡ — ˆ fd„}t        |‰ fi |¤ŽS )z\Helper decorator to rewrite a function so that
    it returns another function from it.
    c                  ó&   •— ˆfd„}t        |fi | ¤ŽS )Nc                 ó   •—  ‰| fi |¤ŽS r   r   )Ústreamr,   Úfs     €r   Ú	processorz.processor.<locals>.new_func.<locals>.processor5   s   ø€ Ù�VÑ&˜vÑ&Ð&r!   r   )r,   r7   r6   s     €r   Únew_funczprocessor.<locals>.new_func4   s   ø€ ô	'ô �yÑ+ FÑ+Ð+r!   r   )r6   r,   r8   s   `  r   r7   r7   /   s   ø€ ô
,ô ˜( AÑ0¨Ñ0Ð0r!   c              #   ól   K  — |dk(  r| D ]  } ||«      –— Œ y t        || || ¬«      D ]  }|–— Œ y ­w)Nr   )Úprogress_barr   )r.   Úfuncr   r   ÚlineÚoutlines         r   Úparallel_or_notr>   =   sN   è ø€ Ø�A‚~Øò 	ˆDÙ�t“*Óñ	ô .Ø�(˜I¸°Yô
ò 	ˆGð ‹Mñ	ùs   ‚24Útokenizez--aggressive-dash-splitsz-azTriggers dash split rules.)r   r   r   z--xml-escapez-xz"Escape special characters for XML.z--protected-patternsz-pzXSpecify file with patters to be protected in tokenisation. Special values: :basic: :web:)r   z--custom-nb-prefixesz-czjSpecify a custom non-breaking prefixes file, add prefixes to the default ones from the specified language.c                 ó\  — t        ||¬«      }|re|dk(  r|j                  }nS|dk(  r|j                  }nAt        |d¬«      5 }	|	j	                  «       D �
cg c]  }
|
j                  «       ‘Œ }}
d d d «       t        |j                  d|||¬«      }t        | |||«      S c c}
w # 1 sw Y   Œ6xY w)N)ÚlangÚ custom_nonbreaking_prefixes_filez:basic:z:web:r   r&   T)Ú
return_strÚaggressive_dash_splitsÚescapeÚprotected_patterns)	r   ÚBASIC_PROTECTED_PATTERNSÚWEB_PROTECTED_PATTERNSÚopenÚ	readlinesÚstripr   r?   r>   )r.   r   r   r   Ú
xml_escaperD   rF   Úcustom_nb_prefixesÚmosesr-   ÚpatternÚmoses_tokenizes               r   Útokenize_filerQ   M   sÀ   € ôH ØÐ8Jô€Eñ Ø Ò*Ø!&×!?Ñ!?ÑØ 7Ò*Ø!&×!=Ñ!=ÑäÐ(°6Ô:ð V¸cØEHÇ]Á]Ã_Ö%U¸' g§m¡m¥oÐ%UÐ"Ð%U÷Vô Ø�‰ØØ5ØØ-ô€Nô ˜8 ^°YÀÓFÐFùò &V÷Vð Vús   ÁB"ÁBÁ+B"ÂB"Â"B+Ú
detokenizez--xml-unescapez$Unescape special characters for XML.c                 ó¨   — t        |¬«      }t        |j                  d|¬«      }t        t	        t        t        j                  | «      «      |||«      S )N)rA   T)rC   Úunescape)r	   r   rR   r>   r)   ÚmapÚstrÚsplit)r.   r   r   r   Úxml_unescaperN   Úmoses_detokenizes          r   Údetokenize_filerZ   �   sJ   € ô   (Ô+€EÜ˜u×/Ñ/¸DÈ<ÔXÐÜÜŒS”—‘˜HÓ%Ó&Ð(8¸)ÀUóð r!   Ú	normalizez--normalize-quote-commasz Normalize quotations and commas.z--normalize-numbersz-dzNormalize number.z--replace-unicode-punctsz2Replace unicode punctuations BEFORE normalization.z--remove-control-charsz.Remove control characters AFTER normalization.c                 óh   — t        |||||¬«      }t        |j                  «      }	t        | |	||«      S )N)Únorm_quote_commasÚnorm_numbersÚpre_replace_unicode_punctÚpost_remove_control_chars)r   r   r[   r>   )
r.   r   r   r   Únormalize_quote_commasÚnormalize_numbersÚreplace_unicode_punctsÚremove_control_charsrN   Úmoses_normalizes
             r   Únormalize_filerf   ©   s?   € ôH !ØØ0Ø&Ø"8Ø"6ô€Eô ˜eŸo™oÓ.€OÜ˜8 _°iÀÓGÐGr!   ztrain-truecasez--modelfilez-mzFilename to save the modelfile.)Úrequiredr   z--is-asrz)A flag to indicate that model is for ASR.z--possibly-use-first-tokenz*Use the first token as part of truecasing.c                 ój   — t        |¬«      }|j                  | ||| ¬«      }|j                  |«       y )N©Úis_asr©Úpossibly_use_first_tokenr   r:   )r
   ÚtrainÚ
save_model)	r.   r   r   r   Ú	modelfilerj   rl   rN   Úmodels	            r   Útrain_truecaserrq   Ý   s?   € ô,  &Ô)€Eà�K‰KØØ!9ØØ�ið	 ó €Eð 
×Ñ�YÕr!   Útruecasez$Filename to save/load the modelfile.z1Use the first token as part of truecase training.c                 ó   — t         j                  j                  |«      s>t        | «      }t	        |¬«      }|j                  |||| ¬«      }	|j                  |«       t	        ||¬«      }
t        |
j                  d¬«      }t        | |||«      S )Nri   rk   )Ú	load_fromrj   T)rC   )
ÚosÚpathÚisfiler   r
   rm   rn   r   rr   r>   )r.   r   r   r   ro   rj   rl   Úiterator_copyÚ	truecaserrp   rN   Úmoses_truecases               r   Útruecase_filer{     s†   € ô. �7‰7�>‰>˜)Ô$Ü  Ó*ˆÜ"¨&Ô1ˆ	Ø—‘ØØ%=ØØ#˜)ð	  ó 
ˆð 	×Ñ˜YÔ'ä Y°vÔ>€EÜ˜UŸ^™^¸Ô=€NÜ˜8 ^°YÀÓFÐFr!   Ú
detruecasez--is-headlinezWhether the file are headlines.c                 ób   — t        «       }t        |j                  d|¬«      }t        | |||«      S )NT)rC   Úis_headline)r   r   r|   r>   )r.   r   r   r   r~   rN   Úmoses_detruecases          r   Údetruecase_filer€   /  s8   € ô Ó€EÜØ×Ñ T°{ôÐô ˜8Ð%5°yÀ%ÓHÐHr!   )&ru   Úcopyr   Ú	functoolsr   r   r'   Úsacremoses.tokenizer   r	   Úsacremoses.truecaser
   r   Úsacremoses.normalizer   Úsacremoses.utilr   ÚdictÚCONTEXT_SETTINGSÚgroupÚoptionÚversion_optionr    ÚintÚ__version__rW   ÚresultcallbackÚresult_callbackr1   r7   r>   ÚcommandrQ   rZ   rf   rq   r{   r€   r   r!   r   ú<module>r‘      sC  ðó 
Ý Ý Ý $ã ç @ß @Ý 5Ý 2á¨4°Ð*:Ô;Ð ð €‡��4Ð*:Ô;Ø€‡�Ø�$ Ð+Xôð €‡�ˆm˜T¨1Ð3FÔGØ€‡�ˆl˜D¨&Ð7RÔSØ€‡�Øˆt˜T¨5Ð7Nôð €×ÑÓñ	ó óó Tó Hóó <ð	ñ ),¨E×,=Ñ,=×,CÑ,CÀCÓ,HÈÑ,KÓ(LÈqÒ(P�#×$Ò$ÐVY×ViÑVi€áÓñ!ó ð!ò1òð  ‡�ˆZÓØ€‡�ØØØØØ	%ôð €‡�ØØØØØ	-ôð €‡�ØØØ	côð
 €‡�ØØØ	uôð
 ñGó óóóóó ð4GðL ‡�ˆ\ÓØ€‡�ØØØØØ	/ôð ñó óó ðð& ‡�ˆ[ÓØ€‡�ØØØØØ	+ôð €‡�Ø˜4¨°tÐBUôð €‡�ØØØØØ	=ôð €‡�ØØØØØ	9ôð ñHó óóóóó ð4Hð4 ‡�ÐÓØ€‡�Ø�4 $Ð-Nôð €‡�ØØØØØ	4ôð €‡�Ø ØØØØ	5ôð ñ ó óóóó ð& ð& ‡�ˆZÓØ€‡�Ø�4 $Ð-Sôð €‡�ØØØØØ	4ôð €‡�Ø ØØØØ	<ôð ñGó óóóó ð&Gð2 ‡�ˆ\ÓØ€‡�ØØØØØ	*ôð ñIó óó ñIr!   