Ë
    ›¾j,G  ã                   óž   — d dl mZ d dlZd dlmZmZ d dlmZ d dlm	Z	 d dl
mZ d dlmZmZ  e«       Z G d„ d	e«      Z G d
„ de«      Zd	dgZy)é    )Úprint_functionN)ÚdefaultdictÚCounter)Úpartial)Úchain)ÚPerluniprops)Úparallelize_preprocessÚgrouperc                   óˆ  ‡ — e Zd ZdZ edj                  ej                  d«      «      «      Z edj                  ej                  d«      «      «      Z	 edj                  ej                  d«      «      «      Z
dˆ fd„	Zdd„Z	 	 	 	 dd„Z	 	 	 	 dd„Z	 	 	 	 dd	„Z	 	 	 	 dd
„Zdd„Zdd„Zed„ «       Zd„ Zd„ Zd„ Zd„ Zˆ xZS )ÚMosesTruecaserzò
    This is a Python port of the Moses Truecaser from
    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/recaser/train-truecaser.perl
    https://github.com/moses-smt/mosesdecoder/blob/master/scripts/recaser/truecase.perl
    Ú ÚLowercase_LetterÚUppercase_Letterc                 ób  •— t         t        | �  «        t        j                  dj                  | j                  | j                  | j                  «      «      | _	        t        j                  d«      | _
        h d£| _        h d£| _        || _        || _        |r| j                  |«      | _        yy)a.  
        :param load_from:
        :type load_from:

        :param is_asr: A flag to indicate that model is for ASR. ASR input has
            no case, make sure it is lowercase, and make sure known are cased
            eg. 'i' to be uppercased even if i is known.
        :type is_asr: bool
        z[{}{}{}]z(<.*(?<=>))(.*)((?=</)[^>]*>)¾   ú!ú.ú:ú?¾   ú&#91;ú&#93;ú&apos;ú&quot;ú"ú'ú(ú[N)Úsuperr   Ú__init__ÚreÚcompileÚformatr   r   ÚTitlecase_LetterÚSKIP_LETTERS_REGEXÚXML_SPLIT_REGXÚSENT_ENDÚDELAYED_SENT_STARTÚencodingÚis_asrÚ_load_modelÚmodel)ÚselfÚ	load_fromr*   r)   Ú	__class__s       €ú]/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/sacremoses/truecase.pyr    zMosesTruecaser.__init__   s™   ø€ ô 	Œn˜dÑ,Ô.ä"$§*¡*Ø×ÑØ×%Ñ% t×'<Ñ'<¸d×>SÑ>Sóó#
ˆÔô !Ÿj™jÐ)HÓIˆÔâ,ˆŒò	#
ˆÔð !ˆŒàˆŒÙØ×)Ñ)¨)Ó4ˆD�Jð ó    c                 ó‚  — d}g }t        |«      D ]¬  \  }}t        j                  d|«      rŒ|| j                  v rŒ,|s|| j                  v rd}Œ?| j
                  j                  |«      sd}Œ]d}|sd}n|r|d   j                  «       rd}n|dk(  rd}d}|dkD  sŒ‹|j                  |j                  «       ||f«       Œ® |S )z�
        This function checks through each tokens in a sentence and returns the
        appropriate weight of each surface token form.
        Tú(<\S[^>]*>)Fr   é   gš™™™™™¹?)	Ú	enumerater!   Úsearchr(   r'   r%   ÚislowerÚappendÚlower)r-   ÚtokensÚpossibly_use_first_tokenÚis_first_wordÚtruecase_weightsÚiÚtokenÚcurrent_word_weights           r0   Úlearn_truecase_weightsz%MosesTruecaser.learn_truecase_weightsD   sí   € ð ˆØÐÜ! &Ó)ò #	U‰HˆAˆuä�y‰y˜¨Ô/Øà˜$×1Ñ1Ñ1Øñ ! U¨d¯m©mÑ%;Ø $�Øà×*Ñ*×1Ñ1°%Ô8Ø %�Øð #$ÐÙ Ø&'Ñ#ñ *ð ˜‘8×#Ñ#Ô%Ø*+Ñ'Ø˜!’VØ*-Ð'à!ˆMà" QÓ&Ø ×'Ñ'¨¯©«¸Ð?RÐ(SÕTðG#	UðH  Ðr1   c                 óø   — t        t        «      }t        | j                  |¬«      }t	        t        ||||¬«      Ž }|D ]  \  }	}
}||	   |
xx   |z  cc<   Œ |r| j                  ||«       | j                  |«      S )aN  
        :param document_iterator: The input document, each outer list is a sentence,
                          the inner list is the list of tokens for each sentence.
        :type document_iterator: iter(list(str))

        :param possibly_use_first_token: When True, on the basis that the first
            word of a sentence is always capitalized; if this option is provided then:
            a) if a sentence-initial token is *not* capitalized, then it is counted, and
            b) if a capitalized sentence-initial token is the only token of the segment,
               then it is counted, but with only 10% of the weight of a normal token.
        :type possibly_use_first_token: bool

        :returns: A dictionary of the best, known objects as values from `_casing_to_model()`
        :rtype: {'best': dict, 'known': Counter}
        )r;   ©Úprogress_bar)r   r   r   rA   r   r	   Ú_save_model_from_casingÚ_casing_to_model)r-   Údocument_iteratorÚsave_tor;   Ú	processesrD   ÚcasingÚtrain_truecaserÚtoken_weightsÚlowercase_tokenÚsurface_tokenÚweights               r0   Ú_trainzMosesTruecaser._trainr   s–   € ô. œWÓ%ˆÜ!Ø×'Ñ'Ø%=ô
ˆô Ü#ØÐ!2°IÈLôð
ˆð 7Dò 	=Ñ2ˆO˜]¨FØ�?Ñ# MÓ2°fÑ<Ô2ð	=ñ Ø×(Ñ(¨°Ô9Ø×$Ñ$ VÓ,Ð,r1   c                 ó^   — d| _         | j                  |||||¬«      | _         | j                   S )z\
        Default duck-type of _train(), accepts list(list(str)) as input documents.
        NrC   )r,   rP   )r-   Ú	documentsrH   r;   rI   rD   s         r0   ÚtrainzMosesTruecaser.trainœ   s;   € ð ˆŒ
Ø—[‘[ØØØ$ØØ%ð !ó 
ˆŒ
ð �z‰zÐr1   c                 óä   — t        || j                  ¬«      5 }d„ |j                  «       D «       }ddd«       d| _        | j	                  ||||¬«      | _        | j                  S # 1 sw Y   Œ7xY w)zj
        Duck-type of _train(), accepts a filename to read as a `iter(list(str))`
        object.
        ©r)   c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­w©N©Úsplit©Ú.0Úlines     r0   ú	<genexpr>z1MosesTruecaser.train_from_file.<locals>.<genexpr>¿   s   è ø€ ò !Ø!%�—
‘
—ñ!ùó   ‚NrC   )Úopenr)   Ú	readlinesr,   rP   )r-   ÚfilenamerH   r;   rI   rD   ÚfinrG   s           r0   Útrain_from_filezMosesTruecaser.train_from_file±   sx   € ô �( T§]¡]Ô3ð 	°sñ!Ø),¯©«ô!Ð÷	ð
 ˆŒ
Ø—[‘[ØØØ$ØØ%ð !ó 
ˆŒ
ð �z‰zÐ÷	ð 	ús   ˜A&Á&A/c                 óŒ   — d„ |j                  «       D «       }d| _        | j                  |||||¬«      | _        | j                  S )zm
        Duck-type of _train(), accepts a file object to read as a `iter(list(str))`
        object.
        c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­wrW   rX   rZ   s     r0   r]   z8MosesTruecaser.train_from_file_object.<locals>.<genexpr>Ù   s   è ø€ ò 
Ø!ˆD�J‰J�Lñ
ùr^   NrC   )r`   r,   rP   )r-   Úfile_objectrH   r;   rI   rD   rG   s          r0   Útrain_from_file_objectz%MosesTruecaser.train_from_file_objectÌ   sT   € ñ
Ø%0×%:Ñ%:Ó%<ô
Ðð ˆŒ
Ø—[‘[ØØØ$ØØ%ð !ó 
ˆŒ
ð �z‰zÐr1   c                 óî  — t        d«      }t        | d«      sJ |«       ‚d}g }| j                  |«      }t        |«      D �]  \  }}	t	        j
                  d|	«      r|j                  |	«       Œ/|	dk(  s|	j                  d«      r|j                  |	«       ŒWt	        j
                  d|	«      j                  «       \  }	}
| j                  r|	j                  «       }	| j                  d   j                  |	j                  «       d«      }|r|r|}	n|r|	| j                  d	   v rn|r|}	|	|
z   }	|j                  |	«       |	| j                  v rd}�Œ|	| j                  vs�Œd
}�Œ" |rdj                  |«      S |S )a  
        Truecase a single sentence / line of text.

        :param text: A single string, i.e. sentence text.
        :type text: str

        :param use_known: Use the known case if a word is a known word but not the first word.
        :type use_known: bool
        zV
Use Truecaser.train() to train a model.
Or use Truecaser('modefile') to load a model.r,   Tr3   ú|z^([^\|]+)(.*)ÚbestNÚknownFú )ÚstrÚhasattrÚ	split_xmlr5   r!   r6   r8   Ú
startswithÚgroupsr*   r9   r,   Úgetr'   r(   Újoin)r-   ÚtextÚ
return_strÚ	use_knownÚcheck_model_messager<   Útruecased_tokensr:   r>   r?   Úother_factorsÚ	best_cases               r0   ÚtruecasezMosesTruecaser.truecaseæ   sƒ  € ô "ð<ó
Ðô �t˜WÔ%Ð:Ð':Ó:Ð%àˆØÐØ—‘ Ó%ˆô " &Ó)ó )	&‰HˆAˆuô �y‰y˜¨Ô/Ø ×'Ñ'¨Ô.Øð
 ˜Š|˜u×/Ñ/°Ô4Ø ×'Ñ'¨Ô.Øô $&§9¡9Ð-=¸uÓ#E×#LÑ#LÓ#NÑ ˆE�=ð �{Š{ØŸ™›�ð Ÿ
™
 6Ñ*×.Ñ.¨u¯{©{«}¸dÓCˆIá¡Ø!‘Ù˜u¨¯
©
°7Ñ(;Ñ;ØÙØ!�ð ˜MÑ)ˆEà×#Ñ# EÔ*ð ˜Ÿ™Ñ%Ø $’Ø˜d×5Ñ5Ó5Ø %’ðS)	&ñX .8ˆs�x‰xÐ(Ó)ÐMÐ=MÐMr1   c              #   óà   K  — t        || j                  ¬«      5 }|D ]8  }| j                  |j                  «       «      }|rdj	                  |«      n|–— Œ: 	 d d d «       y # 1 sw Y   y xY w­w)NrU   rl   )r_   r)   r{   Ústriprs   )r-   ra   ru   rb   r\   rx   s         r0   Útruecase_filezMosesTruecaser.truecase_file*  si   è ø€ Ü�( T§]¡]Ô3ð 	U°sØò U�Ø#'§=¡=°·±³Ó#>Ð á4>�c—h‘hÐ/Ô0ÐDTÓTñU÷	U÷ 	Uñ 	Uüs   ‚A.š>A"Á	A.Á"A+Á'A.c                 ód  — | j                  «       } g }| �ršt        j                  d| «      }t        j                  d| «      }t        j                  d| «      }|rº|j                  «       \  }}t        j                  d| «      rzt	        |«      dkD  rlt        j                  d|d   «      rS|dxx   |z  cc<   t        j                  d|«      }|rB|dxx   |j                  d	«      z  cc<   |j                  d
«      }n|j                  |dz   «       |} n‚|r2|j                  |j                  d	«      «       |j                  d
«      } nN|r2|j                  |j                  d	«      «       |j                  d
«      } nt        dj                  | «      «      ‚|d   j                  «       |d<   | r�Œš|S )a
  
        Python port of split_xml function in Moses' truecaser:
        https://github.com/moses-smt/mosesdecoder/blob/master/scripts/recaser/truecaser.perl

        :param line: Input string, should be tokenized, separated by space.
        :type line: str
        z^\s*(<\S[^>]*>)(.*)$z^\s*([^\s<>]+)(.*)$z^\s*(\S+)(.*)$z^\Sr   z\|$éÿÿÿÿz^(\|+)(.*)$r4   é   rl   zERROR: huh? {})	r}   r!   r6   rq   ÚlenÚgroupr8   Ú	Exceptionr#   )r\   r:   Úhas_xmlÚ
is_non_xmlÚxml_cognatesÚpotential_xmlÚ	line_nextÚ	is_factors           r0   ro   zMosesTruecaser.split_xml1  s€  € ð �z‰z‹|ˆØˆÚä—i‘iÐ 7¸Ó>ˆGäŸ™Ð#9¸4Ó@ˆJäŸ9™9Ð%6¸Ó=ˆLÙØ+2¯>©>Ó+;Ñ(�˜yô —I‘I˜f dÔ+Ü˜F› ašÜŸ	™	 &¨&°©*Ô5à˜2“J -Ñ/“Jä "§	¡	¨.¸)Ó D�IÙ Ø˜r›
 i§o¡o°aÓ&8Ñ8›
Ø$-§O¡O°AÓ$6™	à—M‘MØ%¨Ñ+ôð !‘áØ—‘˜j×.Ñ.¨qÓ1Ô2Ø!×'Ñ'¨Ó*‘ÙØ—‘Ø ×&Ñ& qÓ)ôð $×)Ñ)¨!Ó,‘äÐ 0× 7Ñ 7¸Ó =Ó>Ð>Ø ™×)Ñ)Ó+ˆF�2‰JóK ðL ˆr1   c                 óÂ   — i }t        «       }|D ]G  }||   j                  «       }|d   d   ||<   | j                  rŒ.|dd D ]  \  }}||xx   dz  cc<   Œ ŒI |||dœ}|S )zg

        :returns: A tuple of the (best, known) objects.
        :rtype: tuple(dict, Counter)
        r   r4   N)rj   rk   rJ   )r   Úmost_commonr*   )	r-   rJ   rj   rk   Útoken_lowerr:   r?   Úcountr,   s	            r0   rF   zMosesTruecaser._casing_to_modeld  s‡   € ð ˆÜ“	ˆà!ò 		&ˆKØ˜KÑ(×4Ñ4Ó6ˆFà & q¡	¨!¡ˆD�Ñà—;“;Ø$*¨1¨2 Jò &‘L�E˜5ð ˜%“L AÑ%”Lñ&ð		&ð ¨¸Ñ@ˆØˆr1   c                 óB   — | j                  | j                  d   |«       y )NrJ   )rE   r,   )r-   ra   s     r0   Ú
save_modelzMosesTruecaser.save_modelz  s   € Ø×$Ñ$ T§Z¡Z°Ñ%9¸8ÕDr1   c                 ó¬  — t        |d| j                  ¬«      5 }|D ]¡  }t        ||   j                  «       «      }g }t	        ||   j                  «       «      D ]E  \  }\  }}|dk(  rdj                  |||«      }	ndj                  |||«      }	|j                  |	«       ŒG t        dj                  |«      d|¬«       Œ£ 	 d	d	d	«       y	# 1 sw Y   y	xY w)
a"  
        Outputs the truecaser model file in the same output format as
        https://github.com/moses-smt/mosesdecoder/blob/master/scripts/recaser/train-truecaser.perl

        :param casing: The dictionary of tokens counter from `train()`.
        :type casing: default(Counter)
        ÚwrU   r   z
{} ({}/{})z{} ({})rl   ú
)ÚendÚfileN)
r_   r)   ÚsumÚvaluesr5   rŒ   r#   r8   Úprintrs   )
r-   rJ   ra   Úfoutr?   Útotal_token_countÚtokens_countsr>   rŽ   Ú	out_tokens
             r0   rE   z&MosesTruecaser._save_model_from_casing}  sÝ   € ô �(˜C¨$¯-©-Ô8ð 
	D¸DØò 	D�Ü$'¨¨u©×(<Ñ(<Ó(>Ó$?Ð!Ø "�Ü)2°6¸%±=×3LÑ3LÓ3NÓ)Oò 4Ñ%�A‘~˜˜uØ˜A’vØ$0×$7Ñ$7¸¸uÐFWÓ$X™	à$-×$4Ñ$4°U¸EÐCTÓ$U˜	Ø!×(Ñ(¨Õ3ð4ô �c—h‘h˜}Ó-°4¸dÖCñ	D÷
	D÷ 
	Dñ 
	Dús   ™B'C
Ã
Cc                 ó’  — t        t        «      }t        || j                  ¬«      5 }|D ]v  }|j	                  «       j                  «       }t        |d«      D ]G  \  }}|j                  d«      d   j	                  d«      }t        |«      ||j                  «          |<   ŒI Œx 	 ddd«       | j                  |«      S # 1 sw Y   ŒxY w)zÄ
        Loads pre-trained truecasing file.

        :returns: A dictionary of the best, known objects as values from `_casing_to_model()`
        :rtype: {'best': dict, 'known': Counter}
        rU   r�   ú/r   z()N)
r   r   r_   r)   r}   rY   r
   Úintr9   rF   )r-   ra   rJ   rb   r\   r?   rŽ   s          r0   r+   zMosesTruecaser._load_model‘  sº   € ô œWÓ%ˆÜ�( T§]¡]Ô3ð 	>°sØò >�Ø—z‘z“|×)Ñ)Ó+�Ü$+¨D°!Ó$4ò >‘L�E˜5Ø!ŸK™K¨Ó,¨QÑ/×5Ñ5°dÓ;�EÜ36°u³:�F˜5Ÿ;™;›=Ñ)¨%Ò0ñ>ñ>÷	>ð ×$Ñ$ VÓ,Ð,÷	>ð 	>ús   §A<B=Â=C)NNÚutf8)F)NFr4   F©FF)T)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rm   rs   ÚperlunipropsÚcharsr   r   r$   r    rA   rP   rS   rc   rg   r{   r~   Ústaticmethodro   rF   r�   rE   r+   Ú__classcell__©r/   s   @r0   r   r      sü   ø„ ññ ˜2Ÿ7™7 <×#5Ñ#5Ð6HÓ#IÓJÓKÐÙ˜2Ÿ7™7 <×#5Ñ#5Ð6HÓ#IÓJÓKÐÙ˜2Ÿ7™7 <×#5Ñ#5Ð6HÓ#IÓJÓKÐõ%5óN, ðb Ø!&ØØó(-ðZ Ø!&ØØóð0 Ø!&ØØóð< Ø!&ØØóó4BNóHUð ñ0ó ð0òdò,EòDö(-r1   r   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚMosesDetruecaserc                 ó`   •— t         t        | �  «        h d£| _        h d£| _        h d£| _        y )Nr   r   >#   úal-.+úel-.+ÚaÚasÚatÚbeÚbyÚinÚisÚofÚonÚtoÚandÚanyÚforÚhisÚitsÚnotÚoffÚtheÚwasÚfromÚlastÚthanÚthisÚwereÚwillÚwithÚafterÚtheirÚwhichÚduringÚagainstÚbecauseÚbetween)r   r¬   r    r'   r(   ÚALWAYS_LOWER)r-   r/   s    €r0   r    zMosesDetruecaser.__init__¤  s,   ø€ äÔ Ñ.Ô0Ú,ˆŒò	#
ˆÔò$
ˆÕr1   c                 óx  — g }d}|j                  «       D ]R  }|r|dd j                  «       |dd z   n|}|j                  |«       || j                  v rd}ŒB|| j                  vsŒQd}ŒT |r6|D �cg c]+  }|| j
                  v r|n|dd j                  «       |dd z   ‘Œ- }}|rdj                  |«      S |S c c}w )z¼
        Detruecase the translated files from a model that learnt from truecased
        tokens.

        :param text: A single string, i.e. sentence text.
        :type text: str
        TNr4   Frl   )rY   Úupperr8   r'   r(   rÑ   rs   )r-   rt   Úis_headlineru   Úcased_tokensÚsentence_startr?   s          r0   Ú
detruecasezMosesDetruecaser.detruecaseÚ  sã   € ð ˆØˆà—Z‘Z“\ò 	'ˆEÙ5C�E˜"˜1�I—O‘OÓ%¨¨a¨b¨	Ò1ÈˆEØ×Ñ Ô&Ø˜Ÿ™Ñ%Ø!%‘Ø˜d×5Ñ5Ò5Ø!&‘ð	'ñ ð *öàð  $×"3Ñ"3Ñ3‘¸¸rÀ¸¿¹Ó9JÈUÐSTÐSUÈYÑ9VÑVðˆLð ñ
 *4ˆs�x‰x˜Ó%ÐE¸ÐEùòs   Á00B7r¡   )r¢   r£   r¤   r    r×   r©   rª   s   @r0   r¬   r¬   £  s   ø„ ô4
÷lFr1   r¬   )Ú
__future__r   r!   Úcollectionsr   r   Ú	functoolsr   Ú	itertoolsr   Úsacremoses.corpusr   Úsacremoses.utilr	   r
   r¦   Úobjectr   r¬   Ú__all__© r1   r0   ú<module>rá      sR   ðõ &ã 	ß ,Ý Ý å *ß ;ñ ‹~€ôO-�Vô O-ôdQF�vô QFðh Ð/Ð
0�r1   