Ë
    ›¾jˆ,  ã                   óú   — d dl Z d dlZd dlZddlmZ dZdZdZdZ	dZ
d	„ Zd
„ Zd„ ZdZdd„Zd„ Zd„ ZdZdZ ej(                  de› de› d�«      Z ej(                  d«      Z ej(                  d«      Zd„ Zd„ Zd„ Zy)é    Né   )ÚSortedDatasetÚidÚtextÚ
start_charÚend_charÚmiscc                 ó`   — g }| D ]&  }|d   d   |d   d   }}|j                  ||| «       Œ( |S )Nr   r   éÿÿÿÿr   )Úappend)ÚdocumentÚraw_textÚ	sentencesÚtokensÚ	begin_idxÚend_idxs         úc/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/minisbd/modules/inference.pyÚget_sentencesr      sN   € Ø€IØò 7ˆØ# A™Y |Ñ4°f¸R±jÀÑ6L�7ˆ	Ø×Ñ˜ )¨WÐ5Õ6ð7ð Ðó    c           	      óú  — g }d}| D ]ñ  \  }}}d }|�||j                  t        |dz   |t        |«      z   ft        |i«       |�|d   |d   t        <   |d   |d   t
        <   |D ](  }|j                  t        |dz   ft        |i«       |dz  }Œ* Œ‡t        |«      dk  rŒ–|j                  t        |dz   ft        |i«       |�|d   |d   t        <   |d   |d   t
        <   |dk(  s|dk(  rd|d   t        <   |dz  }Œó |S )Nr   r   r   é   é   zMWT=Yes)r   ÚIDÚlenÚTEXTÚ
START_CHARÚEND_CHARÚMISC)ÚsentenceÚsentÚiÚtokÚpÚposition_infoÚ	expansionÚetoks           r   Úprocess_sentencer'      s4  € Ø€DØ	€AØ!)ò ÑˆˆQ�Øˆ	ØÐ Ø�K‰Kœ˜a ™c 1¤S¨£^Ñ#3Ð4´d¸CÐ@ÔAØÐ(Ø'4°QÑ'7��R‘œÑ$Ø%2°1Ñ%5��R‘œÑ"Ø!ò �Ø—‘œR ! A¡# ¬$°Ð5Ô6Ø�Q‘‘ñô �3‹x˜1Š}ØØ�K‰Kœ˜a ™c˜W¤d¨CÐ0Ô1ØÐ(Ø'4°QÑ'7��R‘œÑ$Ø%2°1Ñ%5��R‘œÑ"Ø�AŠv˜˜ašØ!*��R‘œ‘Ø�‰F‰Að'ð( €Kr   c           
      ó�  — d}d}g }	|�t         j                  d|«      nd}
d}| �| j                  d«      }t        ||«      D �].  \  }}d}g }t        ||«      D �]Ô  \  }}|dk(  r �nÉ|r|dv rd}|d	z  }| �| j                  |«      k(  r|d	z  }||z  }|d	k\  sŒB| �| j	                  |«      }n|}d
|vsJ |«       ‚t        |«      dk  rd}Œt|��d}d}t        j                  |«      D ]ù  }t        |«      dk(  rŒ|r”t        j                  dj                  d„ |D «       «      «      }|j                  |
|«      }|j                  d«      |z
  }|j                  d«      |j                  d«      z
  }|j                  d«      j                  «       }n1	 |
j!                  ||«      |z
  }t        |«      }|j                  «       }|dk  r||z   |t        |«      z
  z   }|||z   z  }Œû ||f} nd} |j)                  ||| f«       d}|dk(  s|dk(  s�Œµ|r�Œ¹|	j)                  t+        |«      «       g }�Œ× t        |«      dkD  rt#        d«      ‚t        |«      s�Œ|	j)                  t+        |«      «       �Œ1 |||	fS # t"        $ rC}t%        d|dz
  «      }t'        t        |
«      |dz   «      }|
|| }t#        d|||fz  «      |‚d}~ww xY w)zÞ
    Decode the predictions into a document of words

    Once everything is fed through the tokenizer model, it's time to decode the predictions
    into actual tokens and sentences that the rest of the pipeline uses
    r   Nú z<UNK>Ú ú<PAD>)ú:ú;é   r   ú	r   z\s*c              3   óF   K  — | ]  }t        j                  |«      –— Œ y ­w)N)ÚreÚescape)Ú.0Úcs     r   ú	<genexpr>z%decode_predictions.<locals>.<genexpr>]   s   è ø€ ÒA]ÐSTÄ"Ç)Á)ÈAÇ,ÑA]ùs   ‚!é   zICould not find |%s| starting from char_offset %d.  Surrounding text: |%s|r   z9Finished processing tokens, but there is still text left!)ÚSPACE_REÚsubÚunit2idÚzipÚnormalize_tokenr   ÚSPACE_SPLIT_REÚsplitr1   ÚcompileÚjoinÚsearchÚstartÚendÚgroupÚlstripÚindexÚ
ValueErrorÚmaxÚminr   r'   )!ÚvocabÚ	orig_textÚall_rawÚ	all_predsÚ	no_ssplitÚskip_newlineÚuse_la_ittb_shorthandÚoffsetÚ	oov_countÚdocr   Úchar_offsetÚUNK_IDÚrawÚpredÚcurrent_tokÚcurrent_sentÚtr#   r"   ÚstÚtok_lenÚpartÚpart_patternÚmatchÚst0ÚpartlenÚ	lstrippedÚeÚ	sub_startÚsub_endr8   r$   s!                                    r   Údecode_predictionsre   .   s4  € ð €FØ€IØ
€Cà+4Ð+@Œ8�<‰<˜˜YÔ'Àd€DØ€KàÐØ—‘˜wÓ'ˆä˜ )Ó,ó <7‰	ˆˆTØˆØˆä˜˜T“Nó 3	&‰DˆAˆqØ�GŠ|Úá$¨¨j©Ø�Ø�a‰KˆFØÐ  U§]¡]°1Ó%5¸Ò%?Ø˜Q‘�	à˜1ÑˆKØ�A‹vØÐ$Ø×/Ñ/°Ó<‘Cà%�CØ 3‘Ð+¨Ó+�Ü�s“8˜q’=Ø"$�KØØÑ(Ø�BØ�GÜ .× 4Ñ 4°[Ó Aò 5˜Ü˜t›9¨š>¨8Ù'Ü+-¯:©:°f·k±kÑA]ÐX\ÔA]Ó6]Ó+^˜LØ$0×$7Ñ$7¸¸kÓ$J˜EØ"'§+¡+¨a£.°;Ñ">˜CØ&+§i¡i°£l°U·[±[À³^Ñ&C˜GØ(-¯©°A«×(=Ñ(=Ó(?™Ið`Ø&*§j¡j°°{Ó&CÀkÑ&Q ô '*¨$£i˜GØ(,¯©«˜IØ š6Ø!,¨sÑ!2°gÄÀIÃÑ6NÑ!O˜BØ# s¨W¡}Ñ4™ð)5ð* &(¨Ð$5‘Mà$(�MØ×#Ñ# S¨!¨]Ð$;Ô<Ø �Ø˜’F˜a 1œf«iØ—J‘JÔ/°Ó=Ô>Ø#%’Lðg3	&ôj ˆ{Ó˜aÒÜÐXÓYÐYÜˆ|ÖØ�J‰JÔ'¨Ó5Ö6ðy<7ð| �f˜cÐ!Ð!øô1 $.ò `Ü,/°°;ÀÑ3CÓ,D 	Ü*-¬c°$«i¸ÀrÑ9IÓ*J Ø&*¨9°WÐ&= Ü&0Ð1|ð  AEð  GRð  TWð  @Xñ  2Xó  'Yð  _`ð  !`ûð	`ús   ÆI9É9	K	Ê>K 	Ë K	iè  c           	      óâ   — | d   }t        t        | j                  dt        «      «      }| j                  dd«      }t        | |||||«      \  }	}
d}| d   }t	        |||
|	|||«      \  }}}|||	|fS )NÚ
batch_sizeÚ
max_seqlenrM   FrN   )rG   ÚMAX_SEQ_LENGTH_DEFAULTÚgetÚpredictre   )ÚconfigÚsessionÚdata_generatorrI   rJ   Úuse_regex_tokensrg   rh   rM   rL   rK   rO   rN   rQ   rP   rR   s                   r   Úoutput_predictionsrp   �   s•   € Ø˜Ñ%€JÜÔ+¨V¯Z©Z¸ÔF\Ó-]Ó^€JØ�j‰j˜ eÓ,€Iä  ¨°.À*ÈjÐZjÓkÑ€Iˆwà!ÐØ˜.Ñ)€LÜ/°°yÀ'È9ÐV_Ðamð  pEó  FÑ€Iˆv�sà�f˜i¨Ð,Ð,r   c           
   #   óÚ   K  — t        dt        | «      |«      D ]I  }t        |t        ||z   t        | «      «      «      D �cg c]  }| |   ‘Œ	 }}| j                  |«      }|–— ŒK y c c}w ­w)Nr   )Úranger   rH   Úcollate)Údatasetrg   r!   ÚjÚbatch_samplesÚbatchs         r   Úget_onnx_batchesrx   Ž   se   è ø€ Ü�1”c˜'“l JÓ/ò ˆÜ-2°1´c¸!¸j¹.Ì#ÈgË,Ó6WÓ-XÖY¨˜ ›ÐYˆÐYà—‘ Ó.ˆà‹ñùÚYùs   ‚>A+Á A&ÁA+c                 ó8  — t        | «      D ��cg c]  \  }}|dk(  sŒ|‘Œ }}}t        |«      dk(  rdt        | «      fg}|S d}g }|D ]  }||k7  r|j                  ||f«       |dz   }Œ! |t        | «      k  r|j                  |t        | «      f«       |S c c}}w )zO
    Return spans of text which don't contain <PAD> and are split by <PAD>
    r+   r   r   )Ú	enumerater   r   )rU   ÚidxÚcharÚpadsÚspansÚprevÚpads          r   Ú
find_spansr�   —   s®   € ô "+¨3£×C‘I�C˜°4¸7³?ŠCÐC€DÑCÜ
ˆ4ƒy�A‚~Ø”S˜“X��ˆð €Lð ˆØˆØò 	ˆCØ�dŠ{Ø—‘˜t S˜kÔ+Ø˜‘7‰Dð	ð ”#�c“(Š?Ø�L‰L˜4¤ S£Ð*Ô,Ø€Lùó Ds
   �B�BaÆ  (?:[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*|"(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21\x23-\x5b\x5d-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])*")@(?:(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?|\[(?:(?:(?:2(?:5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9]))\.){3}(?:(?:2(?:5[0-5]|[0-4][0-9])|1[0-9][0-9]|[1-9]?[0-9])|[a-z0-9-]*[a-z0-9]:(?:[\x01-\x08\x0b\x0c\x0e-\x1f\x21-\x5a\x53-\x7f]|\\[\x01-\x09\x0b\x0c\x0e-\x7f])+)\])a  (?:https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s"]{2,}|www\.[a-zA-Z0-9][a-zA-Z0-9-]+[a-zA-Z0-9]\.[^\s"]{2,}|https?:\/\/(?:www\.|(?!www))[a-zA-Z0-9]+\.[^\s"]{2,}|www\.[a-zA-Z0-9]+\.[^\s"]{2,})|[a-zA-Z0-9]+\.(?:gov|org|edu|net|com|co)(?:\.[^\s"]{2,})z(?:ú|ú)z\sz	( *[^ ]+)c                 ó$  — t        | «      }|D ]  \  }}dj                  | || «      }t        j                  |«      D ]N  }|j	                  «       \  }}t        ||z   ||z   dz
  «      D ]  }	d||	<   Œ	 |||z   dz
     dk(  sŒDd|||z   dz
  <   ŒP Œ� |S )a  
    Update the results of a tokenization batch by checking the raw text against a couple regular expressions

    Currently, emails and urls are handled
    TODO: this might work better as a constraint on the inference

    for efficiency pred is modified in place
    r*   r   r   )r�   r?   ÚMASK_REÚfinditerÚspanrr   )
rU   rV   r~   Ú
span_beginÚspan_endr   r^   Úmatch_beginÚ	match_endr|   s
             r   Úupdate_pred_regexrŒ   ´   sÄ   € ô �s‹O€Eà %ò 
1Ñˆ
�HØ�w‰w�s˜: hÐ/Ó0ˆÜ×%Ñ% dÓ+ò 	1ˆEØ%*§Z¡Z£\Ñ"ˆK˜ô ˜k¨*Ñ4°iÀ
Ñ6JÈ1Ñ6LÓMò �Ø��T’
ðð �I˜jÑ(¨Ñ*Ñ+¨qÓ0Ø/0��Y˜zÑ)¨!Ñ+Ò,ñ	1ð
1ð €Kr   c           
      ó¸  — g }g }t        |«      }t        t        ||«      «      D �]í  \  }	}
t        |
d   «      }t        |
d   d   «      }|
d   D ]  }|j	                  t        |«      «       Œ ||k  r#t        j                  t        ||
«      d¬«      }�nÖdg|z  }dg|z  }|
d   D �cg c]  }|j                  d«      ‘Œ }}t        |«      D �cg c]  }g ‘Œ }}	 t        ||«      D ��cg c]  \  }}t        ||z
  |«      ‘Œ }}}t        |«      }|
d   dd…d|…f   |
d   dd…d|…f   |
d   dd…d|…f   |
d   D �cg c]  }|d| ‘Œ	 c}f}t        j                  t        ||«      d¬«      }t        |«      D ]Š  }t        j                  ||   dk(  ||   dk(  z   «      d   }t        |«      dk  s||   ||   |z
  k\  r||   }nt        j                  |«      dz   }||xx   ||d|…f   gz  cc<   ||xx   |z  cc<   |||<   ŒŒ t        t        ||«      D ��cg c]
  \  }}||k\  ‘Œ c}}«      rn|j!                  ||
«      }
�Œm|D �cg c]  }t        j"                  |d«      ‘Œ }}t        |«      D ]˜  }|	|z  |z   }||   }|j                  d«      } |d|  }|||<   ||   | dz
     dk  rd||   | dz
  <   n||   | dz
     dkD  rd||   | dz
  <   |r"|j	                  t%        |||   d|  «      «       Œ‚|j	                  ||   d|  «       Œš �Œð |j'                  |«      }|j'                  |«      }||fS c c}w c c}w c c}}w c c}w c c}}w c c}w )	z|
    The guts of the prediction method

    Calls predict() over and over until we have predictions for all of the text
    r   r   r.   )Úaxisr+   Nr   r   )r   rz   rx   r   r   ÚlistÚnpÚargmaxÚonnx_predictrE   rr   r:   rH   rG   ÚwhereÚallÚadvance_old_batchÚconcatenaterŒ   Úunsort)!rl   rm   rn   rg   rh   ro   rL   rK   Úsorted_dataÚ	batch_idxrw   Únum_sentencesÚNÚ	paragraphrV   r{   ÚadvÚxÚpara_lengthsÚ_Úidx1ÚensÚenÚbatch1Úpred1ru   Ú
sentbreaksÚadvancer#   Úpar_idxrP   rU   Úpar_lens!                                    r   rk   rk   Í   sÇ  € ð €IØ€Gä Ó/€Kä%Ô&6°{ÀJÓ&OÓPó 6:Ñˆ	�5Ü˜E !™H›ˆÜ��a‘˜‘ÓˆØ˜q™ò 	,ˆIØ�N‰Nœ4 	›?Õ+ð	,ð �
Š?Ü—9‘9œ\¨'°5Ó9ÀÔBŠDð �#˜Ñ%ˆCØ�#˜Ñ%ˆCØ6;¸A±hÖ?°˜AŸG™G GÕ,Ð?ˆLÐ?Ü % mÓ 4Ö5˜1’BÐ5ˆDÐ5ØÜ@CÀCÈÓ@V×W±W°T¸1”s˜1˜t™8 ZÕ0ÐW�ÑWÜ˜“X�Ø˜q™¢! S b S &Ñ)¨5°©8²A°s¸°s°FÑ+;¸UÀ1¹XÂaÈÈ"ÈÀfÑ=MÐ`eÐfgÑ`hÖOiÐ[\ÐPQÐRUÐSUÒPVÒOiÐi�ÜŸ	™	¤,¨w¸Ó"?ÀaÔH�ä˜}Ó-ò 	%�AÜ!#§¡¨5°©8°q©=¸UÀ1¹XÈ¹]Ñ*KÓ!LÈQÑ!O�JÜ˜:“¨!Ò+¨s°1©v¸Àa¹È:Ñ9UÒ/UØ"% a¡&™ä"$§&¡&¨Ó"4°qÑ"8˜à˜“G  a¨¨'¨ kÑ 2Ð3Ñ3“GØ˜“F˜gÑ%“FØ$�C˜’Fð	%ô ´°S¸,Ó1G×H¡g d¨A˜ ›	ÓHÔIØð '×8Ñ8¸¸eÓD�ñ+ ð. 37Ö7¨Q”B—N‘N 1 aÕ(Ð7ˆDÐ7ä˜]Ó+ò 	:ˆGØ Ñ+¨gÑ5ˆFà˜&‘/ˆCØ—i‘i Ó(ˆGØ�h�w�-ˆCØ!ˆG�F‰OØ�G‰}˜W Q™YÑ'¨!Ò+Ø+,��W‘˜g a™iÒ(Ø�g‘˜w q™yÑ)¨AÒ-Ø+,��W‘˜g a™iÑ(ÙØ× Ñ Ô!2°3¸¸W¹ÀhÀwÐ8OÓ!PÕQà× Ñ   g¡¨x°Ð!8Õ9ò	:ðQ6:ðp ×"Ñ" 9Ó-€IØ× Ñ  Ó)€Gà�gÐÐùò] @ùÚ5ùãWùâOiùó Iùò 8s$   Â*L<Ã	MÃ,MÅMÈMÉMc                 óH   — |\  }}}}||dœ}| j                  d |«      }|d   S )N)ÚunitsÚfeaturesr   )Úrun)rm   Úinputsr«   r    r¬   Ú
ort_inputsÚort_outss          r   r’   r’     s4   € Ø"Ñ€Eˆ1ˆh˜Ø ¨hÑ7€JØ�{‰{˜4 Ó,€Hà�A‰;Ðr   )NT)ÚonnxruntimeÚnumpyr�   r1   Údatar   r   r   r   r   r   r   r'   re   ri   rp   rx   r�   ÚEMAIL_RAW_REÚ
URL_RAW_REr>   r…   r7   r<   rŒ   rk   r’   © r   r   ú<module>r·      s²   ðÛ Û Û 	Ý à	€Ø€Ø€
Ø€Ø€òòò4O"ðb Ð ó-òòð& ]€ð e€
à
ˆ"�*‰*�s˜<˜.¨¨*¨°QÐ7Ó
8€Øˆ2�:‰:�eÓ€Ø�—‘˜LÓ)€òò2FóPr   