Ë
    `›¾j‰%  ã                   ó¸  — U d dl Z d dlZd dlZd dlmZ d dlmZ d dlmZm	Z	m
Z
mZmZmZ d dlZd dlmZ ddlmZmZ ddlmZ dd	lmZmZmZmZ d
dlmZmZmZmZ eeeeeedœZ ee!e	de
e   f   f   e"d<   dZ#dZ$ G d„ de!e«      Z% ejL                  d«       eddd¬«       edddd¬«       edddd¬«       ed
dd d!¬«       ed"d#d$d%¬«       edd&d'd(d)¬«       ed"d*d+d,¬«       ed"d-d.d/¬«       ee#d0d1d2 e'e jQ                  «       «      › �¬«       edd3d4d5d¬«       edd6d7d8¬«       edd9d:d;¬«      fd<e!d=ed>e%d?e)d@e*dAee!   dBe*dCe*dDe!dEee   dFee!   dGe*fdH„«       Z+dId
d"dd"d"ddd"dddJœd<ed=ee!ef   d>e!d?e)d@e*dAee!   dBe*dCe*dDe!dEee   dFee!   dGe*dKe*dLee   dMdfdN„Z,dOedPe!dMdfdQ„Z-dOedRedPe!dMdfdS„Z.dTe!dMee!   fdU„Z/dLed<ed=ee!ef   d>e!dDe!dEee   fdV„Z0d<efdW„Z1y)Xé    N)ÚEnum)ÚPath)ÚAnyÚCallableÚIterableÚMappingÚOptionalÚUnion)ÚPrinteré   )ÚDocÚDocBin)Údocs_to_json)Úconll_ner_to_docsÚconllu_to_docsÚiob_to_docsÚjson_to_docsé   )ÚArgÚOptÚappÚwalk_directory)Ú	conllubioÚconlluÚconllÚnerÚiobÚjson.Ú
CONVERTERSÚauto)r   c                   ó   — e Zd ZdZdZy)Ú	FileTypesr   ÚspacyN)Ú__name__Ú
__module__Ú__qualname__r   r#   © ó    ú[/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/spacy/cli/convert.pyr"   r"   *   s   „ Ø€DØ�Er(   r"   ÚconvertzInput file or directoryT)ÚhelpÚexistsú-z!Output directory. '-' for stdout.)r+   Ú
allow_dashr,   r#   z--file-typez-tzType of data to produce)r+   z	--n-sentsz-nz*Number of sentences per doc (0 to disable)Fz--seg-sentsz-szSegment sentences (for -c ner)z--modelz--basez-bzQTrained spaCy pipeline for sentence segmentation to use as base (for --seg-sents)z--morphologyz-mz#Enable appending morphology to tagsz--merge-subtokensz-TzMerge CoNLL-U subtokensz--converterz-czConverter: z	--ner-mapz-nmz6NER tag mapping (as JSON-encoded dict of entity types)z--langz-lz Language (if tokenizer required)z--concatenatez-Cz#Concatenate output to a single fileÚ
input_pathÚ
output_dirÚ	file_typeÚn_sentsÚ	seg_sentsÚmodelÚ
morphologyÚmerge_subtokensÚ	converterÚner_mapÚlangÚconcatenatec                 óô   — t        | «      } |t        d«      k(  rdn|}|dk(  }t        |¬«      }t        ||| «      }t        || ||j                  ||	«       t        | ||j                  |||||||	|
|||¬«       y)aµ  
    Convert files into json or DocBin format for training. The resulting .spacy
    file can be used with the train command and other experiment management
    functions.

    If no output_dir is specified and the output format is JSON, the data
    is written to stdout, so you can pipe them forward to a JSON file:
    $ spacy convert some_file.conllu --file-type json > some_file.json

    DOCS: https://spacy.io/api/cli#convert
    r-   ©Úno_print)r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   ÚsilentÚmsgN)r   r   Ú_get_converterÚverify_cli_argsÚvaluer*   )r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r>   r?   s                 r)   Úconvert_clirC   /   s�   € ôt �jÓ!€JØ*4¼¸S»	Ò*A¡3Àz€JØ˜3Ñ€FÜ
˜6Ô
"€CÜ˜s I¨zÓ:€IÜ�C˜ Z°·±À)ÈWÔUÜØØØ—/‘/ØØØØØ'ØØØØØØör(   r   )r1   r2   r3   r4   r5   r6   r8   r9   r:   r>   r?   r>   r?   Úreturnc                ó®  — t        | «      } |st        |¬«      }|	�t        j                  |	«      nd }	g }t	        | |«      D ][  }|j                  dd¬«      5 }|j                  «       }d d d «       t        |   } ||||||
|||	¬«	      }|j                  ||f«       Œ] |r7t        j                  j                  |D ��cg c]  \  }}|‘Œ	 c}}«      }| |fg}|D ]æ  \  }}|dk(  rt        |«      g}t        |«      }n(t        |d¬«      }t        |«      }|j                  «       }|d	k(  rt!        ||«       Œ]|| k7  r2|j#                  | «      }t        |«      |j%                  d
|› �«      z  }n/t        |«      |j&                  d   z  }|j%                  d
|› �«      }t)        |||«       |j+                  d|› d|› �«       Œè y # 1 sw Y   �Œ^xY wc c}}w )Nr<   Úrzutf-8©Úencoding)r2   r3   Úappend_morphologyr6   r9   r4   r=   r8   r   T)ÚdocsÚstore_user_datar-   ú.éÿÿÿÿzGenerated output file (z documents): )r   r   ÚsrslyÚ	read_jsonr   ÚopenÚreadr   ÚappendÚ	itertoolsÚchainÚfrom_iterabler   Úlenr   Úto_bytesÚ_print_docs_to_stdoutÚrelative_toÚwith_suffixÚpartsÚ_write_docs_to_fileÚgood)r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r>   r?   Ú	doc_filesÚ	input_locÚinfileÚ
input_dataÚfuncrJ   Ú_Úall_docsÚdataÚlen_docsÚdbÚsubpathÚoutput_files                              r)   r*   r*   �   sî  € ô" �jÓ!€JÙÜ˜vÔ&ˆØ*1Ð*=Œe�o‰o˜gÔ&À4€GØ€IÜ# J°	Ó:ò ,ˆ	Ø�^‰^˜C¨'ˆ^Ó2ð 	'°fØŸ™›ˆJ÷	'ô ˜)Ñ$ˆÙØØØØ(Ø+ØØØØô

ˆð 	×Ñ˜) TÐ*Õ+ð!,ñ" Ü—?‘?×0Ñ0Ài×1P¹7¸1¸d²$Ó1PÓQˆØ  (Ð+Ð,ˆ	Ø$ò U‰ˆ	�4Ø˜ÒÜ  Ó&Ð'ˆDÜ˜4“y‰Hä˜T°4Ô8ˆBÜ˜2“wˆHØ—;‘;“=ˆDØ˜ÒÜ! $¨	Õ2à˜JÒ&Ø#×/Ñ/°
Ó;�Ü" :Ó.°×1DÑ1DÀqÈÈÀ_Ó1UÑU‘ä" :Ó.°·±ÀÑ1DÑD�Ø)×5Ñ5¸¸)¸°oÓF�Ü  k°9Ô=Ø�H‰HÐ.¨x¨j¸ÀkÀ]ÐSÕTñ%U÷'	'ñ 	'üó" 2Qs   ÁGÂ?G
ÇG	re   Úoutput_typec                 óŽ   — |dk(  rt        j                  d| «       y t        j                  j                  j                  | «       y )Nr   r-   )rN   Ú
write_jsonÚsysÚstdoutÚbufferÚwrite)re   rj   s     r)   rX   rX   À   s2   € Ø�fÒÜ×Ñ˜˜dÕ#ä�
‰
×Ñ×Ñ Õ%r(   ri   c                 ó  — |j                   j                  «       s|j                   j                  d¬«       |dk(  rt        j                  || «       y |j                  d«      5 }|j                  | «       d d d «       y # 1 sw Y   y xY w)NT)Úparentsr   Úwb)Úparentr,   ÚmkdirrN   rl   rP   rp   )re   ri   rj   Úfile_s       r)   r\   r\   Ç   st   € Ø×Ñ×$Ñ$Ô&Ø×Ñ× Ñ ¨Ð Ô.Ø�fÒÜ×Ñ˜ dÕ+à×Ñ˜dÓ#ð 	 uØ�K‰K˜Ô÷	÷ 	ñ 	ús   Á$A?Á?Bra   c                 óv  — | j                  d«      d d }dddœ}t        j                  d«      }t        j                  d«      }|D ]O  }|j                  «       }|j	                  |«      r|dxx   dz  cc<   |j	                  |«      sŒC|d	xx   dz  cc<   ŒQ |d   dk(  r	|d	   dkD  ry	|d	   dk(  r	|d   dkD  ryy )
Nú
é   r   )r   r   z\S+\|(O|[IB]-\S+)z\S+\s+(O|[IB]-\S+)$r   r   r   )ÚsplitÚreÚcompileÚstripÚsearch)ra   ÚlinesÚformat_guessesÚiob_reÚner_reÚlines         r)   Úautodetect_ner_formatr„   Ñ   sÐ   € à×Ñ˜TÓ" 3 BÐ'€EØ qÑ)€NÜ�Z‰ZÐ,Ó-€FÜ�Z‰ZÐ.Ó/€FØò 'ˆØ�z‰z‹|ˆØ�=‰=˜ÔØ˜5Ó! QÑ&Ó!Ø�=‰=˜ÕØ˜5Ó! QÑ&Ô!ð'ð �eÑ Ò! n°UÑ&;¸aÒ&?ØØ�eÑ Ò! n°UÑ&;¸aÒ&?ØØr(   c                 ó  — |t         vr|dk(  r| j                  d|› d�d¬«       |j                  «       s| j                  d|d¬«       |dk7  r-t        |«      j                  «       s| j                  d|d¬«       |�-t        |«      j                  «       s| j                  d|d¬«       |j	                  «       r.t        ||«      }t        |«      d	k(  r| j                  d
|d¬«       |t        vr| j                  d|› �d¬«       y y )Nr-   zCan't write .z4 data to stdout. Please specify an output directory.r   ©ÚexitszInput file not foundzOutput directory not foundzNER map not foundr   zNo input files in directoryzCan't find converter for )ÚFILE_TYPES_STDOUTÚfailr,   r   Úis_dirr   rV   r   )r?   r/   r0   r1   r7   r8   Ú
input_locss          r)   rA   rA   ä   s  € ð Ô)Ñ)¨j¸CÒ.?Ø�‰Ø˜I˜;Ð&ZÐ[Øð 	ô 	
ð ×ÑÔØ�‰Ð'¨¸1ˆÔ=Ø�SÒ¤ jÓ!1×!8Ñ!8Ô!:Ø�‰Ð-¨zÀˆÔCØÐ¤4¨£=×#7Ñ#7Ô#9Ø�‰Ð$ g°QˆÔ7Ø×ÑÔÜ# J°	Ó:ˆ
Üˆz‹?˜aÒØ�H‰HÐ2°JÀaˆHÔHØœ
Ñ"Ø�‰Ð,¨Y¨KÐ8ÀˆÕBð #r(   c           
      óž  — |j                  «       rŽ|t        k(  rut        |d ¬«      }t        t	        |D �cg c]  }|j
                  dd  ‘Œ c}«      «      }t        |«      dk\  r%dj                  |«      }| j                  d|d¬«       |d   }nt        ||¬«      d   }|t        k(  r|j
                  dd  }|dk(  s|d	k(  r{|j                  d
¬«      5 }|j                  «       }d d d «       t        «      }	|	dk(  r| j                  d«       |	}|S |	d	k(  r| j                  d«       |	}|S | j                  d«       |S c c}w # 1 sw Y   Œ`xY w)N)Úsuffixr   r   ú,z!All input files must be same typer†   r   r   r   Úutf8rG   z'Auto-detected token-per-line NER formatz*Auto-detected sentence-per-line NER formatzgCan't automatically detect NER format. Conversion may not succeed. See https://spacy.io/api/cli#convert)rŠ   ÚAUTOr   ÚlistÚsetr�   rV   Újoinr‰   rP   rQ   r„   ÚinfoÚwarn)
r?   r7   r/   r‹   ÚlocÚ
file_typesÚfile_types_strrv   ra   Úconverter_autodetects
             r)   r@   r@   ÿ   sY  € Ø×ÑÔØœÒÜ'¨
¸4Ô@ˆJÜœc¸ZÖ"H°c 3§:¡:¨a¨b¢>Ò"HÓIÓJˆJÜ�:‹ !Ò#Ø!$§¡¨*Ó!5�Ø—‘Ð<¸nÐTU�ÔVØ# A™‰Jä'¨
¸9ÔEÀaÑHˆJØ”DÒØ×%Ñ% a bÐ)ˆ	Ø�EÒ˜Y¨%Ò/Ø�_‰_ fˆ_Ó-ð 	&°ØŸ™›ˆJ÷	&ä4°ZÓ@ÐØ 5Ò(Ø�H‰HÐ>Ô?Ø,ˆIð Ðð " UÒ*Ø�H‰HÐAÔBØ,ˆIð Ðð �H‰Hð7ôð
 Ðùò3 #I÷	&ð 	&ús   µD>ÃEÅE)2rS   r{   rm   Úenumr   Úpathlibr   Útypingr   r   r   r   r	   r
   rN   Úwasabir   Útokensr   r   Útrainingr   Útraining.convertersr   r   r   r   Ú_utilr   r   r   r   r   ÚstrÚ__annotations__r�   rˆ   r"   ÚcommandÚtupleÚkeysÚintÚboolrC   r*   rX   r\   r„   rA   r@   r'   r(   r)   ú<module>r©      sß  ðÜ Û 	Û 
Ý Ý ß D× Dã Ý ç  Ý #÷ó ÷ 1Ó 0ð  ØØØØØñ:€
ˆG�C˜ # x°¡}Ð"4Ñ5Ð5Ñ6ó ð €ð Ð ô��Tô ð
 €‡�ˆYÓñ ˜#Ð$=ÀdÔKÙØÐ5À$Ètôñ Ø� Ð+Dôñ Ø	ˆ;˜Ð#Oôñ Øˆ}˜dÐ)Iôñ ØØØØØ`ôñ Øˆ~˜tÐ*Oôñ  ØÐ" DÐ/Hôñ Øˆm˜T¨+±e¸J¿O¹OÓ<MÓ6NÐ5OÐ(Pôñ "ØØØØEØôñ Øˆh˜Ð#Eôñ Øˆo˜tÐ*OôñSNàðNð ðNð ðNð ðNð ðNð �C‰=ðNð, ð-Nð2 ð3Nð8 ð9Nð> �d‰^ð?NðL �3‰-ðMNðR òSNó ðNðj ØØØØØ!à"ØØØØ!ò<UØð<Uà�c˜4�iÑ ð<Uð ð	<Uð
 ð<Uð ð<Uð �C‰=ð<Uð ð<Uð ð<Uð ð<Uð �d‰^ð<Uð �3‰-ð<Uð ð<Uð ð<Uð 
�'Ñ	ð<Uð  
ó!<Uð~& ð &°#ð &¸$ó &ð˜cð °ð À3ð È4ó ð cð ¨h°s©mó ð&CØ	ðCàðCð �c˜4�iÑ ðCð ð	Cð
 ðCð �d‰^óCð6¨tô r(   