`j85 ddlZddlmZddlmZmZmZmZmZm Z m Z m Z ddl Z ddl mZddlmZddlmZddlmZdd lmZdd lmZdd lmZmZdd lmZmZdd lm Z ddlm!Z!ddl"m#Z#ddl$m%Z%eee e&e'fefZ(ee'e ee(e(ee&ffZ)ee'ee e&e'fe e&e'fffZ*ee'ee'ee e&e'fe e&e'ffffZ+dee dee'effdZ,dZ-Gdde%Z.de/de e/e/ffdZ0dZ1y)N)Path)AnyCallableDictIterableListOptionalTupleUnion)util)Errors)Language)Matcher)Scorer)IDS)DocSpan)normalize_token_attrsset_token_attrs)Example)SimpleFrozenList)Vocab)Pipeexamplesreturnc d}i}|jtj|dfi||jtj|dfi||jtj|dfd|i||jtj|dfd|i||jtj|dfi||S)Nc.t||jSN)getattrkey)tokenattrs g/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/spacy/pipeline/attributeruler.pymorph_key_getterz/attribute_ruler_score..morph_key_gettersud#'''tagposmorphgetterlemma)updaterscore_token_attrscore_token_attr_per_feat)rkwargsr&resultss r%attribute_ruler_scorer2s(G NN6**8UEfEF NN6**8UEfEF NN'U:JUfU NN(( g &6 :@   NN6**8WGGH Nr'ctSr )r2r'r%make_attribute_ruler_scorerr5/s  r'c FeZdZdZ d%deddedededee dd f d Z d&d Z d d d d d d ee ge e fdeedee edeedeedd f dZdedefdZdefdZdZdeeeeeefeeefffdd fdZdeeeeeeeefeeeffffdd fdZ d'de edededd fdZde edd fdZede efdZ!e"fde ede#fdZ$e"fd e#de eddfd!Z%e"fd"ee&efde edd fd#Z'e"fd"ee&efde eddfd$Z(y )(AttributeRulerzSet token-level attributes for tokens matched by Matcher patterns. Additionally supports importing patterns from tag maps and morph rules. DOCS: https://spacy.io/api/attributeruler F)validatescorervocabnamer8r9rNc||_||_t|j||_||_g|_g|_g|_||_y)aCreate the AttributeRuler. After creation, you can add patterns with the `.initialize()` or `.add_patterns()` methods, or load patterns with `.from_bytes()` or `.from_disk()`. Loading patterns will remove any patterns you've added previously. vocab (Vocab): The vocab. name (str): The pipe name. Defaults to "attribute_ruler". scorer (Optional[Callable]): The scoring method. Defaults to Scorer.score_token_attr for the attributes "tag", "pos", "morph" and "lemma" and Scorer.score_token_attr_per_feat for the attribute "morph". RETURNS (AttributeRuler): The AttributeRuler component. DOCS: https://spacy.io/api/attributeruler#init r8N) r;r:rmatcherr8attrs_attrs_unnormedindicesr9)selfr:r;r8r9s r%__init__zAttributeRuler.__init__:sH0  tzzH=   !# +-"$  r'czt|j|j|_g|_g|_g|_y)zReset all patterns.r=N)rr:r8r>r?r@rArBs r%clearzAttributeRuler.clear[s.tzzDMMB  ! r')nlppatternstag_map morph_rules get_examplesrGrHrIrJc|j|r|j||r|j||r|j|yy)aInitialize the attribute ruler by adding zero or more patterns. Rules can be specified as a sequence of dicts using the `patterns` keyword argument. You can also provide rules using the "tag map" or "morph rules" formats supported by spaCy prior to v3. N)rF add_patternsload_from_tag_mapload_from_morph_rules)rBrKrGrHrIrJs r% initializezAttributeRuler.initializebsE    h '   " "7 +   & &{ 3 r'docc|j} |j|}|j|||S#t$r }||j||g|cYd}~Sd}~wwxYw)zApply the AttributeRuler to a Doc and set all attribute exceptions. doc (Doc): The document to process. RETURNS (Doc): The processed Doc. DOCS: https://spacy.io/api/attributeruler#call N)get_error_handlermatchset_annotations Exceptionr;)rBrQ error_handlermatcheses r%__call__zAttributeRuler.__call__ys`..0  <jjoG  g .J < D3%; ; intr:stringssort)rBrQrXm_idsrYs r%rTzAttributeRuler.matchsr,,s$,G&  a##D) *D!Q 7      s1A"c |D]I\}}}}t||||}|j|}|j|} || } t|| |Ky#t$rnt t j j|jj|j|D cgc]} | jncc} wc} | dwxYw)zModify the document in place)label)rHspanindexN) rr?rA IndexError ValueErrorrE1001formatr>getrdtextr) rBrQrXattr_idmatch_idstartendrer?rfr#ts r%rUzAttributeRuler.set_annotationss-4 0 )GXucUCx8DJJw'ELL)E U  DK /% 0 !LL''!%!1!1$**!=.23aff33#(   sAACB32Cc|jD]\}}d|ig}t|\}}d|vrB|jjj |}|jj ||d<nD|jjj |d}|jj ||d<|j |g|y)zLoad attribute ruler patterns from a tag map. tag_map (dict): The tag map that maps fine-grained tags to coarse-grained tags and morphological features. DOCS: https://spacy.io/api/attributeruler#load_from_morph_rules TAGMORPHN)items_split_morph_attrsr: morphologyaddr_)rBrIr(r?pattern morph_attrsr*s r%rNz AttributeRuler.load_from_tag_maps"--/ 'JCs|nG!3E!: E;e# --11+>!%!3!3E!:g --11%.A!%!3!3E!:g HHgY & 'r'c|D]}||D]}||dg}|||}t|\}}d|vrE|jjj|d}|jj||d<nC|rA|jjj|}|jj||d<|j|g|y)a+Load attribute ruler patterns from morph rules. morph_rules (dict): The morph rules that map token text and fine-grained tags to coarse-grained tags, lemmas and morphological features. DOCS: https://spacy.io/api/attributeruler#load_from_morph_rules )ORTHrsrtN)rvr:rwrxr_)rBrJr(wordryr?rzr*s r%rOz$AttributeRuler.load_from_morph_ruless +C#C( +$(56#C(.%7%>"{e# JJ1155eGnEE%)ZZ%7%7%>E'N  JJ1155kBE%)ZZ%7%7%>E'N'E* + +r'r?rfctt|j}|jj |j j j |||jj|t|j |}|jj||jj|y)aBAdd Matcher patterns for tokens that should be modified with the provided attributes. The token at the specified index within the matched span will be assigned the attributes. patterns (Iterable[List[Dict]]): A list of Matcher patterns. attrs (Dict): The attributes to assign to the target token in the matched span. index (int): The index of the token in the matched span to modify. May be negative to index from the end of the span. Defaults to 0. DOCS: https://spacy.io/api/attributeruler#add N) strlenr?r>rxr:r_r@appendrrA)rBrHr?rfr"s r%rxzAttributeRuler.adds"#djj/" ++//4h? ##E*%djj%8 %  E"r'c6|D]}|jdi|y)acAdd patterns from a list of pattern dicts with the keys as the arguments to AttributeRuler.add. patterns (Iterable[dict]): A list of pattern dicts with the keys as the arguments to AttributeRuler.add (patterns/attrs/index) to add as patterns. DOCS: https://spacy.io/api/attributeruler#add_patterns Nr4)rx)rBrHps r%rMzAttributeRuler.add_patternss" A DHHMqM r'cg}tt|jD]c}i}|jj t |d|d<|j ||d<|j||d<|j|e|S)zAll the added patterns.rrHr?rf) rangerr?r>rkrr@rAr)rB all_patternsirs r%rHzAttributeRuler.patternss s4::' #AA LL,,SV4Q7AjM--a0AgJaAgJ    "  # r'excludecXi}fd|d<fd|d<tj|S)zSerialize the AttributeRuler to a bytestring. exclude (Iterable[str]): String names of serialization fields to exclude. RETURNS (bytes): The serialized object. DOCS: https://spacy.io/api/attributeruler#to_bytes c<jjSN)r)r:to_bytes)rrBsr%z)AttributeRuler.to_bytes..sTZZ%8%8%8%Ir'r:cBtjjSr )srsly msgpack_dumpsrHrEsr%rz)AttributeRuler.to_bytes..s(;(;DMM(Jr'rH)r r)rBr serializes`` r%rzAttributeRuler.to_bytess/ I ' J *}}Y00r' bytes_datacTfd}fd|d}tj||S)a'Load the AttributeRuler from a bytestring. bytes_data (bytes): The data to load. exclude (Iterable[str]): String names of serialization fields to exclude. returns (AttributeRuler): The loaded object. DOCS: https://spacy.io/api/attributeruler#from_bytes cNjtj|yr )rMr msgpack_loads)brBs r% load_patternsz0AttributeRuler.from_bytes..load_patternss   e11!4 5r'c>jj|Sr)r: from_bytes)rrrBs r%rz+AttributeRuler.from_bytes..#stzz44Q4Hr'r:rH)r r)rBrrr deserializes` ` r%rzAttributeRuler.from_bytess/ 6I%   K9 r'pathcNfdfdd}tj||y)zSerialize the AttributeRuler to disk. path (Union[Path, str]): A path to a directory. exclude (Iterable[str]): String names of serialization fields to exclude. DOCS: https://spacy.io/api/attributeruler#to_disk c>jj|Sr)r:to_diskrrrBs r%rz(AttributeRuler.to_disk..4stzz11!W1Er'cDtj|jSr )r write_msgpackrHrrBs r%rz(AttributeRuler.to_disk..5s%"5"5a"Gr'rN)r r)rBrrrs` ` r%rzAttributeRuler.to_disk)s%FG  T9g.r'cTfd}fd|d}tj||S)a(Load the AttributeRuler from disk. path (Union[Path, str]): A path to a directory. exclude (Iterable[str]): String names of serialization fields to exclude. RETURNS (AttributeRuler): The loaded object. DOCS: https://spacy.io/api/attributeruler#from_disk cNjtj|yr )rMr read_msgpackrs r%rz/AttributeRuler.from_disk..load_patternsEs   e003 4r'c>jj|Sr)r: from_diskrs r%rz*AttributeRuler.from_disk..Istzz33Aw3Gr'r)r r)rBrrrrs` ` r%rzAttributeRuler.from_disk9s/ 5H%  t['2 r')attribute_ruler)rN)r))__name__ __module__ __qualname____doc__r2rrboolr rrCrFrrrAttributeRulerPatternType TagMapTypeMorphRulesTyperPrrZrTrUrr r^rNrOMatcherPatternTyperxrMpropertyrrHrbytesrrrrrr4r'r%r7r73s& %:    "  B#'BF(,044xHW,=(=>?4h  4 8$=>? 4 *% 4n-4 4.tfE FFr')2rpathlibrtypingrrrrrr r r rr errorsrlanguagerr>rr9rsymbolsrtokensrrtokens._retokenizerrtrainingrrr:rpiperr^rrrrrr2r5r7dictrvrr4r'r%rs4NNN G#$uS#X345 t&');T3F GG#tE#s(OU38_<== > c4T%S/5c?*J%K KLLMHW$5DcN&!ZTZz $d $uT4Z'8 $Gr'