Ë
    \›¾j‘�  ã                   óT  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZ d dlZd dl	m
Z
 d dlmZ d dlmZ d dlmZ d dlmc mc mZ d dlmZ d dlmZmZmZ d dlmZmZ d d	lmZmZm Z   G d
„ de«      Z! G d„ de«      Z" ejF                  d«      Z$ ejF                  d«      Z% ejF                  d«      jM                  ejN                  «       dZ(dZ)dZ*	 d„ Z+dddddddœZ,ddddœZ-dZ.ddd œZ/d!d"d#d$d%d&œZ0d'„ Z1d(„ Z2d3d)„Z3d*„ Z4d+„ Z5	 	 d4d,„Z6d-„ Z7d.„ Z8d/„ Z9d0„ Z:d3d1„Z;e<d2k(  r e;«        yy)5é    N)ÚEnum)Úloss)Úutils)Ú	CharVocab)ÚTrainer)ÚWVTypeÚExtraVectorsÚ	ModelType)Úadd_peft_argsÚresolve_peft_args)Úformat_confusionÚconfusion_to_accuracyÚconfusion_to_macro_f1c                   ó   — e Zd ZdZdZdZdZy)ÚLossé   é   é   é   N)Ú__name__Ú
__module__Ú__qualname__ÚCROSSÚWEIGHTED_CROSSÚ	LOG_CROSSÚFOCAL© ó    úb/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/stanza/models/classifier.pyr   r      s   „ Ø€EØ€NØ€IØ�Er   r   c                   ó   — e Zd ZdZdZy)Ú
DevScoringÚACCÚWFN)r   r   r   ÚACCURACYÚWEIGHTED_F1r   r   r   r!   r!      s   „ Ø€HØ�Kr   r!   Ústanzazstanza.classifiers.trainerÚelmoformanylangsz#data/sentiment/en_sstplus.train.txtz#data/sentiment/en_sst3roots.dev.txtz$data/sentiment/en_sst3roots.test.txtc                 ó²   — | j                  «       } | syt        j                  | «      } t        | t        «      r| fS t        | t
        «      r| S t        | «      S )z~
    Returns a tuple of sizes to use in FC layers.

    For examples, converts "100" -> (100,)
    "100,200" -> (100,200)
    r   )ÚstripÚastÚliteral_evalÚ
isinstanceÚintÚtuple)Úargs    r   Úconvert_fc_shapesr0   h   sO   € ð �)‰)‹+€CÙØÜ
×
Ñ
˜3Ó
€CÜ�#”sÔØˆvˆÜ�#”uÔØˆ
Ü�‹:Ðr   g-Cëâ6*?g      ð?gü©ñÒMbP?g-Cëâ6
?gü©ñÒMb@?)ÚadamwÚadadeltaÚsgdÚ	adabeliefÚmadgradr3   gê-�™—q=g�íµ ÷Æ°>g:Œ0âŽyE>)r4   r2   r1   gÍÌÌÌÌÌì?)r5   r3   gš™™™™™©?ç-Cëâ6?ç{®Gáz„?gvƒôõ!´>g�íµ ÷ÆÀ>)r1   r2   r3   r4   r5   c            
      óà  — t        j                  «       } | j                  ddddd¬«       | j                  dddd	¬
«       | j                  dt        dd¬«       | j                  dt        dd¬«       | j                  dt        dd¬«       | j                  dt        dd¬«       | j                  dt        dd¬«       | j                  dddd¬
«       | j                  dddd¬ «       | j                  d!t        t        d"¬«       | j                  d#t        t
        d$¬«       | j                  d%t        t        d&¬«       | j                  d'ddd(¬ «       | j                  d)t        d*¬+«       | j                  d,t        d-¬+«       | j                  d.d/„ t        j                  d0d1j                  d2„ t        D «       «      z  ¬«       | j                  d3d4t        j                  d5¬6«       | j                  d7d8t        j                  d9¬6«       | j                  d:d;t        d<¬6«       | j                  d=d>t        d?¬6«       | j                  d@d-t        dA¬6«       | j                  dBdCt        dD¬6«       | j                  dEdFt        dG¬6«       | j                  dHdI„ t        j                   dJd1j                  dK„ t        D «       «      z  ¬«       | j                  dLdt        dM¬6«       | j                  dNdt        dO¬6«       | j                  dPdt        dQ¬6«       | j                  dRdSg dT¢dU¬V«       | j                  dWdt        j                  dX¬6«       | j                  dYdZddd[¬«       | j                  d\dZdd]¬
«       | j                  d^d_„ t"        j$                  d`¬«       | j                  dadbt        dc¬6«       | j                  ddt        dedf¬«       | j                  dgt        dh¬+«       | j                  dit        ddj¬«       | j                  dkt        ddl¬«       | j                  dmt        dndo¬«       | j                  dpdq„ drds¬«       | j                  dtt        dedu¬«       | j                  dvdw„ dxdy¬«       | j                  dzt        dd{¬«       | j                  d|t        dd}¬«       | j                  d~t        dd¬«       | j                  d€t        dd�¬«       | j                  d‚dƒdd„¬
«       | j                  d…d†d‡¬ˆ«       | j                  d‰dŠddd‹¬«       | j                  dŒt        dd�¬«       | j                  dŽt        dd�¬«       | j                  d�d‘d’dd“¬”«       | j                  d•ddd–¬ «       | j                  d—d˜t        d™¬6«       | j                  dšd›t        dœ¬6«       | j                  d�t        dždŸ¬«       | j                  d d’dd¡d¢¬£«       | j                  d¤d¥ddd¦¬§«       | j                  d¨d¥dd©¬
«       | j                  dªt        d«d¬¬«       | j                  d­t        d®d¯¬«       | j                  d°d±ddd²¬«       | j                  d³t        d´dµ¬«       | j                  d¶ddd·¬ «       | j                  d¸ddd¹¬ «       | j                  dºd»dd¹¬
«       | j                  d¼d½ddd¾¬«       | j                  d¿d½dd¾¬
«       | j                  dÀdddÁ¬ «       | j                  dÂdÃdddÄ¬«       | j                  dÅdddÆ¬ «       | j                  dÇddÈ¬É«       | j                  dÊddË¬ˆ«       | j                  dÌdt        dÍ¬6«       t'        | «       t)        j*                  | «       | S )Îz�
    Build the argparse for the classifier.

    Refactored so that other utility scripts can use the same parser if needed.
    z--trainÚtrainTÚ
store_truezTrain the model (default))ÚdestÚdefaultÚactionÚhelpz
--no_trainÚstore_falsezDon't train the model)r;   r=   r>   z--shorthandÚen_ewtz'Treebank shorthand, eg 'en' for English)Útyper<   r>   z--load_nameNz"Name for loading an existing modelz
--save_dirzsaved_models/classifierzRoot dir for saving models.z--save_namezI{shorthand}_{embedding}_{bert_finetuning}_{classifier_type}_classifier.ptzName for saving the modelz--checkpoint_save_namez,File name to save the most recent checkpointz--no_checkpointÚ
checkpointzDon't save checkpointsz--save_intermediate_modelsFz1Save all intermediate models - this can be a lot!)r<   r=   r>   z--train_filez~Input file(s) to train a model from.  Each line is an example.  Should go <label> <tokenized sentence>.  Comma separated list.z
--dev_filez$Input file(s) to use as the dev set.z--test_filez%Input file(s) to use as the test set.z--output_predictionsz,Output predictions when running the test setz--max_epochséd   )rA   r<   z--tické2   z--model_typec                 ó0   — t         | j                  «          S ©N)r
   Úupper©Úxs    r   ú<lambda>z build_argparse.<locals>.<lambda>²   s   € ´yÀÇÁÃÑ7K€ r   zModel type to use.  Options: %sú c              3   ó4   K  — | ]  }|j                   –— Œ y ­wrF   ©Úname©Ú.0rI   s     r   ú	<genexpr>z!build_argparse.<locals>.<genexpr>³   s   è ø€ ÒIdÐUVÈ!Ï&Í&ÑIdùó   ‚z--filter_sizes)r   r   é   z1Filter sizes for the layer after the word vectors)r<   rA   r>   z--filter_channelsiè  z©Number of channels for layers after the word vectors.  Int for same number of channels (scaled by width) for each filter, or tuple/list for exact lengths for each filterz--fc_shapesz400,100z‹Extra fully connected layers to put after the initial filters.  If set to blank, will FC directly from the max pooling to the output layer.z	--dropoutg      à?zDropout value to usez--batch_sizezBatch size when trainingz--batch_single_iteméÈ   z(Items of this size go in their own batchz--dev_eval_batchesiÐ  zURun the dev set after this many train batches.  Set to 0 to only do it once per epochz--dev_eval_scoringc                 ó0   — t         | j                  «          S rF   )r!   rG   rH   s    r   rJ   z build_argparse.<locals>.<lambda>½   s   € ¼ZÈÏÉË	Ñ=R€ r   z?Scoring method to use for choosing the best model.  Options: %sc              3   ó4   K  — | ]  }|j                   –— Œ y ­wrF   rM   rO   s     r   rQ   z!build_argparse.<locals>.<genexpr>¿   s   è ø€ Ò&B°! q§v¥vÑ&BùrR   z--weight_decayz1Weight decay (eg, l2 reg) to use in the optimizerz--learning_ratez%Learning rate to use in the optimizerz
--momentumz Momentum to use in the optimizerz--optimr2   )r2   r5   r3   z\Optimizer type: SGD, Adadelta, or madgrad.  Highly recommend to install madgrad and use that)r<   Úchoicesr>   z--test_remap_labelszºMap of which label each classifier label should map to.  For example, "{0:0, 1:0, 3:1, 4:1}" to map a 5 class sentiment test to a 2 class.  Any labels not mapped will be considered wrongz--forgive_unmapped_labelsÚforgive_unmapped_labelszrWhen remapping labels, such as from 5 class to 2 class, pick a different label if the first guess is not remapped.z--no_forgive_unmapped_labelszxWhen remapping labels, such as from 5 class to 2 class, DON'T pick a different label if the first guess is not remapped.z--lossc                 ó0   — t         | j                  «          S rF   )r   rG   rH   s    r   rJ   z build_argparse.<locals>.<lambda>Î   s   € ´°a·g±g³i±€ r   zCWhether to use regular cross entropy or scale it by 1/log(quantity)z--loss_focal_gammar   zgamma value for a focal lossz--min_train_lenr   z&Filter sentences less than this lengthz--pretrain_max_vocabéÿÿÿÿz--wordvec_pretrain_filez'Exact name of the pretrain file to readz--wordvec_raw_filez*Exact name of the raw wordvec file to readz--wordvec_dirzextern_data/wordveczDirectory of word vectorsz--wordvec_typec                 ó0   — t         | j                  «          S rF   )r   rG   rH   s    r   rJ   z build_argparse.<locals>.<lambda>Ø   s   € ¼ÀÇÁÃ	Ñ9J€ r   Úword2vecz[Different vector types have different options, such as google 300d replacing numbers with #z--extra_wordvec_dimz+Extra dim of word vectors - will be trainedz--extra_wordvec_methodc                 ó0   — t         | j                  «          S rF   )r	   rG   rH   s    r   rJ   z build_argparse.<locals>.<lambda>Ú   s   € ÄÈaÏgÉgËiÑAX€ r   Úsumz8How to train extra dimensions of word vectors, if at allz--extra_wordvec_max_normz+Max norm for initializing the extra vectorsz--charlm_forward_filez$Exact path to use for forward charlmz--charlm_backward_filez%Exact path to use for backward charlmz--charlm_projectionz+Project the charlm values to this dimensionz--char_lowercaseÚchar_lowercasez-Use lowercased characters in character model.z--elmo_modelzextern_data/manyelmo/englishzDirectory with elmo model)r<   r>   z
--use_elmoÚuse_elmoz+Use an elmo model as a source of parametersz--elmo_projectionz$Project elmo to this many dimensionsz--bert_modelz>Use an external bert model (requires the transformers package)z--no_bert_modelÚ
bert_modelÚstore_constzDon't use bert)r;   r=   Úconstr>   z--bert_finetunezFinetune the Bert modelz--bert_learning_rater7   z?Scale the learning rate for transformer finetuning by this muchz--bert_weight_decayr6   z>Scale the weight decay for transformer finetuning by this muchz--bert_hidden_layersr   z;How many layers of hidden state to use from the transformerz--bert_hidden_layers_originalÚbert_hidden_layersz&Use layers 2,3,4 of the Bert embedding)r=   rc   r;   r>   z--bilstmÚbilstmzªUse a bilstm after the inputs, before the convs.  Using bilstm is about as accurate and significantly faster (because of dim reduction) than going straight to the filters)r;   r=   r<   r>   z--no_bilstmz6Don't use a bilstm after the inputs, before the convs.z--bilstm_hidden_dimi,  zDimension of the bilstm to usez--maxpool_widthr   z"Width of the maxpool kernel to usez--no_constituency_backpropÚconstituency_backpropzLWhen using a constituency parser, backprop into the parser's weights if Truez--constituency_modelz7/home/john/stanza_resources/it/constituency/vit_bert.ptzDWhich constituency model to use.  TODO: make this more user friendlyz--constituency_batch_normzJAdd a LayerNorm between the output of the parser and the classifier layersz--constituency_node_attnzZTrue means to make an attn layer out of the tree, with the words as key and nodes as queryz--no_constituency_node_attnÚconstituency_node_attnz--constituency_top_layerÚconstituency_top_layerzrTrue means use the top (ROOT) layer of the constituents.  Otherwise, the next layer down (S, usually) will be usedz--no_constituency_top_layerz--constituency_all_wordsz5Use all word positions in the constituency classifierz--no_constituency_all_wordsÚconstituency_all_wordszNUse the start and end word embeddings as inputs to the constituency classifierz--log_normsz=Log the parameters norms while training.  A very noisy optionz--wandbzStart a wandb session and write the results of training.  Only applies to training.  Use --wandb_name instead to specify a name)r=   r>   z--wandb_namezWName of a wandb session to start when training.  Will default to the dataset short namez--seedzRandom seed for model)ÚargparseÚArgumentParserÚadd_argumentÚstrÚDEFAULT_TRAINÚDEFAULT_DEVÚDEFAULT_TESTr-   r
   ÚCNNÚjoinr*   r+   r0   Úfloatr!   r%   r   r   r   r   Úadd_device_args)Úparsers    r   Úbuild_argparserv   ”   sÉ  € ô ×$Ñ$Ó&€Fà
×Ñ˜	¨¸ÀlÐYtÐÔuØ
×Ñ˜¨7¸=ÐOfÐÔgà
×Ñ˜¬C¸ÐHqÐÔrà
×Ñ˜¬C¸ÐDhÐÔiØ
×Ñ˜¬3Ð8QÐXuÐÔvØ
×Ñ˜¬Cð  :Eð  LgÐô  hà
×ÑÐ0´sÀDÐO}ÐÔ~Ø
×ÑÐ)°À]ÐYqÐÔrà
×ÑÐ4¸eÈLØPð ô Rð ×Ñ˜¬S¼-ð  OOÐô  PØ
×Ñ˜¬3¼ÐJpÐÔqØ
×Ñ˜¬C¼ÐLsÐÔtØ
×ÑÐ.¸Àlð  ZHÐô  IØ
×Ñ˜¬S¸#ÐÔ>Ø
×Ñ˜¤s°BÐÔ7à
×Ñ˜Ñ-KÔU^×UbÑUbØ>ÀÇÁÑIdÔZcÔIdÓAdÑdð ô fð ×ÑÐ(°'Ä×@PÑ@Pð  XKÐô  LØ
×ÑÐ+°TÄ×@PÑ@Pð  XCÐô  DØ
×Ñ˜¨yÔ?Pð  XeÐô  fØ
×Ñ˜¨S´uÐCYÐÔZà
×Ñ˜°¼ÐC]ÐÔ^Ø
×ÑÐ-°sÄÐKuÐÔvØ
×ÑÐ,°dÄð  LcÐô  dØ
×ÑÐ,Ñ3RÔ\f×\rÑ\rØ_Ø!Ÿh™hÑ&B´zÔ&BÓBñCð ô Eð ×ÑÐ(°$¼UÐI|ÐÔ}Ø
×ÑÐ)°4¼eÐJqÐÔrØ
×Ñ˜¨d¼ÐEgÐÔhà
×Ñ˜	¨:Ò?]ð  eCÐô  Dà
×ÑÐ-°tÄ#×BRÑBRð Zð ô [à
×ÑÐ3Ð:SÐ]aÐjvð Rð ô Sà
×ÑÐ6Ð=VÐ_lð Xð ô Yð ×Ñ˜Ñ'@Ì$Ï*É*Øbð ô dà
×ÑÐ,°a¼eÐJhÐÔiØ
×ÑÐ)´¸QØEð ô Gð ×ÑÐ.´SÀ"ÐÔEØ
×ÑÐ1¼ÀTÐPyÐÔzØ
×ÑÐ,´3ÀÐKwÐÔxØ
×Ñ˜¬cÐ;PÐWrÐÔsØ
×ÑÐ(Ñ/JÐT^ð  fCÐô  DØ
×ÑÐ-´CÀÐIvÐÔwØ
×ÑÐ0Ñ7XÐbgð  oiÐô  jØ
×ÑÐ2¼Èð  TAÐô  Bà
×ÑÐ/´cÀ4ÐNtÐÔuØ
×ÑÐ0´sÀDÐOvÐÔwØ
×ÑÐ-´CÀÐLyÐÔzØ
×ÑÐ*Ð1AÈ,ð  ^MÐô  Nà
×Ñ˜Ð0NÐUpÐÔqØ
×Ñ˜¨:¸uÈ\ð  aNÐô  OØ
×ÑÐ+´#¸tÐJpÐÔqà
×Ñ˜¬S¸$ð  FFÐô  GØ
×ÑÐ)°À]ÐZ^ÐeuÐÔvØ
×ÑÐ)°5ÀÐTmÐÔnØ
×ÑÐ.¸Ä5ð  PQÐô  RØ
×ÑÐ-°vÄEð  QQÐô  RØ
×ÑÐ.´SÀ!ð  KHÐô  IØ
×ÑÐ7ÀÐUYÐ`tð  |dÐô  eà
×Ñ˜
¨¸,ÐPTð  \HÐô  IØ
×Ñ˜¨H¸]ð  RJÐô  Kð: ×ÑÐ-´CÀÐKkÐÔlà
×ÑÐ)´¸QÐEiÐÔjà
×ÑÐ4Ð;RÐ\`Ðivð  ~LÐô  MØ
×ÑÐ.´SÐB{ð  CIÐô  JØ
×ÑÐ3¸UÈ<ð  _kÐô  lØ
×ÑÐ2¸EÈ,ð  ^zÐô  {Ø
×ÑÐ5Ð<TÐ]jð  rNÐô  OØ
×ÑÐ2Ð9QÐ[`Ðiuð  }qÐô  rØ
×ÑÐ5Ð<TÐ]jð  rfÐô  gØ
×ÑÐ2¸EÈ,ð  ^UÐô  VØ
×ÑÐ5Ð<TÐ^cÐlyð  AQÐô  Rà
×Ñ˜¨u¸\ð  QPÐô  Qà
×Ñ˜	¨,ð  >Ðô  @Ø
×Ñ˜°ð  <UÐô  Và
×Ñ˜¨$´SÐ?VÐÔWä�&ÔÜ	×Ñ˜&Ô!à€Mr   c           
      óÈ  — ddj                  | j                  D �cg c]  }t        |«      ‘Œ c}«      z  }|d| j                  z  z   }| j                  r8|ddj                  | j                  D �cg c]  }t        |«      ‘Œ c}«      z  z   }t        j                  t        | «      d|| j                  j                  ¬«      }t        j                  d|«       |S c c}w c c}w )NzFS_%sÚ_z_C_%d_z_FC_%s_Ú
classifier)ÚshapeÚclassifier_typezExpanded save_name: %s)rr   Úfilter_sizesrm   Úfilter_channelsÚ	fc_shapesr   Ústandard_model_file_nameÚvarsÚ
model_typerN   ÚloggerÚinfo)ÚargsrI   rz   Úmodel_save_files       r   Úbuild_model_filenamer†   &  sÅ   € Ø�c—h‘h°×0AÑ0AÖB¨1¤ A¥ÒBÓCÑC€EØ�H˜t×3Ñ3Ñ3Ñ3€EØ‡~‚~Ø˜	 C§H¡H¸d¿n¹nÖ-M¸¬c°!­fÒ-MÓ$NÑNÑNˆä×4Ñ4´T¸$³ZÀÐUZÐlp×l{Ñl{÷  mAñ  mAô  B€OÜ
‡K�KÐ(¨/Ô:ØÐùò  Cùò .Ns   ›C
Á/Cc                 óæ  — t        «       }|j                  | «      } t        | t        «       | j                  rd| _        | j                  j                  «       | _        | j                  €%t        j                  | j                  d«      | _        | j                  €%t        j                  | j                  d«      | _        | j                  €%t        j                  | j                  d«      | _        | S )zi
    Add arguments for building the classifier.
    Parses command line args and returns the result.
    TN)rv   Ú
parse_argsr   ÚtloggerÚ
wandb_nameÚwandbÚoptimÚlowerÚweight_decayÚDEFAULT_WEIGHT_DECAYÚgetÚmomentumÚDEFAULT_MOMENTUMÚlearning_rateÚDEFAULT_LEARNING_RATES)r„   ru   s     r   rˆ   rˆ   0  s¹   € ô
 Ó€FØ×Ñ˜TÓ"€DÜ�dœGÔ$à‡‚ØˆŒ
à—‘×!Ñ!Ó#€D„JØ×ÑÐ Ü0×4Ñ4°T·Z±ZÀÓFˆÔØ‡}�}ÐÜ(×,Ñ,¨T¯Z©Z¸Ó>ˆŒØ×ÑÐ!Ü3×7Ñ7¸¿
¹
ÀDÓIˆÔà€Kr   c                 ó,  — | j                  «        t        | j                  «      D ��ci c]  \  }}||“Œ
 }}}t        j                  |d¬«      }g }g }|j                  «       D ]Œ  }||   }	 | |	D �cg c]  }|d   ‘Œ	 c}«      }
t        t        |	«      «      D ]U  }t        j                  |
|   «      }||j                  «          }|j                  |«       |j                  |	|   d   «       ŒW ŒŽ t        j                  ||«      }|S c c}}w c c}w )NT)Ú
keep_indexr   r   )ÚevalÚ	enumerateÚlabelsÚdataÚsort_dataset_by_lenÚkeysÚrangeÚlenÚtorchÚargmaxÚitemÚappendr   Úunsort)ÚmodelÚdatasetrI   ÚyÚindex_label_mapÚdataset_lengthsÚpredictionsÚo_idxÚlengthÚbatchÚoutputÚiÚ	predictedÚpredicted_labels                 r   Údataset_predictionsr±   G  s  € Ø	‡J�J„LÜ*3°E·L±LÓ*A×B¡  A�q˜!‘tÐB€OÑBä×.Ñ.¨wÀ4ÔH€Oà€KØ€EØ!×&Ñ&Ó(ò &ˆØ Ñ'ˆÙ eÖ, ˜˜!›Ò,Ó-ˆÜ”s˜5“zÓ"ò 	&ˆAÜŸ™ V¨A¡YÓ/ˆIØ-¨i¯n©nÓ.>Ñ?ˆOØ×Ñ˜Ô/Ø�L‰L˜˜q™ !™Õ%ñ		&ð&ô —,‘,˜{¨EÓ2€KØÐùó! Cùò -s   ©DÁ2D
c                 óž   — i }|D ]  }i ||<   Œ	 t        | |«      D ]/  \  }}|j                  }||   j                  |d«      dz   ||   |<   Œ1 |S )z}
    Returns a confusion matrix

    First key: gold
    Second key: predicted
    so: confusion_matrix[gold][predicted]
    r   r   )ÚzipÚ	sentimentr�   )r©   r¥   r™   Úconfusion_matrixÚlabelr°   ÚdatumÚexpected_labels           r   Úconfusion_datasetr¹   [  s~   € ð ÐØò %ˆØ"$Ð˜Òð%ô #& k°7Ó";ò yÑˆ˜ØŸ™ˆØ<LÈ^Ñ<\×<`Ñ<`ÐapÐrsÓ<tÐwxÑ<xÐ˜Ñ(¨Ò9ðyð Ðr   c                 óÜ  — | j                  «        |€(t        | j                  «      D ��ci c]  \  }}||“Œ
 }}}d}t        j                  |«      }|j                  «       D ]ü  }	||	   }
|
D �cg c]  }||j                     ‘Œ }} | |
«      }t        t        |«      «      D ]º  }t        j                  ||   «      }|j                  «       }|r€||v r||   }nvd}|rog }t        t        ||   «      «      D ])  }|j                  ||   |   j                  «       |f«       Œ+ |j                  d„ ¬«       |D ]  \  }}||v sŒ||   }d} n |sŒ­|||   k(  sŒ¶|dz   }Œ¼ Œþ |S c c}}w c c}w )aó  
    remap_labels: a dict from old label to new label to use when
    testing a classifier on a dataset with a simpler label set.
    For example, a model trained on 5 class sentiment can be tested
    on a binary distribution with {"0": "0", "1": "0", "3": "1", "4": "1"}

    forgive_unmapped_labels says the following: in the case that the
    model predicts "2" in the above example for remap_labels, instead
    treat the model's prediction as whichever label it gave the
    highest score
    r   Fc                 ó   — | d    S )Nr   r   rH   s    r   rJ   zscore_dataset.<locals>.<lambda>”  s   € °!°A±$°€ r   )ÚkeyTr   )r—   r˜   r™   rš   r›   rœ   r´   r�   rž   rŸ   r    r¡   r¢   Úsort)r¤   r¥   Ú	label_mapÚremap_labelsrX   r¦   rI   Úcorrectr¨   r«   r¬   Úexpected_labelsr­   r®   r¯   r°   ÚfoundÚitemsÚjrx   r¡   s                        r   Úscore_datasetrÅ   n  sš  € ð 
‡J�J„LØÐÜ(1°%·,±,Ó(?×@™f˜q !�Q˜‘TÐ@ˆ	Ñ@Ø€GÜ×.Ñ.¨wÓ7€Oà!×&Ñ&Ó(ò  &ˆà Ñ'ˆØ;@ÖA°a˜9 Q§[¡[Ó1ÐAˆÐAá�u“ˆä”s˜?Ó+Ó,ò 	&ˆAÜŸ™ V¨A¡YÓ/ˆIØ'Ÿn™nÓ.ˆOÙØ" lÑ2Ø&2°?Ñ&C‘Oà!�EÙ.Ø "˜Ü!&¤s¨6°!©9£~Ó!6ò C˜AØ!ŸL™L¨&°©)°A©,×*;Ñ*;Ó*=¸qÐ)AÕBðCàŸ
™
¡˜
Ô7Ø',ò &™G˜A˜tØ# |Ò3Ø2>¸tÑ2D Ø(, Ù %ð	&ñ !Ø à /°!Ñ"4Ó4Ø! A™+‘ñ3	&ð &ðB €NùóK Aùò Bs   «E#Á.E)c           	      ó,  — t        | |«      }t        ||| j                  «      }t        j	                  dj                  t        || j                  «      «      «       t        |«      \  }}t        |«      }t        j	                  d|t        |«      |t        |«      z  fz  «       t        j	                  dj                  |«      «       ||z  }|t        j                  u r|||fS |t        j                  u r|||fS t        dj                  |«      «      ‚)NzDev set confusion matrix:
{}z1Dev set: %d correct of %d examples.  Accuracy: %fúMacro f1: {}zUnknown scoring method {})r±   r¹   r™   r‚   rƒ   Úformatr   r   r   rž   r!   r$   r%   Ú
ValueError)	r¤   Údev_setÚdev_eval_scoringr©   rµ   rÀ   ÚtotalÚmacro_f1Úaccuracys	            r   Úscore_dev_setrÏ   ¤  sþ   € Ü% e¨WÓ5€KÜ(¨°g¸u¿|¹|ÓLÐÜ
‡K�KÐ/×6Ñ6Ô7GÐHXÐZ_×ZfÑZfÓ7gÓhÔiÜ*Ð+;Ó<�N€GˆUÜ$Ð%5Ó6€HÜ
‡K�KÐCØœ#˜g›,¨´#°g³,Ñ(>Ð?ñ@ô Aä
‡K�K�×%Ñ% hÓ/Ô0à˜‰€HØœ:×.Ñ.Ñ.Ø˜ 8Ð+Ð+Ø	œZ×3Ñ3Ñ	3Ø˜ 8Ð+Ð+äÐ4×;Ñ;Ð<LÓMÓNÐNr   c           	      ó˜   — t         j                  j                  | «      \  }}| dj                  di ||j                  |dz  dœ¤Žz   |z   S )zc
    Build an informative intermediate checkpoint name from a base name, epoch #, and accuracy
    z%.E{epoch:04d}-{score_type}{acc:05.2f}rC   )ÚepochÚ
score_typeÚaccr   )ÚosÚpathÚsplitextrÈ   Úvalue)ÚfilenamerÑ   Údev_scoringÚscoreÚrootÚexts         r   Úintermediate_namerÝ   ¶  sw   € ô —‘× Ñ  Ó*�I€Dˆ#ØÐ@Ð9×@Ñ@ñ  JÈUÐbm×bsÑbsð  }Bð  EHñ  }Hñ  DIñ  Jñ  Jð  MPñ  Pð  Pr   c                 óD  — t         j                  d«       d}| j                  «       D ]`  \  }}|j                  «       |j	                  «       z  }||z  }t         j                  d||j                  «       |j	                  «       |«       Œb t         j                  d|«       y )Nz--- Model parameter sizes ---r   z  %s %d %d %dz  Total size: %d)r‚   ÚdebugÚnamed_parametersÚelement_sizeÚnelement)r¤   Ú
total_sizerN   ÚparamÚ
param_sizes        r   Úlog_param_sizesræ   ½  sŒ   € Ü
‡L�LÐ0Ô1Ø€JØ×-Ñ-Ó/ò `‰ˆˆeØ×'Ñ'Ó)¨E¯N©NÓ,<Ñ<ˆ
Ø�jÑ ˆ
Ü�‰�_ d¨E×,>Ñ,>Ó,@À%Ç.Á.ÓBRÐT^Õ_ð`ô ‡L�LÐ# ZÕ0r   c           
      óÆ  — t         j                  t        j                  «       | j                  }| j
                  }t        |j                  «       «      j                  }	t        j                  d|	z  «       t        |«      D �
�ci c]  \  }
}||
“Œ
 }}
}t        |«      D �
�ci c]  \  }
}|t        j                  |
d|	¬«      “Œ  }}
}d„ }|j                  t        j                   k(  r+t        j                  d«       t#        j$                  «       }�n;|j                  t        j&                  k(  rAt        j                  d«       t        j(                  |D �cg c]
  }||d      ‘Œ c}d¬«      }nÝ|j                  t        j*                  k(  rAt        j                  d	«       t        j(                  |D �cg c]
  }||d      ‘Œ c}d
¬«      }n|j                  t        j,                  k(  r>	 ddlm} t        j                  d|j4                  «       d„ } ||j4                  ¬«      }n$t7        dj9                  |j                  «      «      ‚|j;                  |	«       t=        j>                  |«      }| j@                  dkD  rXtC        |||jD                  «      }t        j                  d«       | jF                  � t        j                  d| jF                  «       tI        |«       |jJ                  r•dd l%}|jL                  r|jL                  nd|jN                  z  }|jQ                  ||¬«       |jR                  jU                  dd¬«       |jR                  jU                  dd¬«       |jR                  jU                  dd¬«       |jW                  «       D ].  \  }}|jX                  d   d   }t        j                  d||«       Œ0 |jZ                  rD| j\                  dk(  r5t_        || j\                  |jD                  d«      }| ja                  |d¬«       tc        | j\                  |jd                  «      D �]U  | _.        d}d}t=        jf                  ||jh                  |jj                  «      }|jm                  «        t        j                  d| j\                  «       |jn                  r|jo                  «        t        |«      D �]Š  \  }}| xj@                  d z  c_         t        jq                  d!|| j@                  «       t        jr                  |D �cg c]  }||jt                     ‘Œ c}«      }|jw                  «       D ]  }|jy                  «        Œ  ||«      } ||«      } |||«      } | j{                  «        |jw                  «       D ]  }|j}                  «        Œ || j                  «       z  }|d z   |j€                  z  dk(  s�Œ
||j€                  z  }!t        j                  d"| j\                  d z   |d z   |!«       |jJ                  rjƒ                  d#|!i| j@                  ¬$«       |j„                  dkD  �r|d z   |j„                  z  dk(  rót        j                  d%«       tC        |||jD                  «      \  }"}#}$|jJ                  r jƒ                  |#|$d&œ| j@                  ¬$«       | jF                  �|"| jF                  kD  rF|"| _#        | ja                  |d¬«       t        j                  d'|#|$| j\                  d z   |d z   fz  «       |jm                  «        |jn                  r| j                  jo                  «        ||z  }d}�Œ� ||z  }t        j                  d(| j\                  d z   |fz  «       tC        |||jD                  «      \  }"}#}$|jJ                  r!jƒ                  |#|$|d)œ| j@                  ¬$«       |r | ja                  || j\                  d z   ¬*«       |jZ                  r8t_        || j\                  d z   |jD                  |"«      }| ja                  |d¬«       | jF                  �|"| jF                  kD  s�Œ|"| _#        | ja                  |d¬«       t        j                  d+|#|$| j\                  d z   fz  «       �ŒX |jJ                  rj‡                  «        y y c c}}
w c c}}
w c c}w c c}w # t2        $ r t3        d«      ‚w xY wc c}w ),NzCurrent device: %sF)Úrequires_gradÚdevicec                 ó   — | S rF   r   rH   s    r   rJ   ztrain_model.<locals>.<lambda>Ô  s   €  € r   zCreating CrossEntropyLossz,Creating weighted cross entropy loss w/o logr   )Úlog_dampenedz+Creating weighted cross entropy loss w/ logT)Ú	FocalLossz^focal_loss not installed.  Must `pip install focal_loss_torch` to use the --loss=focal featurezCreating FocalLoss with loss %fc                 ó0   — t        j                  | d¬«      S )Nr   )Údim)rŸ   ÚsoftmaxrH   s    r   rJ   ztrain_model.<locals>.<lambda>ä  s   € ¤E§M¡M°!¸Ô$;€ r   )ÚgammazUnknown loss function {}z&Reloaded model for continued training.zPrevious best score: %.5fz%s_classifier)rN   ÚconfigrÎ   Úmax)ÚsummaryrÍ   Ú
epoch_lossÚminÚlrzoptimizer %s learning rate: %sg        )Úsave_optimizerzStarting epoch %dr   zStarting batch: %d step %dz[%d, %5d] Average loss: %.3fÚ
train_loss)Ústepz---- Interim analysis ----)rÎ   rÍ   zQSaved new best score model!  Accuracy %.5f   Macro F1 %.5f   Epoch %5d   Batch %dz"Finished epoch %d  Total loss %.3f)rÎ   rÍ   rô   )Úepochs_trainedzFSaved new best score model!  Accuracy %.5f   Macro F1 %.5f   Epoch %5d)Dr‰   ÚsetLevelÚloggingÚDEBUGr¤   Ú	optimizerÚnextÚ
parametersré   r‚   rƒ   r˜   rŸ   Útensorr   r   r   ÚnnÚCrossEntropyLossr   Úweighted_cross_entropy_lossr   r   Úfocal_loss.focal_lossrì   ÚImportErrorÚloss_focal_gammarÉ   rÈ   Útorš   r›   Úglobal_steprÏ   rË   Ú
best_scoreræ   r‹   rŠ   Ú	shorthandÚinitÚrunÚdefine_metricrÃ   Úparam_groupsÚsave_intermediate_modelsrú   rÝ   Úsaver�   Ú
max_epochsÚshuffle_datasetÚ
batch_sizeÚbatch_single_itemr9   Ú	log_normsrß   Ústackr´   ÚvaluesÚ	zero_gradÚbackwardrù   r¡   ÚtickÚlogÚdev_eval_batchesÚfinish)%ÚtrainerÚ
model_fileÚcheckpoint_filer„   Ú	train_setrÊ   r™   r¤   rþ   ré   r¦   rI   r¾   Úlabel_tensorsÚprocess_outputsÚloss_functionrì   Útrain_set_by_lenrx   r‹   rŠ   Úopt_nameÚoptÚ
current_lrÚintermediate_fileÚrunning_lossrô   Úshuffled_batchesÚ	batch_numr¬   Úbatch_labelsÚoutputsÚ
batch_lossrø   Ú	dev_scorerÎ   rÍ   s%                                        r   Útrain_modelr2  Æ  sç  € Ü×Ñ”W—]‘]Ô#ð �M‰M€EØ×!Ñ!€Iä�%×"Ñ"Ó$Ó%×,Ñ,€FÜ
‡K�KÐ$ vÑ-Ô.ä$-¨fÓ$5×6™&˜1˜a��A‘Ð6€IÑ6ä#,¨VÓ#4÷6Ù˜!˜Qð œŸ™ Q°eÀFÔKÑKð 6€Mñ 6ñ "€OØ‡y�y”D—J‘JÒÜ�‰Ð/Ô0Ü×+Ñ+Ó-ŠØ	�‰”d×)Ñ)Ò	)Ü�‰ÐBÔCÜ×8Ñ8ÐS\Ö9]Èa¸)ÀAÀaÁD»/Ò9]ÐlqÔr‰Ø	�‰”d—n‘nÒ	$Ü�‰ÐAÔBÜ×8Ñ8ÐS\Ö9]Èa¸)ÀAÀaÁD»/Ò9]ÐlpÔq‰Ø	�‰”d—j‘jÒ	 ð	@Ý7ô 	�‰Ð5°t×7LÑ7LÔMÙ;ˆÙ!¨×(=Ñ(=Ô>‰äÐ3×:Ñ:¸4¿9¹9ÓEÓFÐFØ×Ñ�VÔä×/Ñ/°	Ó:Ðà×Ñ˜QÒä˜% ¨$×*?Ñ*?Ó@ˆÜ�‰Ð<Ô=Ø×ÑÐ)Ü�K‰KÐ3°W×5GÑ5GÔHä�EÔð ‡z‚zÛØ(,¯ª�T—_’_¸_ÈtÏ~É~Ñ=]ˆ
Ø�
‰
˜
¨4ˆ
Ô0Ø�	‰	×Ñ 
°EÐÔ:Ø�	‰	×Ñ 
°EÐÔ:Ø�	‰	×Ñ °eÐÔ<à"Ÿ™Ó*ò L‰ˆ�#Ø×%Ñ% aÑ(¨Ñ.ˆ
Ü�‰Ð4°hÀ
ÕKðLð
 ×$Ò$¨×)?Ñ)?À1Ò)DÜ-¨j¸'×:PÑ:PÐRV×RgÑRgÐilÓmÐØ�‰Ð&°uˆÔ=Ü"'¨×(>Ñ(>ÀÇÁÓ"Pó @SˆÔØˆØˆ
Ü×/Ñ/Ð0@À$Ç/Á/ÐSW×SiÑSiÓjÐà�‰ŒÜ�‰Ð'¨×)?Ñ)?Ô@Ø�>Š>Ø�O‰OÔä )Ð*:Ó ;ó &	#ÑˆI�uà×Ò 1Ñ$ÕÜ�L‰LÐ5°yÀ'×BUÑBUÔVä Ÿ;™;ÈEÖ'RÀq¨°a·k±kÓ(BÒ'RÓSˆLð !×'Ñ'Ó)ò  �Ø—‘•ð ñ ˜E“lˆGÙ% gÓ.ˆGÙ& w°Ó=ˆJØ×ÑÔ!Ø ×'Ñ'Ó)ò �Ø—‘•
ðð ˜JŸO™OÓ-Ñ-ˆLØ˜A‘ §¡Ñ*¨aÔ/Ø)¨D¯I©IÑ5�
Ü—‘Ð:¸G×<RÑ<RÐUVÑ<VÐXaÐdeÑXeÐgqÔrØ—:’:Ø—I‘I˜|¨ZÐ8¸w×?RÑ?R�IÔSØ×(Ñ(¨1Ó,°)¸a±-À4×CXÑCXÑ1XÐ\]Ò1]Ü—K‘KÐ <Ô=Ü4AÀ%ÈÐRV×RgÑRgÓ4hÑ1�I˜x¨Ø—z’zØŸ	™	¨xÀXÑ"NÐU\×UhÑUh˜	ÔiØ×)Ñ)Ð1°YÀ×ASÑASÒ5SØ-6˜Ô*ØŸ™ ZÀ˜ÔFÜŸ™Ð$wð  |Dð  FNð  PW÷  Pfñ  Pfð  ghñ  Phð  jsð  tuñ  juð  {vñ  %vô  wØ—K‘K”MØ—~’~ØŸ™×/Ñ/Ô1Ø˜lÑ*�
Ø"’ðM&	#ðP 	�lÑ"ˆ
ä�‰Ð8¸G×<RÑ<RÐUVÑ<VÐXbÐ;cÑcÔdÜ(5°e¸WÀd×F[ÑF[Ó(\Ñ%ˆ	�8˜XØ�:Š:Ø�I‰I 8¸ÐQ[Ñ\Ðcj×cvÑcvˆIÔwÙØ�L‰L˜¸7×;QÑ;QÐTUÑ;UˆLÔVØ×(Ò(Ü 1°*¸g×>TÑ>TÐWXÑ>XÐZ^×ZoÑZoÐqzÓ {ÐØ�L‰LÐ*¸5ˆLÔAØ×ÑÐ%¨°W×5GÑ5GÔ)GØ!*ˆGÔØ�L‰L˜°EˆLÔ:Ü�K‰KÐ`ÐdlÐnvÐx÷  yOñ  yOð  PQñ  yQð  dRñ  Rö  SðA@SðD ‡z‚zØ�‰�ð ùóq 7ùó6ùò :^ùò :^øô ò 	@ÜÐ~ÓÐð	@üòh (Ss*   Âb0Â##b6Åb<Æ7cÇ.c Ôcãcc           	      ó–
  — t        | «      } t        j                  | j                  «      }t        j                  d|z  «       t        j                  | j                  «       t        | «      }d }| j                  �rt        j                  | j                  | j                  | j                  «      }t        j                  d| j                  z  «       t        j                  dt        t        j                   |«      «      z  «       t"        j%                  t&        j(                  «       t"        j                  d| j*                  «       | j*                  ret        j,                  | j                  || j.                  «      }t"        j                  d|«       n#| j0                  s|r|| _        nt3        d«      ‚d }| j                  rD|�Bt4        j6                  j9                  |«      r#t;        j<                  || | j                  ¬«      }nO| j0                  r-t;        j<                  | j0                  | | j                  ¬«      }nt;        j>                  | «      }|j@                  jC                  «        | j                  rót        jD                  | t        «       t        j                  | jF                  | j                  d ¬«      }t        j                  d	| jF                  «       t        j                  d
t        «      «       t        j                  dt        |«      «       t        jH                  |j@                  jJ                  |«       tM        |||| |||j@                  jJ                  «       | jN                  r|j@                  jO                  «        t        j                  | jP                  | j                  d ¬«      }t        j                  d| jP                  z  «       t        jH                  |j@                  jJ                  |«       | jR                  €×tU        |j@                  |«      }tW        |||j@                  jJ                  «      }	| jX                  rt        j                  d|«       t        j                  dj[                  t]        |	|j@                  jJ                  «      «      «       t_        |	«      \  }
}t        j                  dj[                  ta        |	«      «      «       n8tc        |j@                  || jR                  | jd                  ¬«      }
t        |«      }t        j                  d|
||
|z  fz  «       y )NzUsing random seed: %dzUsing training set: %szTraining set has %d labelszSaving checkpoints: %szCheckpoint filename: %szFNo model provided and not asked to train a model.  This makes no sense)Úload_optimizer)Úmin_lenzUsing dev set: %szTraining set has %d itemszDev set has %d itemszUsing test set: %szList of predictions: %szConfusion matrix:
{}rÇ   )r¿   rX   z2Test set: %d correct of %d examples.  Accuracy: %f)3rˆ   r   Úset_random_seedÚseedr‚   rƒ   Ú
ensure_dirÚsave_dirr†   r9   rš   Úread_datasetÚ
train_fileÚwordvec_typeÚmin_train_lenrž   Údataset_labelsr‰   rû   rü   rý   rB   Úcheckpoint_nameÚcheckpoint_save_nameÚ	load_namerÉ   rÔ   rÕ   Úexistsr   ÚloadÚbuild_new_modelr¤   Úlog_configurationÚlog_training_argsÚdev_fileÚcheck_labelsr™   r2  r  Ú	test_fileÚtest_remap_labelsr±   r¹   Úoutput_predictionsrÈ   r   r   r   rÅ   rX   )r„   r7  Ú	save_namer!  r"  r  rÊ   Útest_setr©   rµ   rÀ   rÌ   s               r   ÚmainrN  K  s¯  € Ü�dÓ€DÜ× Ñ  §¡Ó+€DÜ
‡K�KÐ'¨$Ñ.Ô/ä	×Ñ�T—]‘]Ô#ä$ TÓ*€Ið €OØ‡zƒzÜ×%Ñ% d§o¡o°t×7HÑ7HÈ$×J\ÑJ\Ó]ˆ	Ü�‰Ð,¨t¯©Ñ>Ô?Ü�‰Ð0´3´t×7JÑ7JÈ9Ó7UÓ3VÑVÔWÜ×ÑœŸ™Ô'ä�‰Ð-¨t¯©Ô?Ø�?Š?Ü#×3Ñ3°D·M±MÀ9Èd×NgÑNgÓhˆOÜ�L‰LÐ2°OÕDØ�^Š^ÙØ&ˆD�NäÐeÓfÐfàˆ	à‡z‚z�oÐ1´b·g±g·n±nÀ_Ô6UÜ—,‘,˜°ÀTÇZÁZÔP‰Ø	�ŠÜ—,‘,˜tŸ~™~¨tÀDÇJÁJÔO‰ä×)Ñ)¨$°	Ó:ˆà‡M�M×#Ñ#Ô%à‡z‚zÜ×Ñ ¤fÔ-ä×#Ñ# D§M¡M°4×3DÑ3DÈdÔSˆÜ�‰Ð'¨¯©Ô7Ü�‰Ð/´°Y³Ô@Ü�‰Ð*¬C°«LÔ9Ü×Ñ˜'Ÿ-™-×.Ñ.°Ô8ä�G˜Y¨¸¸yÈ'ÐSZ×S`ÑS`×SgÑSgÔhà‡~‚~Ø�‰×ÑÔ!Ü× Ñ  §¡°×1BÑ1BÈDÔQ€HÜ
‡K�KÐ$ t§~¡~Ñ5Ô6Ü×Ñ�g—m‘m×*Ñ*¨HÔ5à×ÑÐ%Ü)¨'¯-©-¸ÓBˆÜ,¨[¸(ÀGÇMÁM×DXÑDXÓYÐØ×"Ò"Ü�K‰KÐ1°;Ô?Ü�‰Ð+×2Ñ2Ô3CÐDTÐV]×VcÑVc×VjÑVjÓ3kÓlÔmÜ.Ð/?Ó@‰ˆ�Ü�‰�N×)Ñ)Ô*?Ð@PÓ*QÓRÕSä §¡¨xØ-1×-CÑ-CØ8<×8TÑ8TôVˆô �H“ˆÜ
‡K�KÐDØ˜% ¨5¡Ð1ñ2õ 3r   Ú__main__rF   )NNF)=rj   r*   rü   rÔ   ÚrandomÚreÚenumr   rŸ   Útorch.nnr  Ústanza.models.commonr   r   Ústanza.models.pos.vocabr   Ústanza.models.classifiers.dataÚmodelsÚclassifiersrš   Ú!stanza.models.classifiers.trainerr   Ústanza.models.classifiers.utilsr   r	   r
   Ú stanza.models.common.peft_configr   r   Ústanza.utils.confusionr   r   r   r   r!   Ú	getLoggerr‚   r‰   rû   ÚWARNINGrn   ro   rp   r0   r”   ÚDEFAULT_LEARNING_EPSÚDEFAULT_LEARNING_RHOr’   r�   rv   r†   rˆ   r±   r¹   rÅ   rÏ   rÝ   ræ   r2  rN  r   r   r   r   ú<module>ra     sg  ðÛ Û 
Û Û 	Û Û 	Ý ã Ý å %Ý &Ý -ç -Ó -Ý 5ß KÑ Kß Mç aÑ aôˆ4ô ô�ô ð 
ˆ×	Ñ	˜8Ó	$€Ø
ˆ'×
Ñ
Ð8Ó
9€à €× Ñ Ð$Ó %× .Ñ .¨w¯©Ô ?à3€Ø1€Ø3€ð;òzðL %+¸ÀEÐX_Ðlrð  |Añ  CÐ Ø&+¸ÈÑNÐ ØÐ Ø #¨CÑ1Ð Ø"&°FÀ4ÐV\ÐimÑoÐ òPòdóò.ò(ð& -1Ø=Bó4òlOò$Pò1òCóJF3ðP ˆzÒÙ…Fð r   