Ë
    \›¾jK?  ã                   ó>  — d Z ddlZddlmZ ddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlZddlZddlmZmZmZ ddlmZ ddlmZ ddlmZ  ej0                  d	«      Zd
„ Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z dd„Z!dd„Z"d„ Z#dd„Z$d„ Z%d„ Z&d„ Z'd„ Z(e)dk(  r e"«        yy)zR
Entry point for training and evaluating a character-level neural language model.
é    N)Úcopy)ÚGeneratorType)Úbuild_charlm_vocabÚCharacterLanguageModelÚCharacterLanguageModelTrainer)Ú	CharVocab)Úutils)Ú_training_loggingÚstanzac                 óz   — t        | t        j                  «      r| j                  «       S t	        d„ | D «       «      S )zJWraps hidden states in new Tensors,
    to detach them from their history.c              3   ó2   K  — | ]  }t        |«      –— Œ y ­w©N)Úrepackage_hidden)Ú.0Úvs     ú^/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/stanza/models/charlm.pyú	<genexpr>z#repackage_hidden.<locals>.<genexpr>   s   è ø€ Ò4¨QÔ% a×(Ñ4ùs   ‚)Ú
isinstanceÚtorchÚTensorÚdetachÚtuple)Úhs    r   r   r      s/   € ô �!”U—\‘\Ô"Ø�x‰x‹zÐäÑ4°!Ô4Ó4Ð4ó    c                 ó    — | j                  d«      |z  }| j                  dd||z  «      } | j                  |d«      } | j                  |«      } | S )Nr   éÿÿÿÿ)ÚsizeÚnarrowÚviewÚto)ÚdataÚbszÚdeviceÚnbatchs       r   Úbatchifyr%       sL   € à�Y‰Y�q‹\˜SÑ €Fà�;‰;�q˜!˜V c™\Ó*€Dà�9‰9�S˜"Ó€DØ�7‰7�6‹?€DØ€Kr   c                 ó®   — t        || j                  d«      dz
  |z
  «      }| d d …|||z   …f   }| d d …|dz   |dz   |z   …f   j                  d«      }||fS )Né   r   )Úminr   Úreshape)ÚsourceÚiÚseq_lenr!   Útargets        r   Ú	get_batchr.   *   sj   € Ü�'˜6Ÿ;™; q›>¨AÑ-°Ñ1Ó2€GØ’!�Q�q˜‘y�[�.Ñ!€DØ’A�q˜‘s˜1˜Q™3˜w™;�Ð&Ñ'×/Ñ/°Ó3€FØ�ˆ<Ðr   c                 óâ   — t        j                  | «      5 }|j                  «       }d d d «       |d   j                  «      }|dk(  r|d d d…   }t	        j
                  |«      S # 1 sw Y   Œ?xY w)NÚcharÚbackwardr   )r	   Úopen_read_textÚreadÚmapr   Útensor)ÚfilenameÚvocabÚ	directionÚfinr!   Úidxs         r   Ú	load_filer;   0   sk   € Ü	×	Ñ	˜hÓ	'ð ¨3Ø�x‰x‹zˆ÷ð �‰-×
Ñ
˜DÓ
!€CØ�JÒ c©$¨B¨$¡i Ü�<‰<˜ÓÐ÷ð ús   –A%Á%A.c              #   ó^  K  — t         j                  j                  | «      ryt        t        j                  | «      «      }|D ]U  }t
        j                  dj                  |«      «       t        t         j                  j                  | |«      ||«      }|–— ŒW y t        | ||«      }|–— y ­w)NzLoading data from {})
ÚosÚpathÚisdirÚsortedÚlistdirÚloggerÚinfoÚformatr;   Újoin)r>   r7   r8   Ú	filenamesr6   r!   s         r   Ú	load_datarG   8   s‡   è ø€ Ü	‡w�w‡}�}�TÔÜœ2Ÿ:™: dÓ+Ó,ˆ	Ø!ò 	ˆHÜ�K‰KÐ.×5Ñ5°hÓ?Ô@ÜœRŸW™WŸ\™\¨$°Ó9¸5À)ÓLˆDØ‹Jñ	ô
 ˜˜u iÓ0ˆØ‹
ùs   ‚B+B-c                  ó  — t        j                  t         j                  ¬«      } | j                  dt        d¬«       | j                  dt        d¬«       | j                  dt        d¬«       | j                  d	t        d
¬«       | j                  ddddg¬«       | j                  ddddgd¬«       | j                  ddddd¬«       | j                  ddddd¬«       | j                  dt
        dd¬«       | j                  dt
        d d!¬«       | j                  d"t
        d#d$¬«       | j                  d%t        d&d'¬«       | j                  d(t        d)d*¬«       | j                  d+t        d,d-¬«       | j                  d.t
        dd/¬«       | j                  d0t
        d1d2¬«       | j                  d3t
        d4d5¬«       | j                  d6t        d7d8¬«       | j                  d9t        d:d;¬«       | j                  d<t        d7d=¬«       | j                  d>t
        d#d?¬«       | j                  d@t        d,dA¬«       | j                  dBt        d,dC¬«       | j                  dDt
        dEdF¬«       | j                  dGt
        d4dH¬«       | j                  dIt
        dJdK¬«       | j                  dLt        d dM¬«       | j                  dNt        d dO¬«       | j                  dPt        d dQ¬«       | j                  dRdSdTdU¬V«       | j                  dWt        dXdY¬«       | j                  dZd[d\¬]«       t        j                  | «       | j                  d^t
        d_¬`«       | j                  dad[db¬]«       | j                  dcd dd¬e«       | S )fN)Úformatter_classz--train_filezInput plaintext file)ÚtypeÚhelpz--train_dirz>If non-empty, load from directory with multiple training filesz--eval_filez)Input plaintext file for the dev/test setz--shorthandzUD treebank shorthandz--modeÚtrainÚpredict)ÚdefaultÚchoicesz--directionÚforwardr1   z"Forward or backward language model)rN   rO   rK   z	--forwardÚstore_constr8   zTrain a forward language model)ÚactionÚdestÚconstrK   z
--backwardzTrain a backward language modelz--char_emb_diméd   zDimension of unit embeddings)rJ   rN   rK   z--char_hidden_dimi   zDimension of hidden unitsz--char_num_layersr'   z#Layers of RNN in the language modelz--char_dropoutgš™™™™™©?zDropout probabilityz--char_unit_dropoutgñhãˆµøä>z1Randomly set an input char to UNK during trainingz--char_rec_dropoutç        zRecurrent dropout probabilityz--batch_sizezBatch size to usez--bptt_sizeéú   z%Sequence length to consider at a timez--epochsé2   z#Total epochs to train the model forz--max_grad_normg      Ð?z Maximum gradient norm to clip toz--lr0é   zInitial learning ratez--annealzHAnneal the learning rate by this amount when dev performance deterioratez
--patiencez(Patience for annealing the learning ratez--weight_decayzWeight decayz
--momentumzMomentum for SGD.z--cutoffiè  zJFrequency cutoff for char vocab. By default we assume a very large corpus.z--report_stepsz#Update step interval to report lossz--eval_stepsi † zKUpdate step interval to run eval on dev; set to -1 to eval after each epochz--save_namezFile name to save the modelz--vocab_save_namezFile name to save the vocabz--checkpoint_save_namez,File name to save the most recent checkpointz--no_checkpointÚ
checkpointÚstore_falsezDon't save checkpoints)rS   rR   rK   z
--save_dirzsaved_models/charlmzDirectory to save models inz	--summaryÚ
store_truez&Use summary writer to record progress.)rR   rK   z--seediÒ  )rJ   rN   z--wandbzStart a wandb session and write the results of training.  Only applies to training.  Use --wandb_name instead to specify a namez--wandb_namezWName of a wandb session to start when training.  Will default to the dataset short name)rN   rK   )	ÚargparseÚArgumentParserÚArgumentDefaultsHelpFormatterÚadd_argumentÚstrÚintÚfloatr	   Úadd_device_args)Úparsers    r   Úbuild_argparserf   C   s¸  € Ü×$Ñ$´X×5[Ñ5[Ô\€FØ
×Ñ˜¬SÐ7MÐÔNØ
×Ñ˜¬CÐ6vÐÔwØ
×Ñ˜¬CÐ6aÐÔbØ
×Ñ˜¬CÐ6MÐÔNà
×Ñ˜¨'¸GÀYÐ;OÐÔPØ
×Ñ˜¨yÀ9ÈjÐBYð  aEÐô  FØ
×Ñ˜¨MÀÐS\ð  dDÐô  EØ
×Ñ˜¨]ÀÐT^ð  fGÐô  Hà
×ÑÐ(¬s¸CÐFdÐÔeØ
×ÑÐ+´#¸tÐJeÐÔfØ
×ÑÐ+´#¸qÐGlÐÔmØ
×ÑÐ(¬u¸dÐI^ÐÔ_Ø
×ÑÐ-´EÀ4ð  OBÐô  CØ
×ÑÐ,´5À#ÐLkÐÔlà
×Ñ˜¬S¸#ÐDWÐÔXØ
×Ñ˜¬C¸ÐCjÐÔkØ
×Ñ˜
¬°bÐ?dÐÔeØ
×ÑÐ)´¸tÐJlÐÔmØ
×Ñ˜¤e°QÐ=TÐÔUØ
×Ñ˜
¬¸ð  DNÐô  OØ
×Ñ˜¬3¸Ð@jÐÔkØ
×ÑÐ(¬u¸cÈÐÔWØ
×Ñ˜¬5¸#ÐDWÐÔXØ
×Ñ˜
¬°dð  BNÐô  Oà
×ÑÐ(¬s¸BÐEjÐÔkØ
×Ñ˜¬S¸&ð  HUÐô  VØ
×Ñ˜¬C¸ÐDaÐÔbØ
×ÑÐ+´#¸tÐJgÐÔhØ
×ÑÐ0´sÀDÐO}ÐÔ~Ø
×ÑÐ)°À]ÐYqÐÔrØ
×Ñ˜¬3Ð8MÐTqÐÔrØ
×Ñ˜¨LÐ?gÐÔhÜ	×Ñ˜&Ô!Ø
×Ñ˜¤s°DÐÔ9à
×Ñ˜	¨,ð  >Ðô  @Ø
×Ñ˜°ð  <UÐô  VØ€Mr   c                 ó’   — | d   r| d   }ndj                  | d   | d   «      }t        j                  j                  | d   |«      }|S )NÚ	save_namez{}_{}_charlm.ptÚ	shorthandr8   Úsave_dir)rD   r=   r>   rE   )Úargsrh   Ú
model_files      r   Úbuild_model_filenamerm   p   sO   € ØˆKÒØ˜Ñ%‰	à%×,Ñ,¨T°+Ñ->ÀÀ[Ñ@QÓRˆ	Ü—‘—‘˜d :Ñ.°	Ó:€JØÐr   c                 óz   — t        «       }|j                  | ¬«      } | j                  rd| _        t	        | «      } | S )N©rk   T)rf   Ú
parse_argsÚ
wandb_nameÚwandbÚvars)rk   re   s     r   rp   rp   x   s9   € ÜÓ€Fà×Ñ $ÐÓ'€Dà‡‚ØˆŒ
ä�‹:€DØ€Kr   c                 ó  — t        | ¬«      } t        j                  | d   «       t        j	                  dj                  | d   | d   «      «       t        j                  | d   «       | d   dk(  rt        | «       y t        | «       y )Nro   Úseedz4Running {} character-level language model in {} moder8   Úmoderj   rL   )	rp   r	   Úset_random_seedrB   rC   rD   Ú
ensure_dirrL   Úevaluatero   s    r   Úmainrz   ƒ   sr   € Ü˜4Ô €Dä	×Ñ˜$˜v™,Ô'ä
‡K�KÐF×MÑMÈdÐS^ÑN_ÐaeÐflÑamÓnÔoä	×Ñ�T˜*Ñ%Ô&àˆF�|�wÒÜˆd�ä��r   c                 óL  — |j                  «        t        |j                  «       «      j                  }d}d}t	        |t
        «      r%t        |«      }t        |«      dk(  sJ d«       ‚|d   }t        || d   |«      }t        j                  «       5  t        d|j                  d«      dz
  | d   «      D ]Á  }	t        ||	| d   «      \  }}
t        |j                  d«      «      D �	cg c]  }	|j                  d«      ‘Œ }}	|j                  |||«      \  }}} ||j                  dt        |d   «      «      |
«      }t!        |«      }||j                  d«      |j"                  j%                  «       z  z  }ŒÃ 	 ddd«       ||j                  d«      z  S c c}	w # 1 sw Y   Œ"xY w)	z0
    Run an evaluation over entire dataset.
    Nr   r'   z!Only support single dev/test fileÚ
batch_sizeÚ	bptt_sizer   r0   )ÚevalÚnextÚ
parametersr#   r   r   ÚlistÚlenr%   r   Úno_gradÚranger   r.   rP   r   r   r!   Úitem)rk   r7   r!   ÚmodelÚ	criterionr#   ÚhiddenÚ
total_lossÚbatchesr+   r-   ÚlensÚoutputÚdecodedÚlosss                  r   Úevaluate_epochr�   ‘   s{  € ð 
‡J�J„LÜ�%×"Ñ"Ó$Ó%×,Ñ,€FØ€FØ€JÜ�$œÔ&Ü�D‹zˆÜ�4‹y˜AŠ~ÐBÐBÓBˆ~Ø�A‰wˆÜ�t˜T ,Ñ/°Ó8€GÜ	�‰‹ñ 	:Ü�q˜'Ÿ,™, q›/¨AÑ-¨t°KÑ/@ÓAò 	:ˆAÜ$ W¨a°°kÑ1BÓC‰LˆD�&Ü*/°·	±	¸!³Ó*=Ö> Q�D—I‘I˜a•LÐ>ˆDÐ>à&+§m¡m°D¸$ÀÓ&GÑ#ˆF�F˜GÙ˜WŸ\™\¨"¬c°%¸±-Ó.@ÓAÀ6ÓJˆDä% fÓ-ˆFØ˜$Ÿ)™) A›,¨¯©¯©Ó)9Ñ9Ñ9‰Jñ	:÷	:ð ˜Ÿ™ Q›Ñ'Ð'ùò ?÷	:ð 	:ús    ÂAFÃ&FÃ>A:FÆFÆF#c                 óx  — t        j                   «       }t        | |||j                  |j                  «      }	t	        j
                  |	«      }
t        t        j                   «       |z
  «      }t        || «      }|j                  j                  |	«       t        || «      }||k7  rt        j                  d|«       t        j                  dj                  |j                  ||	|
«      «       |�|	|k  rC|	}|j                  |d¬«       t        j                  dj                  |j                  «      «       |r<|j                  d|	|j                  ¬«       |j                  d|
|j                  ¬«       |rA|j                  |d	¬«       t        j                  d
j                  |j                  «      «       |	|
|fS )z`
    Run an evaluation over entire dataset, print progress and save the model if necessary.
    zUpdating learning rate to %fzY| eval checkpoint @ global step {:10d} | time elapsed {:6d}s | loss {:5.2f} | ppl {:8.2f}F)Úfullz#new best model saved at step {:10d}Údev_loss)Úglobal_stepÚdev_pplTz#new checkpoint saved at step {:10d})Útimer�   r†   r‡   ÚmathÚexprb   Úget_current_lrÚ	schedulerÚsteprB   rC   rD   r“   ÚsaveÚ
add_scalar)rk   r7   r!   ÚtrainerÚ	best_lossrl   Úcheckpoint_fileÚwriterÚ
start_timerŽ   ÚpplÚelapsedÚprevious_lrÚ
current_lrs                 r   Úevaluate_and_saver¦   ª   su  € ô —‘“€JÜ˜$  t¨W¯]©]¸G×<MÑ<MÓN€DÜ
�(‰(�4‹.€CÜ”$—)‘)“+ 
Ñ*Ó+€Gä  ¨$Ó/€KØ×Ñ×Ñ˜4Ô Ü ¨Ó.€JØ�jÒ Ü�‰Ð2°JÔ?Ü
‡K�KØc×jÑjØ×ÑØØØó		
ôð Ð˜D 9Ò,Øˆ	Ø�‰�Z eˆÔ,Ü�‰Ð9×@Ñ@À×ATÑATÓUÔVÙØ×Ñ˜* d¸×8KÑ8KÐÔLØ×Ñ˜) S°g×6IÑ6IÐÔJÙØ�‰�_¨4ˆÔ0Ü�‰Ð9×@Ñ@À×ATÑATÓUÔVà��iÐÐr   c                 ód   — | j                   j                  «       j                  d|d   g«      d   S )NÚ_last_lrÚlr0r   )r™   Ú
state_dictÚget)r�   rk   s     r   r˜   r˜   Í   s/   € Ø×Ñ×'Ñ'Ó)×-Ñ-¨j¸4À¹;¸-ÓHÈÑKÐKr   c                 ó^   — dt        j                  t        j                  | d„ d¬«      «      iS )Nr0   c                 ó   — | S r   © )ÚstorageÚlocs     r   ú<lambda>z!load_char_vocab.<locals>.<lambda>Ñ   s   € ÐZa€ r   T)Úweights_only)r   Úload_state_dictr   Úload)Ú
vocab_files    r   Úload_char_vocabr¶   Ð   s'   € Ø”I×-Ñ-¬e¯j©j¸ÑEaÐptÔ.uÓvÐwÐwr   c                 ó  — t        j                  | t        «       t        | «      }| d   �| d   dz   | d   z   ndj	                  | d   | d   «      }| d   rt        j
                  | d   || d   «      }nd }t        j                  j                  |«      r!t        j                  d«       t        |«      }n[t        j                  d	«       d
t        | d   €| d   n| d   | d   ¬«      i}t        j                  |d
   j                  «       |«       t        j                  dj	                  t        |d
   «      «      «       |rPt        j                  j                  |«      r1t        j                  d|z  «       t!        j"                  | |d¬«      }nt!        j$                  | |«      }d }| d   rHddlm} | d   �dj	                  | d   | d   «      ndj	                  | d   | d   | d   «      } ||¬«      }d}	| d   dkD  rd}	| d   ridd l}
| d   r| d   n| d   ›d| d   ›d �}|
j-                  || ¬!«       |
j.                  j1                  d"d#¬$«       |
j.                  j1                  d%d#¬$«       t3        |j4                  j7                  «       «      j8                  }d }|j:                  }t=        || d&   d'z   «      D �]  |_        | d   �| d   }n| d   }t?        ||| d   «      }tA        | d(   || d   «      }|D �]€  }tC        || d)   |«      }d }d*}tE        jF                  |jI                  d'«      d'z
  | d+   z  «      }d,\  }}||jI                  d'«      d'z
  d'z
  k  sŒe|j4                  jK                  «        |xjL                  d'z  c_&        tO        jN                  «       }tP        jR                  jS                  «       d-k  r| d+   n| d+   d.z  }tU        d/tW        tP        jR                  jY                  |d/«      «      «      }t[        |tW        | d+   d0z  «      «      }t]        |||«      \  }}t=        |jI                  d«      «      D �cg c]  }|jI                  d'«      ‘Œ }}|j^                  ja                  «        |j4                  jc                  |||«      \  }}} |je                  | jg                  d1t        |d
   «      «      |«      }!||!jh                  jk                  «       z  }|!jm                  «        t        jn                  j                   jq                  |jr                  | d2   «       |j^                  ju                  «        tw        |«      }|d'z   | d3   z  dk(  r”|| d3   z  }"tO        jN                  «       |z
  }#t        j                  d4j	                  |j:                  |d'z   ||#| d3   z  |"tE        jx                  |"«      «      «       | d   r
j{                  d5|"i|jL                  ¬6«       d*}|d'z  }|z  }|	r[|jL                  | d   z  dk(  rFt}        | |||||||«      \  }$}%}| d   r+
j{                  |%|t        || «      d7œ|jL                  ¬6«       ||jI                  d'«      d'z
  d'z
  k  r�Œ�Œƒ |	r|j:                  | d&   k(  s�ŒØt}        | |||||||«      \  }$}%}| d   s�Œõ
j{                  |%|t        || «      d7œ|jL                  ¬6«       �Œ" |r|j�                  «        | d   r
jƒ                  «        y c c}w )8NÚvocab_save_namerj   ú/z{}/{}_vocab.ptri   rZ   Úcheckpoint_save_namezLoading existing vocab filezBuilding and saving vocabr0   Ú	train_dirÚ
train_fileÚcutoff)r½   z"Training model with vocab size: {}zLoading existing checkpoint: %sT)ÚfinetuneÚsummaryr   )ÚSummaryWriterrh   z{}/{}_summaryz{}/{}_{}_charlm_summaryr8   )Úlog_dirFÚ
eval_stepsrr   rq   Ú_Ú_charlm)ÚnameÚconfigrž   r(   )r¿   r¢   Úepochsr'   Ú	eval_filer|   rV   r}   )r   r   gffffffî?g       @rY   g333333ó?r   Úmax_grad_normÚreport_stepszS| epoch {:5d} | {:5d}/{:5d} batches | sec/batch {:.6f} | loss {:5.2f} | ppl {:8.2f}Ú
train_loss)rš   )r¢   rž   Úlr)Br	   Úlog_training_argsrB   rm   rD   Úcheckpoint_namer=   r>   ÚexistsrC   r¶   r   r   r›   rª   r‚   r   r´   Úfrom_new_modelÚtorch.utils.tensorboardrÀ   rr   ÚinitÚrunÚdefine_metricr   r†   r€   r#   Úepochr„   rG   r;   r%   r–   Úceilr   rL   r“   r•   ÚnpÚrandomÚmaxrb   Únormalr(   r.   Ú	optimizerÚ	zero_gradrP   r‡   r   r!   r…   r1   ÚnnÚclip_grad_norm_Úparamsrš   r   r—   Úlogr¦   r˜   ÚcloseÚfinish)&rk   rl   rµ   rŸ   r7   r�   r    rÀ   Úsummary_dirÚeval_within_epochrr   rq   r#   rž   Ústart_epochÚ
train_pathÚ
train_dataÚdev_dataÚ
data_chunkrŠ   rˆ   r‰   Útotal_batchesÚ	iterationr+   r¡   Úbpttr,   r!   r-   r‹   rŒ   r�   rŽ   Úcur_lossr£   rÃ   r¢   s&                                         r   rL   rL   Ó   sø  € Ü	×Ñ˜D¤&Ô)Ü% dÓ+€JàEIÐJ[ÑE\ÐEh��jÑ! CÑ'¨$Ð/@Ñ*AÒAØ×$Ñ$ T¨*Ñ%5°t¸KÑ7HÓIð ð ˆLÒÜ×/Ñ/°°ZÑ0@À*ÈdÐSiÑNjÓk‰àˆä	‡w�w‡~�~�jÔ!Ü�‰Ð1Ô2Ü 
Ó+‰ä�‰Ð/Ô0ØÔ+À$À{ÑBSÐB[¨D°Ò,>ÐaeÐfqÑarÐ{ð  AIñ  |Jô  Kð  LˆÜ�
‰
�5˜‘=×+Ñ+Ó-¨zÔ:Ü
‡K�KÐ4×;Ñ;¼CÀÀfÁÓ<NÓOÔPáœ2Ÿ7™7Ÿ>™>¨/Ô:Ü�‰Ð5¸ÑGÔHÜ/×4Ñ4°T¸?ÐUYÔZ‰ä/×>Ñ>¸tÀUÓKˆà€FØˆI‚Ý9ØUYÐZeÑUfÐUr�o×,Ñ,¨T°*Ñ-=¸tÀKÑ?PÔQØ*×1Ñ1°$°zÑ2BÀDÈÑDUÐW[Ð\gÑWhÓið 	á {Ô3ˆð ÐØˆLÑ˜AÒØ ÐàˆG‚}ÛØ+/°Ò+=�T˜,Ò'ÐUYÐZeÓUfÐhlÐmxÓhyÐCzˆ
Ø�
‰
˜
¨4ˆ
Ô0Ø�	‰	×Ñ °UÐÔ;Ø�	‰	×Ñ ¨uÐÔ5ä�'—-‘-×*Ñ*Ó,Ó-×4Ñ4€Fà€IØ—-‘-€KÜ˜{¨D°©N¸1Ñ,<Ó=ó FˆŒà�ÑÐ(Ø˜kÑ*‰Jà˜lÑ+ˆJÜ˜z¨5°$°{Ñ2CÓDˆ
Ü˜T +Ñ.°°t¸KÑ7HÓIˆð %ó 6	GˆJÜ˜z¨4°Ñ+=¸vÓFˆGØˆFØˆJÜ ŸI™I w§|¡|°A£¸Ñ':¸dÀ;Ñ>OÑ&OÓPˆMØ‰LˆI�qà�g—l‘l 1“o¨Ñ)¨AÑ-Ó-Ø—‘×#Ñ#Ô%Ø×#Ò# qÑ(Õ#Ü!ŸY™Y›[�
Ü,.¯I©I×,<Ñ,<Ó,>ÀÒ,E�t˜KÒ(È4ÐP[ÑK\Ð^`ÑK`�ä˜a¤¤R§Y¡Y×%5Ñ%5°d¸AÓ%>Ó!?Ó@�ä˜g¤s¨4°Ñ+<¸sÑ+BÓ'CÓD�Ü(¨°!°WÓ=‘��fÜ.3°D·I±I¸a³LÓ.AÖB¨˜Ÿ	™	 !�ÐB�ÐBà×!Ñ!×+Ñ+Ô-Ø*1¯-©-×*?Ñ*?ÀÀdÈFÓ*SÑ'�˜ Ø×(Ñ(¨¯©°b¼#¸eÀF¹mÓ:LÓ)MÈvÓV�Ø˜dŸi™iŸn™nÓ.Ñ.�
Ø—‘”ä—‘—‘×.Ñ.¨w¯~©~¸tÀOÑ?TÔUØ×!Ñ!×&Ñ&Ô(ä)¨&Ó1�à ‘M T¨.Ñ%9Ñ9¸QÒ>Ø)¨D°Ñ,@Ñ@�HÜ"Ÿi™i›k¨JÑ6�GÜ—K‘KØm×tÑtØ#ŸM™MØ%¨™MØ)Ø# d¨>Ñ&:Ñ:Ø$Ü ŸH™H XÓ.óô	ð ˜G’}ØŸ	™	 <°Ð":À×ATÑAT˜	ÔUØ!$�Jà˜Q‘�	Ø�W‘�ñ %¨×)<Ñ)<¸tÀLÑ?QÑ)QÐUVÒ)VÜ(9¸$ÀÀxÐQXÐZcÐeoð  rAð  CIó  )JÑ%�A�s˜IØ˜G’}ØŸ	™	¨#¸IÌ^Ð\cÐeiÓMjÑ"kÐry÷  sFñ  sF˜	ô  Gð_ �g—l‘l 1“o¨Ñ)¨AÑ-Ö-ð6	Gñr ! G§M¡M°T¸(±^Ô$CÜ 1°$¸¸xÈÐR[Ð]gÐixð  {Aó  !BÑˆAˆs�IØ�GŒ}Ø—	‘	 #°IÄ^ÐT[Ð]aÓEbÑcÐjq×j}Ñj}�	Ö~ðMFñP Ø�‰ŒØˆG‚}Ø�‰ŒØ
ùòc Cs   Ñ\
c                 óp  — t        | «      }t        j                  |«      j                  | d   «      }|j                  }t        | d   || d   «      }t        j                  j                  «       }t        | ||||«      }t        j                  dj                  |t        j                  |«      «      «       y )Nr#   rÈ   r8   z)| best model | loss {:5.2f} | ppl {:8.2f})rm   r   r´   r    r7   rG   r   rÝ   ÚCrossEntropyLossr�   rB   rC   rD   r–   r—   )rk   rl   r†   r7   r!   r‡   rŽ   s          r   ry   ry   S  s�   € Ü% dÓ+€Jä"×'Ñ'¨
Ó3×6Ñ6°t¸H±~ÓF€EØ�K‰K€EÜ�T˜+Ñ&¨¨t°KÑ/@ÓA€DÜ—‘×)Ñ)Ó+€Iä˜$  t¨U°IÓ>€DÜ
‡K�KØ3×:Ñ:ØÜ�H‰H�T‹Nó	
ôð r   Ú__main__r   )*Ú__doc__r]   r   ÚloggingÚlzmar–   r=   rØ   r•   Útypesr   Únumpyr×   r   Ústanza.models.common.char_modelr   r   r   Ústanza.models.common.vocabr   Ústanza.models.commonr	   Ústanza.modelsr
   Ú	getLoggerrB   r   r%   r.   r;   rG   rf   rm   rp   rz   r�   r¦   r˜   r¶   rL   ry   Ú__name__r®   r   r   ú<module>rü      s³   ðñó Ý Û Û Û Û 	Û Û Ý Û Û ç uÑ uÝ 0Ý &Ý +à	ˆ×	Ñ	˜8Ó	$€ò5òòòò	ò+òZó	óò(ó2! òFLòxò~ò@ð" ˆzÒÙ…Fð r   