Ë
    ›¾jå  ã                   óÊ  — d dl Z d dlZd dlmZmZmZmZmZmZm	Z	m
Z
mZmZmZ d dlZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z, ddl-m.Z.m/Z/m0Z0 ddl1m2Z2  e
de¬	«      Z3 e
d
e,¬	«      Z4 ejj                  dejl                  z  «      Z7d ejj                  d«      z  Z8d ejj                  dejl                  z  «      z  Z9 G d„ d«      Z:	 dedededede!deeef   fd„Z;dede!dedefd„Z<dede=de=de=de=f
d„Z>d„ Z?d„ Z@d ed!ed"ed#ed$edeeef   fd%„ZAd0d&„ZBd'e3de3fd(„ZCd'e3de3fd)„ZDd*e:dedefd+„ZEd*e:dedefd,„ZFd-ed.efd/„ZGy)1é    N)ÚAnyÚIteratorÚListÚOptionalÚSequenceÚTupleÚTypeÚTypeVarÚUnionÚcastÚoverloadé   )ÚArray1dÚArray2dÚArray3dÚArray4dÚArrayXdÚ	BatchableÚDeviceTypesÚDTypesÚDTypesFloatÚ	DTypesIntÚFloats1dÚFloats2dÚFloats3dÚFloats4dÚFloatsXdÚ	FloatsXdTÚ	GeneratorÚInts1dÚInts2dÚInts3dÚInts4dÚIntsXdÚList2dÚListXdÚPaddedÚShapeÚSizedGeneratorÚXpÚ_Floats)Úget_array_moduleÚis_xp_arrayÚto_numpyé   )ÚCBlasÚArrayT)ÚboundÚFloatsTç       @ç      ð?c                   óD  — e Zd ZU dZeed<   eZeed<   	 déde	de
ddfd	„Zdefd
„Zddœd„Zdddœdee
ef   dedede
def
d„Zdddœdee
ef   dededede
defd„Zd„ Zde
dee
   fd„Zddœdede
dee   defd„Zddœd ede
dee   defd!„Z	 	 	 dêd"ed#ed$ee   d%ed&edefd'„Zd(ed)e
defd*„Zd(ed+ed,e defd-„Z!e"	 	 	 dëd(e#e   d.ee$   d/e
d0e
def
d1„«       Z%e"	 	 	 dëd(e#e   d.ee$   d/e
d0e
def
d2„«       Z%e"	 	 	 dëd(e&d.ee$   d/e
d0e
de'f
d3„«       Z%e"	 	 	 dëd(e(d.ee$   d/e
d0e
de)f
d4„«       Z%e"	 	 	 dëd(e*e)   d.ee$   d/e
d0e
de)f
d5„«       Z%	 	 	 dëd(e*e)   d.ee$   d/e
d0e
de)f
d6„Z%e"	 dìd(eded/e
de#e   fd7„«       Z+e"dìd(eded/e
de#e   fd8„«       Z+e"dìd(e'ded/e
de&fd9„«       Z+e"dìd(e)ded/e
de(fd:„«       Z+dìd(e)ded/e
de(fd;„Z+e"díd<e#e,   de-fd=„«       Z.e"díd<e#e   de/fd>„«       Z.	 díd<ee#e,   e#e   f   de0fd?„Z.d@e0de#e
   de&fdA„Z1d<e&de2fdB„Z3d@e2de&fdC„Z4dDe5dEee6   de7fdF„Z8dGdHdIœdJe
d.ee9   dKede fdL„Z:dGdHdIœdJe
dMe
d.ee9   dKedef
dN„Z;dGdHdIœdJe
dMe
dOe
d.ee9   dKede/fdP„Z<dGdHdIœdJe
dMe
dOe
dQe
d.ee9   dKede=fdR„Z>dGdHdIœdDe5d.ee9   dKede7fdS„Z?dTdHdIœdJe
d.ee@   dKedefdU„ZAdTdHdIœdJe
dMe
d.ee@   dKede,f
dV„ZBdTdHdIœdJe
dMe
dOe
d.ee@   dKede-fdW„ZCdTdHdIœdJe
dMe
dOe
dQe
d.ee@   dKedeDfdX„ZEdTdHdIœdDe5d.ee@   dKedeFfdY„ZGdGdHdIœdDe5d.ee$   dKedeHfdZ„ZId[e)dJe
deJfd\„ZKd[e)dJe
dMe
de'fd]„ZLd[e)dJe
dMe
dOe
de0f
d^„ZMd[e)dJe
dMe
dOe
dQe
deNfd_„ZOd[e7dJe
de fd`„ZPd[e7dJe
dMe
defda„ZQd[e7dJe
dMe
dOe
de/f
db„ZRd[e7dJe
dMe
dOe
dQe
de=fdc„ZSd[e7dDe5de7fdd„ZTd[eFdJe
defde„ZUd[eFdJe
dMe
de,fdf„ZVd[eFdJe
dMe
dOe
de-f
dg„ZWd[eFdJe
dMe
dOe
dQe
deDfdh„ZXd[eFdDe5deFfdi„ZYd[eZdDe5deZfdj„Z[dGdkœdlee=e*e*e*e*e6            f   d.ee$   de=fdm„Z\dGdkœdlee/e*e*e*e6         f   d.ee$   de/fdn„Z]dGdkœdleee*e*e6      f   d.ee$   defdo„Z^dGdkœdlee e*e6   f   d.ee$   de fdp„Z_dGdkœdlee7e*eH   f   d.ee$   de7fdq„Z`dTdkœdleee*e
   f   d.ee$   defdr„ZadTdkœdlee,e*e*e
      f   d.ee$   de,fds„ZbdTdkœdlee-e*e*e*e
         f   d.ee$   de-fdt„ZcdTdkœdleeDe*e*e*e*e
            f   d.ee$   deDfdu„ZddTdkœdleeFe*eH   f   d.ee$   deFfdv„Zeddkœdlee)e*e)   e*eH   f   d.ee$   de)fdw„ZfdîdleZd.ee$   deZfdx„Zgddyœd(ehdzedehfd{„Ziddyœd ehd|ehdzedehfd}„Zjddyœd|ehdzedehfd~„Zkddyœd|eldzedelfd„Zmddd€d�œd"eldzed‚e
dƒe6delf
d„„Znddd…œd†ededzed‚e
def
d‡„Zodd€dˆœd|eld eld‚e
dƒe6delf
d‰„Zpd ed|ededefdŠ„Zqd‹e dŒe/d�e/d(edŽedereerf   fd�„Zsd‹e dŒe/d�e/d(edŽedefd�„Ztd eded‹e d‘erderee f   f
d’„Zud(e/deree,f   fd“„Zvd ed”e,d•e
de/fd–„Zwdïd(edzedefd—„Zx	 dïd ed|edzedefd˜„Zy	 	 	 	 	 dðd(ehd™e6dše6d›e6dœe6dzedehfd�„Zz	 	 	 	 	 dðd ehd(ehd™e6dše6d›e6dœe6dzedehfdž„Z{dñd(ehdŸe6dzedehfd „Z|	 dñd ehd(ehdŸe6dzedehf
d¡„Z}dïd(ehdzedehfd¢„Z~	 dïd ehd(ehdzedehfd£„Zdïd(ehdzedehfd¤„Z€	 dïd ehd(ehdzedehfd¥„Z�dïd(ehdzedehfd¦„Z‚	 dïd ehd(ehd|ehdzedehf
d§„Zƒdïd(ehdzedehfd¨„Z„	 dïd ehd(ehdzedehfd©„Z…dïd(ehdzedehfdª„Z†	 dïd ehd(ehdzedehfd«„Z‡dïd(ehdzedehfd¬„Zˆ	 dïd ehd(ehdzedehfd­„Z‰d(ehdehfd®„ZŠd(ehdehfd¯„Z‹dïd(ehdzedehfd°„ZŒ	 dïd ehd(ehdzedehfd±„Z�dïd(ehdzedehfd²„ZŽ	 dïd ehd(ehdzedehfd³„Z�	 dòd(ehd´e6dzedehfdµ„Z�	 	 dòd ehd(ed´e6dzedehf
d¶„Z‘	 dód·eld¸eld¹e
dºe6ddf
d»„Z’	 dôd¸e d¼e d½e d¾e d¿e6dÀe6dÁe6dÂe6dÃe6dere e e e f   fdÄ„Z“d¼eld´e6delfdÅ„Z”dÆeldÇelde6fdÈ„Z•d(ededefdÉ„Z–d(ededereef   fdÊ„Z—d(ededereef   fdË„Z˜d(ededefdÌ„Z™d(edederee,f   fdÍ„ZšdÎedÏedefdÐ„Z›dÑedÒedefdÓ„ZœdÔededefdÕ„Z�dÖededefd×„ZždØed”e,dedefdÙ„ZŸdÚedÛe
de,fdÜ„Z dŸe
dÝedefdÞ„Z¡	 dõdße
dàe
dáe
d$ee   def
dâ„Z¢dãedäe,defdå„Z£dãe7däeFdæe7de7fdç„Z¤dè„ Z¥y)öÚOpsÚbaseÚnameÚxpéÿÿÿÿÚdevice_typeÚ	device_idÚreturnNc                 ó    — || _         || _        y ©N)r<   r=   )Úselfr<   r=   Úkwargss       ú\/home/spazio/kingdom_calls/work5/transenv/lib/python3.12/site-packages/thinc/backends/ops.pyÚ__init__zOps.__init__@   s   € ð 'ˆÔØ"ˆ�ó    c                 óH   — t        | «      j                  › d�}t        |«      ‚)zReturn C BLAS function table.z" does not provide C BLAS functions)ÚtypeÚ__name__ÚNotImplementedError)rA   Úerrs     rC   Úcblasz	Ops.cblasF   s%   € ä�d“×$Ñ$Ð%Ð%GÐHˆÜ! #Ó&Ð&rE   )Ú
byte_orderc                ó¸   — t        |t        j                  «      r6|r2|j                  j	                  |«      }t        j
                  ||¬«      }|S t        d«      ‚)N©Údtypez,Cannot convert non-numpy from base Ops class)Ú
isinstanceÚnumpyÚndarrayrO   ÚnewbyteorderÚasarrayÚ
ValueError)rA   ÚdatarL   rO   s       rC   r.   zOps.to_numpyK   sH   € Ü�dœEŸM™MÔ*ÙØŸ
™
×/Ñ/°
Ó;�Ü—}‘} T°Ô7�ØˆKäÐKÓLÐLrE   Fr/   )ÚshuffleÚbufferÚsizeÚsequencerW   rX   c                óR  ‡ ‡‡‡‡‡— t        ‰d«      sdt        ‰«      › �}t        |«      ‚‰ j                  t	        ‰«      t        |t        «      rt        j                  |«      n|«      Št        j                  t	        ‰«      «      Šˆˆˆ ˆˆˆfd„}t        |t	        ‰«      «      S )a—  Iterate slices from a sequence, optionally shuffled. Slices
        may be either views or copies of the underlying data.

        The `size` argument may be either an integer, or a sequence of integers.
        If a sequence, a new size is drawn before every output.

        If shuffle is True, shuffled batches are produced by first generating
        an index array, shuffling it, and then using it to slice into the
        sequence.

        An internal queue of `buffer` items is accumulated before being each
        output. Buffering is useful for some devices, to allow the
        network to run asynchronously without blocking on every batch.
        Ú__len__z-Can't minibatch data. Expected sequence, got c            
   3   ó$  •K  — ‰rt         j                  j                  ‰«       g } d}‰D ]S  }t        |«      }| j	                  ‰j                  ‰‰|||z    «      «       t        | «      ‰k\  r| E d {  –—†  g } ||z  }ŒU | E d {  –—†  y 7 Œ7 Œ­w©Nr   ©rQ   ÚrandomrW   ÚintÚappendÚ
_get_batchÚlen)	ÚqueueÚirY   rX   ÚindicesrA   rZ   rW   Úsizess	      €€€€€€rC   Ú_iter_itemsz"Ops.minibatch.<locals>._iter_itemsu   s™   øè ø€ ÙÜ—‘×$Ñ$ WÔ-ØˆEØˆAØò �Ü˜4“y�Ø—‘˜TŸ_™_¨X°w¸qÀ1ÀtÁ8Ð7LÓMÔNÜ�u“: Ò'Ø$×$Ð$Ø�EØ�T‘	‘ðð ×Ñð %øð ús$   ƒA0BÁ3BÁ4BÂBÂBÂB)ÚhasattrrG   rU   Ú_get_batch_sizesrd   rP   ra   Ú	itertoolsÚrepeatrQ   Úaranger)   )	rA   rY   rZ   rW   rX   rJ   ri   rg   rh   s	   ` ```  @@rC   Ú	minibatchzOps.minibatchT   s‹   ý€ ô, �x Ô+ØAÄ$ÀxÃ.ÐAQÐRˆCÜ˜S“/Ð!Ø×%Ñ%Ü�‹M´ZÀÄcÔ5Jœ9×+Ñ+¨DÔ1ÐPTó
ˆô —,‘,œs 8›}Ó-ˆ÷
	ñ 	ô ˜k¬3¨u«:Ó6Ð6rE   Úothersc                óÆ  ‡ ‡‡‡
‡‡— |ft        |«      z   Št        d„ ‰D «       «      s9dj                  ‰D �cg c]  }t        |«      › ‘Œ c}«      }d|› �}t	        |«      ‚‰ j                  t        |«      t        |t        «      rt        j                  |«      n|«      Št        j                  t        |«      «      Š
ˆˆ
ˆ ˆˆˆfd„}	t        |	t        ‰«      «      S c c}w )zyMinibatch one or more sequences of data, and yield
        lists with one batch per sequence. See ops.minibatch.
        c              3   ó4   K  — | ]  }t        |d «      –— Œ y­w)r\   N)rj   ©Ú.0Úseqs     rC   ú	<genexpr>z!Ops.multibatch.<locals>.<genexpr>“   s   è ø€ Ò@¨s”7˜3 	×*Ñ@ùó   ‚z, z/Can't multibatch data. Expected sequences, got c               3   ó^  •K  — ‰	rt         j                  j                  ‰«       g } d}‰
D ]p  }t        |«      }‰|||z    }| j	                  g «       ‰D ]&  }| d   j	                  ‰j                  ||«      «       Œ( t        | «      ‰k\  r| E d {  –—†  g } ||z  }Œr | E d {  –—†  y 7 Œ7 Œ­w©Nr   r;   r_   )re   rf   rY   Ú	idx_batchrZ   rX   rg   rA   Ú	sequencesrW   rh   s        €€€€€€rC   ri   z#Ops.multibatch.<locals>._iter_itemsœ   sÁ   øè ø€ ÙÜ—‘×$Ñ$ WÔ-ØˆEØˆAØò 	�Ü˜4“y�Ø# A¨¨D©Ð1�	Ø—‘˜RÔ Ø )ò K�HØ˜"‘I×$Ñ$ T§_¡_°X¸yÓ%IÕJðKä�u“: Ò'Ø$×$Ð$Ø�EØ�T‘	‘ð	ð ×Ñð %øð ús$   ƒBB-ÂB)ÂB-Â#B+Â$B-Â+B-)ÚtupleÚallÚjoinrG   rU   rk   rd   rP   ra   rl   rm   rQ   rn   r)   )rA   rY   rZ   rW   rX   rp   ru   ÚvaluesrJ   ri   rg   r{   rh   s   `  ``     @@@rC   Ú
multibatchzOps.multibatch…   sÁ   ý€ ð �K¤%¨£-Ñ/ˆ	ÜÑ@°iÔ@Ô@Ø—Y‘Y¸)ÖD°3¤4¨£9 +¡ÒDÓEˆFØCÀFÀ8ÐLˆCÜ˜S“/Ð!Ø×%Ñ%Ü�‹M´ZÀÄcÔ5Jœ9×+Ñ+¨DÔ1ÐPTó
ˆô —,‘,œs 8›}Ó-ˆ÷	ñ 	ô" ˜k¬3¨u«:Ó6Ð6ùò3  Es   ·Cc                 ó  ‡— t        ‰t        «      r|D �cg c]  }‰|   ‘Œ	 }}n*t        ‰t        «      rt        ˆfd„|D «       «      }n‰|   }t        |«      r*| j	                  | j
                  j                  |«      «      }|S c c}w )Nc              3   ó(   •K  — | ]	  }‰|   –— Œ y ­wr@   © )rt   rf   rZ   s     €rC   rv   z!Ops._get_batch.<locals>.<genexpr>³   s   øè ø€ Ò8¨1˜8 A�;Ñ8ùs   ƒ)rP   Úlistr|   r-   Ú	as_contigr:   rT   )rA   rZ   rg   rf   Úsubseqs    `   rC   rc   zOps._get_batch¯   su   ø€ Ü�h¤Ô%Ø+2Ö3 a�h˜q“kÐ3ˆFÑ3Ü˜¤%Ô(ÜÓ8°Ô8Ó8‰Fà˜gÑ&ˆFÜ�vÔØ—^‘^ D§G¡G§O¡O°FÓ$;Ó<ˆFØˆùò 4s   –BÚlengthrh   c                 óh   — g }d}||k  r(|j                  t        |«      «       ||d   z  }||k  rŒ(|S ry   )rb   Únext)rA   r‡   rh   Úoutputrf   s        rC   rk   zOps._get_batch_sizesº   s@   € ØˆØˆØ�&ŠjØ�M‰Mœ$˜u›+Ô&Ø�˜‘‰OˆAð �&‹jð ˆrE   )Úlengthsru   ÚnWr‹   c                ó`  — |dk(  sJ ‚|dk(  sJ ‚|j                   d   }|j                   d   }| j                  ||dz  dz   |«      }| j                  |d|  d||«      ||d…d|…f<   ||dd…|f<   | j                  ||d d||«      |d| …|dz   d…f<   | j                  |||d|z  dz   z  «      S )a  Given an (M, N) sequence of vectors, return an (M, N*(nW*2+1))
        sequence. The new sequence is constructed by concatenating nW preceding
        and succeeding vectors onto each column in the sequence, to extract a
        window of features.
        r/   Nr   r   r;   )ÚshapeÚalloc3fÚ	reshape3fÚ	reshape2f)rA   ru   rŒ   r‹   ÚBÚIÚcolss          rC   Úseq2colzOps.seq2colÂ   sÕ   € ð �QŠwˆˆwØ˜$ŠÐˆØ�I‰I�a‰LˆØ�I‰I�a‰LˆØ�|‰|˜A  Q¡¨¡
¨QÓ/ˆàŸ™¨¨D¨b¨S¨	°2°r¸1Ó=ˆˆR‰S�#�2�#ˆX‰ØˆŠQ�ˆU‰Ø#Ÿ~™~¨c°"°#¨h¸¸BÀÓBˆˆTˆrˆcˆT�2˜‘6‘8ˆ^ÑØ�~‰~˜d A q¨A°©F°Q©JÑ'7Ó8Ð8rE   ÚdYc          
      ó‚  — |dk(  sJ ‚|dk(  sJ ‚|dz  dz   }|j                   d   }|j                   d   |z  }| j                  ||«      }| j                  ||||«      }|d| xxx | j                  ||d…d|…f   d|«      z  ccc ||dd…|f   z  }||dxxx | j                  |d| …|dz   d…f   d|«      z  ccc |S )zÔThe reverse/backward operation of the `seq2col` function: calculate
        the gradient of the original `(M, N)` sequence, as a function of the
        gradient of the output `(M, N*(nW*2+1))` sequence.
        r/   Nr   r   r;   )rŽ   Úalloc2fr�   r‘   )	rA   r–   rŒ   r‹   ÚnFr’   r“   ÚdXÚdY3ds	            rC   Úbackprop_seq2colzOps.backprop_seq2colÖ   sá   € ð �QŠwˆˆwØ˜$ŠÐˆØ�!‰V�a‰ZˆØ�H‰H�Q‰KˆØ�H‰H�Q‰K˜2Ñˆà�\‰\˜!˜QÓˆØ�~‰~˜b ! R¨Ó+ˆØ
ˆ4ˆRˆC‹�D—N‘N 4¨©¨S¨b¨S¨¡>°2°qÓ9Ñ9‹Ø
ˆd’1�b�5‰kÑˆØ
ˆ2ˆ3‹�4—>‘> $ t¨¨ t¨R°!©V©X ~Ñ"6¸¸AÓ>Ñ>‹Øˆ	rE   ÚxÚyÚoutÚtrans1Útrans2c                 ó¶   — |r|j                   }|r|j                   }|€| j                  j                  ||«      S | j                  j                  |||¬«       |S )zPerform General Matrix Multiplication (GeMM) and optionally store
        the result in the specified output variable.
        ©rŸ   )ÚTr:   Údot)rA   r�   rž   rŸ   r    r¡   s         rC   ÚgemmzOps.gemmë   sQ   € ñ Ø—‘ˆAÙØ—‘ˆAØˆ;Ø—7‘7—;‘;˜q !Ó$Ð$à�G‰G�K‰K˜˜1 #ˆKÔ&ØˆJrE   ÚXÚrepsc                 ó:   — | j                   j                  ||«      S r@   )r:   Útile)rA   r§   r¨   s      rC   rª   zOps.tile   s   € Ø�w‰w�|‰|˜A˜tÓ$Ð$rE   ÚWÚbc                 ó8   — | j                  ||d¬«      }||z  }|S )zVApply a weights layer and a bias to some inputs, i.e.
        Y = X @ W.T + b
        T)r¡   )r¦   )rA   r§   r«   r¬   ÚYs        rC   Úaffinez
Ops.affine  s&   € ð �I‰I�a˜ 4ˆIÓ(ˆØ	ˆQ‰ˆØˆrE   rO   ÚpadÚndim_if_emptyc                  ó   — y r@   rƒ   ©rA   r§   rO   r°   r±   s        rC   ÚflattenzOps.flatten  s   € ð rE   c                  ó   — y r@   rƒ   r³   s        rC   r´   zOps.flatten  s   € ð rE   c                  ó   — y r@   rƒ   r³   s        rC   r´   zOps.flatten  ó   € ð rE   c                  ó   — y r@   rƒ   r³   s        rC   r´   zOps.flatten(  r·   rE   c                  ó   — y r@   rƒ   r³   s        rC   r´   zOps.flatten1  r·   rE   c           	      óº  — |�t        |«      dk(  r| j                  d|z  |xs d¬«      S t        |d   «      }|d   j                  }|D �cg c]  }|j                  dk7  sŒ|‘Œ }}t        |«      dk(  r| j                  ||xs d¬«      S t        |«      dk\  r–g }|D ]P  }|j                  |j                  |f|j                  dd z   |j                  ¬«      «       |j                  |«       ŒR |j                  |j                  |fj                  dd z   |j                  ¬«      «       |}|j                  |«      }	|�|j                  |	|¬«      }	|	S c c}w )z.Flatten a list of arrays into one large array.Nr   ©r   ÚfrN   r/   )rd   Úallocr,   rŽ   rY   ra   rb   ÚzerosrO   ÚconcatenaterT   )
rA   r§   rO   r°   r±   r:   Úshape_if_emptyr�   ÚpaddedÚresults
             rC   r´   zOps.flatten:  sG  € ð ˆ9œ˜A› !šØ—:‘:˜d ]Ñ2¸%º,À3�:ÓGÐGÜ˜a ™dÓ#ˆØ˜1™Ÿ™ˆØÖ)�1˜QŸV™V q›[ŠQÐ)ˆÐ)Üˆq‹6�QŠ;Ø—:‘:˜n°E²L¸S�:ÓAÐAÜˆs‹8�qŠ=ØˆFØò !�Ø—‘˜bŸh™h¨ v°·±¸¸°Ñ';À1Ç7Á7˜hÓKÔLØ—‘˜aÕ ð!ð �M‰M˜"Ÿ(™( C 6¨A¯G©G°A°B¨KÑ#7¸q¿w¹w˜(ÓGÔHØˆAØ—‘ Ó"ˆØÐØ—Z‘Z ¨e�ZÓ4ˆFØˆùò *s   ÁEÁ!Ec                  ó   — y r@   rƒ   ©rA   r§   r‹   r°   s       rC   Ú	unflattenzOps.unflattenU  s   € ð rE   c                  ó   — y r@   rƒ   rÄ   s       rC   rÅ   zOps.unflattenZ  s   € ØSVrE   c                  ó   — y r@   rƒ   rÄ   s       rC   rÅ   zOps.unflatten]  ó   € ØNQrE   c                  ó   — y r@   rƒ   rÄ   s       rC   rÅ   zOps.unflattenb  rÈ   rE   c                 ó.  — t        |«      }|dkD  rt        j                  |dkD  ||z   d«      }| j                  j	                  |t        j
                  |«      «      dd }|dkD  r|D �cg c]  }||d ‘Œ	 }}t        |«      t        |«      k(  sJ ‚|S c c}w )z˜The reverse/backward operation of the `flatten` function: unflatten
        a large array into a list of arrays according to the given lengths.
        r   Nr;   )r.   rQ   Úwherer:   ÚsplitÚcumsumrd   )rA   r§   r‹   r°   ÚunflatÚas         rC   rÅ   zOps.unflattene  s“   € ô
 ˜7Ó#ˆà�Š7Ü—k‘k '¨A¡+¨w¸©}¸aÓ@ˆGØ—‘—‘˜q¤%§,¡,¨wÓ"7Ó8¸¸"Ð=ˆØ�Š7Ø'-Ö. !�a˜˜’gÐ.ˆFÐ.ä�6‹{œc '›lÒ*Ð*Ð*àˆùò	 /s   Á)BÚseqsc                  ó   — y r@   rƒ   ©rA   rÐ   Úround_tos      rC   r°   zOps.padv  s   € Ø=@rE   c                  ó   — y r@   rƒ   rÒ   s      rC   r°   zOps.pady  s   € ØADrE   c                 ó`  — |dk  rt        d|› �«      ‚|st        d«      ‚t        t        d„ |D «       «      «      dk7  rt        d«      ‚t        t        d„ |D «       «      «      dk7  rt        d«      ‚t        t        d„ |D «       «      «      dk7  rt        d	«      ‚t        d
„ |D «       «      }|| |z  z  }t        |«      |f|d   j                  dd z   }t        t        | j                  ||d   j                  ¬«      «      }t        |«      D ]  \  }}|||d|j                  d   …f<   Œ |S )zßPerform padding on a list of arrays so that they each have the same
        length, by taking the maximum dimension across each axis. This only
        works on non-empty sequences with the same `ndim` and `dtype`.
        r/   z.Rounding for padding must at least be 1, was: zCannot pad empty sequencec              3   ó4   K  — | ]  }|j                   –— Œ y ­wr@   )Úndimrs   s     rC   rv   zOps.pad.<locals>.<genexpr>‹  s   è ø€ Ò, �3—8•8Ñ,ùrw   z)Cannot pad sequences with different ndimsc              3   ó4   K  — | ]  }|j                   –— Œ y ­wr@   rN   rs   s     rC   rv   zOps.pad.<locals>.<genexpr>�  s   è ø€ Ò- �3—9•9Ñ-ùrw   z*Cannot pad sequences with different dtypesc              3   ó:   K  — | ]  }|j                   d d –— Œ y­w)r/   N)rŽ   rs   s     rC   rv   zOps.pad.<locals>.<genexpr>�  s   è ø€ Ò1 S�3—9‘9˜Q˜R”=Ñ1ùs   ‚z4Cannot pad sequences that differ on other dimensionsc              3   ó2   K  — | ]  }t        |«      –— Œ y ­wr@   )rd   rs   s     rC   rv   zOps.pad.<locals>.<genexpr>’  s   è ø€ Ò3 sœ#˜cŸ(Ñ3ùs   ‚r   NrN   )
rU   rd   ÚsetÚmaxrŽ   r   r   r½   rO   Ú	enumerate)rA   rÐ   rÓ   Úmax_seq_lenÚfinal_shaperŠ   rf   Úarrs           rC   r°   zOps.pad|  s>  € ð �aŠ<ÜØ@ÀÀ
ÐKóð ñ
 ÜÐ8Ó9Ð9ÜŒsÑ, tÔ,Ó,Ó-°Ò2ÜÐHÓIÐIÜŒsÑ-¨Ô-Ó-Ó.°!Ò3ÜÐIÓJÐJÜŒsÑ1¨DÔ1Ó1Ó2°aÒ7ÜÐSÓTÐTäÑ3¨dÔ3Ó3ˆð 	˜�| hÑ.Ñ.ˆÜ˜4“y +Ð.°°a±·±¸q¸rÐ1BÑBˆÜœw¨¯
©
°;ÀdÈ1ÁgÇmÁm¨
Ó(TÓUˆÜ “oò 	,‰FˆAˆsà(+ˆF�1�n˜Ÿ	™	 !™�nÐ$Ò%ð	,ð ˆrE   rÁ   c                 ó|   — g }t        |«      D ]  \  }}|j                  ||d|…f   «       Œ t        t        |«      S )z–The reverse/backward operation of the `pad` function: transform an
        array back into a list of arrays, each with their original length.
        N)rÝ   rb   r   r%   )rA   rÁ   r‹   rŠ   rf   r‡   s         rC   Úunpadz	Ops.unpad�  sG   € ð ˆÜ" 7Ó+ò 	.‰IˆAˆvØ�M‰M˜&  G V G Ñ,Õ-ð	.ä”F˜FÓ#Ð#rE   c                 óÒ  — |sLt        | j                  ddd«      | j                  d«      | j                  d«      | j                  d«      «      S t        |«      dk(  r˜| j	                  |d   |d   j
                  d   d|d   j
                  d   «      }| j                  dg|j
                  d   z  «      }| j                  |j
                  d   g«      }| j                  dg«      }t        ||||«      S t        |«      D ��cg c]  \  }}t        |«      |f‘Œ }}}|j                  d¬«       |D �	�cg c]  \  }	}|‘Œ	 }
}	}|D �	�cg c]  \  }	}|	‘Œ	 }}	}t        |D �cg c]  }|j
                  d   ‘Œ c}«      }t        |«      }|d   j
                  d   }t        t        |
D �cg c]  }||   ‘Œ	 c}«      }| j                  |«      }|j
                  |||fk(  s
J |||f«       ‚| j                  |j                  d«      «      }|j
                  |||fk(  sJ ‚t        |«      D �cg c]  }d‘Œ }}t        |«      }t        |«      D ]'  }|r|||dz
     k\  r|dz  }|r|||dz
     k\  rŒ|||<   Œ) t!        |«      t!        |«      k(  sJ ‚t        || j                  |«      | j                  |«      | j                  |
«      «      S c c}}w c c}}	w c c}}	w c c}w c c}w c c}w )z4Pack a sequence of 2d arrays into a Padded datatype.r   r/   T)Úreverse©r/   r   r   )r'   r�   Úalloc1ird   Úreshape3rŽ   Ú	asarray1irÝ   ÚsortrÜ   r   r%   r°   r…   Ú	transposeÚrangeÚsum)rA   rÐ   rV   Ú	size_at_tr‹   rg   rf   ru   Úlengths_indicesr‡   Úindices_Úlengths_ÚnSÚnBÚnOrà   Ú_Úbatch_size_at_t_Úcurrent_sizeÚts                       rC   Úlist2paddedzOps.list2padded¦  sÇ  € áÜØ—‘˜Q  1Ó% t§|¡|°A£¸¿¹ÀQ»ÈÏÉÐVWËóð ô �‹Y˜!Š^Ø—=‘=  a¡¨$¨q©'¯-©-¸Ñ*:¸A¸tÀA¹w¿}¹}ÈQÑ?OÓPˆDØŸ™¨ s¨T¯Z©Z¸©]Ñ':Ó;ˆIØ—n‘n d§j¡j°¡m _Ó5ˆGØ—n‘n a SÓ)ˆGÜ˜$ 	¨7°GÓ<Ð<Ü7@À³×G©V¨Q°œC ›H aš=ÐGˆÑGØ×Ñ TÐÔ*Ø'6×7™)˜& !’AÐ7ˆÑ7Ø,;×<™y˜v q’FÐ<ˆÑ<Ü¨$Ö/ 3�#—)‘)˜A“,Ò/Ó0ˆÜ�‹YˆØ�!‰W�]‰]˜1Ñˆô ”F¨hÖ7¨˜T !›WÒ7Ó8ˆØ—x‘x “~ˆØ�y‰y˜R  R˜LÒ(Ð6¨2¨r°2¨,Ó6Ð(Ø�n‰n˜SŸ]™]¨9Ó5Ó6ˆØ�y‰y˜R  R˜LÒ(Ð(Ð(ä',¨R£yÖ1 !šAÐ1ÐÐ1Ü˜8“}ˆÜ�r“ò 	/ˆAÙ 1¨°ÀÑ1AÑ(BÒ#BØ Ñ!�ñ  1¨°ÀÑ1AÑ(BÓ#Bà".Ð˜QÒð	/ô �8‹}¤Ð$4Ó 5Ò5Ð5Ð5ÜØØ�N‰NÐ+Ó,Ø�N‰N˜8Ó$Ø�N‰N˜8Ó$ó	
ð 	
ùó/ Hùã7ùÛ<ùÚ/ùò 8ùò 2s$   ÄKÄ4KÅKÅ KÆ(K
È)	K$c                 ó\  — |j                   }t        |j                  «      }t        |j                  «      }dgt	        |«      z  }| j                  |j                  d«      «      }t        |j                  d   «      D ]  }||dt        ||   «      …f   |||   <   Œ t        t        |«      S )z;Unpack a Padded datatype to a list of 2-dimensional arrays.Nrå   r   )rV   r.   rg   r‹   rd   r…   rê   rë   rŽ   ra   r   r%   )rA   rÁ   rV   rg   r‹   Úunpaddedrf   s          rC   Úpadded2listzOps.padded2listÐ  s�   € à�{‰{ˆÜ˜6Ÿ>™>Ó*ˆÜ˜6Ÿ>™>Ó*ˆØ-1¨F´S¸³\Ñ,Aˆà�~‰~˜dŸn™n¨YÓ7Ó8ˆÜ�t—z‘z !‘}Ó%ò 	>ˆAØ#'¨Ð+<¬S°¸±«_Ð+<Ð(<Ñ#=ˆH�W˜Q‘ZÒ ð	>ä”F˜HÓ%Ð%rE   rŽ   Údropc                 ó  — |�|dk  r| j                   j                  |d¬«      S |dk\  r| j                  |«      S | j                   j                  j	                  dd|«      }||k\  d|z
  z  }t        t        | j                  |d¬«      «      S )a  Create a random mask for applying dropout, with a certain percent of
        the mask (defined by `drop`) will contain zeros. The neurons at those
        positions will be deactivated during training, resulting in a more
        robust network and less overfitting.
        r   r¼   rN   r5   ç        Úfloat32)r:   ÚonesÚalloc_fr`   Úuniformr   r   rT   )rA   rŽ   rü   Ú	coinflipsÚmasks        rC   Úget_dropout_maskzOps.get_dropout_maskÜ  s…   € ð ˆ<˜4 1š9Ø—7‘7—<‘< ¨S�<Ó1Ð1Ø�SŠ[Ø—<‘< Ó&Ð&Ø—G‘G—N‘N×*Ñ*¨3°°UÓ;ˆ	Ø˜TÑ! c¨D¡jÑ1ˆÜ”H˜dŸl™l¨4°y˜lÓAÓBÐBrE   rÿ   T©rO   r¾   Úd0r¾   c                óH   — t        t        | j                  |f||¬«      «      S ©Nr  )r   r   r½   ©rA   r  rO   r¾   s       rC   Úalloc1fzOps.alloc1fê  s"   € ô ”H˜dŸj™j¨"¨°eÀ5˜jÓIÓJÐJrE   Úd1c                óJ   — t        t        | j                  ||f||¬«      «      S r	  )r   r   r½   ©rA   r  r  rO   r¾   s        rC   r˜   zOps.alloc2fó  s$   € ô ”H˜dŸj™j¨"¨b¨¸Àe˜jÓLÓMÐMrE   Úd2c                óL   — t        t        | j                  |||f||¬«      «      S r	  )r   r   r½   ©rA   r  r  r  rO   r¾   s         rC   r�   zOps.alloc3fý  s&   € ô ”H˜dŸj™j¨"¨b°"¨¸UÈ%˜jÓPÓQÐQrE   Úd3c          	      óN   — t        t        | j                  ||||f||¬«      «      S r	  )r   r   r½   ©rA   r  r  r  r  rO   r¾   s          rC   Úalloc4fzOps.alloc4f  s)   € ô ”H˜dŸj™j¨"¨b°"°bÐ)9ÀÈe˜jÓTÓUÐUrE   c                óF   — t        t        | j                  |||¬«      «      S r	  )r   r   r½   ©rA   rŽ   rO   r¾   s       rC   r  zOps.alloc_f  s    € ô ”H˜dŸj™j¨°eÀ5˜jÓIÓJÐJrE   Úint32c                óH   — t        t        | j                  |f||¬«      «      S r	  )r   r    r½   r
  s       rC   ræ   zOps.alloc1i  s"   € ô ”F˜DŸJ™J¨ u°EÀ˜JÓGÓHÐHrE   c                óJ   — t        t        | j                  ||f||¬«      «      S r	  )r   r!   r½   r  s        rC   Úalloc2izOps.alloc2i&  s$   € ô ”F˜DŸJ™J¨¨B x°uÀE˜JÓJÓKÐKrE   c                óL   — t        t        | j                  |||f||¬«      «      S r	  )r   r"   r½   r  s         rC   Úalloc3izOps.alloc3i0  s&   € ô ”F˜DŸJ™J¨¨B° |¸5È˜JÓNÓOÐOrE   c          	      óN   — t        t        | j                  ||||f||¬«      «      S r	  )r   r#   r½   r  s          rC   Úalloc4izOps.alloc4i;  s)   € ô ”F˜DŸJ™J¨¨B°°BÐ'7¸uÈE˜JÓRÓSÐSrE   c                óF   — t        t        | j                  |||¬«      «      S r	  )r   r$   r½   r  s       rC   Úalloc_izOps.alloc_iG  s    € ô ”F˜DŸJ™J u°EÀ˜JÓGÓHÐHrE   c                ó    — t        |t        «      r|f}|r| j                  j                  ||¬«      S | j                  j	                  ||¬«      S )z%Allocate an array of a certain shape.rN   )rP   ra   r:   r¾   Úemptyr  s       rC   r½   z	Ops.allocP  sF   € ô �eœSÔ!Ø�HˆEáØ—7‘7—=‘= ¨e�=Ó4Ð4à—7‘7—=‘= ¨e�=Ó4Ð4rE   Úarrayc                 óD   — t        t        | j                  ||f«      «      S r@   )r   r   Úreshape©rA   r$  r  s      rC   Úreshape1zOps.reshape1`  s   € Ü”G˜TŸ\™\¨%°"°Ó7Ó8Ð8rE   c                 óF   — t        t        | j                  |||f«      «      S r@   )r   r   r&  ©rA   r$  r  r  s       rC   Úreshape2zOps.reshape2c  s   € Ü”G˜TŸ\™\¨%°"°b°Ó:Ó;Ð;rE   c           	      óH   — t        t        | j                  ||||f«      «      S r@   )r   r   r&  ©rA   r$  r  r  r  s        rC   rç   zOps.reshape3f  s    € Ü”G˜TŸ\™\¨%°"°b¸"°Ó>Ó?Ð?rE   c           
      óJ   — t        t        | j                  |||||f«      «      S r@   )r   r   r&  ©rA   r$  r  r  r  r  s         rC   Úreshape4zOps.reshape4i  s#   € Ü”G˜TŸ\™\¨%°"°b¸"¸bÐ1AÓBÓCÐCrE   c                 óD   — t        t        | j                  ||f«      «      S r@   )r   r   r&  r'  s      rC   Ú	reshape1fzOps.reshape1fl  s   € Ü”H˜dŸl™l¨5°2°%Ó8Ó9Ð9rE   c                 óF   — t        t        | j                  |||f«      «      S r@   )r   r   r&  r*  s       rC   r‘   zOps.reshape2fo  s   € Ü”H˜dŸl™l¨5°2°r°(Ó;Ó<Ð<rE   c           	      óH   — t        t        | j                  ||||f«      «      S r@   )r   r   r&  r-  s        rC   r�   zOps.reshape3fr  s    € Ü”H˜dŸl™l¨5°2°r¸2°,Ó?Ó@Ð@rE   c           
      óJ   — t        t        | j                  |||||f«      «      S r@   )r   r   r&  r/  s         rC   Ú	reshape4fzOps.reshape4fu  s%   € ô ”H˜dŸl™l¨5°2°r¸2¸rÐ2BÓCÓDÐDrE   c                 ó&   — | j                  ||«      S r@   ©r&  ©rA   r$  rŽ   s      rC   Ú	reshape_fzOps.reshape_fz  ó   € Ø�|‰|˜E 5Ó)Ð)rE   c                 óD   — t        t        | j                  ||f«      «      S r@   )r   r    r&  r'  s      rC   Ú	reshape1izOps.reshape1i}  s   € Ü”F˜DŸL™L¨°°Ó6Ó7Ð7rE   c                 óF   — t        t        | j                  |||f«      «      S r@   )r   r!   r&  r*  s       rC   Ú	reshape2izOps.reshape2i€  s   € Ü”F˜DŸL™L¨°°R°Ó9Ó:Ð:rE   c           	      óH   — t        t        | j                  ||||f«      «      S r@   )r   r"   r&  r-  s        rC   Ú	reshape3izOps.reshape3iƒ  s    € Ü”F˜DŸL™L¨°°R¸°Ó=Ó>Ð>rE   c           
      óJ   — t        t        | j                  |||||f«      «      S r@   )r   r#   r&  r/  s         rC   Ú	reshape4izOps.reshape4i†  s#   € Ü”F˜DŸL™L¨°°R¸¸RÐ0@ÓAÓBÐBrE   c                 ó&   — | j                  ||«      S r@   r8  r9  s      rC   Ú	reshape_izOps.reshape_i‰  r;  rE   c                 óf   — t        |t        «      r|f}t        t        |j	                  |«      «      S )zReshape an array.)rP   ra   r   r1   r&  r9  s      rC   r&  zOps.reshapeŒ  s)   € ä�eœSÔ!Ø�HˆEÜ”F˜EŸM™M¨%Ó0Ó1Ð1rE   rN   rV   c                óD   — t        t        | j                  ||¬«      «      S ©NrN   )r   r   rT   ©rA   rV   rO   s      rC   Ú	asarray4fzOps.asarray4f’  ó   € ô ”H˜dŸl™l¨4°u˜lÓ=Ó>Ð>rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r   rT   rI  s      rC   Ú	asarray3fzOps.asarray3fš  rK  rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r   rT   rI  s      rC   Ú	asarray2fzOps.asarray2f¢  rK  rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r   rT   rI  s      rC   Ú	asarray1fzOps.asarray1fª  rK  rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r   rT   rI  s      rC   Ú	asarray_fzOps.asarray_f²  rK  rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r    rT   rI  s      rC   rè   zOps.asarray1iº  ó   € ô ”F˜DŸL™L¨°U˜LÓ;Ó<Ð<rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r!   rT   rI  s      rC   Ú	asarray2izOps.asarray2i¿  ó   € ô ”F˜DŸL™L¨°U˜LÓ;Ó<Ð<rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r"   rT   rI  s      rC   Ú	asarray3izOps.asarray3iÇ  rX  rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r#   rT   rI  s      rC   Ú	asarray4izOps.asarray4iÏ  rX  rE   c                óD   — t        t        | j                  ||¬«      «      S rH  )r   r$   rT   rI  s      rC   Ú	asarray_izOps.asarray_i×  rU  rE   c                óR  — t        || j                  j                  «      r2|€|S |j                  |k(  r|S | j                  j	                  ||¬«      S t        |d«      r|j                  «       S |�| j                  j                  ||¬«      S | j                  j                  |«      S )z,Ensure a given array is of the correct type.rN   rQ   )rP   r:   rR   rO   rT   rj   rQ   r$  rI  s      rC   rT   zOps.asarrayÜ  sŽ   € ô �d˜DŸG™GŸO™OÔ,Øˆ}Ø�Ø—‘˜uÒ$Ø�à—w‘w—‘ t°5�Ó9Ð9Ü�T˜7Ô#à—:‘:“<ÐØÐØ—7‘7—=‘= ¨U�=Ó3Ð3à—7‘7—=‘= Ó&Ð&rE   c                 óŽ   — |j                   d   r|d|j                  fv r|S |�d|ini } | j                  j                  |fi |¤ŽS )zÚAllow the backend to make a contiguous copy of an array.
        Implementations of `Ops` do not have to make a copy or make it
        contiguous if that would not improve efficiency for the execution engine.
        ÚC_CONTIGUOUSNrO   )ÚflagsrO   r:   Úascontiguousarray)rA   rV   rO   rB   s       rC   r…   zOps.as_contigò  sR   € ð
 �:‰:�nÒ%¨%°D¸$¿*¹*Ð3EÑ*EØˆKØ%*Ð%6�'˜5Ñ!¸BˆØ(ˆt�w‰w×(Ñ(¨Ñ8°Ñ8Ð8rE   ©Úinplacere  c                ó  — |rI| j                   j                  |dd|¬«      }| j                   j                  | |¬«       |dz  }|dz  }|S | j                   j                  |dd«      }dd| j                   j                  | «      z   z  S )Nç      4Àç      4@r£   r5   ç      ð¿)r:   ÚclipÚexp©rA   r§   re  s      rC   ÚsigmoidzOps.sigmoidü  s€   € Ùà—‘—‘˜Q  t°�Ó3ˆAØ�G‰G�K‰K˜˜ ˆKÔ"Ø�‰HˆAØ�$‰JˆAØˆHà—‘—‘˜Q  tÓ,ˆAØ˜# §¡§¡¨Q¨B£Ñ/Ñ0Ð0rE   r®   c                óf   — |r| j                  |d¬«       ||z  }|S || j                  ||¬«      z  S )NTrd  )Údsigmoid©rA   r–   r®   re  s       rC   Úbackprop_sigmoidzOps.backprop_sigmoid  s=   € ñ Ø�M‰M˜! TˆMÔ*Ø�‰GˆAØˆHà˜Ÿ™ a°˜Ó9Ñ9Ð9rE   c                ó*   — |r
|d|z
  z  }|S |d|z
  z  S )Nr/   r5   rƒ   ©rA   r®   re  s      rC   ro  zOps.dsigmoid  s%   € ÙØ��Q‘‰JˆAØˆHà˜˜a™‘=Ð rE   c                ó8   — |r|dz  }|dz  }|dz  }|S d|dz  z
  S )Nr   ri  r5   r/   rƒ   rs  s      rC   Údtanhz	Ops.dtanh  s2   € ÙØ�!‰GˆAØ�‰IˆAØ�‰HˆAØˆHà�q˜!‘t‘8ˆOrE   r5   )re  ÚaxisÚtemperaturerv  rw  c                óÂ   — |dk7  r||z  }| j                   j                  ||d¬«      }||z
  }| j                   j                  |«      }||j                  |d¬«      z  }|S ©Nr5   T)rv  Úkeepdims)r:   rÜ   rk  rì   )rA   r�   re  rv  rw  ÚmaxesÚshiftedÚnew_xs           rC   ÚsoftmaxzOps.softmax"  sd   € ð ˜#ÒØ�K‘ˆAØ—‘—‘˜A D°4�Ó8ˆØ�e‘)ˆØ—‘—‘˜GÓ$ˆØ�—‘ ¨t�Ó4Ñ4ˆØˆrE   )re  rv  ÚXsc                ó  — |j                   dk\  rd|j                   › �}t        |«      ‚| j                  j                  |dd«      }| j                  j	                  |«      }| j                  | j                  ||«      |«      }||z  }|S )Né   z)Softmax currently only supports 2d. Got: rg  rh  )r×   rI   r:   rj  rk  Úbackprop_reduce_sumÚ
reduce_sum)rA   r  r‹   re  rv  rJ   r}  Úsummeds           rC   Úsoftmax_sequenceszOps.softmax_sequences2  s|   € ð �7‰7�aŠ<Ø=¸b¿g¹g¸YÐGˆCÜ% cÓ*Ð*à�W‰W�\‰\˜"˜e TÓ*ˆØ—‘—‘˜B“ˆØ×)Ñ)¨$¯/©/¸%ÀÓ*IÈ7ÓSˆØ�‰ˆØˆrE   )rv  rw  c                óV   — |dk7  r||z  }||z  }|||j                  |d¬«      z  z  }|S ry  ©rì   )rA   r®   r–   rv  rw  rš   s         rC   Úbackprop_softmaxzOps.backprop_softmax?  s@   € ð ˜#ÒØ�kÑ!ˆBà�‰VˆØ
ˆa�"—&‘&˜d¨T�&Ó2Ñ2Ñ2ˆØˆ	rE   c                 ód   — ||z  }| j                  | j                  ||«      |«      }|||z  z  }|S r@   )r‚  rƒ  )rA   r–   r®   r‹   rš   Úsum_dXs         rC   Úbackprop_softmax_sequenceszOps.backprop_softmax_sequencesI  s=   € ð �‰VˆØ×)Ñ)¨$¯/©/¸"¸gÓ*FÈÓPˆØ
ˆa�&‰jÑˆØˆ	rE   ÚparamsÚH0ÚC0rí   c                 ó¦   — |j                   |j                   k(  sJ ‚|j                   d   |j                   d   k(  sJ ‚t        |||||«      \  }}||fS ©Nr/   )rŽ   Úlstm_forward_training)rA   rŒ  r�  rŽ  r§   rí   r®   Ú	fwd_states           rC   r‘  zOps.lstm_forward_trainingQ  sX   € ð �x‰x˜2Ÿ8™8Ò#Ð#Ð#Ø�x‰x˜‰{˜bŸh™h q™kÒ)Ð)Ð)Ü,¨V°R¸¸QÀ	ÓJ‰ˆˆ9Ø�)ˆ|ÐrE   c                 ó*   — t        |||||«      \  }}|S r@   )r‘  )rA   rŒ  r�  rŽ  r§   rí   r®   rô   s           rC   Úlstm_forward_inferencezOps.lstm_forward_inference^  s   € ô % V¨R°°Q¸	ÓB‰ˆˆ1ØˆrE   r’  c                 ó,   — t        ||||«      \  }}||fS r@   )Úbackprop_lstm)rA   r–   r‹   rŒ  r’  rš   Úd_paramss          rC   r–  zOps.backprop_lstmi  s"   € ô % R¨°&¸)ÓD‰ˆˆHØ�8ˆ|ÐrE   c                 óN   — |j                  d¬«      }|j                  d¬«      |fS )Nr;   ©rv  )ÚargmaxrÜ   )rA   r§   Úwhichs      rC   Úmaxoutz
Ops.maxouto  s'   € Ø—‘˜b�Ó!ˆØ�u‰u˜"ˆu‹~˜uÐ$Ð$rE   r›  ÚPc           	      ó  — | j                  |j                  d   |j                  d   ||j                  ¬«      }t        |j                  d   «      D ]1  }t        |j                  d   «      D ]  }|||f   ||||||f   f<   Œ Œ3 |S )Nr   r/   rN   )r�   rŽ   rO   rë   )rA   r–   r›  r�  rš   r¬   Úos          rC   Úbackprop_maxoutzOps.backprop_maxouts  sŒ   € Ø�\‰\˜"Ÿ(™( 1™+ r§x¡x°¡{°A¸R¿X¹Xˆ\ÓFˆÜ�r—x‘x ‘{Ó#ò 	1ˆAÜ˜2Ÿ8™8 A™;Ó'ò 1�Ø(*¨1¨a¨4©��1�a˜˜q !˜t™Ð$Ò%ñ1ð	1ð ˆ	rE   c                 ó*   — |s||dkD  z  S ||dkD  z  }|S r^   rƒ   rl  s      rC   ÚreluzOps.reluz  s%   € ÙØ˜˜A™‘;Ðà��Q‘‰JˆAØˆHrE   c                 ó*   — |s||dkD  z  S ||dkD  z  }|S r^   rƒ   rp  s       rC   Úbackprop_reluzOps.backprop_relu�  s'   € ñ Ø˜˜Q™‘<ÐØ
ˆa�!‰e‰ˆØˆ	rE   ÚslopeÚoffsetÚmin_valÚmax_valc                 ó¢   — |r)||z  }||z  }| j                   j                  ||||¬«      S ||z  |z   }| j                   j                  |||«      S )Nr£   )r:   rj  )rA   r§   r¥  r¦  r§  r¨  re  rŸ   s           rC   Úclipped_linearzOps.clipped_linear‰  sY   € ñ Ø�‰JˆAØ�‰KˆAØ—7‘7—<‘<  7¨G¸�<Ó;Ð;Ø�%‰i˜&Ñ ˆØ�w‰w�|‰|˜C ¨'Ó2Ð2rE   c                 óZ  — ||z
  |z  }||z
  |z  }	| j                   j                  |«      j                  |j                  «      }| j                   j                  d«      j                  |j                  «      }
| j                   j	                  ||k  ||	k  z  ||
«      }|r||z  }|S ||z  S )Nrþ   )r:   Úfloat64ÚastyperO   rË   )rA   r–   r§   r¥  r¦  r§  r¨  re  ÚlowÚhighÚzerorš   s               rC   Úbackprop_clipped_linearzOps.backprop_clipped_linear™  s¡   € ð ˜Ñ 5Ñ(ˆØ˜&Ñ  EÑ)ˆØ—‘—‘ Ó&×-Ñ-¨a¯g©gÓ6ˆØ�w‰w�‰˜sÓ#×*Ñ*¨1¯7©7Ó3ˆØ�W‰W�]‰]˜C !™G¨¨D©Ñ1°5¸$Ó?ˆÙØ�"‰HˆBØˆIØ�B‰wˆrE   Únc                 ó*   — | j                  |||¬«      S ©N)r¨  re  ©rª  )rA   r§   r²  re  s       rC   Úrelu_kz
Ops.relu_k­  s   € Ø×"Ñ" 1¨a¸Ð"ÓAÐArE   c                 ó,   — | j                  ||||¬«      S r´  ©r±  )rA   r–   r§   r²  re  s        rC   Úbackprop_relu_kzOps.backprop_relu_k°  s   € ð ×+Ñ+¨B°¸1ÀgÐ+ÓNÐNrE   c                 ó,   — | j                  |dd|¬«      S )Nçš™™™™™É?ç      à?)r¥  r¦  re  rµ  rl  s      rC   Úhard_sigmoidzOps.hard_sigmoidµ  s   € Ø×"Ñ" 1¨C¸ÀWÐ"ÓMÐMrE   c                 ó,   — | j                  ||dd¬«      S )Nr»  r¼  )r¥  r¦  r¸  ©rA   r–   r§   re  s       rC   Úbackprop_hard_sigmoidzOps.backprop_hard_sigmoid¸  s   € ð ×+Ñ+¨B°¸ÀSÐ+ÓIÐIrE   c                 ó,   — | j                  |dd|¬«      S )Nri  r5   )r§  r¨  re  rµ  rl  s      rC   Ú	hard_tanhzOps.hard_tanh½  s   € Ø×"Ñ" 1¨d¸CÈÐ"ÓQÐQrE   c                 ó,   — | j                  ||dd¬«      S )Nri  r5   )r§  r¨  r¸  r¿  s       rC   Úbackprop_hard_tanhzOps.backprop_hard_tanhÀ  s   € ð ×+Ñ+¨B°¸4ÈÐ+ÓMÐMrE   c                 ó^   — |r|| j                  |«      z  }|S || j                  |«      z  }|S r@   ©rm  ©rA   r§   re  rŸ   s       rC   Úswishz	Ops.swishÅ  s3   € ÙØ�—‘˜a“Ñ ˆAØˆHØ�$—,‘,˜q“/Ñ!ˆØˆ
rE   c                 óV   — || j                  |«      d|z
  z  z   }|r||z  }|S ||z  }|S r�  rÆ  )rA   r–   r§   r®   re  rŸ   s         rC   Úbackprop_swishzOps.backprop_swishÌ  s>   € ð �—‘˜Q“ 1 q¡5Ñ)Ñ)ˆÙØ�!‰GˆBØˆIØ�1‰fˆØˆ
rE   c                 ó^   — |r|| j                  |«      z  }|S || j                  |«      z  }|S r@   )r½  rÇ  s       rC   Ú
hard_swishzOps.hard_swish×  s9   € ÙØ�×"Ñ" 1Ó%Ñ%ˆAØˆHØ�$×#Ñ# AÓ&Ñ&ˆØˆ
rE   c                 óN   — |dz  dz   }d||dkD  <   d||dk  <   |r||z  }|S ||z  S )Ngš™™™™™Ù?r¼  r5   g      @r   g      Àrƒ   ©rA   r–   r§   re  rš   s        rC   Úbackprop_hard_swishzOps.backprop_hard_swishÞ  sE   € ð �‰W�s‰]ˆØˆˆ1ˆs‰7‰Øˆˆ1ˆt‰8‰ÙØ�"‰HˆBØˆIØ�B‰wˆrE   c                 ór   — |r|| j                  |dz   «      dz  z  }|S || j                  |dz   «      dz  z  S )Nr�  é   )r¶  rl  s      rC   Úhard_swish_mobilenetzOps.hard_swish_mobilenetê  sC   € ÙØ�—‘˜Q ™UÓ# aÑ'Ñ'ˆAØˆHØ�D—K‘K  A¡Ó&¨Ñ*Ñ+Ð+rE   c                 óT   — d|dz  dz   z  }d||dkD  <   d||dk  <   |r||z  }|S ||z  S )NgUUUUUUÅ?r4   ç      @r5   r   g      Àrƒ   rÎ  s        rC   Úbackprop_hard_swish_mobilenetz!Ops.backprop_hard_swish_mobilenetð  sJ   € ð ˜˜C™ #™Ñ&ˆØˆˆ1ˆs‰7‰Øˆˆ1ˆt‰8‰ÙØ�"‰HˆBØˆIØ�B‰wˆrE   c                 ó¶   — | j                   j                  |«      }|dz  }| j                   j                  ||¬«       ||z  }|dz  }|dz  }|r||z  }|S ||z  S )Nr5   r£   r/   r¼  ©r:   ÚsquareÚsqrt)rA   r§   re  Útmps       rC   ÚdishzOps.dishû  sf   € Ø�g‰g�n‰n˜QÓˆØˆs‰
ˆØ�‰�‰�S˜cˆÔ"Ø�#‰gˆØˆq‰ˆØˆs‰
ˆÙØ�‰HˆAØˆHà�s‘7ˆNrE   c                 óÎ   — | j                   j                  |«      }|dz   }|| j                   j                  |«      z  }d|z  |z  }||dz  z  }||z  }|dz  }|r||z  }|S ||z  S )Nr5   r¼  g      ø?r×  )rA   r–   r§   re  Úx_sqÚx_sq_plus_oneÚderivÚseconds           rC   Úbackprop_dishzOps.backprop_dish  s   € ð �w‰w�~‰~˜aÓ ˆØ˜s™
ˆØ�D—G‘G—L‘L Ó/Ñ/ˆØ�q‘˜4‘ˆØ�- Ñ$Ñ$ˆØ�‰ˆØ�‰ˆÙØ�%‰KˆBØˆIà˜‘:ÐrE   c                 ó`  — | j                   j                  |«      }| j                   j                  |«      }d}d}d}d}d}d}dd||z  z   z  }	d||	z  |z   |	z  |z   |	z  |z   |	z  |z   |	z  | j                   j                  | |z  «      z  z
  }
||
z  }|j	                  |j
                  «      }|S )Ng~ZìÆ OÐ?gi<Ì15Ò¿gWáºU¾ö?g9LW@÷¿g-¡UB„ûð?gŒ{=©@÷Ô?r5   )r:   ÚsignÚabsrk  r­  rO   )rA   r§   rã  Úa1Úa2Úa3Úa4Úa5Úpr÷   rž   rŸ   s               rC   ÚerfzOps.erf  sË   € à�w‰w�|‰|˜A‹ˆØ�G‰G�K‰K˜‹NˆàˆØˆØˆØˆØˆØˆà�3˜˜Q™‘;ÑˆØ�r˜A‘v ‘{ aÑ'¨2Ñ-°Ñ2°RÑ7¸1Ñ<¸rÑAÀQÑFÈÏÉÏÉØˆB�‰FóJ
ñ 
ñ 
ˆð �Q‰hˆØ�j‰j˜Ÿ™Ó!ˆØˆ
rE   c                 ó~   — | j                   j                  |dd«      }d| j                   j                  |«      z  dz  S )Nrg  rh  r/   r   )r:   rj  Úcosh)rA   r§   s     rC   Úsechsqz
Ops.sechsq.  s4   € à�G‰G�L‰L˜˜E 4Ó(ˆØ�D—G‘G—L‘L “OÑ#¨Ñ)Ð)rE   c           
      ó"  — d| j                   j                  t        |d| j                   j                  |d«      z  z   z  «      z   }|dz  }|j	                  |j
                  «      }|r||z  }|S | j                   j                  |«      }||z  }|S )Nr5   g÷Hmâä¦?r�  r¼  )r:   ÚtanhÚSQRT2PIÚpowerr­  rO   r$  )rA   r§   re  rÚ  r®   s        rC   Úgelu_approxzOps.gelu_approx3  sƒ   € Ø�D—G‘G—L‘L¤¨A°¸4¿7¹7¿=¹=ÈÈAÓ;NÑ0NÑ,NÑ!OÓPÑPˆØˆs‰
ˆØ�j‰j˜Ÿ™Ó!ˆÙØ�‰HˆAØˆHà—‘—‘˜aÓ ˆAØ�‰HˆAØˆHrE   c                 ó\  — t        t        | j                  |j                  «      «      }| j                  j                  |d«      }d| j                  j                  d|z  d|z  z   «      z  }|d|z  d|z  z   | j                  d|z  d|z  z   «      z  z  }|dz  }||z  }|r||z  }|S ||z  S )Nr�  r¼  g„¡iäND¢?g¸ŸFˆé?gFržVvf«?g¢ÀDˆÙ?)r   r   r  rŽ   r:   rò  rð  rî  )rA   r–   r§   re  rš   ÚXp3rÚ  s          rC   Úbackprop_gelu_approxzOps.backprop_gelu_approx?  s¼   € ô ”)˜TŸ\™\¨!¯'©'Ó2Ó3ˆØ�g‰g�m‰m˜A˜qÓ!ˆØ�D—G‘G—L‘L ¨S¡°8¸a±<Ñ!?Ó@Ñ@ˆØ�	˜C‘ (¨Q¡,Ñ.°$·+±+Ø˜‰O˜h¨™lÑ*ó3
ñ 
ñ 	
ˆð 	ˆs‰
ˆØ
ˆc‰	ˆÙØ�"‰HˆBØˆIØ�B‰wˆrE   c                 ó6   — t        | |«      }|r||z  }|S ||z  S r@   )Úgaussian_cdf)rA   r§   re  Úcdfs       rC   ÚgeluzOps.geluO  s(   € ä˜4 Ó#ˆÙØ�‰HˆAØˆHØ�3‰wˆrE   c                 óV   — t        | |«      |t        | |«      z  z   }|r||z  }|S ||z  S r@   )rø  Úgaussian_pdfrÎ  s        rC   Úbackprop_geluzOps.backprop_geluW  s;   € ô ˜$ Ó" Q¬°d¸AÓ)>Ñ%>Ñ>ˆÙØ�"‰HˆBØˆIØ�B‰wˆrE   Ú	thresholdc           	      óþ   — || j                   j                  | j                   j                  d| j                   j                  |«      z   «      «      z  }| j                   j	                  ||k\  ||«      }|r||d d  |S |S ©Nr5   )r:   rð  Úlogrk  rË   )rA   r§   rþ  re  rÚ  r®   s         rC   ÚmishzOps.misha  si   € ð �$—'‘'—,‘,˜tŸw™wŸ{™{¨3°·±·±¸Q³Ñ+?Ó@ÓAÑAˆØ�G‰G�M‰M˜!˜y™.¨!¨SÓ1ˆÙØˆA‰aˆDØˆHàˆHrE   c                 ó  — |j                   |j                   k7  r'd|j                   › d|j                   › �}t        |«      ‚t        |«      }||k  }||   }||   }	d|dz   z  }
|
d|j                  d|z  «      z  z  }
|
|j                  d|z  «      z  }
|
|j                  |«      d|z  dz   z  z  }
|j                  |«      dz   }||z  }|dz  }|	|j                  |«      |
z  |dz  z  z  }|r|}n|j	                  |«      }|||<   |S )	Nú!arrays have incompatible shapes: ú and g      @r5   r4   rÔ  ç      @r   )rŽ   rU   r,   rk  Úcopy)rA   r–   r§   rþ  re  Úmsgr:   rg   ÚXsubÚdYsubÚomegaÚdeltaÚdXsubrŸ   s                 rC   Úbackprop_mishzOps.backprop_mishl  s%  € ð �8‰8�q—w‘wÒØ5°b·h±h°Z¸uÀQÇWÁWÀIÐNˆCÜ˜S“/Ð!ä˜aÓ ˆØ�i‘-ˆØ�‰zˆØ�7‘ˆØ�t˜c‘zÑ"ˆØ��r—v‘v˜c D™jÓ)Ñ)Ñ)ˆØ�—‘˜˜d™
Ó#Ñ#ˆØ�—‘˜“ #¨¡*°Ñ!3Ñ4Ñ4ˆØ—‘�t“˜sÑ"ˆØ�‰ˆØ�‰ˆØ˜"Ÿ&™& ›,¨Ñ.°5¸!±8Ñ<Ñ=ˆáØ‰Cà—'‘'˜"“+ˆCØˆˆG‰Øˆ
rE   ÚemaÚweightsr÷   Ú	max_decayc                 óD   — d|z   d|z   z  }||kD  r|}|d|z
  ||z
  z  z  }y )Nr5   g      $@r/   rƒ   )rA   r  r  r÷   r  Údecays         rC   Úupdate_averageszOps.update_averages‹  s:   € ð �q‘˜T A™XÑ&ˆØ�9ÒØˆEØ��E‘	˜c G™mÑ,Ñ,‰rE   ÚgradientÚmom1Úmom2Úbeta1Úbeta2ÚepsÚ
learn_rateÚmod_ratec
                 óð   — t        ||«       t        ||«       t        ||«       ||z  }||z  }||d|z
  z  z  }|||z  d|z
  z  z  }||||	| j                  j                  |«      z  |z   z  z  z  }||||fS r   )Ú_check_compatible_shaper:   rÙ  )
rA   r  r  r  r  r  r  r  r  r  s
             rC   ÚadamzOps.adam”  s�   € ô 	  ¨Ô2Ü ¨Ô.Ü ¨Ô.ð 	�‰ˆØ�‰ˆØ�˜C %™KÑ(Ñ(ˆØ�˜8Ñ# s¨U¡{Ñ3Ñ3ˆð 	�: ¨°D·G±G·L±LÀÓ4FÑ)FÈÑ)LÑ!MÑNÑNˆØ˜ $¨Ð,Ð,rE   c                 ól   — t        |«      }|j                  j                  |«      }||k\  r|||z  z  }|S r@   )r,   ÚlinalgÚnorm)rA   r  rþ  r:   Ú	grad_norms        rC   Úclip_gradientzOps.clip_gradient®  s:   € ä˜hÓ'ˆØ—I‘I—N‘N 8Ó,ˆ	Ø˜	Ò!Ø˜	 IÑ-Ñ-ˆHØˆrE   Úy_trueÚy_predc                 óž   — | j                   j                  |dz   «      }||z  d|z
  | j                   j                  d|z
  dz   «      z  z   }| S )Ng:Œ0âŽyE>r/   )r:   r  )rA   r%  r&  Úlog_ypÚlosss        rC   ÚloglosszOps.logloss¶  sL   € à—‘—‘˜V d™]Ó+ˆØ˜‘ A¨¡J°$·'±'·+±+¸qÀ6¹zÈTÑ>QÓ2RÑ#RÑRˆØˆuˆrE   c                 óB  — | j                  |j                  d   |j                  d   d¬«      }d}t        |«      D ]`  \  }}|dk  rt        d|› �«      ‚||z   |j                  d   kD  rt	        d«      ‚|r!||||z    j                  d¬«      ||<   ||z  }Œ\d||<   Œb |S ©	Nr   r/   F©r¾   ú'all sequence lengths must be >= 0, got ú)lengths must sum up to the number of rowsr™  rþ   )r˜   rŽ   rÝ   rU   Ú
IndexErrorrì   ©rA   r§   r‹   r®   Ústartrf   r‡   s          rC   rƒ  zOps.reduce_sum¼  s¼   € Ø�L‰L˜Ÿ™ qÑ)¨1¯7©7°1©:¸UˆLÓCˆØˆÜ" 7Ó+ò 		‰IˆAˆvØ˜ŠzÜ Ð#JÈ6È(Ð!SÓTÐTØ˜‘ !§'¡'¨!¡*Ò,Ü Ð!LÓMÐMÙØ˜ ¨¡Ð0×4Ñ4¸!Ð4Ó<��!‘Ø˜‘‘à��!’ð		ð ˆrE   c                 ó~  — |j                   dk(  r!| j                  d|j                  d   «      |fS | j                  j	                  |dkD  «      st        d«      ‚| j                  |j                  d   dz   d¬«      }d|d<   |j                  «       |dd  |d   |j                  d   k7  rt        d«      ‚||d d    |fS )Nr   r/   ú all sequence lengths must be > 0Fr-  r;   r/  )	rY   r˜   rŽ   r:   r}   rU   ræ   rÍ   r0  )rA   r§   r‹   Ústarts_endss       rC   Úreduce_firstzOps.reduce_firstË  s¾   € Ø�<‰<˜1ÒØ—<‘<  1§7¡7¨1¡:Ó.°Ð7Ð7Ø�w‰w�{‰{˜7 Q™;Ô'ÜÐ?ÓAÐAØ—l‘l 7§=¡=°Ñ#3°aÑ#7¸u�lÓEˆØˆ�A‰Ø!Ÿ.™.Ó*ˆ�A�BˆØ�r‰?˜aŸg™g a™jÒ(ÜÐHÓIÐIà�˜S˜bÐ!Ñ" KÐ/Ð/rE   c                 ó.  — |j                   dk(  r!| j                  d|j                  d   «      |fS | j                  j	                  |dkD  «      st        d«      ‚|j                  «       dz
  }|d   dz   |j                  d   k7  rt        d«      ‚||   |fS )Nr   r/   r4  r;   r/  )rY   r˜   rŽ   r:   r}   rU   rÍ   r0  )rA   r§   r‹   Úlastss       rC   Úreduce_lastzOps.reduce_lastØ  s�   € Ø�<‰<˜1ÒØ—<‘<  1§7¡7¨1¡:Ó.°Ð7Ð7Ø�w‰w�{‰{˜7 Q™;Ô'ÜÐ?ÓAÐAØ—‘Ó  1Ñ$ˆØ�‰9�q‰=˜AŸG™G A™JÒ&ÜÐHÓIÐIØ�‰x˜ˆÐrE   c                 óB  — | j                  |j                  d   |j                  d   d¬«      }d}t        |«      D ]`  \  }}|dk  rt        d|› �«      ‚||z   |j                  d   kD  rt	        d«      ‚|r||||z    j                  d¬«      ||<   nd||<   ||z  }Œb |S r,  )r˜   rŽ   rÝ   rU   r0  Úmeanr1  s          rC   Úreduce_meanzOps.reduce_meanâ  s¼   € Ø�L‰L˜Ÿ™ qÑ)¨1¯7©7°1©:¸UˆLÓCˆØˆÜ" 7Ó+ò 		‰IˆAˆvØ˜ŠzÜ Ð#JÈ6È(Ð!SÓTÐTØ˜‘ !§'¡'¨!¡*Ò,Ü Ð!LÓMÐMÙØ˜ ¨¡Ð0×5Ñ5¸1Ð5Ó=��!’à��!‘Ø�V‰O‰Eð		ð ˆrE   c                 óâ  — | j                  |j                  d   |j                  d   |j                  d¬«      }| j                  |j                  d   |j                  d   d¬«      }d}t	        |«      D ]u  \  }}|dk  rt        d|› �«      ‚||z   |j                  d   kD  rt        d«      ‚|r6||||z    j                  d¬«      ||<   ||||z    j                  d¬«      ||<   ||z  }Œw ||fS )	Nr   r/   Fr  r-  ú&all sequence lengths must be > 0, got r/  r™  )	r˜   rŽ   rO   r  rÝ   rU   r0  rš  rÜ   )rA   r§   r‹   r®   r›  r2  rf   r‡   s           rC   Ú
reduce_maxzOps.reduce_maxñ  s  € Ø�L‰L˜Ÿ™ qÑ)¨1¯7©7°1©:¸Q¿W¹WÈEˆLÓRˆØ—‘˜WŸ]™]¨1Ñ-¨q¯w©w°q©zÀ�ÓGˆØˆÜ" 7Ó+ò 	‰IˆAˆvØ˜Š{Ü Ð#IÈ&ÈÐ!RÓSÐSØ˜‘ !§'¡'¨!¡*Ò,Ü Ð!LÓMÐMÙØ˜U U¨V¡^Ð4×;Ñ;ÀÐ;ÓC��a‘Ø˜ ¨¡Ð0×4Ñ4¸!Ð4Ó<��!‘Ø�V‰O‰Eð	ð �%ˆxˆrE   Úd_firstsr5  c                 ó0  — |j                   dk(  r,| j                  d|j                  d   |j                  d¬«      S |j                   dk(  rt	        d«      ‚| j                  t        |d   «      |j                  d   |j                  d¬«      }|||d d <   |S )Nr   r/   Tr  z starts_ends must not have size 1r;   )rY   r˜   rŽ   rO   rU   ra   )rA   r@  r5  rš   s       rC   Úbackprop_reduce_firstzOps.backprop_reduce_first   sœ   € ð ×Ñ˜qÒ Ø—<‘<  8§>¡>°!Ñ#4¸H¿N¹NÐRV�<ÓWÐWØ×Ñ Ò"ÜÐ?ÓAÐAØ�\‰\Ü�˜B‘Ó  (§.¡.°Ñ"3¸8¿>¹>ÐQUð ó 
ˆð  (ˆˆ;�s˜ÐÑØˆ	rE   Úd_lastsr8  c                 óü   — |j                   dk(  r,| j                  d|j                  d   |j                  d¬«      S | j                  t	        |d   «      dz   |j                  d   |j                  d¬«      }|||<   |S )Nr   r/   Tr  r;   )rY   r˜   rŽ   rO   ra   )rA   rC  r8  rš   s       rC   Úbackprop_reduce_lastzOps.backprop_reduce_last  sx   € Ø�:‰:˜Š?Ø—<‘<  7§=¡=°Ñ#3¸7¿=¹=ÐPT�<ÓUÐUØ�\‰\Ü��b‘	‹N˜QÑ §¡¨aÑ 0¸¿¹ÈTð ó 
ˆð ˆˆ5‰	Øˆ	rE   Úd_sumsc                 óê   — | j                  |j                  «       |j                  d   |j                  d¬«      }d}t	        |«      D ](  \  }}|dk  rt        d|› �«      ‚||   ||||z    ||z  }Œ* |S ©Nr/   Fr  r   r.  ©r˜   rì   rŽ   rO   rÝ   rU   )rA   rF  r‹   rš   r2  rf   r‡   s          rC   r‚  zOps.backprop_reduce_sum  s�   € Ø�\‰\Ø�K‰K‹M˜6Ÿ<™<¨™?°&·,±,Àeð ó 
ˆð ˆÜ" 7Ó+ò 	‰IˆAˆvØ˜ŠzÜ Ð#JÈ6È(Ð!SÓTÐTØ)/°©ˆBˆu�u˜v‘~Ð&Ø�V‰O‰Eð		ð
 ˆ	rE   Úd_meansc                 óð   — | j                  |j                  «       |j                  d   |j                  d¬«      }d}t	        |«      D ]+  \  }}|dk  rt        d|› �«      ‚||   |z  ||||z    ||z  }Œ- |S rH  rI  )rA   rJ  r‹   rš   r2  rf   r‡   s          rC   Úbackprop_reduce_meanzOps.backprop_reduce_mean"  s“   € Ø�\‰\Ø�K‰K‹M˜7Ÿ=™=¨Ñ+°7·=±=Èð ó 
ˆð ˆÜ" 7Ó+ò 	‰IˆAˆvØ˜ŠzÜ Ð#JÈ6È(Ð!SÓTÐTØ)0°©°fÑ)<ˆBˆu�u˜v‘~Ð&Ø�V‰O‰Eð		ð
 ˆ	rE   Úd_maxesc                 óD  — | j                  |j                  «       |j                  d   |j                  ¬«      }d}t	        |«      D ]V  \  }}|dk  rt        d|› �«      ‚| j                  j                  ||||z    ||   j                  d«      ||   d«       ||z  }ŒX |S )Nr/   rN   r   r>  )r/   r;   )	r˜   rì   rŽ   rO   rÝ   rU   r:   Úput_along_axisr&  )rA   rM  r›  r‹   rš   r2  rf   r‡   s           rC   Úbackprop_reduce_maxzOps.backprop_reduce_max.  s®   € ð �\‰\˜'Ÿ+™+›-¨¯©°qÑ)9ÀÇÁˆ\ÓOˆØˆÜ" 7Ó+ò 	‰IˆAˆvØ˜Š{Ü Ð#IÈ&ÈÐ!RÓSÐSà�G‰G×"Ñ"Ø�5˜5 6™>Ð*¨E°!©H×,<Ñ,<¸WÓ,EÀwÈqÁzÐSTôð �V‰O‰Eð	ð ˆ	rE   ÚidsÚseedc                 ó€   — ddl m}  |«       }| j                  |j                  |j	                  |d¬«      |«      «      S )zcHash a sequence of 64-bit keys into a table with 4 32-bit keys, using
        murmurhash3.
        r/   ©ÚNumpyOpsÚuint64rN   )Ú	numpy_opsrU  rW  ÚhashrT   )rA   rQ  rR  rU  rW  s        rC   rX  zOps.hash=  s=   € õ 	(á“Jˆ	Ø�~‰~Ø�N‰N˜9×,Ñ,¨S¸Ð,ÓAÀ4ÓHó
ð 	
rE   Úkeysc           	      ó€   — ddl m}  |«       }| j                  |j                  ||j	                  |d¬«      «      «      S )Nr/   rT  rV  rN   )rW  rU  rè   ÚngramsrT   )rA   r²  rY  rU  rW  s        rC   r[  z
Ops.ngramsH  s=   € Ý'á“Jˆ	Ø�~‰~Ø×Ñ˜Q 	× 1Ñ 1°$¸hÐ 1Ó GÓHó
ð 	
rE   ÚNÚDÚperiodc                 ób   — ddl m}  |«       }| j                  |j                  ||||«      «      S )Nr/   rT  )rW  rU  rO  Úposition_encode)rA   r\  r]  r^  rŸ   rU  rW  s          rC   r`  zOps.position_encodeP  s.   € õ 	(á“Jˆ	Ø�~‰~˜i×7Ñ7¸¸1¸fÀcÓJÓKÐKrE   Útablerg   c                 ó,   — ||   j                  d¬«      S )Nr/   r™  r‡  )rA   ra  rg   s      rC   Ú
gather_addzOps.gather_addY  s   € Ø�W‰~×!Ñ! qÐ!Ó)Ð)rE   r   c                 óP   — | j                   j                  j                  |||«      S r@   )r:   ÚaddÚat)rA   ra  rg   r   s       rC   Úscatter_addzOps.scatter_add\  s   € ð �w‰w�{‰{�~‰~˜e W¨fÓ5Ð5rE   c                 ó˜   — | j                  ||d   j                  ¬«      }t        |«      D ]  \  }}|||d|j                  d   …f<   Œ |S )z8Maybe don't need this? Just a quicky to get Jax working.r   rN   N)r½   rO   rÝ   rŽ   )rA   rŽ   r  rŠ   rf   r�   s         rC   Úinsert_intozOps.insert_intoa  sR   € à—‘˜E¨¨A©¯©�Ó5ˆÜ˜b“Mò 	(‰DˆAˆqØ&'ˆF�1�l˜Ÿ™ ™
�l�?Ò#ð	(àˆrE   )Úcpur;   )NFF)Nr   r   r»   )r/   r@   )F)r5   rþ   rþ   r5   F)r  F)rh  F)g§èH.ÿï?)r5   )i'  N)¦rH   Ú
__module__Ú__qualname__r9   ÚstrÚ__annotations__rQ   r:   r*   r   ra   rD   r0   rK   r.   r   r   r   Úboolr)   ro   r€   rc   r   rk   r   r   r    r•   rœ   r¦   rª   r   r¯   r   r   r   r´   r%   r   r&   r   r   rÅ   r!   r"   r°   r   r   râ   r'   rø   rû   r(   Úfloatr   r  r   r  r˜   r�   r   r  r  r   ræ   r  r  r#   r  r$   r!  r   r½   r   r(  r+  rç   r   r0  r2  r‘   r�   r6  r:  r=  r?  rA  rC  rE  r1   r&  rJ  rM  rO  rQ  rS  rè   rW  rZ  r\  r^  rT   r…   r   rm  rq  ro  r3   ru  r~  r…  rˆ  r‹  r   r‘  r”  r–  rœ  r   r¢  r¤  rª  r±  r¶  r¹  r½  rÀ  rÂ  rÄ  rÈ  rÊ  rÌ  rÏ  rÒ  rÕ  rÛ  rá  rë  rî  ró  rö  rú  rý  r  r  r  r  r$  r*  rƒ  r6  r9  r<  r?  rB  rE  r‚  rL  rP  rX  r[  r`  rc  rg  ri  rƒ   rE   rC   r7   r7   <   sñ  … Ø€Dˆ#ÓØ€BˆƒNð BDñ#Ø&ð#Ø;>ð#à	ó#ð'�uó 'ð
 ,0ô Mð Øò/7à�C˜�NÑ#ð/7ð ð/7ð
 ð/7ð ð/7ð 
ó/7ðl Øò(7à�C˜�NÑ#ð(7ð ð(7ð ð	(7ð
 ð(7ð ð(7ð 
ó(7òT	ð sð °8¸C±=ó ð FJò9Øð9Ø!$ð9Ø2:¸6Ñ2Bð9à	ó9ð* EIòØðØ #ðØ19¸&Ñ1Aðà	óð2 #'ØØñàðð ðð �hÑð	ð
 ðð ðð 
óð*%�hð % cð %¨hó %ð˜ð  Xð °(ð ¸xó ð ð #'ØØñà�‰>ðð ˜Ñðð ð	ð
 ðð 
òó ðð ð #'ØØñà�‰<ðð ˜Ñðð ð	ð
 ðð 
òó ðð ð #'ØØñàðð ˜Ñðð ð	ð
 ðð 
òó ðð ð #'ØØñàðð ˜Ñðð ð	ð
 ðð 
òó ðð ð #'ØØñà�GÑðð ˜Ñðð ð	ð
 ðð 
òó ðð #'ØØñà�GÑðð ˜Ñðð ð	ð
 ðð 
óð6 à78ñØðØ$*ðØ14ðà	ˆh‰òó ðð ÙV˜6ÐV¨FÐV¸ÐVÀTÈ&Á\ÒVó ØVàÙQ˜7ÐQ¨VÐQ¸#ÐQÀfÒQó ØQð ÙQ˜7ÐQ¨VÐQ¸#ÐQÀfÒQó ØQñ˜7ð ¨Vð ¸#ð Àfó ð" Ù@˜˜V™Ð@°VÒ@ó Ø@àÙD˜˜X™ÐD°xÒDó ØDð CDñØ˜$˜v™,¨¨X©Ð6Ñ7ðà	óðB$˜Gð $¨d°3©ið $¸Fó $ð(
 ð (
¨6ó (
ðT
& &ð 
&¨Vó 
&ðC eð C°8¸E±?ð CÀxó Cð$ (1ØòKàðKð ˜Ñ$ð	Kð
 ðKð 
óKð (1ØòNàðNð ðNð
 ˜Ñ$ðNð ðNð 
óNð  (1Øò	Ràð	Rð ð	Rð ð		Rð ˜Ñ$ð	Rð ð	Rð 
ó	Rð$ (1Øò
Vàð
Vð ð
Vð ð	
Vð
 ð
Vð ˜Ñ$ð
Vð ð
Vð 
ó
Vð  (1ØòKàðKð ˜Ñ$ð	Kð
 ðKð 
óKð &-ØòIàðIð ˜	Ñ"ð	Ið
 ðIð 
óIð &-ØòLàðLð ðLð
 ˜	Ñ"ðLð ðLð 
óLð  &-Øò	Pàð	Pð ð	Pð ð		Pð ˜	Ñ"ð	Pð ð	Pð 
ó	Pð$ &-Øò
Tàð
Tð ð
Tð ð	
Tð
 ð
Tð ˜	Ñ"ð
Tð ð
Tð 
ó
Tð  &-ØòIàðIð ˜	Ñ"ð	Ið
 ðIð 
óIð #,Øò5àð5ð ˜Ñð	5ð
 ð5ð 
ó5ð 9˜gð 9¨3ð 9°7ó 9ð<˜gð <¨3ð <°Cð <¸Gó <ð@˜gð @¨3ð @°Cð @¸Sð @ÀWó @ðD˜gð D¨3ð D°Cð D¸Sð DÀcð DÈgó Dð:˜xð :¨Sð :°Xó :ð=˜xð =¨Sð =°cð =¸hó =ðA˜xð A¨Sð A°cð A¸sð AÀxó AðEØðEØ#&ðEØ,/ðEØ58ðEØ>AðEà	óEð
*˜xð *°ð *¸(ó *ð8˜vð 8¨3ð 8°6ó 8ð;˜vð ;¨3ð ;°Cð ;¸Fó ;ð?˜vð ?¨3ð ?°Cð ?¸Sð ?ÀVó ?ðC˜vð C¨3ð C°Cð C¸Sð CÀcð CÈfó Cð*˜vð *¨eð *¸ó *ð2˜Vð 2¨Eð 2°fó 2ð #,ò	?à�H˜h x°¸À%¹Ñ0IÑ'JÑKÐKÑLð?ð ˜Ñð	?ð
 
ó?ð #,ò	?à�H˜h x°¸±Ñ'@ÑAÐAÑBð?ð ˜Ñð	?ð
 
ó?ð #,ò	?à�H˜h x°¡Ñ7Ð7Ñ8ð?ð ˜Ñð	?ð
 
ó?ð #,ò	?à�H˜h u™oÐ-Ñ.ð?ð ˜Ñð	?ð
 
ó?ð #,ò	?à�H˜h s™mÐ+Ñ,ð?ð ˜Ñð	?ð
 
ó?ð PWò=Ø˜& (¨3¡-Ð/Ñ0ð=Ø<DÀVÑ<Lð=à	ó=ð #*ò	=à�F˜H X¨c¡]Ñ3Ð3Ñ4ð=ð ˜Ñð	=ð
 
ó=ð #*ò	=à�F˜H X¨h°s©mÑ%<Ñ=Ð=Ñ>ð=ð ˜Ñð	=ð
 
ó=ð #*ò	=à�F˜H X¨h°xÀ±}Ñ.EÑ%FÑGÐGÑHð=ð ˜Ñð	=ð
 
ó=ð PWò=Ø˜& (¨3¡-Ð/Ñ0ð=Ø<DÀVÑ<Lð=à	ó=ð #'ò	'à�G˜X gÑ.°¸±Ð=Ñ>ð'ð ˜Ñð	'ð
 
ó'ñ,9˜fð 9¨X°fÑ-=ð 9Èó 9ð 8=ò 
1˜ð 
1°ð 
1Àó 
1ð ?Dò:Øð:Ø )ð:Ø7;ð:à	ó:ð 9>ò !˜)ð !°ð !À)ó !ð 49ò �wð ¨Dð ¸Wó ð ØØ òàðð ð	ð
 ðð ðð 
óð" AFÐSUòØðØ%+ðØ9=ðØMPðà	óð 79ÈsòØðØ%ðØ03ðØGLðà	óðØðØ'ðØ28ðà	óðàðð ðð ð	ð
 ðð ðð 
ˆx˜ˆÑ	óð	àð	ð ð	ð ð		ð
 ð	ð ð	ð 
ó	ðØðØ%+ðØ5=ðØJOðà	ˆx˜Ð!Ñ	"óð%˜ð % U¨8°VÐ+;Ñ%<ó %ð (ð °6ð ¸cð Àhó ñ�hð ¨ð ¸(ó ð :?ñØðØ'ðØ26ðà	óð ØØØØñ3àð3ð ð3ð ð	3ð
 ð3ð ð3ð ð3ð 
ó3ð( ØØØØñàðð ðð ð	ð
 ðð ðð ðð ðð 
óñ(B˜	ð B eð B¸Dð BÈYó Bð LQñOØðOØ )ðOØ.3ðOØDHðOà	óOñ
N˜ið N°$ð NÀ9ó Nð <AñJØðJØ )ðJØ48ðJà	óJñ
R˜9ð R¨tð RÀ	ó Rð <AñNØðNØ )ðNØ48ðNà	óNñ
�yð ¨4ð ¸Ió ð JOñØðØ )ðØ.7ðØBFðà	óñ˜Ið °ð Àó ð <Añ	Øð	Ø )ð	Ø48ð	à	ó	ñ, ið ,¸$ð ,È9ó ,ð <Añ	Øð	Ø )ð	Ø48ð	à	ó	ñ�ið ¨$ð ¸9ó ð <AñØðØ )ðØ48ðà	óð$�Yð  9ó ð(*˜	ð * ió *ñ

˜Yð 
°ð 
À)ó 
ð <AñØðØ )ðØ48ðà	óñ �ið ¨$ð ¸9ó ð <AñØðØ )ðØ48ðà	óð FKñ	Øð	Ø',ð	Ø>Bð	à	ó	ð  Øñàðð ðð ð	ð
 ðð 
óð@ JPñ-Øð-Ø%,ð-Ø14ð-ØAFð-à	ó-ð& ñ-àð-ð ð-ð ð	-ð
 ð-ð ð-ð ð-ð ð-ð ð-ð ð-ð 
ˆx˜ 8¨XÐ5Ñ	6ó-ð4 gð ¸%ð ÀGó ð˜gð ¨wð ¸5ó ð˜Hð ¨vð ¸(ó ð0˜hð 0°ð 0¸EÀ(ÈFÐBRÑ<Só 0ð˜Xð °ð ¸5ÀÈ6ÐAQÑ;Ró ð˜Xð °ð ¸8ó ð˜Hð ¨vð ¸%ÀÈ&Ð@PÑ:Qó ðØ ðØ/5ðà	óð¨Hð ¸Vð Èó ð
¨(ð 
¸Vð 
Èó 
ð
¨Hð 
¸vð 
È(ó 
ðØðØ(.ðØ9?ðà	óð	
˜ð 	
 cð 	
¨fó 	
ð
˜ð 
 6ð 
¨fó 
ð NRñLØðLØðLØ&)ðLØ8@ÀÑ8JðLà	óLð* ð *°6ð *¸hó *ð6Øð6Ø(.ð6Ø8@ð6à	ó6ó
rE   r7   rŒ  Úc_initÚh_initr§   r‹   r>   c                 ó´  — t        | «      }|j                  \  }}}|j                  \  }	}
|d   }t        t        |j	                  |||j                  d   |dz  fd¬«      «      }t        t        |j	                  |||j                  d   |fd¬«      «      }t        t        |j	                  |||j                  d   |fd¬«      «      }t        t
        |j	                  ||fd¬«      «      }t        t
        |j	                  ||fd¬«      «      }g }d}|D ]  }|j                  |||z   f«       ||z  }Œ d}|}t        |«      D �]Š  }|j                  d   }
t        |«      D �]  }|||f   j                  d|f«      }|||f   j                  d|f«      }t        | |||
|«      \  }}t        |«      \  }}}|||fxx   |j                  ||j                  «      z  cc<   |||fxx   |z  cc<   |dk(  r|n
t        |«      D �]|  \  }}|d ||z
   }|d ||z
   }|||||…f   }|d |j                  d    }||j                  ||j                  «      z  }t        t        |j                  d|df«      «      }t        |d d …d d …df   «      }t        |d d …d d …df   «      }t        |d d …d d …df   «      } |j!                  |d d …d d …df   «      }!||z  }"|"||!z  z  }"|j#                  ||| |!f«      j                  dd|f«      j%                  d	«      j                  d|dz  f«      }t'        ||| j                  d   z   «      }|j!                  |"«      | z  |||||…f<   ||||||…f<   |"|||||…f<   |"}|||||…f   }�Œ �Œ! t        t
        ||   j%                  d
«      j                  |	df«      «      }#|dk(  r|j)                  |#«      }#|#}�Œ� #||||ffS )Nr   é   r¼   rN   r/   r;   r   r�  )r   r   r/   rå   )r,   rŽ   r   r   r¾   r   rb   rë   r&  Ú_split_weightsÚ_transpose_weightsr¥   r¤   Úreversedr   rm  rð  Úhstackrê   Úminrc  )$rŒ  rq  rr  r§   r‹   r:   ÚdepthÚdirsró   r\  ÚnIÚ
batch_sizeÚGr®   ÚCÚYt2ÚCt2rg   r2  Úparams_iÚorig_Xrf   ÚdÚlayer_paramsÚWxÚWhÚbiasÚendÚGt3ÚGt3_ÚhfÚhiÚhoÚhcÚCt3ÚHs$                                       rC   r‘  r‘  �  s%  € ô 
˜&Ó	!€BØ—l‘l�O€Eˆ4�Ø�G‰G�E€A€rØ˜‘€JäŒX�r—x‘x ¨¨a¯g©g°a©j¸"¸q¹&Ð AÈ�xÓMÓN€AÜŒX�r—x‘x ¨¨a¯g©g°a©j¸"Ð =ÀS�xÓIÓJ€AÜŒX�r—x‘x ¨¨a¯g©g°a©j¸"Ð =ÀS�xÓIÓJ€AÜ
Œx˜Ÿ™ :¨rÐ"2¸#˜Ó>Ó
?€CÜ
Œx˜Ÿ™ :¨rÐ"2¸#˜Ó>Ó
?€Cà€GØ€EØò ˆ
Ø�‰˜˜u zÑ1Ð2Ô3Ø�Ñ‰ðð €HØ€FÜ�5‹\ó .ˆØ�W‰W�Q‰ZˆÜ�t“ó (	)ˆAð ˜˜A˜‘,×&Ñ&¨¨2 wÓ/ˆCØ˜˜A˜‘,×&Ñ&¨¨2 wÓ/ˆCÜ%3°F¸A¸rÀ2ÀxÓ%PÑ"ˆL˜(Ü-¨lÓ;‰LˆB��DØˆa�ˆd‹G�r—v‘v˜a §¡“Ñ&‹GØˆa�ˆd‹G�t‰O‹GØ)*¨aª™g´X¸gÓ5Fó )‘
��sà˜-˜C %™KÐ(�Ø˜-˜C %™KÐ(�à˜˜1˜e C˜i˜Ñ(�Ø˜.˜CŸI™I a™LÐ)�Ø�r—v‘v˜c 2§4¡4Ó(Ñ(�ÜœH c§k¡k°2°r¸1°+Ó&>Ó?�Ü˜T¢!¢Q¨ '™]Ó+�Ü˜T¢!¢Q¨ '™]Ó+�Ü˜T¢!¢Q¨ '™]Ó+�Ø—W‘W˜T¢!¢Q¨ '™]Ó+�Ø˜3‘h�Ø�r˜B‘w‘�ð —I‘I˜r 2 r¨2Ð.Ó/ß‘W˜b ! R˜[Ó)ß‘Y˜yÓ)ß‘W˜b " q¡&˜\Ó*ð	 ô ˜#˜u r§x¡x°¡{Ñ2Ó3�Ø%'§W¡W¨S£\°BÑ%6��!�Q˜˜c˜	�/Ñ"Ø%(��!�Q˜˜c˜	�/Ñ"Ø%(��!�Q˜˜c˜	�/Ñ"à�Ø˜˜1˜e C˜i˜Ñ(’ò?)ð(	)ôR ”˜1˜Q™4Ÿ>™>¨)Ó4×<Ñ<¸aÀ¸WÓEÓFˆØ�1Š9Ø×$Ñ$ QÓ'ˆAØŠð].ð^ ˆq�!�Q˜ÐÐÐrE   r–   r’  c                 óè  — t        |«      }|\  }}}}|j                  \  }	}
}}|j                  d   }|d   }t        t        |j	                  ||f|j
                  ¬«      «      }t        t        |j	                  ||dz  f|j
                  ¬«      «      }t        t        |j	                  |j                  d   f|j
                  ¬«      «      }d}g }t        |	«      D ]b  }|j                  g «       |dk(  r|n||
z  }t        |
«      D ]5  }t        |||||«      \  }}t        |«      }|d   j                  ||f«       Œ7 Œd d}g }t        |	«      D ]b  }|j                  g «       |dk(  r|n||
z  }t        |
«      D ]5  }t        |||||«      \  }}t        |«      }|d   j                  ||f«       Œ7 Œd g }d}|D ]  }|j                  |||z   f«       ||z  }Œ |gt        |	dz
  «      D �cg c]5  }t        t        ||   j                  d«      j                  |df«      «      ‘Œ7 c}z   }|D �cg c];  }|j	                  |j                  d   |j                  d   f|j
                  ¬«      ‘Œ= }}t        t        |	«      «      D �]~  }t        t        | j                  ||
|f«      j                  d«      «      }||   }||   }|
dk\  r|j                  |«      }t        |
«      D �]  }||   |   d   \  } }!}"||   |   d   \  }#}$}%|dk(  r|d   \  }&}'|d d }(|(j!                  «        n|d   \  }&}'|dd  }(|(D ]}  \  })}*t#        |*|)z
  |'|&z
  «      }+t%        |||&|&|+z   …f   ||&|&|+z    ||||&|&|+z   …f   ||||&|&|+z   …f   ||||)|)|+z   …f   «      \  },}-|||)|)|+z   …fxx   |,|!z  z  cc<   |-||)|)|+z    |)}&|*}'Œ |#|j&                  |z  z  }#|$|j&                  |||f   z  z  }$|%|j)                  d¬«      z  }%||| z  z  }�Œ |} �Œ� j                  d   j                  d   k(  sJ ‚g }.|D ]&  }|D ]  \  }/}0|.j                  t+        |/«      «       Œ! Œ( ||j-                  |.«      fS c c}w c c}w )	Nr/   r   rN   rt  r;   rå   r   r™  )r,   rŽ   r   r   r¾   rO   r   rë   rb   ru  rv  rê   r&  rw  r   rc  rä   ry  Úbackprop_lstm_gatesr¤   rì   Ú_untranspose_unsplit_weightsr¿   )1r–   r‹   rŒ  r’  r:   r®   r~  r  r§   rz  r{  r\  ró   r|  r}  ÚdCÚdGr—  r‚  Úall_layer_paramsrf   Ún_inputsr„  r…  Úall_layer_gradsÚlayer_gradsrg   r2  r  ÚdXsr›   rš   r†  r‡  rˆ  ÚdWxÚdWhÚd_biasÚstart_t3Úend_t3Úlayer_indicesÚstart_t2Úend_t2rY   ÚdGt3ÚdCt2Ú
grad_partsÚ	dir_gradsrô   s1                                                    rC   r–  r–  Ö  sF  € Ü	˜&Ó	!€Bð �J€A€qˆ!ˆQØŸ™Ñ€Eˆ4��BØ	
�‰�‰€BØ˜‘€Jä	Œh˜Ÿ™ ! R °·±˜Ó8Ó	9€BÜ	Œh˜Ÿ™ ! R¨!¡V °A·G±G˜Ó<Ó	=€BÜ”H˜bŸh™h¨¯©°Q©Ð'9ÀÇÁ˜hÓNÓO€Hð €HØTVÐÜ�5‹\ò BˆØ×Ñ Ô#Ø˜aš‘2 b¨4¡iˆÜ�t“ò 	BˆAÜ%3°F¸A¸rÀ8ÈXÓ%VÑ"ˆL˜(Ü-¨lÓ;ˆLØ˜RÑ ×'Ñ'¨°xÐ(@ÕAñ	BðBð €HØSU€OÜ�5‹\ò @ˆØ×Ñ˜rÔ"Ø˜aš‘2 b¨4¡iˆÜ�t“ò 	@ˆAÜ$2°8¸QÀÀHÈhÓ$WÑ!ˆK˜Ü,¨[Ó9ˆKØ˜BÑ×&Ñ&¨°XÐ'>Õ?ñ	@ð@ð €GØ€EØò ˆ
Ø�‰˜˜u zÑ1Ð2Ô3Ø�Ñ‰ðð ˆä�u˜q‘yÓ!öàô 	ŒX�q˜‘t—~‘~ iÓ0×8Ñ8¸!¸R¸ÓAÕBòñ 
€Bð GIÖ
IÀˆ2�8‰8�Q—W‘W˜Q‘Z §¡¨¡Ð,°A·G±Gˆ8Õ<Ð
I€CÐ
Iä”e˜E“lÓ#ó 'ˆÜ”H˜bŸj™j¨!¨T°2¨Ó7×AÑAÀ)ÓLÓMˆØ�‰VˆØˆq‰EˆØ�1Š9Ø×'Ñ'¨Ó-ˆDÜ�t“ó  	ˆAØ+¨AÑ.¨qÑ1°!Ñ4‰LˆB��DØ.¨qÑ1°!Ñ4°QÑ7ÑˆC��fØ�AŠvØ#*¨2¡;Ñ �˜&Ø '¨¨ �Ø×%Ñ%Õ'à#*¨1¡:Ñ �˜&Ø '¨¨ �Ø$1ò  Ñ �˜&Ü˜6 HÑ,¨f°xÑ.?Ó@�Ü0Ø˜˜H x°$¡Ð6Ð6Ñ7Ø�x (¨T¡/Ð2Ø�a˜˜H x°$¡Ð6Ð6Ñ7Ø�a˜˜H x°$¡Ð6Ð6Ñ7Ø�a˜˜H x°$¡Ð6Ð6Ñ7ó‘
��dð �Q˜ 8¨d¡?Ð2Ð2Ó3°t¸b±yÑ@Ó3à15��8˜h¨™oÐ.Ø#�Ø‘ð ð  �2—4‘4˜!‘8‰OˆCà�2—4‘4˜!˜A˜q˜D™'‘>Ñ!ˆCà�b—f‘f !�f“nÑ$ˆFà�"�r‘'‰MŠBðA 	ðB ŠðO'ðP �8‰8�A‰;˜!Ÿ'™' !™*Ò$Ð$Ð$Ø€JØ&ò GˆØ'ò 	G‰LˆI�qØ×ÑÔ:¸9ÓEÕFñ	GðGð ˆr�~‰~˜jÓ)Ð)Ð)ùògùò Js   Ç.:Q*È1A Q/rf   ró   r|  r‚  c                 óN  — d|z  |z  }d|z  }d|z  |z  }d|z  }| |||z    j                  d|z  |f«      }	||z  }| |||z    j                  d|z  f«      }
||z  }| |||z    j                  d|z  |f«      }||z  }| |||z    j                  d|z  f«      }||z  }|	|
f||ff|fS )Nrt  r8  )rŒ  rf   ró   r|  r‚  ÚWx_sizeÚbx_sizeÚWh_sizeÚbh_sizer†  Úbxr‡  Úbhs                rC   ru  ru  <  sú   € Ø�"‰f�r‰k€GØ�"‰f€GØ�"‰f�r‰k€GØ�"‰f€GØ	�˜8 gÑ-Ð	.×	6Ñ	6¸¸B¹À°|Ó	D€BØ�Ñ€HØ	�˜8 gÑ-Ð	.×	6Ñ	6¸¸B¹°yÓ	A€BØ�Ñ€HØ	�˜8 gÑ-Ð	.×	6Ñ	6¸¸B¹À°|Ó	D€BØ�Ñ€HØ	�˜8 gÑ-Ð	.×	6Ñ	6¸¸B¹°yÓ	A€BØ�Ñ€HØ�ˆH�r˜2�hÐ Ð)Ð)rE   c                 ó‚  — | \  \  }}\  }}t        |«      }|j                  dd|j                  d   f«      }|j                  d«      j                  d|j                  d   f«      }|j                  d«      j                  d«      j                  d«      }|j                  dd|j                  d   f«      }|j                  d«      j                  d|j                  d   f«      }|j                  d«      j                  d«      j                  d«      }|j                  } ||«      } ||«      } ||«      |z   }|||fS )Nrt  r;   rå   )rt  r;   ©r/   r   ©r;   )r,   r&  rŽ   rê   rc  )rŒ  r†  r­  r‡  r®  r:   Úascontigrˆ  s           rC   rv  rv  L  s%  € ð  Ñ�H€Rˆ‰hˆr�2Ü	˜"Ó	€BØ	�‰�Q˜˜BŸH™H R™LÐ)Ó	*€BØ	�‰�iÓ	 ×	(Ñ	(¨"¨b¯h©h°r©lÐ);Ó	<€BØ	�‰�GÓ	×	&Ñ	& vÓ	.×	6Ñ	6°uÓ	=€BØ	�‰�Q˜˜BŸH™H R™LÐ)Ó	*€BØ	�‰�iÓ	 ×	(Ñ	(¨"¨b¯h©h°r©lÐ);Ó	<€BØ	�‰�GÓ	×	&Ñ	& vÓ	.×	6Ñ	6°uÓ	=€BØ×#Ñ#€HÙ	�"‹€BÙ	�"‹€BÙ�B‹<˜"Ñ€DØˆr�4ˆ<ÐrE   c                 ó*  — | \  }}}t        |«      }|j                  d   }|j                  d   }|j                  dd|f«      j                  d«      j                  d|f«      }|j                  dd|f«      j                  d«      j                  d|f«      }|j                  d«      j                  d«      j                  d«      }|j	                  |j                  d¬	«      }|j                  |j                  «       ||j                  «       |f«      S )
Nr/   r;   rt  rå   )r;   rt  r°  r±  r¼   rN   )r,   rŽ   r&  rê   r¾   r¿   Úravel)rŒ  r†  r‡  rˆ  r:   ró   r|  r¾   s           rC   r”  r”  ^  së   € Ø�L€BˆˆDÜ	˜"Ó	€BØ	�‰�!‰€BØ	�‰�!‰€BØ	�‰�R˜˜B�KÓ	 ×	*Ñ	*¨9Ó	5×	=Ñ	=¸rÀ2¸hÓ	G€BØ	�‰�R˜˜B�KÓ	 ×	*Ñ	*¨9Ó	5×	=Ñ	=¸rÀ2¸hÓ	G€BØ�<‰<˜Ó ×*Ñ*¨6Ó2×:Ñ:¸5ÓA€DØ�H‰H�T—Z‘Z sˆHÓ+€EØ�>‰>˜2Ÿ8™8›: t¨R¯X©X«Z¸Ð?Ó@Ð@rE   ÚdYt3ÚdCt3rŠ  r�  r�  c                 ó¾  — t        | «      }|j                  |dd¬«      \  }}}}	|j                  d   |j                  d   cxk(  r&|j                  d   cxk(  r|	j                  d   k(  sJ ‚ J ‚|j                  d   | j                  d   cxk(  r9|j                  d   cxk(  r&|j                  d   cxk(  r|j                  d   k(  sJ ‚ J ‚|j                  |«      }
| |
z  }| |z  }||t	        |
«      z  z  }||	z  }||z  }||z  }||z  }|t	        |	«      z  }|t        |«      z  }|t        |«      z  }|t        |«      z  }|j                  ||||fd¬«      }||fS )Nrt  r;   r™  r   )r,   rÌ   rŽ   rð  ru  ro  r¿   )rµ  r¶  rŠ  r�  r�  r:   rŒ  r�  rŽ  r�  ÚtanhCt3Úd_hoÚ	d_tanhCt3Úd_hiÚd_hcÚd_hfr¥  Úd_At3_hcÚd_At3_hoÚd_At3_hiÚd_At3_hfÚdAt3s                         rC   r“  r“  j  sg  € ô 
˜$Ó	€BØ—X‘X˜c 1¨2�XÓ.�N€BˆˆB�Ø�8‰8�A‰;˜"Ÿ(™( 1™+ÔC¨¯©°!©ÔC¸¿¹À¹ÒCÐCÑCÐCÐCØ�8‰8�A‰;˜$Ÿ*™* Q™-ÔX¨4¯:©:°a©=ÔX¸C¿I¹IÀa¹LÔXÈCÏIÉIÐVWÉLÒXÐXÑXÐXÐXØ�g‰g�c‹l€Gà�'‰>€DØ�r‘	€IàˆIœ˜g›Ñ&Ñ&€Dà�"‰9€DØ�"‰9€Dà�#‰:€DØ�"‰9€DØ”e˜B“iÑ€HØ”h˜r“lÑ"€HØ”h˜r“lÑ"€HØ”h˜r“lÑ"€HØ�>‰>˜8 X¨x¸ÐBÈˆ>ÓL€DØ�ˆ:ÐrE   c                 ón   — t        | «      }|j                  | dd«      } dd|j                  |  «      z   z  S )Nrg  rh  r5   )r,   rj  rk  )r§   rŸ   r:   s      rC   rm  rm  †  s:   € Ü	˜!Ó	€Bð 	�‰��5˜$Ó€AØ�#˜Ÿ™ ˜r›
Ñ"Ñ#Ð#rE   r®   c                 ó   — | d| z
  z  S r   rƒ   ©r®   s    rC   ro  ro  Ž  s   € Ø��a‘‰=ÐrE   c                 ó   — d| dz  z
  S )Nr/   r   rƒ   rÅ  s    rC   ru  ru  ’  s   € Øˆq�!‰t‰8€OrE   Úopsc                 ó>   — dd| j                  t        |z  «      z   z  S )z6Gaussian CDF for distribution with mean 0 and stdev 1.r¼  r5   )rë  Ú	INV_SQRT2©rÇ  r§   s     rC   rø  rø  –  s   € à�#˜Ÿ™¤	¨A¡Ó.Ñ.Ñ/Ð/rE   c                 óR   — t         | j                  j                  d|z  |z  «      z  S )z6Gaussian PDF for distribution with mean 0 and stdev 1.g      à¿)ÚINV_SQRT_2PIr:   rk  rÊ  s     rC   rü  rü  ›  s"   € ä˜#Ÿ&™&Ÿ*™* T¨A¡X°¡\Ó2Ñ2Ð2rE   ÚuÚvc                 ó„   — | j                   |j                   k7  r'd| j                   › d|j                   › �}t        |«      ‚y )Nr  r  )rŽ   rU   )rÍ  rÎ  r  s      rC   r  r     s;   € Ø‡w�w�!—'‘'ÒØ1°!·'±'°¸%ÀÇÁ¸yÐIˆÜ˜‹oÐð rE   r@   )Hrl   ÚmathÚtypingr   r   r   r   r   r   r	   r
   r   r   r   rQ   Útypesr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   Úutilr,   r-   r.   rK   r0   r1   r3   rÙ  Úpirñ  rÉ  rÌ  r7   r‘  r–  ra   ru  rv  r”  r“  rm  ro  ru  rø  rü  r  rƒ   rE   rC   ú<module>rÕ     s  ðÛ Û ÷÷ ÷ ñ ó ÷÷ ÷ ÷ ÷ ÷ ÷ õ ÷> ;Ñ :Ý á	� Ô	)€Ù
�) 7Ô
+€Ø
ˆ$�)‰)�C˜$Ÿ'™'‘MÓ
"€Ø�)�$—)‘)˜C“.Ñ €	Ø�Y�T—Y‘Y˜s T§W¡W™}Ó-Ñ-€÷jñ jðZ)#ðLD ØðD Ø&ðD Ø08ðD Ø=EðD ØPVðD à
ˆ8�Uˆ?ÑóD ðNc*�hð c*¨ð c*¸ð c*Èeó c*ðL*˜8ð *¨ð *°ð *¸#ð *Èó *ò ò$	AðØ
ðØ"ðØ)1ðØ8@ðØGOðà
ˆ8�XÐÑóó8$ð�ð ˜6ó ðˆVð ˜ó ð0�cð 0˜ið 0¨Ió 0ð
3�cð 3˜ið 3¨Ió 3ð
˜xð ¨Hô rE   