Ë
    T^(h÷ò  ã                   ó   — d Z ddlZddlmZ ddlmZmZmZ ddlZddl	Zddlm
Z
 ddlmZmZmZ ddlmZ dd	lmZ dd
lmZmZmZmZmZmZ ddlmZ ddlmZmZmZ ddl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(  e&jR                  e*«      Z+dZ,dZ-dZ.da/d„ Z0d„ Z1dQd„Z2dQd„Z3dQd„Z4d„ Z5 G d„ dejl                  jn                  «      Z8 G d„ dejl                  jn                  «      Z9 G d„ d«      Z:dRd„Z;d „ Z<	 	 	 dSd!„Z= G d"„ d#e
j|                  «      Z? G d$„ d%e
j|                  «      Z@ G d&„ d'e
j|                  «      ZA G d(„ d)e
j|                  «      ZB G d*„ d+e
j|                  «      ZC G d,„ d-e
j|                  «      ZD G d.„ d/e
j|                  «      ZE G d0„ d1e
j|                  «      ZF G d2„ d3e
j|                  «      ZG G d4„ d5e
j|                  «      ZH G d6„ d7e
j|                  «      ZI G d8„ d9e«      ZJd:ZKd;ZL e"d<eK«       G d=„ d>eJ«      «       ZM e"d?eK«       G d@„ dAeJ«      «       ZN G dB„ dCe
j|                  «      ZO e"dDeK«       G dE„ dFeJ«      «       ZP e"dGeK«       G dH„ dIeJ«      «       ZQ e"dJeK«       G dK„ dLeJ«      «       ZR e"dMeK«       G dN„ dOeJ«      «       ZSg dP¢ZTy)TzPyTorch MRA model.é    N)ÚPath)ÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELoss)Úloadé   )ÚACT2FN)Ú"BaseModelOutputWithCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úapply_chunking_to_forwardÚ find_pruneable_heads_and_indicesÚprune_linear_layer)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚis_ninja_availableÚis_torch_cuda_availableÚloggingé   )Ú	MraConfigzuw-madison/mra-base-512-4r   ÚAutoTokenizerc                  óÂ   ‡— t        t        «      j                  «       j                  j                  j                  dz  dz  Šˆfd„}  | g d¢«      }t	        d|d¬«      ay )NÚkernelsÚmrac                 ó4   •— | D �cg c]  }‰|z  ‘Œ	 c}S c c}w ©N© )ÚfilesÚfileÚ
src_folders     €úb/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/mra/modeling_mra.pyÚappend_rootz&load_cuda_kernels.<locals>.append_root?   s   ø€ Ø.3Ö4 d�
˜TÓ!Ò4Ð4ùÒ4s   †)zcuda_kernel.cuzcuda_launch.cuztorch_extension.cppÚcuda_kernelT)Úverbose)r   Ú__file__ÚresolveÚparentr   Úmra_cuda_kernel)r+   Ú	src_filesr)   s     @r*   Úload_cuda_kernelsr3   ;   sQ   ø€ ä”h“×'Ñ'Ó)×0Ñ0×7Ñ7×>Ñ>ÀÑJÈUÑR€Jô5ñ ÒWÓX€Iä˜=¨)¸TÔB�Oó    c                 óN  — t        | j                  «       «      dk7  rt        d«      ‚t        |j                  «       «      dk7  rt        d«      ‚| j                  d«      dk7  rt        d«      ‚| j                  d«      dk7  rt        d«      ‚| j                  d	¬
«      j                  j                  dd	«      }|j                  «       }|j                  «       }|j                  «       }t        j                  ||||«      \  }}|j                  dd	«      dd…dd…ddd…f   }||fS )z8
    Computes maximum values for softmax stability.
    é   z.sparse_qk_prod must be a 4-dimensional tensor.é   ú'indices must be a 2-dimensional tensor.é    z>The size of the second dimension of sparse_qk_prod must be 32.r   z=The size of the third dimension of sparse_qk_prod must be 32.éþÿÿÿ©ÚdiméÿÿÿÿN)
ÚlenÚsizeÚ
ValueErrorÚmaxÚvaluesÚ	transposeÚ
contiguousÚintr1   Ú	index_max)Úsparse_qk_prodÚindicesÚquery_num_blockÚkey_num_blockÚ
index_valsÚmax_valsÚmax_vals_scatters          r*   Ú
sparse_maxrN   G   s  € ô ˆ>×ÑÓ Ó! QÒ&ÜÐIÓJÐJä
ˆ7�<‰<‹>Ó˜aÒÜÐBÓCÐCà×Ñ˜1Ó Ò#ÜÐYÓZÐZà×Ñ˜1Ó Ò#ÜÐXÓYÐYà×#Ñ#¨Ð#Ó+×2Ñ2×<Ñ<¸RÀÓD€JØ×&Ñ&Ó(€Jà�k‰k‹m€GØ× Ñ Ó"€Gä!0×!:Ñ!:¸:ÀwÐP_ÐanÓ!oÑ€HÐØ'×1Ñ1°"°bÓ9º!ºQÀÂa¸-ÑHÐàÐ%Ð%Ð%r4   c                 ó  — t        | j                  «       «      dk7  rt        d«      ‚t        |j                  «       «      dk7  rt        d«      ‚| j                  d   |j                  d   k7  rt        d«      ‚| j                  \  }}||z  }t	        j
                  |j                  d«      t        j                  |j                  ¬«      }| j                  |||«      } | |dd…df   ||z  j                  «       dd…f   } | S )zN
    Converts attention mask to a sparse mask for high resolution logits.
    r7   z$mask must be a 2-dimensional tensor.r8   r   zBmask and indices must have the same size in the zero-th dimension.©ÚdtypeÚdeviceN)	r>   r?   r@   ÚshapeÚtorchÚarangeÚlongrR   Úreshape)ÚmaskrH   Ú
block_sizeÚ
batch_sizeÚseq_lenÚ	num_blockÚ	batch_idxs          r*   Úsparse_maskr^   c   sà   € ô ˆ4�9‰9‹;Ó˜1ÒÜÐ?Ó@Ð@ä
ˆ7�<‰<‹>Ó˜aÒÜÐBÓCÐCà‡z�z�!�}˜Ÿ™ aÑ(Ò(ÜÐ]Ó^Ð^àŸ*™*Ñ€J�Ø˜:Ñ%€Iä—‘˜WŸ\™\¨!›_´E·J±JÀwÇ~Á~ÔV€IØ�<‰<˜
 I¨zÓ:€DØ�	š!˜T˜'Ñ" W¨yÑ%8×$>Ñ$>Ó$@Â!ÐCÑD€Dà€Kr4   c                 ój  — | j                  «       \  }}}|j                  «       \  }}}||z  dk7  rt        d«      ‚||z  dk7  rt        d«      ‚| j                  |||z  ||«      j                  dd«      } |j                  |||z  ||«      j                  dd«      }t	        | j                  «       «      dk7  rt        d«      ‚t	        |j                  «       «      dk7  rt        d«      ‚t	        |j                  «       «      d	k7  rt        d
«      ‚| j                  d«      dk7  rt        d«      ‚|j                  d«      dk7  rt        d«      ‚| j                  «       } |j                  «       }|j                  «       }|j                  «       }t        j                  | ||j                  «       «      S )z7
    Performs Sampled Dense Matrix Multiplication.
    r   zTquery_size (size of first dimension of dense_query) must be divisible by block_size.úPkey_size (size of first dimension of dense_key) must be divisible by block_size.r=   r:   r6   z+dense_query must be a 4-dimensional tensor.ú)dense_key must be a 4-dimensional tensor.r7   r8   r   r9   z.The third dimension of dense_query must be 32.z,The third dimension of dense_key must be 32.)	r?   r@   rW   rC   r>   rD   rE   r1   Úmm_to_sparse)	Údense_queryÚ	dense_keyrH   rY   rZ   Ú
query_sizer<   Ú_Úkey_sizes	            r*   rb   rb   z   s¡  € ð #.×"2Ñ"2Ó"4Ñ€J�
˜CØ —~‘~Ó'Ñ€A€x�à�JÑ !Ò#ÜÐoÓpÐpà�*Ñ Ò!ÜÐkÓlÐlà×%Ñ% j°*À
Ñ2JÈJÐX[Ó\×fÑfÐgiÐkmÓn€KØ×!Ñ! *¨h¸*Ñ.DÀjÐRUÓV×`Ñ`ÐacÐegÓh€Iä
ˆ;×ÑÓÓ !Ò#ÜÐFÓGÐGä
ˆ9�>‰>ÓÓ Ò!ÜÐDÓEÐEä
ˆ7�<‰<‹>Ó˜aÒÜÐBÓCÐCà×Ñ˜Ó˜bÒ ÜÐIÓJÐJà‡~�~�aÓ˜BÒÜÐGÓHÐHà×(Ñ(Ó*€KØ×$Ñ$Ó&€Ià�k‰k‹m€GØ× Ñ Ó"€Gä×'Ñ'¨°YÀÇÁÃÓNÐNr4   c                 óB  — |j                  «       \  }}}||z  dk7  rt        d«      ‚| j                  d«      |k7  rt        d«      ‚| j                  d«      |k7  rt        d«      ‚|j                  |||z  ||«      j                  dd«      }t	        | j                  «       «      d	k7  rt        d
«      ‚t	        |j                  «       «      d	k7  rt        d«      ‚t	        |j                  «       «      dk7  rt        d«      ‚|j                  d«      dk7  rt        d«      ‚| j                  «       } |j                  «       }|j                  «       }|j                  «       }t        j                  | |||«      }|j                  dd«      j                  |||z  |«      }|S )zP
    Performs matrix multiplication of a sparse matrix with a dense matrix.
    r   r`   r7   zQThe size of the second dimension of sparse_query must be equal to the block_size.r   zPThe size of the third dimension of sparse_query must be equal to the block_size.r=   r:   r6   ú,sparse_query must be a 4-dimensional tensor.ra   r8   r9   z8The size of the third dimension of dense_key must be 32.)	r?   r@   rW   rC   r>   rD   rE   r1   Úsparse_dense_mm)	Úsparse_queryrH   rd   rI   rY   rZ   rg   r<   Údense_qk_prods	            r*   rj   rj   ¢   s‘  € ð !*§¡Ó 0Ñ€J�˜#à�*Ñ Ò!ÜÐkÓlÐlà×Ñ˜Ó˜zÒ)ÜÐlÓmÐmà×Ñ˜Ó˜zÒ)ÜÐkÓlÐlà×!Ñ! *¨h¸*Ñ.DÀjÐRUÓV×`Ñ`ÐacÐegÓh€Iä
ˆ<×ÑÓÓ 1Ò$ÜÐGÓHÐHä
ˆ9�>‰>ÓÓ Ò!ÜÐDÓEÐEä
ˆ7�<‰<‹>Ó˜aÒÜÐBÓCÐCà‡~�~�aÓ˜BÒÜÐSÓTÐTà×*Ñ*Ó,€Là�k‰k‹m€GØ× Ñ Ó"€GØ×$Ñ$Ó&€Iä#×3Ñ3°LÀ'È9ÐVeÓf€MØ!×+Ñ+¨B°Ó3×;Ñ;¸JÈÐZdÑHdÐfiÓj€MØÐr4   c                 ó`   — | |z  |z  t        j                  | |d¬«      z   j                  «       S )NÚfloor©Úrounding_mode)rT   ÚdivrV   )rH   Údim_1_blockÚdim_2_blocks      r*   Útranspose_indicesrt   Ê   s.   € Ø�{Ñ" kÑ1´E·I±I¸gÀ{ÐbiÔ4jÑj×pÑpÓrÐrr4   c                   ó>   — e Zd Zed„ «       Zed„ «       Zedd„«       Zy)ÚMraSampledDenseMatMulc                 óV   — t        ||||«      }| j                  |||«       || _        |S r%   )rb   Úsave_for_backwardrY   )Úctxrc   rd   rH   rY   rG   s         r*   ÚforwardzMraSampledDenseMatMul.forwardÏ   s1   € ä% k°9¸gÀzÓRˆØ×Ñ˜k¨9°gÔ>Ø#ˆŒØÐr4   c                 ó  — | j                   \  }}}| j                  }|j                  d«      |z  }|j                  d«      |z  }t        |||«      }t	        |j                  dd«      |||«      }	t	        ||||«      }
|
|	d d fS ©Nr   r=   r:   )Úsaved_tensorsrY   r?   rt   rj   rC   )ry   Úgradrc   rd   rH   rY   rI   rJ   Ú	indices_TÚgrad_keyÚ
grad_querys              r*   ÚbackwardzMraSampledDenseMatMul.backwardÖ   sŽ   € à*-×*;Ñ*;Ñ'ˆ�Y Ø—^‘^ˆ
Ø%×*Ñ*¨1Ó-°Ñ;ˆØ!Ÿ™ qÓ)¨ZÑ7ˆÜ% g¨ÀÓNˆ	Ü" 4§>¡>°"°bÓ#9¸9ÀkÐS`ÓaˆÜ$ T¨7°I¸ÓOˆ
Ø˜8 T¨4Ð/Ð/r4   c                 ó2   — t         j                  | |||«      S r%   )rv   Úapply)rc   rd   rH   rY   s       r*   Úoperator_callz#MraSampledDenseMatMul.operator_callá   s   € ä$×*Ñ*¨;¸	À7ÈJÓWÐWr4   N©r9   ©Ú__name__Ú
__module__Ú__qualname__Ústaticmethodrz   r‚   r…   r&   r4   r*   rv   rv   Î   s>   „ Øñó ðð ñ0ó ð0ð òXó ñXr4   rv   c                   ó<   — e Zd Zed„ «       Zed„ «       Zed„ «       Zy)ÚMraSparseDenseMatMulc                 óV   — t        ||||«      }| j                  |||«       || _        |S r%   )rj   rx   rI   )ry   rk   rH   rd   rI   rG   s         r*   rz   zMraSparseDenseMatMul.forwardç   s2   € ä(¨°wÀ	È?Ó[ˆØ×Ñ˜l¨G°YÔ?Ø-ˆÔØÐr4   c                 óü   — | j                   \  }}}| j                  }|j                  d«      |j                  d«      z  }t        |||«      }t	        |j                  dd«      |||«      }t        |||«      }	|	d |d fS r|   )r}   rI   r?   rt   rj   rC   rb   )
ry   r~   rk   rH   rd   rI   rJ   r   r€   r�   s
             r*   r‚   zMraSparseDenseMatMul.backwardî   s†   € à+.×+<Ñ+<Ñ(ˆ�g˜yØ×-Ñ-ˆØ!Ÿ™ qÓ)¨\×->Ñ->¸rÓ-BÑBˆÜ% g¨ÀÓNˆ	Ü" <×#9Ñ#9¸"¸bÓ#AÀ9ÈdÐTaÓbˆÜ! $¨	°7Ó;ˆ
Ø˜4 ¨4Ð/Ð/r4   c                 ó2   — t         j                  | |||«      S r%   )r�   r„   )rk   rH   rd   rI   s       r*   r…   z"MraSparseDenseMatMul.operator_callø   s   € ä#×)Ñ)¨,¸ÀÈOÓ\Ð\r4   Nr‡   r&   r4   r*   r�   r�   æ   s>   „ Øñó ðð ñ0ó ð0ð ñ]ó ñ]r4   r�   c                   ó   — e Zd Zed„ «       Zy)ÚMraReduceSumc                 óB  — | j                  «       \  }}}}t        | j                  «       «      dk7  rt        d«      ‚t        |j                  «       «      dk7  rt        d«      ‚| j                  «       \  }}}}|j                  «       \  }}| j                  d¬«      j	                  ||z  |«      } t        j                  |j                  d«      t
        j                  |j                  ¬«      }t        j                  ||d¬	«      j                  «       |d d …d f   |z  z   j	                  ||z  «      }	t        j                  ||z  |f| j                  | j                  ¬«      }
|
j                  d|	| «      j	                  |||«      }|j	                  |||z  «      }|S )
Nr6   ri   r7   r8   r;   r   rP   rn   ro   )r?   r>   r@   ÚsumrW   rT   rU   rV   rR   rq   ÚzerosrQ   Ú	index_add)rk   rH   rI   rJ   rZ   r\   rY   rf   r]   Úglobal_idxesÚtempÚoutputs               r*   r…   zMraReduceSum.operator_callþ   sy  € à/;×/@Ñ/@Ó/BÑ,ˆ
�I˜z¨1äˆ|× Ñ Ó"Ó# qÒ(ÜÐKÓLÐLäˆw�|‰|‹~Ó !Ò#ÜÐFÓGÐGà*×/Ñ/Ó1Ñˆˆ1ˆj˜!Ø '§¡£Ñˆ
�Ià#×'Ñ'¨AÐ'Ó.×6Ñ6°zÀIÑ7MÈzÓZˆä—L‘L §¡¨a£¼¿
¹
È7Ï>É>ÔZˆ	ä�I‰I�g˜}¸GÔD×IÑIÓKÈiÒXYÐ[_ÐX_ÑN`ÐcrÑNrÑrß
‰'�*˜yÑ(Ó
)ð 	ô �{‰{Ø˜/Ñ)¨:Ð6¸l×>PÑ>PÐYe×YlÑYlô
ˆð —‘  <°Ó>×FÑFÀzÐSbÐdnÓoˆà—‘ 
¨O¸jÑ,HÓIˆØˆr4   N)rˆ   r‰   rŠ   r‹   r…   r&   r4   r*   r’   r’   ý   s   „ Øñó ñr4   r’   c                 ó&  — | j                  «       \  }}}||z  }d}	|�Â|j                  |||«      j                  d¬«      }
| j                  ||||«      j                  d¬«      |
dd…dd…df   dz   z  }|j                  ||||«      j                  d¬«      |
dd…dd…df   dz   z  }|�×|j                  ||||«      j                  d¬«      |
dd…dd…df   dz   z  }	n¢|t        j                  ||t        j
                  | j                  ¬«      z  }
| j                  ||||«      j                  d¬«      }|j                  ||||«      j                  d¬«      }|�$|j                  ||||«      j                  d¬«      }	t        j                  ||j                  dd«      «      t        j                  |«      z  }|j                  dd¬«      j                  }|�0|d	|
dd…ddd…f   |
dd…dd…df   z  d
k  j                  «       z  z
  }||
||	fS )z/
    Compute low resolution approximation.
    Nr=   r;   r:   ç�íµ ÷Æ°>rP   T)r<   Úkeepdimsç     ˆÃ@g      à?)r?   rW   r”   rT   ÚonesÚfloatrR   ÚmeanÚmatmulrC   ÚmathÚsqrtrA   rB   )ÚqueryÚkeyrY   rX   ÚvaluerZ   r[   Úhead_dimÚnum_block_per_rowÚ	value_hatÚtoken_countÚ	query_hatÚkey_hatÚlow_resolution_logitÚlow_resolution_logit_row_maxs                  r*   Úget_low_resolution_logitr¯     sN  € ð %*§J¡J£LÑ!€J�˜à :Ñ-Ðà€IØÐØ—l‘l :Ð/@À*ÓM×QÑQÐVXÐQÓYˆØ—M‘M *Ð.?ÀÈXÓV×ZÑZÐ_aÐZÓbØšš1˜d˜
Ñ# dÑ*ñ
ˆ	ð —+‘+˜jÐ*;¸ZÈÓR×VÑVÐ[]ÐVÓ^Øšš1˜d˜
Ñ# dÑ*ñ
ˆð ÐØŸ™ jÐ2CÀZÐQYÓZ×^Ñ^ÐceÐ^ÓfØšAšq $˜JÑ'¨$Ñ.ñ‰Ið !¤5§:¡:¨jÐ:KÔSX×S^ÑS^Ðgl×gsÑgsÔ#tÑtˆØ—M‘M *Ð.?ÀÈXÓV×[Ñ[Ð`bÐ[Ócˆ	Ø—+‘+˜jÐ*;¸ZÈÓR×WÑWÐ\^ÐWÓ_ˆØÐØŸ™ jÐ2CÀZÐQYÓZ×_Ñ_ÐdfÐ_ÓgˆIä Ÿ<™<¨	°7×3DÑ3DÀRÈÓ3LÓMÔPT×PYÑPYÐZbÓPcÑcÐà#7×#;Ñ#;ÀÈTÐ#;Ó#R×#YÑ#YÐ àÐà  3¨;²q¸$Â°zÑ+BÀ[ÒQRÒTUÐW[ÐQ[ÑE\Ñ+\Ð`cÑ*c×)jÑ)jÓ)lÑ#lÑlð 	ð   Ð.JÈIÐUÐUr4   c                 ó¨  — | j                   \  }}}|dkD  rf|dz  }t        j                  ||| j                  ¬«      }	t        j                  t        j
                  |	| ¬«      |¬«      }
| |
ddd…dd…f   dz  z   } |dkD  r:| dd…d|…dd…f   dz   | dd…d|…dd…f<   | dd…dd…d|…f   dz   | dd…dd…d|…f<   t        j                  | j                  |d«      |ddd	¬
«      }|j                  }|dk(  rE|j                  j                  d¬«      j                  }| |dd…ddf   k\  j                  «       }||fS |dk(  rd}||fS t        |› d�«      ‚)zZ
    Compute the indices of the subset of components to be used in the approximation.
    r   r7   ©rR   )ÚdiagonalNg     ˆ³@r=   TF)r<   ÚlargestÚsortedÚfullr;   Úsparsez# is not a valid approx_model value.)rS   rT   rž   rR   ÚtrilÚtriuÚtopkrW   rH   rB   ÚminrŸ   r@   )r­   Ú
num_blocksÚapprox_modeÚinitial_prior_first_n_blocksÚinitial_prior_diagonal_n_blocksrZ   Útotal_blocks_per_rowrf   ÚoffsetÚ	temp_maskÚdiagonal_maskÚ
top_k_valsrH   Ú	thresholdÚhigh_resolution_masks                  r*   Úget_block_idxesrÆ   B  s¬  € ð +?×*DÑ*DÑ'€JÐ$ aà&¨Ò*Ø0°AÑ5ˆÜ—J‘JÐ3Ð5IÐRf×RmÑRmÔnˆ	ÜŸ
™
¤5§:¡:¨iÀ6À'Ô#JÐU[Ô\ˆØ3°mÀDÊ!ÊQÀJÑ6OÐRUÑ6UÑUÐà# aÒ'à ¢Ð$AÐ%AÐ$AÂ1Ð!DÑEÈÑKð 	šQÐ =Ð!=Ð =ºqÐ@ÑAð !¢¢AÐ'DÐ(DÐ'DÐ!DÑEÈÑKð 	šQ¢Ð#@Ð$@Ð#@Ð@ÑAô —‘Ø×$Ñ$ Z°Ó4°jÀbÐRVÐ_dô€Jð × Ñ €Gà�fÒØ×%Ñ%×)Ñ)¨bÐ)Ó1×8Ñ8ˆ	Ø 4¸	Â!ÀTÈ4À-Ñ8PÑ P×WÑWÓYÐð Ð(Ð(Ð(ð 
˜Ò	 Ø#Ðð Ð(Ð(Ð(ô ˜K˜=Ð(KÐLÓMÐMr4   c	                 óÈ  — t         €#t        j                  | «      j                  «       S | j	                  «       \  }	}
}}|	|
z  }||z  dk7  rt        d«      ‚||z  }| j                  |||«      } |j                  |||«      }|j                  |||«      }|�-| |dd…dd…df   z  } ||dd…dd…df   z  }||dd…dd…df   z  }|dk(  rt        | ||||«      \  }}}}nA|dk(  r1t        j                  «       5  t        | |||«      \  }}}}ddd«       nt        d«      ‚t        j                  «       5  z
  }t        |||||«      \  }}ddd«       t        j                  | ||¬«      t        j                  |«      z  }t        ||||«      \  }}||z
  }|�"|dd	t!        ||«      dd…dd…dd…df   z
  z  z
  }t        j"                  |«      }t$        j                  ||||«      }t&        j                  ||||«      }|dk(  �ryt        j"                  z
  dz  z
  «      dd…ddd…f   z  }t        j(                  |«      dd…dd…ddd…f   j+                  d	d	|d	«      j                  |||«      }|j-                  d
¬«      dd…dd…df   j+                  d	d	|«      j                  ||«      }|j+                  d	d	|«      j                  ||«      |z
  } |�| |z  } t        j"                  | | dk  j/                  «       z  «      }!||!dd…dd…df   z  }||!z  }t        j"                  |  | dkD  j/                  «       z  «      }"||"dd…dd…df   z  }||"z  }||z   |dd…dd…df   |dd…dd…df   z   dz   z  }#n#|dk(  r||dd…dd…df   dz   z  }#nt        d«      ‚|�|#|dd…dd…df   z  }#|#j                  |	|
||«      }#|#S # 1 sw Y   �ŒµxY w# 1 sw Y   �ŒŽxY w)z0
    Use Mra to approximate self-attention.
    Nr   z4sequence length must be divisible by the block_size.rµ   r¶   z&approx_mode must be "full" or "sparse")rY   r�   r   r=   r;   r›   z-config.approx_mode must be "full" or "sparse")r1   rT   Ú
zeros_likeÚrequires_grad_r?   r@   rW   r¯   Úno_gradÚ	ExceptionrÆ   rv   r…   r¢   r£   rN   r^   Úexpr�   r’   r¡   Úrepeatr”   rŸ   )$r¤   r¥   r¦   rX   r»   r¼   rY   r½   r¾   rZ   Únum_headr[   r§   Ú
meta_batchr¨   r­   rª   r®   r©   rf   Úlow_resolution_logit_normalizedrH   rÅ   Úhigh_resolution_logitrL   rM   Úhigh_resolution_attnÚhigh_resolution_attn_outÚhigh_resolution_normalizerÚlow_resolution_attnÚlow_resolution_attn_outÚlow_resolution_normalizerÚlog_correctionÚlow_resolution_corrÚhigh_resolution_corrÚcontext_layers$                                       r*   Úmra2_attentionrÜ   h  s¹  € ô ÐÜ×Ñ Ó&×5Ñ5Ó7Ð7à.3¯j©j«lÑ+€J�˜' 8Ø˜hÑ&€Jà�Ñ˜qÒ ÜÐOÓPÐPà :Ñ-Ðà�M‰M˜* g¨xÓ8€EØ
�+‰+�j '¨8Ó
4€CØ�M‰M˜* g¨xÓ8€EàÐØ˜šQ¢ 4˜ZÑ(Ñ(ˆØ�DššA˜t˜Ñ$Ñ$ˆØ˜šQ¢ 4˜ZÑ(Ñ(ˆà�fÒÜUmØ�3˜
 D¨%óV
ÑRÐ˜kÐ+GÉð 
˜Ò	 Ü�]‰]‹_ñ 	ÜQiØ�s˜J¨óRÑNÐ  +Ð/KÈQ÷	ð 	ô
 Ð@ÓAÐAä	�‰‹ñ 
Ø*>ÐA]Ñ*]Ð'Ü(7Ø+ØØØ(Ø+ó)
Ñ%ˆÐ%÷
ô 2×?Ñ?Øˆs�G¨
ð @ó ä�	‰	�(ÓñÐô ",Ð,AÀ7ÐL]Ð_pÓ!qÑ€HÐØ1Ð4DÑDÐØÐØ 5¸¸qÄ;ÈtÐU\ÓC]Ò^_ÒabÒdeÐgkÐ^kÑClÑ?lÑ8mÑ mÐÜ Ÿ9™9Ð%:Ó;ÐÜ3×AÑAØ˜g uÐ.?ó Ðô ".×!;Ñ!;Ø˜gÐ'8Ð:Kó"Ðð �fÓä�I‰IÐ*Ð-IÑIÈCÐRfÑLfÑfÓgØš!˜T¢1˜*Ñ%ñ&ð 	ô �L‰LÐ,¨iÓ8ººA¸tÂQ¸ÑGß‰V�A�q˜* aÓ(ß‰W�Z ¨(Ó3ð 	 ð  ×#Ñ#¨Ð#Ó+ªAªq°$¨JÑ7×>Ñ>¸qÀ!ÀZÓP×XÑXÐYcÐelÓmð 	"ð 6×<Ñ<¸QÀÀ:ÓN×VÑVÐWaÐcjÓkÐnvÑvˆØÐØ+¨dÑ2ˆNä#Ÿi™i¨¸.ÈAÑ:M×9TÑ9TÓ9VÑ(VÓWÐØ"9Ð<OÒPQÒSTÐVZÐPZÑ<[Ñ"[ÐØ$=Ð@SÑ$SÐ!ä$Ÿy™y¨.¨¸NÈQÑ<N×;UÑ;UÓ;WÑ)WÓXÐØ#;Ð>RÒSTÒVWÐY]ÐS]Ñ>^Ñ#^Ð Ø%?ÐBVÑ%VÐ"à1Ð4KÑKØ&¢qª!¨T zÑ2Ð5NÊqÒRSÐUYÈzÑ5ZÑZÐ]aÑañ
‰ð 
˜Ò	 Ø0Ð4NÊqÒRSÐUYÈzÑ4ZÐ]aÑ4aÑb‰äÐGÓHÐHàÐØ%¨ªQ²°4¨ZÑ(8Ñ8ˆà!×)Ñ)¨*°hÀÈÓR€MàÐ÷S	ñ 	ú÷
ñ 
ús   Ã7O
Ä3OÏ
OÏO!c                   ó*   ‡ — e Zd ZdZˆ fd„Zdd„Zˆ xZS )ÚMraEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                 óp  •— t         ‰| �  «        t        j                  |j                  |j
                  |j                  ¬«      | _        t        j                  |j                  dz   |j
                  «      | _	        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        | j#                  dt%        j&                  |j                  «      j)                  d«      dz   «       t+        |dd«      | _        | j#                  dt%        j.                  | j0                  j3                  «       t$        j4                  | j0                  j6                  ¬	«      d
¬«       y )N)Úpadding_idxr7   ©ÚepsÚposition_ids)r   r=   Úposition_embedding_typeÚabsoluteÚtoken_type_idsrP   F)Ú
persistent)ÚsuperÚ__init__r   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚmax_position_embeddingsÚposition_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsÚ	LayerNormÚlayer_norm_epsÚDropoutÚhidden_dropout_probÚdropoutÚregister_bufferrT   rU   ÚexpandÚgetattrrä   r•   rã   r?   rV   rR   ©ÚselfÚconfigÚ	__class__s     €r*   ré   zMraEmbeddings.__init__Þ  s?  ø€ Ü‰ÑÔÜ!Ÿ|™|¨F×,=Ñ,=¸v×?QÑ?QÐ_e×_rÑ_rÔsˆÔÜ#%§<¡<°×0NÑ0NÐQRÑ0RÐTZ×TfÑTfÓ#gˆÔ Ü%'§\¡\°&×2HÑ2HÈ&×J\ÑJ\Ó%]ˆÔ"ô Ÿ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒð 	×Ñ˜^¬U¯\©\¸&×:XÑ:XÓ-Y×-`Ñ-`ÐahÓ-iÐlmÑ-mÔnÜ'.¨vÐ7PÐR\Ó']ˆÔ$Ø×ÑØÜ�K‰K˜×)Ñ)×.Ñ.Ó0¼¿
¹
È4×K\ÑK\×KcÑKcÔdØð 	õ 	
r4   c                 óT  — |�|j                  «       }n|j                  «       d d }|d   }|€| j                  d d …d |…f   }|€st        | d«      r-| j                  d d …d |…f   }|j	                  |d   |«      }|}n:t        j                  |t
        j                  | j                  j                  ¬«      }|€| j                  |«      }| j                  |«      }	||	z   }
| j                  dk(  r| j                  |«      }|
|z  }
| j                  |
«      }
| j                  |
«      }
|
S )Nr=   r   ræ   r   rP   rå   )r?   rã   Úhasattrræ   rù   rT   r•   rV   rR   rî   rò   rä   rð   ró   r÷   )rü   Ú	input_idsræ   rã   Úinputs_embedsÚinput_shapeÚ
seq_lengthÚbuffered_token_type_idsÚ buffered_token_type_ids_expandedrò   Ú
embeddingsrð   s               r*   rz   zMraEmbeddings.forwardò  s=  € ØÐ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKà  ‘^ˆ
àÐØ×,Ñ,ªQ°°°¨^Ñ<ˆLð
 Ð!Ü�tÐ-Ô.Ø*.×*=Ñ*=ºaÀÀ*À¸nÑ*MÐ'Ø3J×3QÑ3QÐR]Ð^_ÑR`ÐblÓ3mÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSW×SdÑSd×SkÑSkÔ!l�àÐ Ø ×0Ñ0°Ó;ˆMØ $× :Ñ :¸>Ó JÐà"Ð%:Ñ:ˆ
Ø×'Ñ'¨:Ò5Ø"&×":Ñ":¸<Ó"HÐØÐ-Ñ-ˆJØ—^‘^ JÓ/ˆ
Ø—\‘\ *Ó-ˆ
ØÐr4   )NNNN©rˆ   r‰   rŠ   Ú__doc__ré   rz   Ú__classcell__©rþ   s   @r*   rÞ   rÞ   Û  s   ø„ ÙQô
÷( r4   rÞ   c                   ó.   ‡ — e Zd Zdˆ fd„	Zd„ Zdd„Zˆ xZS )ÚMraSelfAttentionc                 ó¶  •— t         ‰| �  «        |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚t        d u}t        «       rt        «       r|s	 t        «        |j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t!        j"                  |j                  | j                  «      | _        t!        j"                  |j                  | j                  «      | _        t!        j"                  |j                  | j                  «      | _        t!        j*                  |j,                  «      | _        |�|n|j0                  | _        |j2                  dz  |j4                  z  | _        t9        | j6                  t        |j2                  dz  dz  «      «      | _        |j:                  | _        |j<                  | _        |j>                  | _        y # t        $ r#}t        j                  d|› �«       Y d }~�ŒÌd }~ww xY w)	Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)zGCould not load the custom kernel for multi-scale deformable attention: r9   r7   ) rè   ré   rì   Únum_attention_headsr   r@   r1   r   r   r3   rË   ÚloggerÚwarningrE   Úattention_head_sizeÚall_head_sizer   ÚLinearr¤   r¥   r¦   rõ   Úattention_probs_dropout_probr÷   rä   rï   Úblock_per_rowr\   rº   r¼   r½   r¾   )rü   rý   rä   Úkernel_loadedÚerþ   s        €r*   ré   zMraSelfAttention.__init__  sû  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ô
 (¨tÐ3ˆÜ"Ô$Ô);Ô)=ÁmðnÜ!Ô#ð $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
Ü—9‘9˜V×/Ñ/°×1CÑ1CÓDˆŒÜ—Y‘Y˜v×1Ñ1°4×3EÑ3EÓFˆŒ
ä—z‘z &×"EÑ"EÓFˆŒà'>Ð'JÑ#ÐPV×PnÑPnð 	Ô$ð !×8Ñ8¸BÑ>À&×BVÑBVÑVˆŒÜ˜TŸ^™^¬S°&×2PÑ2PÐTVÑ2VÐ[\Ñ1\Ó-]Ó^ˆŒà!×-Ñ-ˆÔØ,2×,OÑ,OˆÔ)Ø/5×/UÑ/UˆÕ,øô+ ò nÜ—‘Ð!hÐijÐhkÐl×mÒmûðnús   Á=
H, È,	IÈ5IÉIc                 ó    — |j                  «       d d | j                  | j                  fz   } |j                  |Ž }|j	                  dddd«      S )Nr=   r   r7   r   r   )r?   r  r  ÚviewÚpermute)rü   ÚlayerÚnew_layer_shapes      r*   Útranspose_for_scoresz%MraSelfAttention.transpose_for_scores9  sO   € ØŸ*™*›, s¨Ð+¨t×/GÑ/GÈ×IaÑIaÐ.bÑbˆØ�—
‘
˜OÐ,ˆØ�}‰}˜Q  1 aÓ(Ð(r4   c           
      óÜ  — | j                  |«      }| j                  | j                  |«      «      }| j                  | j                  |«      «      }| j                  |«      }|j	                  «       \  }}}	}
d|dz  z   }|j                  «       j                  d|d«      j                  ||z  |	«      j                  «       }d}|
|k  r±|||	||
z
  f}t        j                  |t        j                  ||j                  ¬«      gd¬«      }t        j                  |t        j                  ||j                  ¬«      gd¬«      }t        j                  |t        j                  ||j                  ¬«      gd¬«      }t        |j                  «       |j                  «       |j                  «       |j                  «       | j                  | j                   | j"                  | j$                  ¬«      }|
|k  r|d d …d d …d d …d |
…f   }|j                  |||	|
«      }|j'                  d	d
dd«      j)                  «       }|j	                  «       d d | j*                  fz   } |j,                  |Ž }|f}|S )Nç      ð?r�   r   r9   r±   r=   r;   )r¼   r½   r¾   r   r7   r   r:   )r¤   r   r¥   r¦   r?   ÚsqueezerÍ   rW   rE   rT   Úcatr•   rR   rÜ   rŸ   r\   r¼   r½   r¾   r  rD   r  r  )rü   Úhidden_statesÚattention_maskÚmixed_query_layerÚ	key_layerÚvalue_layerÚquery_layerrZ   Ú	num_headsr[   r§   Úgpu_warp_sizeÚpad_sizerÛ   Únew_context_layer_shapeÚoutputss                   r*   rz   zMraSelfAttention.forward>  sD  € Ø ŸJ™J }Ó5Ðà×-Ñ-¨d¯h©h°}Ó.EÓFˆ	Ø×/Ñ/°·
±
¸=Ó0IÓJˆØ×/Ñ/Ð0AÓBˆà3>×3CÑ3CÓ3EÑ0ˆ
�I˜w¨ð ˜~°Ñ7Ñ7ˆà×"Ñ"Ó$×+Ñ+¨A¨y¸!Ó<×DÑDÀZÐR[ÑE[Ð]dÓe×iÑiÓkð 	ð ˆà�mÒ#Ø! 9¨g°}ÀxÑ7OÐOˆHäŸ)™) [´%·+±+¸hÈ{×OaÑOaÔ2bÐ$cÐikÔlˆKÜŸ	™	 9¬e¯k©k¸(È9×K[ÑK[Ô.\Ð"]ÐceÔfˆIÜŸ)™) [´%·+±+¸hÈ{×OaÑOaÔ2bÐ$cÐikÔlˆKä&Ø×ÑÓØ�O‰OÓØ×ÑÓØ× Ñ Ó"Ø�N‰NØ×(Ñ(Ø)-×)JÑ)JØ,0×,PÑ,Pô	
ˆð �mÒ#Ø)ª!ªQ²°9°H°9Ð*<Ñ=ˆMà%×-Ñ-¨j¸)ÀWÈhÓWˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ*˜×*Ñ*Ð,CÐDˆà Ð"ˆàˆr4   r%   )rˆ   r‰   rŠ   ré   r   rz   r
  r  s   @r*   r  r    s   ø„ õ!VòF)÷
0r4   r  c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )ÚMraSelfOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t        j                  |j                  «      | _
        y ©Nrá   )rè   ré   r   r  rì   Údenseró   rô   rõ   rö   r÷   rû   s     €r*   ré   zMraSelfOutput.__init__s  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r4   r%  Úinput_tensorÚreturnc                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r%   ©r4  r÷   ró   ©rü   r%  r5  s      r*   rz   zMraSelfOutput.forwardy  ó7   € ØŸ
™
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ }°|Ñ'CÓDˆØÐr4   ©rˆ   r‰   rŠ   ré   rT   ÚTensorrz   r
  r  s   @r*   r1  r1  r  ó1   ø„ ô>ð U§\¡\ð ÀÇÁð ÐRW×R^ÑR^÷ r4   r1  c                   ó.   ‡ — e Zd Zdˆ fd„	Zd„ Zdd„Zˆ xZS )ÚMraAttentionc                 ó„   •— t         ‰| �  «        t        ||¬«      | _        t	        |«      | _        t        «       | _        y )N)rä   )rè   ré   r  rü   r1  r™   ÚsetÚpruned_heads)rü   rý   rä   rþ   s      €r*   ré   zMraAttention.__init__�  s3   ø€ Ü‰ÑÔÜ$ VÐE\Ô]ˆŒ	Ü# FÓ+ˆŒÜ›EˆÕr4   c                 ó>  — t        |«      dk(  ry t        || j                  j                  | j                  j                  | j
                  «      \  }}t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _        t        | j                  j                  |«      | j                  _	        t        | j                  j                  |d¬«      | j                  _        | j                  j                  t        |«      z
  | j                  _        | j                  j                  | j                  j                  z  | j                  _        | j
                  j                  |«      | _        y )Nr   r   r;   )r>   r   rü   r  r  rB  r   r¤   r¥   r¦   r™   r4  r  Úunion)rü   ÚheadsÚindexs      r*   Úprune_headszMraAttention.prune_heads‡  s  € Üˆu‹:˜Š?ØÜ7Ø�4—9‘9×0Ñ0°$·)±)×2OÑ2OÐQU×QbÑQbó
‰ˆˆuô
 -¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ*¨4¯9©9¯=©=¸%Ó@ˆ�	‰	ŒÜ,¨T¯Y©Y¯_©_¸eÓDˆ�	‰	ŒÜ.¨t¯{©{×/@Ñ/@À%ÈQÔOˆ�‰Ôð )-¯	©	×(EÑ(EÌÈEË
Ñ(Rˆ�	‰	Ô%Ø"&§)¡)×"?Ñ"?À$Ç)Á)×B_ÑB_Ñ"_ˆ�	‰	ÔØ ×-Ñ-×3Ñ3°EÓ:ˆÕr4   c                 óf   — | j                  ||«      }| j                  |d   |«      }|f|dd  z   }|S ©Nr   r   )rü   r™   )rü   r%  r&  Úself_outputsÚattention_outputr/  s         r*   rz   zMraAttention.forward™  s@   € Ø—y‘y °Ó?ˆØŸ;™; |°A¡¸ÓFÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr4   r%   )rˆ   r‰   rŠ   ré   rG  rz   r
  r  s   @r*   r?  r?  €  s   ø„ õ"ò;÷$r4   r?  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMraIntermediatec                 ó  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        |j                  t        «      rt        |j                     | _        y |j                  | _        y r%   )rè   ré   r   r  rì   Úintermediate_sizer4  Ú
isinstanceÚ
hidden_actÚstrr   Úintermediate_act_fnrû   s     €r*   ré   zMraIntermediate.__init__¢  s]   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3KÑ3KÓLˆŒ
Ü�f×'Ñ'¬Ô-Ü'-¨f×.?Ñ.?Ñ'@ˆDÕ$à'-×'8Ñ'8ˆDÕ$r4   r%  r6  c                 óJ   — | j                  |«      }| j                  |«      }|S r%   )r4  rS  ©rü   r%  s     r*   rz   zMraIntermediate.forwardª  s&   € ØŸ
™
 =Ó1ˆØ×0Ñ0°Ó?ˆØÐr4   r;  r  s   @r*   rM  rM  ¡  s#   ø„ ô9ð U§\¡\ð °e·l±l÷ r4   rM  c                   ón   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  dej
                  fd„Zˆ xZS )Ú	MraOutputc                 ó(  •— t         ‰| �  «        t        j                  |j                  |j
                  «      | _        t        j                  |j
                  |j                  ¬«      | _        t        j                  |j                  «      | _        y r3  )rè   ré   r   r  rO  rì   r4  ró   rô   rõ   rö   r÷   rû   s     €r*   ré   zMraOutput.__init__²  s`   ø€ Ü‰ÑÔÜ—Y‘Y˜v×7Ñ7¸×9KÑ9KÓLˆŒ
ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆŒÜ—z‘z &×"<Ñ"<Ó=ˆ�r4   r%  r5  r6  c                 ór   — | j                  |«      }| j                  |«      }| j                  ||z   «      }|S r%   r8  r9  s      r*   rz   zMraOutput.forward¸  r:  r4   r;  r  s   @r*   rW  rW  ±  r=  r4   rW  c                   ó,   ‡ — e Zd Zˆ fd„Zdd„Zd„ Zˆ xZS )ÚMraLayerc                 óÔ   •— t         ‰| �  «        |j                  | _        d| _        t	        |«      | _        |j                  | _        t        |«      | _        t        |«      | _
        y ©Nr   )rè   ré   Úchunk_size_feed_forwardÚseq_len_dimr?  Ú	attentionÚadd_cross_attentionrM  ÚintermediaterW  r™   rû   s     €r*   ré   zMraLayer.__init__À  sW   ø€ Ü‰ÑÔØ'-×'EÑ'EˆÔ$ØˆÔÜ% fÓ-ˆŒØ#)×#=Ñ#=ˆÔ Ü+¨FÓ3ˆÔÜ Ó'ˆ�r4   c                 ó¢   — | j                  ||«      }|d   }|dd  }t        | j                  | j                  | j                  |«      }|f|z   }|S rI  )r`  r   Úfeed_forward_chunkr^  r_  )rü   r%  r&  Úself_attention_outputsrK  r/  Úlayer_outputs          r*   rz   zMraLayer.forwardÉ  sc   € Ø!%§¡°¸~Ó!NÐØ1°!Ñ4Ðà(¨¨Ð,ˆä0Ø×#Ñ# T×%AÑ%AÀ4×CSÑCSÐUeó
ˆð  �/ GÑ+ˆàˆr4   c                 óL   — | j                  |«      }| j                  ||«      }|S r%   )rb  r™   )rü   rK  Úintermediate_outputrf  s       r*   rd  zMraLayer.feed_forward_chunkÖ  s,   € Ø"×/Ñ/Ð0@ÓAÐØ—{‘{Ð#6Ð8HÓIˆØÐr4   r%   )rˆ   r‰   rŠ   ré   rz   rd  r
  r  s   @r*   r[  r[  ¿  s   ø„ ô(óör4   r[  c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )Ú
MraEncoderc                 óÐ   •— t         ‰| �  «        || _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _	        y c c}w )NF)
rè   ré   rý   r   Ú
ModuleListÚrangeÚnum_hidden_layersr[  r  Úgradient_checkpointing)rü   rý   rf   rþ   s      €r*   ré   zMraEncoder.__init__Ý  sN   ø€ Ü‰ÑÔØˆŒÜ—]‘]¼eÀF×D\ÑD\Ó>]Ö#^¸¤H¨VÕ$4Ò#^Ó_ˆŒ
Ø&+ˆÕ#ùò $_s   ½A#c                 ó6  — |rdnd }t        | j                  «      D ]Q  \  }}|r||fz   }| j                  r*| j                  r| j	                  |j
                  ||«      }	n	 |||«      }	|	d   }ŒS |r||fz   }|st        d„ ||fD «       «      S t        ||¬«      S )Nr&   r   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr%   r&   )Ú.0Úvs     r*   ú	<genexpr>z%MraEncoder.forward.<locals>.<genexpr>   s   è ø€ ÒX˜qÈ!É-œÑXùs   ‚Š)Úlast_hidden_stater%  )Ú	enumerater  ro  ÚtrainingÚ_gradient_checkpointing_funcÚ__call__Útupler   )
rü   r%  r&  Ú	head_maskÚoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚiÚlayer_moduleÚlayer_outputss
             r*   rz   zMraEncoder.forwardã  sÁ   € ñ #7™B¸DÐä(¨¯©Ó4ò 	-‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!à×*Ò*¨t¯}ª}Ø $× AÑ AØ ×)Ñ)Ø!Ø"ó!‘ñ !-¨]¸NÓ K�à)¨!Ñ,‰Mð	-ñ  Ø 1°]Ð4DÑ DÐáÜÑX ]Ð4EÐ$FÔXÓXÐXÜ1Ø+Ø+ô
ð 	
r4   )NNFT)rˆ   r‰   rŠ   ré   rz   r
  r  s   @r*   rj  rj  Ü  s   ø„ ô,ð ØØ"Ø÷!
r4   rj  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMraPredictionHeadTransformc                 óh  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        |j                  t        «      rt        |j                     | _
        n|j                  | _
        t        j                  |j                  |j                  ¬«      | _        y r3  )rè   ré   r   r  rì   r4  rP  rQ  rR  r   Útransform_act_fnró   rô   rû   s     €r*   ré   z#MraPredictionHeadTransform.__init__	  s{   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü�f×'Ñ'¬Ô-Ü$*¨6×+<Ñ+<Ñ$=ˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™ f×&8Ñ&8¸f×>SÑ>SÔTˆ�r4   r%  r6  c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r%   )r4  r…  ró   rU  s     r*   rz   z"MraPredictionHeadTransform.forward  s4   € ØŸ
™
 =Ó1ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆØÐr4   r;  r  s   @r*   rƒ  rƒ    s$   ø„ ôUð U§\¡\ð °e·l±l÷ r4   rƒ  c                   ó*   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zˆ xZS )ÚMraLMPredictionHeadc                 óH  •— t         ‰| �  «        t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        t	        j                  t        j                  |j                  «      «      | _        | j                  | j                  _        y )NF)Úbias)rè   ré   rƒ  Ú	transformr   r  rì   rë   ÚdecoderÚ	ParameterrT   r•   rŠ  rû   s     €r*   ré   zMraLMPredictionHead.__init__  sm   ø€ Ü‰ÑÔÜ3°FÓ;ˆŒô —y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒä—L‘L¤§¡¨V×->Ñ->Ó!?Ó@ˆŒ	ð !ŸI™Iˆ�‰Õr4   c                 ó:   — | j                   | j                  _         y r%   )rŠ  rŒ  ©rü   s    r*   Ú_tie_weightsz MraLMPredictionHead._tie_weights(  s   € Ø ŸI™Iˆ�‰Õr4   c                 óJ   — | j                  |«      }| j                  |«      }|S r%   )r‹  rŒ  rU  s     r*   rz   zMraLMPredictionHead.forward+  s$   € ØŸ™ }Ó5ˆØŸ™ ]Ó3ˆØÐr4   )rˆ   r‰   rŠ   ré   r�  rz   r
  r  s   @r*   rˆ  rˆ    s   ø„ ô&ò&ör4   rˆ  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚMraOnlyMLMHeadc                 óB   •— t         ‰| �  «        t        |«      | _        y r%   )rè   ré   rˆ  Úpredictionsrû   s     €r*   ré   zMraOnlyMLMHead.__init__3  s   ø€ Ü‰ÑÔÜ.¨vÓ6ˆÕr4   Úsequence_outputr6  c                 ó(   — | j                  |«      }|S r%   )r•  )rü   r–  Úprediction_scoress      r*   rz   zMraOnlyMLMHead.forward7  s   € Ø ×,Ñ,¨_Ó=ÐØ Ð r4   r;  r  s   @r*   r“  r“  2  s#   ø„ ô7ð! u§|¡|ð !¸¿¹÷ !r4   r“  c                   ó"   — e Zd ZdZeZdZdZd„ Zy)ÚMraPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    r#   Tc                 ó  — t        |t        j                  «      rm|j                  j                  j                  d| j                  j                  ¬«       |j                  �%|j                  j                  j                  «        yyt        |t        j                  «      rz|j                  j                  j                  d| j                  j                  ¬«       |j                  �2|j                  j                  |j                     j                  «        yyt        |t        j                  «      rJ|j                  j                  j                  «        |j                  j                  j                  d«       yy)zInitialize the weightsg        )r    ÚstdNr"  )rP  r   r  ÚweightÚdataÚnormal_rý   Úinitializer_rangerŠ  Úzero_rê   rà   ró   Úfill_)rü   Úmodules     r*   Ú_init_weightsz MraPreTrainedModel._init_weightsG  s  € ä�fœbŸi™iÔ(ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ä˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)ð .r4   N)	rˆ   r‰   rŠ   r	  r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingr¤  r&   r4   r*   rš  rš  =  s   „ ñð
 €LØÐØ&*Ð#ó*r4   rš  aF  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`MraConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
ak	  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z]The bare MRA Model transformer outputting raw hidden-states without any specific head on top.c                   óp  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Z eej                  d«      «       e
eee¬«      	 	 	 	 	 	 	 	 ddeej                      deej                      d	eej                      d
eej                      deej                      deej                      dee   dee   deeef   fd„«       «       Zˆ xZS )ÚMraModelc                 ó’   •— t         ‰| �  |«       || _        t        |«      | _        t        |«      | _        | j                  «        y r%   )rè   ré   rý   rÞ   r  rj  ÚencoderÚ	post_initrû   s     €r*   ré   zMraModel.__init__—  s;   ø€ Ü‰Ñ˜Ô ØˆŒä'¨Ó/ˆŒÜ! &Ó)ˆŒð 	�‰Õr4   c                 ó.   — | j                   j                  S r%   ©r  rî   r�  s    r*   Úget_input_embeddingszMraModel.get_input_embeddings¡  s   € Ø�‰×.Ñ.Ð.r4   c                 ó&   — || j                   _        y r%   r®  )rü   r¦   s     r*   Úset_input_embeddingszMraModel.set_input_embeddings¤  s   € Ø*/ˆ�‰Õ'r4   c                 ó˜   — |j                  «       D ]7  \  }}| j                  j                  |   j                  j	                  |«       Œ9 y)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        N)Úitemsr«  r  r`  rG  )rü   Úheads_to_pruner  rE  s       r*   Ú_prune_headszMraModel._prune_heads§  sE   € ð
 +×0Ñ0Ó2ò 	C‰LˆE�5Ø�L‰L×Ñ˜uÑ%×/Ñ/×;Ñ;¸EÕBñ	Cr4   úbatch_size, sequence_length©Ú
checkpointÚoutput_typer¥  r  r&  ræ   rã   r{  r  r|  r}  r6  c	                 ó¬  — |�|n| j                   j                  }|�|n| j                   j                  }|�|�t        d«      ‚|�#| j	                  ||«       |j                  «       }	n!|�|j                  «       d d }	nt        d«      ‚|	\  }
}|�|j                  n|j                  }|€t        j                  |
|f|¬«      }|€pt        | j                  d«      r4| j                  j                  d d …d |…f   }|j                  |
|«      }|}n&t        j                  |	t        j                  |¬«      }| j                  ||	«      }| j!                  || j                   j"                  «      }| j                  ||||¬«      }| j%                  |||||¬«      }|d	   }|s	|f|d
d  z   S t'        ||j(                  |j*                  |j,                  ¬«      S )NzDYou cannot specify both input_ids and inputs_embeds at the same timer=   z5You have to specify either input_ids or inputs_embedsr±   ræ   rP   )r  rã   ræ   r  )r&  r{  r|  r}  r   r   )ru  r%  Ú
attentionsÚcross_attentions)rý   r|  Úuse_return_dictr@   Ú%warn_if_padding_and_no_attention_maskr?   rR   rT   rž   r   r  ræ   rù   r•   rV   Úget_extended_attention_maskÚget_head_maskrn  r«  r   r%  r»  r¼  )rü   r  r&  ræ   rã   r{  r  r|  r}  r  rZ   r  rR   r  r  Úextended_attention_maskÚembedding_outputÚencoder_outputsr–  s                      r*   rz   zMraModel.forward¯  sþ  € ð$ %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ø×6Ñ6°yÀ.ÔQØ#Ÿ.™.Ó*‰KØÐ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
�JØ%.Ð%:�×!Ò!À×@TÑ@TˆàÐ!Ü"ŸZ™Z¨*°jÐ)AÈ6ÔRˆNàÐ!Ü�t—‘Ð(8Ô9Ø*.¯/©/×*HÑ*HÊÈKÈZÈKÈÑ*XÐ'Ø3J×3QÑ3QÐR\Ð^hÓ3iÐ0Ø!A‘ä!&§¡¨[ÄÇ
Á
ÐSYÔ!Z�ð 15×0PÑ0PÐQ_ÐalÓ0mÐð ×&Ñ& y°$·+±+×2OÑ2OÓPˆ	àŸ?™?ØØ%Ø)Ø'ð	 +ó 
Ðð Ÿ,™,ØØ2ØØ!5Ø#ð 'ó 
ˆð *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;ä1Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô	
ð 	
r4   )NNNNNNNN)rˆ   r‰   rŠ   ré   r¯  r±  rµ  r   ÚMRA_INPUTS_DOCSTRINGÚformatr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCr   rT   r<  Úboolr   r   rz   r
  r  s   @r*   r©  r©  ’  s  ø„ ô
ò/ò0òCñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø6Ø$ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø/3Ø&*ñJ
à˜EŸL™LÑ)ðJ
ð ! §¡Ñ.ðJ
ð ! §¡Ñ.ð	J
ð
 ˜uŸ|™|Ñ,ðJ
ð ˜EŸL™LÑ)ðJ
ð   §¡Ñ-ðJ
ð ' t™nðJ
ð ˜d‘^ðJ
ð 
ˆuÐ8Ð8Ñ	9òJ
óó gôJ
r4   r©  z1MRA Model with a `language modeling` head on top.c                   ó’  ‡ — e Zd ZddgZˆ fd„Zd„ Zd„ Z eej                  d«      «       e
eee¬«      	 	 	 	 	 	 	 	 	 ddeej                      d	eej                      d
eej                      deej                      deej                      deej                      deej                      dee   dee   deeef   fd„«       «       Zˆ xZS )ÚMraForMaskedLMzcls.predictions.decoder.weightzcls.predictions.decoder.biasc                 ó„   •— t         ‰| �  |«       t        |«      | _        t	        |«      | _        | j                  «        y r%   )rè   ré   r©  r#   r“  Úclsr¬  rû   s     €r*   ré   zMraForMaskedLM.__init__  s4   ø€ Ü‰Ñ˜Ô ä˜FÓ#ˆŒÜ! &Ó)ˆŒð 	�‰Õr4   c                 óB   — | j                   j                  j                  S r%   )rÌ  r•  rŒ  r�  s    r*   Úget_output_embeddingsz$MraForMaskedLM.get_output_embeddings  s   € Ø�x‰x×#Ñ#×+Ñ+Ð+r4   c                 ó„   — || j                   j                  _        |j                  | j                   j                  _        y r%   )rÌ  r•  rŒ  rŠ  )rü   Únew_embeddingss     r*   Úset_output_embeddingsz$MraForMaskedLM.set_output_embeddings  s,   € Ø'5ˆ�‰×ÑÔ$Ø$2×$7Ñ$7ˆ�‰×ÑÕ!r4   r¶  r·  r  r&  ræ   rã   r{  r  Úlabelsr|  r}  r6  c
           
      ó˜  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }
|
d   }| j                  |«      }d}|�Ft	        «       } ||j                  d| j                   j                  «      |j                  d«      «      }|	s|f|
dd z   }|�|f|z   S |S t        |||
j                  |
j                  ¬«      S )a£  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
        N©r&  ræ   rã   r{  r  r|  r}  r   r=   r   ©ÚlossÚlogitsr%  r»  )
rý   r½  r#   rÌ  r	   r  rë   r   r%  r»  )rü   r  r&  ræ   rã   r{  r  rÒ  r|  r}  r/  r–  r˜  Úmasked_lm_lossÚloss_fctr™   s                   r*   rz   zMraForMaskedLM.forward  s÷   € ð0 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—(‘(ØØ)Ø)Ø%ØØ'Ø!5Ø#ð ó 	
ˆð " !™*ˆØ ŸH™H _Ó5ÐàˆØÐÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNáØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r4   ©	NNNNNNNNN)rˆ   r‰   rŠ   Ú_tied_weights_keysré   rÎ  rÑ  r   rÄ  rÅ  r   rÆ  r   rÇ  r   rT   r<  rÈ  r   r   rz   r
  r  s   @r*   rÊ  rÊ    s+  ø„ à:Ð<ZÐ[Ðôò,ò8ñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø"Ø$ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø)-Ø/3Ø&*ñ0
à˜EŸL™LÑ)ð0
ð ! §¡Ñ.ð0
ð ! §¡Ñ.ð	0
ð
 ˜uŸ|™|Ñ,ð0
ð ˜EŸL™LÑ)ð0
ð   §¡Ñ-ð0
ð ˜Ÿ™Ñ&ð0
ð ' t™nð0
ð ˜d‘^ð0
ð 
ˆu�nÐ$Ñ	%ò0
óó gô0
r4   rÊ  c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )ÚMraClassificationHeadz-Head for sentence-level classification tasks.c                 ó4  •— t         ‰| �  «        t        j                  |j                  |j                  «      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _
        || _        y r%   )rè   ré   r   r  rì   r4  rõ   rö   r÷   Ú
num_labelsÚout_projrý   rû   s     €r*   ré   zMraClassificationHead.__init__S  sg   ø€ Ü‰ÑÔÜ—Y‘Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z‘z &×"<Ñ"<Ó=ˆŒÜŸ	™	 &×"4Ñ"4°f×6GÑ6GÓHˆŒàˆ�r4   c                 óê   — |d d …dd d …f   }| j                  |«      }| j                  |«      }t        | j                  j                     |«      }| j                  |«      }| j                  |«      }|S )Nr   )r÷   r4  r   rý   rQ  rà  )rü   ÚfeaturesÚkwargsÚxs       r*   rz   zMraClassificationHead.forward[  se   € Ø’Q˜š1�WÑˆØ�L‰L˜‹OˆØ�J‰J�q‹MˆÜ�4—;‘;×)Ñ)Ñ*¨1Ó-ˆØ�L‰L˜‹OˆØ�M‰M˜!ÓˆØˆr4   r  r  s   @r*   rÝ  rÝ  P  s   ø„ Ù7ôör4   rÝ  z‘MRA Model transformer with a sequence classification/regression head on top (a linear layer on top of
    the pooled output) e.g. for GLUE tasks.c                   ó~  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚMraForSequenceClassificationc                 ó¦   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        |«      | _        | j                  «        y r%   )rè   ré   rß  r©  r#   rÝ  Ú
classifierr¬  rû   s     €r*   ré   z%MraForSequenceClassification.__init__k  sA   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ˜FÓ#ˆŒÜ/°Ó7ˆŒð 	�‰Õr4   r¶  r·  r  r&  ræ   rã   r{  r  rÒ  r|  r}  r6  c
           
      ó  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }
|
d   }| j                  |«      }d}|��‡| j                   j                  €�| j
                  dk(  rd| j                   _        nl| j
                  dkD  rL|j                  t        j                  k(  s|j                  t        j                  k(  rd| j                   _        nd| j                   _        | j                   j                  dk(  rIt        «       }| j
                  dk(  r& ||j                  «       |j                  «       «      }nŒ |||«      }n‚| j                   j                  dk(  r=t        «       } ||j                  d| j
                  «      |j                  d«      «      }n,| j                   j                  dk(  rt        «       } |||«      }|	s|f|
dd z   }|�|f|z   S |S t        |||
j                   |
j"                  ¬	«      S )
a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        NrÔ  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr=   rÕ  )rý   r½  r#   rè  Úproblem_typerß  rQ   rT   rV   rE   r
   r#  r	   r  r   r   r%  r»  )rü   r  r&  ræ   rã   r{  r  rÒ  r|  r}  r/  r–  r×  rÖ  rÙ  r™   s                   r*   rz   z$MraForSequenceClassification.forwardt  s×  € ð0 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—(‘(ØØ)Ø)Ø%ØØ'Ø!5Ø#ð ó 	
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÑØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨Ó/�ÙØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r4   rÚ  )rˆ   r‰   rŠ   ré   r   rÄ  rÅ  r   rÆ  r   rÇ  r   rT   r<  rÈ  r   r   rz   r
  r  s   @r*   ræ  ræ  e  s"  ø„ ôñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø,Ø$ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø)-Ø/3Ø&*ñA
à˜EŸL™LÑ)ðA
ð ! §¡Ñ.ðA
ð ! §¡Ñ.ð	A
ð
 ˜uŸ|™|Ñ,ðA
ð ˜EŸL™LÑ)ðA
ð   §¡Ñ-ðA
ð ˜Ÿ™Ñ&ðA
ð ' t™nðA
ð ˜d‘^ðA
ð 
ˆuÐ.Ð.Ñ	/òA
óó gôA
r4   ræ  zšMRA Model with a multiple choice classification head on top (a linear layer on top of
    the pooled output and a softmax) e.g. for RocStories/SWAG tasks.c                   ó~  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚMraForMultipleChoicec                 ó  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        t	        j
                  |j                  d«      | _        | j                  «        y r]  )
rè   ré   r©  r#   r   r  rì   Úpre_classifierrè  r¬  rû   s     €r*   ré   zMraForMultipleChoice.__init__Ä  s_   ø€ Ü‰Ñ˜Ô ä˜FÓ#ˆŒÜ Ÿi™i¨×(:Ñ(:¸F×<NÑ<NÓOˆÔÜŸ)™) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr4   z(batch_size, num_choices, sequence_lengthr·  r  r&  ræ   rã   r{  r  rÒ  r|  r}  r6  c
           
      ó�  — |	�|	n| j                   j                  }	|�|j                  d   n|j                  d   }
|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�!|j                  d|j	                  d«      «      nd}|�1|j                  d|j	                  d«      |j	                  d«      «      nd}| j                  ||||||||	¬«      }|d   }|dd…df   }| j                  |«      } t        j                  «       |«      }| j                  |«      }|j                  d|
«      }d}|�t        «       } |||«      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )aJ  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
            num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
            `input_ids` above)
        Nr   r=   r:   rÔ  r   rÕ  )rý   r½  rS   r  r?   r#   rñ  r   ÚReLUrè  r	   r   r%  r»  )rü   r  r&  ræ   rã   r{  r  rÒ  r|  r}  Únum_choicesr/  Úhidden_stateÚpooled_outputr×  Úreshaped_logitsrÖ  rÙ  r™   s                      r*   rz   zMraForMultipleChoice.forwardÎ  sü  € ð0 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆØ,5Ð,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆà>GÐ>S�I—N‘N 2 y§~¡~°bÓ'9Ô:ÐY]ˆ	ØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØM[ÐMg˜×,Ñ,¨R°×1DÑ1DÀRÓ1HÔIÐmqˆØGSÐG_�|×(Ñ(¨¨\×->Ñ->¸rÓ-BÔCÐeiˆð Ð(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —(‘(ØØ)Ø)Ø%ØØ'Ø!5Ø#ð ó 	
ˆð ˜q‘zˆØ$¢Q¨ TÑ*ˆØ×+Ñ+¨MÓ:ˆØ!œŸ™›	 -Ó0ˆØ—‘ Ó/ˆà Ÿ+™+ b¨+Ó6ˆàˆØÐÜ'Ó)ˆHÙ˜O¨VÓ4ˆDáØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r4   rÚ  )rˆ   r‰   rŠ   ré   r   rÄ  rÅ  r   rÆ  r   rÇ  r   rT   r<  rÈ  r   r   rz   r
  r  s   @r*   rï  rï  ¾  s"  ø„ ôñ +Ð+?×+FÑ+FÐGqÓ+rÓsÙØ&Ø-Ø$ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø)-Ø/3Ø&*ñ@
à˜EŸL™LÑ)ð@
ð ! §¡Ñ.ð@
ð ! §¡Ñ.ð	@
ð
 ˜uŸ|™|Ñ,ð@
ð ˜EŸL™LÑ)ð@
ð   §¡Ñ-ð@
ð ˜Ÿ™Ñ&ð@
ð ' t™nð@
ð ˜d‘^ð@
ð 
ˆuÐ/Ð/Ñ	0ò@
óó tô@
r4   rï  z˜MRA Model with a token classification head on top (a linear layer on top of
    the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks.c                   ó~  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚMraForTokenClassificationc                 ó,  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  «      | _        | j                  «        y r%   )rè   ré   rß  r©  r#   r   rõ   rö   r÷   r  rì   rè  r¬  rû   s     €r*   ré   z"MraForTokenClassification.__init__  si   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä˜FÓ#ˆŒÜ—z‘z &×"<Ñ"<Ó=ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr4   r¶  r·  r  r&  ræ   rã   r{  r  rÒ  r|  r}  r6  c
           
      óÂ  — |	�|	n| j                   j                  }	| j                  ||||||||	¬«      }
|
d   }| j                  |«      }| j	                  |«      }d}|�Êt        «       }|�Œ|j                  d«      dk(  }|j                  d| j                  «      }t        j                  ||j                  d«      t        j                  |j                  «      j                  |«      «      } |||«      }n2 ||j                  d| j                  «      |j                  d«      «      }|	s|f|
dd z   }|�|f|z   S |S t        |||
j                  |
j                  ¬«      S )zÛ
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        NrÔ  r   r=   r   rÕ  )rý   r½  r#   r÷   rè  r	   r  rß  rT   ÚwhereÚtensorÚignore_indexÚtype_asr   r%  r»  )rü   r  r&  ræ   rã   r{  r  rÒ  r|  r}  r/  r–  r×  rÖ  rÙ  Úactive_lossÚactive_logitsÚactive_labelsr™   s                      r*   rz   z!MraForTokenClassification.forward(  sh  € ð, &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—(‘(ØØ)Ø)Ø%ØØ'Ø!5Ø#ð ó 	
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÐÜ'Ó)ˆHàÐ)Ø,×1Ñ1°"Ó5¸Ñ:�Ø &§¡¨B°·±Ó @�Ü %§¡Ø §¡¨R£´%·,±,¸x×?TÑ?TÓ2U×2]Ñ2]Ð^dÓ2eó!�ñ   ¨}Ó=‘á §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR�áØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r4   rÚ  )rˆ   r‰   rŠ   ré   r   rÄ  rÅ  r   rÆ  r   rÇ  r   rT   r<  rÈ  r   r   rz   r
  r  s   @r*   rù  rù    s  ø„ ô	ñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø)Ø$ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø)-Ø/3Ø&*ñ9
à˜EŸL™LÑ)ð9
ð ! §¡Ñ.ð9
ð ! §¡Ñ.ð	9
ð
 ˜uŸ|™|Ñ,ð9
ð ˜EŸL™LÑ)ð9
ð   §¡Ñ-ð9
ð ˜Ÿ™Ñ&ð9
ð ' t™nð9
ð ˜d‘^ð9
ð 
ˆuÐ+Ð+Ñ	,ò9
óó gô9
r4   rù  zÒMRA Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).c                   óž  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
¬«      	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     deej                     dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚMraForQuestionAnsweringc                 óò   •— t         ‰| �  |«       d|_        |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        | j                  «        y )Nr7   )
rè   ré   rß  r©  r#   r   r  rì   Ú
qa_outputsr¬  rû   s     €r*   ré   z MraForQuestionAnswering.__init__p  s[   ø€ Ü‰Ñ˜Ô àˆÔØ ×+Ñ+ˆŒä˜FÓ#ˆŒÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰Õr4   r¶  r·  r  r&  ræ   rã   r{  r  Ústart_positionsÚend_positionsr|  r}  r6  c           
      óî  — |
�|
n| j                   j                  }
| j                  |||||||	|
¬«      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      }|j                  d«      }d}|�·|�µt        |j                  «       «      dkD  r|j                  d«      }t        |j                  «       «      dkD  r|j                  d«      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|
s||f|dd z   }|�|f|z   S |S t        ||||j                  |j                  ¬	«      S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        NrÔ  r   r   r=   r;   )rþ  r7   )rÖ  Ústart_logitsÚ
end_logitsr%  r»  )rý   r½  r#   r  Úsplitr#  r>   r?   Úclampr	   r   r%  r»  )rü   r  r&  ræ   rã   r{  r  r  r  r|  r}  r/  r–  r×  r
  r  Ú
total_lossÚignored_indexrÙ  Ú
start_lossÚend_lossr™   s                         r*   rz   zMraForQuestionAnswering.forward|  s­  € ð: &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—(‘(ØØ)Ø)Ø%ØØ'Ø!5Ø#ð ó 	
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/ˆØ×'Ñ'¨Ó+ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r4   )
NNNNNNNNNN)rˆ   r‰   rŠ   ré   r   rÄ  rÅ  r   rÆ  r   rÇ  r   rT   r<  rÈ  r   r   rz   r
  r  s   @r*   r  r  j  s9  ø„ ô
ñ +Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø0Ø$ôð -1Ø15Ø15Ø/3Ø,0Ø04Ø26Ø04Ø/3Ø&*ñF
à˜EŸL™LÑ)ðF
ð ! §¡Ñ.ðF
ð ! §¡Ñ.ð	F
ð
 ˜uŸ|™|Ñ,ðF
ð ˜EŸL™LÑ)ðF
ð   §¡Ñ-ðF
ð " %§,¡,Ñ/ðF
ð   §¡Ñ-ðF
ð ' t™nðF
ð ˜d‘^ðF
ð 
ˆuÐ2Ð2Ñ	3òF
óó gôF
r4   r  )rÊ  rï  r  ræ  rù  r[  r©  rš  r†   )NN)r9   r   r   )Ur	  r¢   Úpathlibr   Útypingr   r   r   rT   Útorch.utils.checkpointr   Útorch.nnr   r	   r
   Útorch.utils.cpp_extensionr   Úactivationsr   Úmodeling_outputsr   r   r   r   r   r   Úmodeling_utilsr   Úpytorch_utilsr   r   r   Úutilsr   r   r   r   r   r   Úconfiguration_mrar   Ú
get_loggerrˆ   r  rÆ  rÇ  Ú_TOKENIZER_FOR_DOCr1   r3   rN   r^   rb   rj   rt   ÚautogradÚFunctionrv   r�   r’   r¯   rÆ   rÜ   ÚModulerÞ   r  r1  r?  rM  rW  r[  rj  rƒ  rˆ  r“  rš  ÚMRA_START_DOCSTRINGrÄ  r©  rÊ  rÝ  ræ  rï  rù  r  Ú__all__r&   r4   r*   ú<module>r$     sí  ðñ ã Ý ß )Ñ )ã Û Ý ß AÑ AÝ *å !÷÷ õ .ß lÑ l÷÷ õ )ð 
ˆ×	Ñ	˜HÓ	%€à1Ð Ø€Ø$Ð ð €ò	Cò&ó8ó.%OóP%òPsôX˜EŸN™N×3Ñ3ô Xô0]˜5Ÿ>™>×2Ñ2ô ]÷.ñ ó:%VòP#)ðZ Ø!"Ø$%ópôf7�B—I‘Iô 7ôtY�r—y‘yô Yôz�B—I‘Iô ô�2—9‘9ô ôB�b—i‘iô ô �—	‘	ô ôˆr�y‰yô ô:(
�—‘ô (
ôX §¡ô ô$˜"Ÿ)™)ô ô0!�R—Y‘Yô !ô*˜ô *ð6	Ð ð,Ð ñ^ ØcØóôi
Ð!ó i
ó	ði
ñX ÐMÐObÓcôI
Ð'ó I
ó dðI
ôZ˜BŸI™Iô ñ* ð/àóô
Q
Ð#5ó Q
óð
Q
ñh ðHàóô
Q
Ð-ó Q
óð
Q
ñh ðPàóô
K
Ð 2ó K
óð
K
ñ\ ðhàóô
Y
Ð0ó Y
óð
Y
òx	�r4   