Ë
    S^(h ã                  ó  — d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
ZddlZddlmZ ddlmZmZmZmZmZ ddlmZmZmZmZmZmZmZmZmZ dd	l m!Z!m"Z"m#Z# dd
l$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*  e(jV                  e,«      Z-dZ.dZ/ G d„ dej`                  jb                  «      Z2 G d„ dej`                  jb                  «      Z3 G d„ dej`                  jb                  «      Z4 G d„ dej`                  jb                  «      Z5 G d„ dej`                  jb                  «      Z6 G d„ dej`                  jb                  «      Z7 G d„ dej`                  jb                  «      Z8 G d„ dej`                  jb                  «      Z9 G d„ d ej`                  jb                  «      Z: G d!„ d"ej`                  jb                  «      Z;d#„ Z<d$„ Z=d%„ Z>d&„ Z?d'„ Z@ G d(„ d)ej`                  jb                  «      ZA G d*„ d+ej`                  jb                  «      ZB G d,„ d-ej`                  jb                  «      ZC G d.„ d/ej`                  jb                  «      ZD G d0„ d1ej`                  jb                  «      ZE G d2„ d3ej`                  jb                  «      ZF G d4„ d5e«      ZGd6ZHd7ZI e&d8eH«       G d9„ d:eG«      «       ZJ e&d;eH«       G d<„ d=eGe«      «       ZK e&d>eH«       G d?„ d@eGe«      «       ZL e&dAeH«       G dB„ dCeGe«      «       ZM e&dDeH«       G dE„ dFeGe«      «       ZNg dG¢ZOy)HzTF 2.0 DeBERTa model.é    )ÚannotationsN)ÚDictÚOptionalÚSequenceÚTupleÚUnioné   )Úget_tf_activation)ÚTFBaseModelOutputÚTFMaskedLMOutputÚTFQuestionAnsweringModelOutputÚTFSequenceClassifierOutputÚTFTokenClassifierOutput)	ÚTFMaskedLanguageModelingLossÚTFModelInputTypeÚTFPreTrainedModelÚTFQuestionAnsweringLossÚTFSequenceClassificationLossÚTFTokenClassificationLossÚget_initializerÚkerasÚunpack_inputs)Úcheck_embeddings_within_boundsÚ
shape_listÚstable_softmax)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )ÚDebertaConfigr!   zkamalkraj/deberta-basec                  óD   ‡ — e Zd Zdˆ fd„Zddd„Zedd„«       Zd	d„Zˆ xZS )
ÚTFDebertaContextPoolerc                óÊ   •— t        ‰| �  di |¤Ž t        j                  j	                  |j
                  d¬«      | _        t        |j                  d¬«      | _	        || _
        y )NÚdense©ÚnameÚdropout© )ÚsuperÚ__init__r   ÚlayersÚDenseÚpooler_hidden_sizer%   ÚTFDebertaStableDropoutÚpooler_dropoutr(   Úconfig©Úselfr1   ÚkwargsÚ	__class__s      €úm/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/deberta/modeling_tf_deberta.pyr+   zTFDebertaContextPooler.__init__9   sO   ø€ Ü‰ÑÑ"˜6Ò"Ü—\‘\×'Ñ'¨×(AÑ(AÈÐ'ÓPˆŒ
Ü-¨f×.CÑ.CÈ)ÔTˆŒØˆ�ó    c                óª   — |d d …df   }| j                  ||¬«      }| j                  |«      } t        | j                  j                  «      |«      }|S )Nr   ©Útraining)r(   r%   r
   r1   Úpooler_hidden_act)r3   Úhidden_statesr:   Úcontext_tokenÚpooled_outputs        r6   ÚcallzTFDebertaContextPooler.call?   sT   € ð &¢a¨ dÑ+ˆØŸ™ ]¸X˜ÓFˆØŸ
™
 =Ó1ˆØHÔ)¨$¯+©+×*GÑ*GÓHÈÓWˆØÐr7   c                ó.   — | j                   j                  S ©N)r1   Úhidden_size©r3   s    r6   Ú
output_dimz!TFDebertaContextPooler.output_dimH   s   € à�{‰{×&Ñ&Ð&r7   c                óô  — | j                   ry d| _         t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒexY w# 1 sw Y   y xY w)NTr%   r(   )
ÚbuiltÚgetattrÚtfÚ
name_scoper%   r'   Úbuildr1   r.   r(   ©r3   Úinput_shapes     r6   rJ   zTFDebertaContextPooler.buildL   sÈ   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ OØ—
‘
× Ñ  $¨¨d¯k©k×.LÑ.LÐ!MÔN÷Oä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷Oð Oú÷)ð )ús   Á3C"Â<C.Ã"C+Ã.C7©r1   r!   ©F©r:   Úbool)ÚreturnÚintrA   )	Ú__name__Ú
__module__Ú__qualname__r+   r?   ÚpropertyrD   rJ   Ú__classcell__©r5   s   @r6   r#   r#   8   s&   ø„ õôð ò'ó ð'÷	)r7   r#   c                  ó,   ‡ — e Zd ZdZdˆ fd„	Zdd„Zˆ xZS )ÚTFDebertaXSoftmaxa>  
    Masked Softmax which is optimized for saving memory

    Args:
        input (`tf.Tensor`): The input tensor that will apply softmax.
        mask (`tf.Tensor`): The mask matrix where 0 indicate that element will be ignored in the softmax calculation.
        dim (int): The dimension that will apply softmax
    c                ó2   •— t        ‰| �  di |¤Ž || _        y ©Nr)   )r*   r+   Úaxis)r3   r]   r4   r5   s      €r6   r+   zTFDebertaXSoftmax.__init__b   s   ø€ Ü‰ÑÑ"˜6Ò"Øˆ�	r7   c                ó’  — t        j                  t        j                  |t         j                  «      «      }t        j                  |t        j                  t        d«      | j                  ¬«      |«      }t        t        j                  |t         j                  ¬«      | j                  «      }t        j                  |d|«      }|S )Nz-inf©Údtypeç        )
rH   Úlogical_notÚcastrP   ÚwhereÚfloatÚcompute_dtyper   Úfloat32r]   )r3   ÚinputsÚmaskÚrmaskÚoutputs        r6   r?   zTFDebertaXSoftmax.callf   s}   € Ü—‘œrŸw™w t¬R¯W©WÓ5Ó6ˆÜ—‘˜%¤§¡¬¨v«¸d×>PÑ>PÔ!QÐSYÓZˆÜ¤§¡¨´b·j±jÔ AÀ4Ç9Á9ÓMˆÜ—‘˜%  fÓ-ˆØˆr7   )éÿÿÿÿ)rh   ú	tf.Tensorri   rm   )rS   rT   rU   Ú__doc__r+   r?   rW   rX   s   @r6   rZ   rZ   X   s   ø„ ñõ÷r7   rZ   c                  óP   ‡ — e Zd ZdZˆ fd„Zej                  d„ «       Zddd„Zˆ xZ	S )r/   z
    Optimized dropout module for stabilizing the training

    Args:
        drop_prob (float): the dropout probabilities
    c                ó2   •— t        ‰| �  di |¤Ž || _        y r\   )r*   r+   Ú	drop_prob)r3   rq   r4   r5   s      €r6   r+   zTFDebertaStableDropout.__init__v   s   ø€ Ü‰ÑÑ"˜6Ò"Ø"ˆ�r7   c                ó  ‡ ‡‡— t        j                  dt         j                  j                  j                  j                  d‰ j                  z
  ¬«      j                  t        |«      ¬«      z
  t         j                  «      Št        j                  dd‰ j                  z
  z  ‰ j                  ¬«      Š‰ j                  dkD  r9t        j                  ‰t        j                  d‰ j                  ¬«      |«      ‰z  }ˆˆˆ fd„}||fS )	z~
        Applies dropout to the inputs, as vanilla dropout, but also scales the remaining elements up by 1/drop_prob.
        r    g      ð?)Úprobs)Úsample_shaper_   r   ra   c                ó˜   •— ‰j                   dkD  r9t        j                  ‰t        j                  d‰j                  ¬«      | «      ‰z  S | S )Nr   ra   r_   )rq   rH   rd   rc   rf   )Úupstreamri   Úscaler3   s    €€€r6   Úgradz-TFDebertaStableDropout.xdropout.<locals>.gradˆ   s>   ø€ Ø�~‰~ Ò!Ü—x‘x ¤b§g¡g¨c¸×9KÑ9KÔ&LÈhÓWÐZ_Ñ_Ð_à�r7   )rH   rc   ÚcompatÚv1ÚdistributionsÚ	Bernoullirq   Úsampler   rP   Úconvert_to_tensorrf   rd   )r3   rh   rx   ri   rw   s   `  @@r6   ÚxdropoutzTFDebertaStableDropout.xdropoutz   sÎ   ú€ ô
 �w‰wØÜ�i‰i�l‰l×(Ñ(×2Ñ2¸¸t¿~¹~Ñ9MÐ2ÓN×UÑUÔcmÐntÓcuÐUÓvñwä�G‰Gó
ˆô
 ×$Ñ$ S¨A°·±Ñ,>Ñ%?Àt×GYÑGYÔZˆØ�>‰>˜AÒÜ—X‘X˜d¤B§G¡G¨C°t×7IÑ7IÔ$JÈFÓSÐV[Ñ[ˆFö	 ð �tˆ|Ðr7   c                ó,   — |r| j                  |«      S |S rA   )r   )r3   rh   r:   s      r6   r?   zTFDebertaStableDropout.call�   s   € ÙØ—=‘= Ó(Ð(Øˆr7   rN   )rh   rm   r:   rm   )
rS   rT   rU   rn   r+   rH   Úcustom_gradientr   r?   rW   rX   s   @r6   r/   r/   n   s1   ø„ ñô#ð ×Ññó ð÷*ð r7   r/   c                  ó6   ‡ — e Zd ZdZdˆ fd„	Zˆ fd„Zdd„Zˆ xZS )ÚTFDebertaLayerNormzBLayerNorm module in the TF style (epsilon inside the square root).c                ó@   •— t        ‰| �  di |¤Ž || _        || _        y r\   )r*   r+   ÚsizeÚeps)r3   r…   r†   r4   r5   s       €r6   r+   zTFDebertaLayerNorm.__init__™   s!   ø€ Ü‰ÑÑ"˜6Ò"ØˆŒ	Øˆ�r7   c                óú   •— | j                  | j                  gt        j                  «       d¬«      | _        | j                  | j                  gt        j
                  «       d¬«      | _        t        ‰| �!  |«      S )NÚweight)ÚshapeÚinitializerr'   Úbias)	Ú
add_weightr…   rH   Úones_initializerÚgammaÚzeros_initializerÚbetar*   rJ   )r3   rL   r5   s     €r6   rJ   zTFDebertaLayerNorm.buildž   s^   ø€ Ø—_‘_¨D¯I©I¨;ÄB×DWÑDWÓDYÐ`h�_ÓiˆŒ
Ø—O‘O¨4¯9©9¨+Ä2×CWÑCWÓCYÐ`f�OÓgˆŒ	Ü‰w‰}˜[Ó)Ð)r7   c                ó.  — t        j                  |dgd¬«      }t        j                  t        j                  ||z
  «      dgd¬«      }t         j                  j	                  || j
                  z   «      }| j                  ||z
  z  |z  | j                  z   S )Nrl   T)r]   Úkeepdims)rH   Úreduce_meanÚsquareÚmathÚsqrtr†   rŽ   r�   )r3   ÚxÚmeanÚvarianceÚstds        r6   r?   zTFDebertaLayerNorm.call£   ss   € Ü�~‰~˜a r d°TÔ:ˆÜ—>‘>¤"§)¡)¨A°©HÓ"5¸R¸DÈ4ÔPˆÜ�g‰g�l‰l˜8 d§h¡hÑ.Ó/ˆØ�z‰z˜Q ™XÑ&¨Ñ,¨t¯y©yÑ8Ð8r7   )gê-�™—q=)r—   rm   rQ   rm   ©rS   rT   rU   rn   r+   rJ   r?   rW   rX   s   @r6   rƒ   rƒ   –   s   ø„ ÙLõô
*÷
9r7   rƒ   c                  ó2   ‡ — e Zd Zdˆ fd„Zddd„Zdd„Zˆ xZS )ÚTFDebertaSelfOutputc                ó*  •— t        ‰| �  di |¤Ž t        j                  j	                  |j
                  d¬«      | _        t        j                  j                  |j                  d¬«      | _	        t        |j                  d¬«      | _        || _        y )Nr%   r&   Ú	LayerNorm©Úepsilonr'   r(   r)   )r*   r+   r   r,   r-   rB   r%   ÚLayerNormalizationÚlayer_norm_epsrŸ   r/   Úhidden_dropout_probr(   r1   r2   s      €r6   r+   zTFDebertaSelfOutput.__init__«   sq   ø€ Ü‰ÑÑ"˜6Ò"Ü—\‘\×'Ñ'¨×(:Ñ(:ÀÐ'ÓIˆŒ
ÜŸ™×8Ñ8À×AVÑAVÐ]hÐ8ÓiˆŒÜ-¨f×.HÑ.HÈyÔYˆŒØˆ�r7   c                óv   — | j                  |«      }| j                  ||¬«      }| j                  ||z   «      }|S )Nr9   ©r%   r(   rŸ   ©r3   r<   Úinput_tensorr:   s       r6   r?   zTFDebertaSelfOutput.call²   s;   € ØŸ
™
 =Ó1ˆØŸ™ ]¸X˜ÓFˆØŸ™ }°|Ñ'CÓDˆØÐr7   c                óî  — | j                   ry d| _         t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒÖxY w# 1 sw Y   ŒqxY w# 1 sw Y   y xY w©NTr%   rŸ   r(   )rF   rG   rH   rI   r%   r'   rJ   r1   rB   rŸ   r(   rK   s     r6   rJ   zTFDebertaSelfOutput.build¸   s)  € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ HØ—
‘
× Ñ  $¨¨d¯k©k×.EÑ.EÐ!FÔG÷Hä�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ LØ—‘×$Ñ$ d¨D°$·+±+×2IÑ2IÐ%JÔK÷Lä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷Hð Hú÷Lð Lú÷)ð )úó$   Á3EÂ<3EÄ-E+ÅEÅE(Å+E4rM   rN   rO   rA   ©rS   rT   rU   r+   r?   rJ   rW   rX   s   @r6   r�   r�   ª   s   ø„ õô÷)r7   r�   c                  óZ   ‡ — e Zd Zdˆ fd„Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFDebertaAttentionc                óz   •— t        ‰| �  di |¤Ž t        |d¬«      | _        t	        |d¬«      | _        || _        y )Nr3   r&   rk   r)   )r*   r+   Ú"TFDebertaDisentangledSelfAttentionr3   r�   Údense_outputr1   r2   s      €r6   r+   zTFDebertaAttention.__init__È   s7   ø€ Ü‰ÑÑ"˜6Ò"Ü6°vÀFÔKˆŒ	Ü/°¸XÔFˆÔØˆ�r7   c           	     ó~   — | j                  |||||||¬«      }|€|}| j                  |d   ||¬«      }	|	f|dd  z   }
|
S )N©r<   Úattention_maskÚquery_statesÚrelative_posÚrel_embeddingsÚoutput_attentionsr:   r   ©r<   r¨   r:   r    )r3   r±   )r3   r¨   r´   rµ   r¶   r·   r¸   r:   Úself_outputsÚattention_outputrk   s              r6   r?   zTFDebertaAttention.callÎ   su   € ð —y‘yØ&Ø)Ø%Ø%Ø)Ø/Øð !ó 
ˆð ÐØ'ˆLØ×,Ñ,Ø& q™/¸Èxð -ó 
Ðð #Ð$ |°A°BÐ'7Ñ7ˆàˆr7   c                óÆ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒexY w# 1 sw Y   y xY w)NTr3   r±   )rF   rG   rH   rI   r3   r'   rJ   r±   rK   s     r6   rJ   zTFDebertaAttention.buildë   sµ   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó&Ð2Ü—‘˜tŸy™yŸ~™~Ó.ñ &Ø—	‘	—‘ Ô%÷&ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ð .ð ;÷&ð &ú÷.ð .úó   ÁCÂ%CÃCÃC rM   ©NNNFF)r¨   rm   r´   rm   rµ   úOptional[tf.Tensor]r¶   r¿   r·   r¿   r¸   rP   r:   rP   rQ   úTuple[tf.Tensor]rA   r¬   rX   s   @r6   r®   r®   Ç   sq   ø„ õð -1Ø,0Ø.2Ø"'Øðàðð "ðð *ð	ð
 *ðð ,ðð  ðð ðð 
ó÷:	.r7   r®   c                  ó0   ‡ — e Zd Zdˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFDebertaIntermediatec                óT  •— t        ‰| �  di |¤Ž t        j                  j	                  |j
                  t        |j                  «      d¬«      | _        t        |j                  t        «      r"t        |j                  «      | _        || _        y |j                  | _        || _        y )Nr%   ©ÚunitsÚkernel_initializerr'   r)   )r*   r+   r   r,   r-   Úintermediate_sizer   Úinitializer_ranger%   Ú
isinstanceÚ
hidden_actÚstrr
   Úintermediate_act_fnr1   r2   s      €r6   r+   zTFDebertaIntermediate.__init__ø   sŒ   ø€ Ü‰ÑÑ"˜6Ò"ä—\‘\×'Ñ'Ø×*Ñ*¼Èv×OgÑOgÓ?hÐovð (ó 
ˆŒ
ô �f×'Ñ'¬Ô-Ü'8¸×9JÑ9JÓ'KˆDÔ$ð ˆ�ð (.×'8Ñ'8ˆDÔ$Øˆ�r7   c                óL   — | j                  |¬«      }| j                  |«      }|S ©N©rh   )r%   rÌ   ©r3   r<   s     r6   r?   zTFDebertaIntermediate.call  s(   € ØŸ
™
¨-˜
Ó8ˆØ×0Ñ0°Ó?ˆàÐr7   c                ó(  — | j                   ry d| _         t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   y xY w)NTr%   )	rF   rG   rH   rI   r%   r'   rJ   r1   rB   rK   s     r6   rJ   zTFDebertaIntermediate.build  s}   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ HØ—
‘
× Ñ  $¨¨d¯k©k×.EÑ.EÐ!FÔG÷Hð Hð 4÷Hð Hús   Á3BÂBrM   ©r<   rm   rQ   rm   rA   r¬   rX   s   @r6   rÂ   rÂ   ÷   s   ø„ õó÷Hr7   rÂ   c                  ó2   ‡ — e Zd Zdˆ fd„Zddd„Zdd„Zˆ xZS )ÚTFDebertaOutputc                óR  •— t        ‰| �  di |¤Ž t        j                  j	                  |j
                  t        |j                  «      d¬«      | _        t        j                  j                  |j                  d¬«      | _        t        |j                  d¬«      | _        || _        y )Nr%   rÄ   rŸ   r    r(   r&   r)   )r*   r+   r   r,   r-   rB   r   rÈ   r%   r¢   r£   rŸ   r/   r¤   r(   r1   r2   s      €r6   r+   zTFDebertaOutput.__init__  s†   ø€ Ü‰ÑÑ"˜6Ò"ä—\‘\×'Ñ'Ø×$Ñ$¼È×IaÑIaÓ9bÐipð (ó 
ˆŒ
ô Ÿ™×8Ñ8À×AVÑAVÐ]hÐ8ÓiˆŒÜ-¨f×.HÑ.HÈyÔYˆŒØˆ�r7   c                óx   — | j                  |¬«      }| j                  ||¬«      }| j                  ||z   «      }|S )NrÏ   r9   r¦   r§   s       r6   r?   zTFDebertaOutput.call  s=   € ØŸ
™
¨-˜
Ó8ˆØŸ™ ]¸X˜ÓFˆØŸ™ }°|Ñ'CÓDˆàÐr7   c                óî  — | j                   ry d| _         t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒÖxY w# 1 sw Y   ŒqxY w# 1 sw Y   y xY wrª   )rF   rG   rH   rI   r%   r'   rJ   r1   rÇ   rŸ   rB   r(   rK   s     r6   rJ   zTFDebertaOutput.build&  s)  € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ NØ—
‘
× Ñ  $¨¨d¯k©k×.KÑ.KÐ!LÔM÷Nä�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ LØ—‘×$Ñ$ d¨D°$·+±+×2IÑ2IÐ%JÔK÷Lä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷Nð Nú÷Lð Lú÷)ð )úr«   rM   rN   )r<   rm   r¨   rm   r:   rP   rQ   rm   rA   r¬   rX   s   @r6   rÔ   rÔ     s   ø„ õô÷)r7   rÔ   c                  óZ   ‡ — e Zd Zdˆ fd„Z	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFDebertaLayerc                ó�   •— t        ‰| �  di |¤Ž t        |d¬«      | _        t	        |d¬«      | _        t        |d¬«      | _        y )NÚ	attentionr&   Úintermediaterk   r)   )r*   r+   r®   rÛ   rÂ   rÜ   rÔ   Úbert_outputr2   s      €r6   r+   zTFDebertaLayer.__init__6  s?   ø€ Ü‰ÑÑ"˜6Ò"ä+¨F¸ÔEˆŒÜ1°&¸~ÔNˆÔÜ*¨6¸ÔAˆÕr7   c           	     óž   — | j                  |||||||¬«      }|d   }	| j                  |	¬«      }
| j                  |
|	|¬«      }|f|dd  z   }|S )N)r¨   r´   rµ   r¶   r·   r¸   r:   r   ©r<   r¹   r    )rÛ   rÜ   rÝ   )r3   r<   r´   rµ   r¶   r·   r¸   r:   Úattention_outputsr»   Úintermediate_outputÚlayer_outputÚoutputss                r6   r?   zTFDebertaLayer.call=  sˆ   € ð !ŸN™NØ&Ø)Ø%Ø%Ø)Ø/Øð +ó 
Ðð -¨QÑ/ÐØ"×/Ñ/Ð>NÐ/ÓOÐØ×'Ñ'Ø-Ð<LÐW_ð (ó 
ˆð  �/Ð$5°a°bÐ$9Ñ9ˆàˆr7   c                ó’  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   Œ¿xY w# 1 sw Y   ŒqxY w# 1 sw Y   y xY w)NTrÛ   rÜ   rÝ   )	rF   rG   rH   rI   rÛ   r'   rJ   rÜ   rÝ   rK   s     r6   rJ   zTFDebertaLayer.buildY  s	  € Ø�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4˜¨Ó-Ð9Ü—‘˜t×/Ñ/×4Ñ4Ó5ñ -Ø× Ñ ×&Ñ& tÔ,÷-ð -ð :÷+ð +ú÷.ð .ú÷-ð -ús$   ÁD%Â%D1Ã?D=Ä%D.Ä1D:Ä=ErM   r¾   ©r<   rm   r´   rm   rµ   r¿   r¶   r¿   r·   r¿   r¸   rP   r:   rP   rQ   rÀ   rA   r¬   rX   s   @r6   rÙ   rÙ   5  sr   ø„ õBð -1Ø,0Ø.2Ø"'Øðà ðð "ðð *ð	ð
 *ðð ,ðð  ðð ðð 
ó÷8-r7   rÙ   c                  ót   ‡ — e Zd Zdˆ fd„Zdd„Zd„ Zd„ Zd	d„Z	 	 	 	 	 	 d
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZ	S )ÚTFDebertaEncoderc                óL  •— t        ‰| �  di |¤Ž t        |j                  «      D �cg c]  }t	        |d|› �¬«      ‘Œ c}| _        t        |dd«      | _        || _        | j                  r4t        |dd«      | _	        | j                  dk  r|j                  | _	        y y y c c}w )	Nzlayer_._r&   Úrelative_attentionFÚmax_relative_positionsrl   r    r)   )r*   r+   ÚrangeÚnum_hidden_layersrÙ   ÚlayerrG   ré   r1   rê   Úmax_position_embeddings)r3   r1   r4   Úir5   s       €r6   r+   zTFDebertaEncoder.__init__i  sŸ   ø€ Ü‰ÑÑ"˜6Ò"äKPÐQW×QiÑQiÓKjÖkÀa”n V°H¸Q¸C°.ÖAÒkˆŒ
Ü")¨&Ð2FÈÓ"NˆÔØˆŒØ×"Ò"Ü*1°&Ð:RÐTVÓ*WˆDÔ'Ø×*Ñ*¨QÒ.Ø.4×.LÑ.L�Õ+ð /ð #ùò ls   ¨B!c                ó¾  — | j                   ry d| _         | j                  rY| j                  d| j                  dz  | j                  j
                  gt        | j                  j                  «      ¬«      | _        t        | dd «      �K| j                  D ];  }t        j                  |j                  «      5  |j                  d «       d d d «       Œ= y y # 1 sw Y   ŒIxY w)NTzrel_embeddings.weighté   ©r'   r‰   rŠ   rí   )rF   ré   rŒ   rê   r1   rB   r   rÈ   r·   rG   rí   rH   rI   r'   rJ   )r3   rL   rí   s      r6   rJ   zTFDebertaEncoder.buildt  sÃ   € Ø�:Š:ØØˆŒ
Ø×"Ò"Ø"&§/¡/Ø,Ø×2Ñ2°QÑ6¸¿¹×8OÑ8OÐPÜ+¨D¯K©K×,IÑ,IÓJð #2ó #ˆDÔô
 �4˜ $Ó'Ð3ØŸ™ò &�Ü—]‘] 5§:¡:Ó.ñ &Ø—K‘K Ô%÷&ð &ñ&ð 4÷&ð &ús   Â5CÃC	c                ó>   — | j                   r| j                  }|S d }|S rA   )ré   r·   )r3   r·   s     r6   Úget_rel_embeddingz"TFDebertaEncoder.get_rel_embeddingƒ  s*   € Ø04×0GÒ0G˜×,Ñ,ˆØÐð NRˆØÐr7   c                óˆ  — t        t        |«      «      dk  r}t        j                  t        j                  |d«      d«      }|t        j                  t        j                  |d«      d«      z  }t        j
                  |t        j                  «      }|S t        t        |«      «      dk(  rt        j                  |d«      }|S )Nrñ   r    éþÿÿÿrl   r	   )Úlenr   rH   Úexpand_dimsÚsqueezerc   Úuint8)r3   r´   Úextended_attention_masks      r6   Úget_attention_maskz#TFDebertaEncoder.get_attention_mask‡  s›   € ÜŒz˜.Ó)Ó*¨aÒ/Ü&(§n¡n´R·^±^ÀNÐTUÓ5VÐXYÓ&ZÐ#Ø4´r·~±~ÄbÇjÁjÐQhÐjlÓFmÐoqÓ7rÑrˆNÜŸW™W ^´R·X±XÓ>ˆNð Ðô ”˜NÓ+Ó,°Ò1ÜŸ^™^¨N¸AÓ>ˆNàÐr7   c                óŽ   — | j                   r8|€6|�t        |«      d   nt        |«      d   }t        |t        |«      d   «      }|S )Nrö   )ré   r   Úbuild_relative_position)r3   r<   rµ   r¶   Úqs        r6   Úget_rel_poszTFDebertaEncoder.get_rel_pos‘  sO   € Ø×"Ò" |Ð';Ø0<Ð0H”
˜<Ó(¨Ò,ÌjÐYfÓNgÐhjÑNkˆAÜ2°1´jÀÓ6OÐPRÑ6SÓTˆLØÐr7   c	           
     ó  — |rdnd }	|rdnd }
| j                  |«      }| j                  |||«      }t        |t        «      r|d   }n|}| j	                  «       }t        | j                  «      D ]i  \  }}|r|	|fz   }	 ||||||||¬«      }|d   }|�8|}t        |t        «      r(|dz   t        | j                  «      k  r||dz      nd }n|}|sŒa|
|d   fz   }
Œk |r|	|fz   }	|st        d„ ||	|
fD «       «      S t        ||	|
¬«      S )Nr)   r   r³   r    c              3  ó&   K  — | ]	  }|€Œ|–— Œ y ­wrA   r)   )Ú.0Úvs     r6   ú	<genexpr>z(TFDebertaEncoder.call.<locals>.<genexpr>Í  s   è ø€ Òh˜qÐZ[ÑZgœÑhùs   ‚Š©Úlast_hidden_stater<   Ú
attentions)
rü   r   rÉ   r   rô   Ú	enumeraterí   r÷   Útupler   )r3   r<   r´   rµ   r¶   r¸   Úoutput_hidden_statesÚreturn_dictr:   Úall_hidden_statesÚall_attentionsÚnext_kvr·   rï   Úlayer_moduleÚlayer_outputss                   r6   r?   zTFDebertaEncoder.call—  sT  € ñ #7™B¸DÐÙ0™°dˆà×0Ñ0°Ó@ˆØ×'Ñ'¨°|À\ÓRˆä�m¤XÔ.Ø# AÑ&‰Gà#ˆGà×/Ñ/Ó1ˆä(¨¯©Ó4ò 	F‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!á(Ø%Ø-Ø)Ø)Ø-Ø"3Ø!ôˆMð *¨!Ñ,ˆMàÐ'Ø,�Ü˜m¬XÔ6Ø67¸!±e¼cÀ$Ç*Á*»oÒ6M˜m¨A°©EÒ2ÐSW‘Gà'�â Ø!/°=ÀÑ3CÐ2EÑ!E‘ð/	Fñ4  Ø 1°]Ð4DÑ DÐáÜÑh ]Ð4EÀ~Ð$VÔhÓhÐhä Ø+Ð;LÐYgô
ð 	
r7   rM   rA   )NN)NNFFTF)r<   rm   r´   rm   rµ   r¿   r¶   r¿   r¸   rP   r  rP   r  rP   r:   rP   rQ   ú*Union[TFBaseModelOutput, Tuple[tf.Tensor]])
rS   rT   rU   r+   rJ   rô   rü   r   r?   rW   rX   s   @r6   rç   rç   h  sŽ   ø„ õ	Mó&òòóð -1Ø,0Ø"'Ø%*Ø Øð:
à ð:
ð "ð:
ð *ð	:
ð
 *ð:
ð  ð:
ð #ð:
ð ð:
ð ð:
ð 
4÷:
r7   rç   c                ó’  — t        j                  | t         j                  ¬«      }t        j                  |t         j                  ¬«      }|dd…df   t        j                  t        j                  |ddg«      | dg«      z
  }|d| …dd…f   }t        j
                  |d¬«      }t        j                  |t         j                  «      S )aæ  
    Build relative position according to the query and key

    We assume the absolute position of query \(P_q\) is range from (0, query_size) and the absolute position of key
    \(P_k\) is range from (0, key_size), The relative positions from query to key is \(R_{q \rightarrow k} = P_q -
    P_k\)

    Args:
        query_size (int): the length of query
        key_size (int): the length of key

    Return:
        `tf.Tensor`: A tensor with shape [1, query_size, key_size]

    r_   Nr    rl   r   ©r]   )rH   rë   Úint32ÚtileÚreshaperø   rc   Úint64)Ú
query_sizeÚkey_sizeÚq_idsÚk_idsÚrel_pos_idss        r6   rþ   rþ   Ô  s”   € ô  �H‰H�Z¤r§x¡xÔ0€EÜ�H‰H�X¤R§X¡XÔ.€EØš˜4˜‘.¤2§7¡7¬2¯:©:°e¸aÀ¸WÓ+EÈ
ÐTUÀÓ#WÑW€KØ˜k˜z˜kª1˜nÑ-€KÜ—.‘. °1Ô5€KÜ�7‰7�;¤§¡Ó)Ð)r7   c                óš   — t        |«      d   t        |«      d   t        |«      d   t        |«      d   g}t        j                  | |«      S )Nr   r    rñ   rl   ©r   rH   Úbroadcast_to)Úc2p_posÚquery_layerr¶   Úshapess       r6   Úc2p_dynamic_expandr$  ì  sP   € ä�;Ó Ñ"Ü�;Ó Ñ"Ü�;Ó Ñ"Ü�<Ó  Ñ$ð	€Fô �?‰?˜7 FÓ+Ð+r7   c                óš   — t        |«      d   t        |«      d   t        |«      d   t        |«      d   g}t        j                  | |«      S )Nr   r    rö   r  )r!  r"  Ú	key_layerr#  s       r6   Úp2c_dynamic_expandr'  ö  sP   € ä�;Ó Ñ"Ü�;Ó Ñ"Ü�9Ó˜bÑ!Ü�9Ó˜bÑ!ð	€Fô �?‰?˜7 FÓ+Ð+r7   c                ó„   — t        |«      d d t        | «      d   t        |«      d   gz   }t        j                  | |«      S )Nrñ   rö   r  )Ú	pos_indexÚp2c_attr&  r#  s       r6   Úpos_dynamic_expandr+     sC   € Ü˜Ó   !Ð$¬
°9Ó(=¸bÑ(AÄ:ÈiÓCXÐY[ÑC\Ð']Ñ]€FÜ�?‰?˜9 fÓ-Ð-r7   c                óz  — |dk  rt        j                  | «      |z   }|t        j                  | «      dz
  k7  rˆt        j                  | «      dz
  |z
  }t        j                  t        j                  t        j                  | «      «      |d¬«      }t        j                  | |¬«      } t        j                  ||¬«      }nd}t        j
                  | dt        j                  | «      d   f«      }t        j
                  |dt        j                  |«      d   f«      }t        j                  ||d¬«      }t        j
                  |t        j                  |«      «      }|dk7  rVt        j                  t        j                  t        j                  | «      «      | d¬«      }t        j                  ||¬«      }|S )Nr   r    r  ©Úpermrl   )Ú
batch_dims)rH   ÚrankÚrollrë   Ú	transposer  r‰   Úgather)r—   ÚindicesÚgather_axisÚpre_rollÚpermutationÚflat_xÚflat_indicesÚgathereds           r6   Útorch_gatherr;    s=  € Ø�Q‚Ü—g‘g˜a“j ;Ñ.ˆà”b—g‘g˜a“j 1‘nÒ$Ü—7‘7˜1“: ‘> KÑ/ˆÜ—g‘gœbŸh™h¤r§w¡w¨q£zÓ2°HÀ1ÔEˆÜ�L‰L˜ Ô-ˆÜ—,‘,˜w¨[Ô9‰àˆä�Z‰Z˜˜B¤§¡¨£¨B¡Ð0Ó1€FÜ—:‘:˜g¨¬B¯H©H°WÓ,=¸bÑ,AÐ'BÓC€LÜ�y‰y˜ ¸!Ô<€HÜ�z‰z˜(¤B§H¡H¨WÓ$5Ó6€Hà�1‚}Ü—g‘gœbŸh™h¤r§w¡w¨q£zÓ2°X°IÀAÔFˆÜ—<‘< ¨{Ô;ˆà€Or7   c                  ól   ‡ — e Zd ZdZdˆ fd„Zdd„Zd	d„Z	 	 	 	 	 d
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zd„ Zˆ xZ	S )r°   a  
    Disentangled self-attention module

    Parameters:
        config (`str`):
            A model config class instance with the configuration to build a new model. The schema is similar to
            *BertConfig*, for more details, please refer [`DebertaConfig`]

    c                óü  •— t        ‰| �  di |¤Ž |j                  |j                  z  dk7  r&t	        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  j                  | j                  dz  t        |j                  «      dd¬«      | _        |j                  �|j                  ng | _        t        |d	d«      | _        t        |d
d«      | _        | j"                  rŠt        j                  j                  | j                  t        |j                  «      dd¬«      | _        t        j                  j                  | j                  t        |j                  «      dd¬«      | _        t)        d¬«      | _        | j                   rót        |dd«      | _        | j,                  dk  r|j.                  | _        t1        |j2                  d¬«      | _        d| j                  v rEt        j                  j                  | j                  t        |j                  «      dd¬«      | _        d| j                  v rDt        j                  j                  | j                  t        |j                  «      d¬«      | _        t1        |j:                  d¬«      | _        || _        y )Nr   zThe hidden size (z6) is not a multiple of the number of attention heads (ú)r	   Úin_projF©rÆ   r'   Úuse_biasré   Útalking_headÚhead_logits_projÚhead_weights_projrl   r  rê   r    Úpos_dropoutr&   Úc2pÚpos_projÚp2cÚ
pos_q_proj)rÆ   r'   r(   r)   ) r*   r+   rB   Únum_attention_headsÚ
ValueErrorrR   Úattention_head_sizeÚall_head_sizer   r,   r-   r   rÈ   r?  Úpos_att_typerG   ré   rB  rC  rD  rZ   Úsoftmaxrê   rî   r/   r¤   rE  rG  rI  Úattention_probs_dropout_probr(   r1   r2   s      €r6   r+   z+TFDebertaDisentangledSelfAttention.__init__(  s¥  ø€ Ü‰ÑÑ"˜6Ò"Ø×Ñ × :Ñ :Ñ:¸aÒ?ÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔÜ—|‘|×)Ñ)Ø×Ñ Ñ"Ü.¨v×/GÑ/GÓHØØð	 *ó 
ˆŒð 4:×3FÑ3FÐ3R˜F×/Ò/ÐXZˆÔä")¨&Ð2FÈÓ"NˆÔÜ# F¨N¸EÓBˆÔà×ÒÜ$)§L¡L×$6Ñ$6Ø×(Ñ(Ü#2°6×3KÑ3KÓ#LØ'Øð	 %7ó %ˆDÔ!ô &+§\¡\×%7Ñ%7Ø×(Ñ(Ü#2°6×3KÑ3KÓ#LØ(Øð	 &8ó &ˆDÔ"ô )¨bÔ1ˆŒà×"Ò"Ü*1°&Ð:RÐTVÓ*WˆDÔ'Ø×*Ñ*¨QÒ.Ø.4×.LÑ.L�Ô+Ü5°f×6PÑ6PÐWdÔeˆDÔØ˜×)Ñ)Ñ)Ü %§¡× 2Ñ 2Ø×&Ñ&Ü'6°v×7OÑ7OÓ'PØ#Ø"ð	 !3ó !�”ð ˜×)Ñ)Ñ)Ü"'§,¡,×"4Ñ"4Ø×&Ñ&¼?È6×KcÑKcÓ;dÐkwð #5ó #�”ô .¨f×.QÑ.QÐXaÔbˆŒØˆ�r7   c                óJ  — | j                   ry d| _         | j                  d| j                  t        j                  j                  «       ¬«      | _        | j                  d| j                  t        j                  j                  «       ¬«      | _        t        | dd «      �dt        j                  | j                  j                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �Mt        j                  | j                   j                  «      5  | j                   j                  d «       d d d «       t        | dd «      �Mt        j                  | j"                  j                  «      5  | j"                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j$                  j                  «      5  | j$                  j                  d «       d d d «       t        | d	d «      �Mt        j                  | j&                  j                  «      5  | j&                  j                  d «       d d d «       t        | d
d «      �bt        j                  | j(                  j                  «      5  | j(                  j                  | j                  j                  g«       d d d «       t        | dd «      �ct        j                  | j*                  j                  «      5  | j*                  j                  | j                  j                  g«       d d d «       y y # 1 sw Y   �ŒRxY w# 1 sw Y   �ŒxY w# 1 sw Y   �Œ¸xY w# 1 sw Y   �ŒkxY w# 1 sw Y   �ŒxY w# 1 sw Y   Œ»xY w# 1 sw Y   y xY w)NTÚq_biasrò   Úv_biasr?  r(   rC  rD  rE  rG  rI  )rF   rŒ   rM  r   ÚinitializersÚZerosrR  rS  rG   rH   rI   r?  r'   rJ   r1   rB   r(   rC  rD  rE  rG  rI  rK   s     r6   rJ   z(TFDebertaDisentangledSelfAttention.builda  sÙ  € Ø�:Š:ØØˆŒ
Ø—o‘oØ $×"4Ñ"4Ä5×CUÑCU×C[ÑC[ÓC]ð &ó 
ˆŒð —o‘oØ $×"4Ñ"4Ä5×CUÑCU×C[ÑC[ÓC]ð &ó 
ˆŒô �4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ JØ—‘×"Ñ" D¨$°·±×0GÑ0GÐ#HÔI÷Jä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ä�4Ð+¨TÓ2Ð>Ü—‘˜t×4Ñ4×9Ñ9Ó:ñ 2Ø×%Ñ%×+Ñ+¨DÔ1÷2ä�4Ð,¨dÓ3Ð?Ü—‘˜t×5Ñ5×:Ñ:Ó;ñ 3Ø×&Ñ&×,Ñ,¨TÔ2÷3ä�4˜¨Ó-Ð9Ü—‘˜t×/Ñ/×4Ñ4Ó5ñ -Ø× Ñ ×&Ñ& tÔ,÷-ä�4˜ TÓ*Ð6Ü—‘˜tŸ}™}×1Ñ1Ó2ñ ?Ø—‘×#Ñ# T§[¡[×%<Ñ%<Ð$=Ô>÷?ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ AØ—‘×%Ñ% t§{¡{×'>Ñ'>Ð&?Ô@÷Að Að 9÷#Jñ Jú÷)ñ )ú÷2ñ 2ú÷3ñ 3ú÷-ñ -ú÷?ð ?ú÷Að AúsT   Ã	3MÄ:MÆM&Ç.M3ÉN Ê"1NÌ1NÍMÍM#Í&M0Í3M=Î N
ÎNÎN"c                óœ   — t        |«      d d | j                  dgz   }t        j                  ||¬«      }t        j                  |g d¢¬«      S )Nrl   ©Útensorr‰   ©r   rñ   r    r	   r-  )r   rJ  rH   r  r2  )r3   rX  r‰   s      r6   Útranspose_for_scoresz7TFDebertaDisentangledSelfAttention.transpose_for_scores�  sF   € Ü˜6Ó" 3 BÐ'¨4×+CÑ+CÀRÐ*HÑHˆä—‘ 6°Ô7ˆô �|‰|˜FªÔ6Ð6r7   c           	     ó¸  — |€>| j                  |«      }t        j                  | j                  |«      dd¬«      \  }	}
}�n€d„ }t        j                  t        j                  | j                   j
                  d   «      | j                  dz  d¬«      }t        j                  | j                  d¬«      }t        j                  d«      D ]Œ  }t        j                  | j                  | j                  ¬«      }t        j                  | j                  «      D ]  }|j                  |||dz  |z      «      }Œ |j                  ||j                  «       «      }ŒŽ dgdz  } ||d   |d   |«      } ||d   |d   |«      } ||d	   |d	   |«      }| j                  |«      }	| j                  |«      }
| j                  |«      }|	| j                  | j                  dddd…f   «      z   }	|| j                  | j                  dddd…f   «      z   }d}dt        | j                  «      z   }t!        j"                  t%        |	«      d   |z  «      }|	|z  }	t        j&                  |	t        j                  |
g d
¢«      «      }| j(                  r(| j+                  ||¬«      }| j-                  |	|
|||«      }|�||z   }| j.                  r=t        j                  | j1                  t        j                  |g d¢«      «      g d¢«      }| j3                  ||«      }| j5                  ||¬«      }| j.                  r=t        j                  | j7                  t        j                  |g d¢«      «      g d¢«      }t        j&                  ||«      }t        j                  |g d¢«      }t%        |«      }|dd |d   |d   z  gz   }t        j8                  ||«      }|r||f}|S |f}|S )a¤  
        Call the module

        Args:
            hidden_states (`tf.Tensor`):
                Input states to the module usually the output from previous layer, it will be the Q,K and V in
                *Attention(Q,K,V)*

            attention_mask (`tf.Tensor`):
                An attention mask matrix of shape [*B*, *N*, *N*] where *B* is the batch size, *N* is the maximum
                sequence length in which element [i,j] = *1* means the *i* th token in the input can attend to the *j*
                th token.

            return_att (`bool`, *optional*):
                Whether return the attention matrix.

            query_states (`tf.Tensor`, *optional*):
                The *Q* state in *Attention(Q,K,V)*.

            relative_pos (`tf.Tensor`):
                The relative position encoding between the tokens in the sequence. It's of shape [*B*, *N*, *N*] with
                values ranging in [*-max_relative_positions*, *max_relative_positions*].

            rel_embeddings (`tf.Tensor`):
                The embedding of relative distances. It's a tensor of shape [\(2 \times
                \text{max_relative_positions}\), *hidden_size*].


        Nr	   rl   )Únum_or_size_splitsr]   c                ój   — t        j                  || d¬«      }|�|t        j                  |«      z  }|S )NT)Útranspose_b)rH   Úmatmulr2  )ÚwÚbr—   Úouts       r6   Úlinearz7TFDebertaDisentangledSelfAttention.call.<locals>.linear·  s0   € Ü—i‘i  1°$Ô7�Ø�=Øœ2Ÿ<™<¨›?Ñ*�CØ�
r7   r   )r`   r…   r    rñ   ©r   r    r	   rñ   r9   )r   rñ   r	   r    )r   r	   r    rñ   rY  rö   )r?  rH   ÚsplitrZ  r2  rˆ   rJ  ÚTensorArrayr`   rë   ÚwriteÚconcatrR  rS  r÷   rN  r•   r–   r   r_  ré   rE  Údisentangled_att_biasrB  rC  rO  r(   rD  r  )r3   r<   r´   rµ   r¶   r·   r¸   r:   Úqpr"  r&  Úvalue_layerrc  ÚwsÚqkvwÚkÚqkvw_insiderï   Úqkvbrÿ   r  Úrel_attÚscale_factorrw   Úattention_scoresÚattention_probsÚcontext_layerÚcontext_layer_shapeÚnew_context_layer_shaperã   s                                 r6   r?   z'TFDebertaDisentangledSelfAttention.call‰  s§  € ðN ÐØ—‘˜mÓ,ˆBÜ24·(±(Ø×)Ñ)¨"Ó-À!È"ô3Ñ/ˆK˜¢Kò
ô —‘Ü—‘˜TŸ\™\×0Ñ0°Ñ3Ó4È×IaÑIaÐdeÑIeÐlmôˆBô —>‘>¨¯
©
¸Ô;ˆDÜ—X‘X˜a“[ò ;�Ü Ÿn™n°4·:±:ÀD×D\ÑD\Ô]�ÜŸ™ $×":Ñ":Ó;ò F�AØ"-×"3Ñ"3°A°r¸!¸a¹%À!¹)±}Ó"E‘KðFà—z‘z ! [×%7Ñ%7Ó%9Ó:‘ð	;ð
 �6˜A‘:ˆDá�t˜A‘w  Q¡¨Ó6ˆAÙ�t˜A‘w  Q¡¨Ó7ˆAÙ�t˜A‘w  Q¡¨Ó7ˆAØ×3Ñ3°AÓ6ˆKØ×1Ñ1°!Ó4ˆIØ×3Ñ3°AÓ6ˆKà! D×$=Ñ$=¸d¿k¹kÈ$ÐPTÒVWÈ-Ñ>XÓ$YÑYˆØ! D×$=Ñ$=¸d¿k¹kÈ$ÐPTÒVWÈ-Ñ>XÓ$YÑYˆàˆàœ3˜t×0Ñ0Ó1Ñ1ˆÜ—	‘	œ* [Ó1°"Ñ5¸ÑDÓEˆØ! EÑ)ˆäŸ9™9 [´"·,±,¸yÊ,Ó2WÓXÐØ×"Ò"Ø!×-Ñ-¨nÀxÐ-ÓPˆNØ×0Ñ0°¸iÈÐWeÐgsÓtˆGàÐØ/°'Ñ9Ðà×ÒÜ!Ÿ|™|Ø×%Ñ%¤b§l¡lÐ3CÂ\Ó&RÓSÒUaó Ðð Ÿ,™,Ð'7¸ÓHˆØŸ,™, À˜,ÓJˆØ×ÒÜ Ÿl™lØ×&Ñ&¤r§|¡|°OÂ\Ó'RÓSÒUaóˆOô Ÿ	™	 /°;Ó?ˆÜŸ™ ]²LÓAˆÜ(¨Ó7Ðð
 #6°c°rÐ":Ð>QÐRTÑ>UÐXkÐlnÑXoÑ>oÐ=pÑ"pÐÜŸ
™
 =Ð2IÓJˆÙ6G�= /Ð2ˆØˆð O\ÐM]ˆØˆr7   c           
     ó0  — |€&t        |«      d   }t        |t        |«      d   «      }t        |«      }t        |«      dk(  r+t        j                  t        j                  |d«      d«      }nJt        |«      dk(  rt        j                  |d«      }n%t        |«      dk7  rt        dt        |«      › �«      ‚t        j                  t        j                  t        j                  t        |«      d   t        |«      d   «      | j                  «      t        j                  «      }t        j                  || j                  |z
  | j                  |z   …d d …f   d«      }d}	d| j                  v r‹| j                  |«      }
| j                  |
«      }
t        j                  |t        j                  |
g d	¢«      «      }t        j                   ||z   d|dz  dz
  «      }t#        |t%        |||«      d
«      }|	|z  }	d| j                  v �rŽ| j'                  |«      }| j                  |«      }|t        j(                  j+                  t        j                  t        |«      d
   |z  | j,                  ¬«      «      z  }t        |«      d   t        |«      d   k7  r%t        t        |«      d   t        |«      d   «      }n|}t        j                   | |z   d|dz  dz
  «      }t        j                  |t        j                  |g d	¢«      «      }t        j                  t#        |t/        |||«      d
«      g d	¢«      }t        |«      d   t        |«      d   k7  r;t        j                  |d d …d d …d d …df   d
«      }t#        |t1        |||«      d«      }|	|z  }	|	S )Nrö   rñ   r   r	   r    é   z2Relative position ids must be of dim 2 or 3 or 4. rF  rd  rl   rH  r_   )r   rþ   r÷   rH   rø   rK  rc   ÚminimumÚmaximumrê   r  rN  rG  rZ  r_  r2  Úclip_by_valuer;  r$  rI  r•   r–   rf   r'  r+  )r3   r"  r&  r¶   r·   rr  rÿ   Úshape_list_posÚatt_spanÚscoreÚpos_key_layerÚc2p_attr!  Úpos_query_layerÚr_posÚp2c_posr*  r)  s                     r6   ri  z8TFDebertaDisentangledSelfAttention.disentangled_att_biasø  s@  € ØÐÜ˜;Ó'¨Ñ+ˆAÜ2°1´jÀÓ6KÈBÑ6OÓPˆLÜ# LÓ1ˆÜˆ~Ó !Ò#ÜŸ>™>¬"¯.©.¸ÀqÓ*IÈ1ÓM‰LÜ�Ó  AÒ%ÜŸ>™>¨,¸Ó:‰Lä�Ó  AÒ%ÜÐQÔRUÐVdÓReÐQfÐgÓhÐhä—7‘7Ü�J‰JÜ—
‘
œ: kÓ2°2Ñ6¼
À9Ó8MÈbÑ8QÓRÐTX×ToÑToóô �H‰Hó	
ˆô Ÿ™Ø˜4×6Ñ6¸ÑAÀD×D_ÑD_ÐbjÑDjÐjÒlmÐmÑnÐpqó
ˆð ˆð �D×%Ñ%Ñ%Ø ŸM™M¨.Ó9ˆMØ ×5Ñ5°mÓDˆMÜ—i‘i ¬R¯\©\¸-ÊÓ-VÓWˆGÜ×&Ñ& |°hÑ'>ÀÀ8ÈaÁ<ÐRSÑCSÓTˆGÜ" 7Ô,>¸wÈÐUaÓ,bÐdfÓgˆGØ�WÑˆEð �D×%Ñ%Ò%Ø"Ÿo™o¨nÓ=ˆOØ"×7Ñ7¸ÓHˆOØœrŸw™wŸ|™|Ü—‘œ
 ?Ó3°BÑ7¸,ÑFÈd×N`ÑN`Ôaó ñ ˆOô ˜+Ó& rÑ*¬j¸Ó.CÀBÑ.GÒGÜ/´
¸9Ó0EÀbÑ0IÌ:ÐV_ÓK`ÐacÑKdÓe‘à$�Ü×&Ñ&¨ v°Ñ'8¸!¸XÈ¹\ÈAÑ=MÓNˆGÜ—i‘i 	¬2¯<©<¸ÊÓ+VÓWˆGÜ—l‘lÜ˜WÔ&8¸À+ÈyÓ&YÐ[]Ó^Ò`lóˆGô ˜+Ó& rÑ*¬j¸Ó.CÀBÑ.GÒGÜŸN™N¨<ºº1ºaÀ¸
Ñ+CÀRÓH�	Ü& wÔ0BÀ9ÈgÐW`Ó0aÐceÓf�Ø�WÑˆEàˆr7   rM   rA   )rX  rm   rQ   rm   r¾   rå   )
rS   rT   rU   rn   r+   rJ   rZ  r?   ri  rW   rX   s   @r6   r°   r°     s�   ø„ ñõ7órAó@7ð -1Ø,0Ø.2Ø"'Øðmà ðmð "ðmð *ð	mð
 *ðmð ,ðmð  ðmð ðmð 
ómö^7r7   r°   c                  óZ   ‡ — e Zd ZdZˆ fd„Zdd„Z	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚTFDebertaEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.c                óH  •— t        ‰| �  di |¤Ž || _        t        |d|j                  «      | _        |j                  | _        |j                  | _        t        |dd«      | _        |j                  | _        | j
                  |j                  k7  rEt        j                  j                  |j                  t        |j                  «      dd¬«      | _        t        j                  j                  |j                  d¬«      | _        t#        |j$                  d	¬
«      | _        y )NÚembedding_sizeÚposition_biased_inputTÚ
embed_projFr@  rŸ   r    r(   r&   r)   )r*   r+   r1   rG   rB   rˆ  rî   r‰  rÈ   r   r,   r-   r   rŠ  r¢   r£   rŸ   r/   r¤   r(   r2   s      €r6   r+   zTFDebertaEmbeddings.__init__5  sò   ø€ Ü‰ÑÑ"˜6Ò"àˆŒÜ% fÐ.>À×@RÑ@RÓSˆÔØ!×-Ñ-ˆÔØ'-×'EÑ'EˆÔ$Ü%,¨VÐ5LÈdÓ%SˆÔ"Ø!'×!9Ñ!9ˆÔØ×Ñ &×"4Ñ"4Ò4Ü#Ÿl™l×0Ñ0Ø×"Ñ"Ü#2°6×3KÑ3KÓ#LØ!Øð	 1ó ˆDŒOô Ÿ™×8Ñ8À×AVÑAVÐ]hÐ8ÓiˆŒÜ-¨f×.HÑ.HÈyÔYˆ�r7   c                óú  — t        j                  d«      5  | j                  d| j                  j                  | j
                  gt        | j                  «      ¬«      | _        d d d «       t        j                  d«      5  | j                  j                  dkD  rM| j                  d| j                  j                  | j
                  gt        | j                  «      ¬«      | _
        nd | _
        d d d «       t        j                  d«      5  | j                  rC| j                  d| j                  | j                  gt        | j                  «      ¬«      | _        nd | _        d d d «       | j                  ry d| _        t!        | d	d «      �dt        j                  | j"                  j$                  «      5  | j"                  j'                  d d | j                  j                  g«       d d d «       t!        | d
d «      �Mt        j                  | j(                  j$                  «      5  | j(                  j'                  d «       d d d «       t!        | dd «      �[t        j                  | j*                  j$                  «      5  | j*                  j'                  d d | j
                  g«       d d d «       y y # 1 sw Y   �ŒQxY w# 1 sw Y   �ŒÓxY w# 1 sw Y   �ŒlxY w# 1 sw Y   ŒóxY w# 1 sw Y   Œ¥xY w# 1 sw Y   y xY w)NÚword_embeddingsrˆ   rò   Útoken_type_embeddingsr   Ú
embeddingsÚposition_embeddingsTrŸ   r(   rŠ  )rH   rI   rŒ   r1   Ú
vocab_sizerˆ  r   rÈ   rˆ   Útype_vocab_sizer�  r‰  rî   rB   r�  rF   rG   rŸ   r'   rJ   r(   rŠ  rK   s     r6   rJ   zTFDebertaEmbeddings.buildH  s|  € Ü�]‰]Ð,Ó-ñ 	ØŸ/™/ØØ—{‘{×-Ñ-¨t×/BÑ/BÐCÜ+¨D×,BÑ,BÓCð *ó ˆDŒK÷	ô �]‰]Ð2Ó3ñ 	2Ø�{‰{×*Ñ*¨QÒ.Ø-1¯_©_Ø%ØŸ;™;×6Ñ6¸×8KÑ8KÐLÜ /°×0FÑ0FÓ Gð .=ó .�Õ*ð .2�Ô*÷	2ô �]‰]Ð0Ó1ñ 	0Ø×)Ò)Ø+/¯?©?Ø%Ø×7Ñ7¸×9IÑ9IÐJÜ /°×0FÑ0FÓ Gð ,;ó ,�Õ(ð ,0�Ô(÷	0ð �:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ LØ—‘×$Ñ$ d¨D°$·+±+×2IÑ2IÐ%JÔK÷Lä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ IØ—‘×%Ñ% t¨T°4×3FÑ3FÐ&GÔH÷Ið Ið 9÷I	ñ 	ú÷	2ñ 	2ú÷	0ñ 	0ú÷Lð Lú÷)ð )ú÷Ið IúsJ   –AJ2Â A.J?ÄAKÆ43KÈ%K%É?)K1Ê2J<Ê?K	ËKËK"Ë%K.Ë1K:c                ó.  — |€|€t        d«      ‚|�At        || j                  j                  «       t	        j
                  | j                  |¬«      }t        |«      dd }|€t	        j                  |d¬«      }|€/t	        j                  t	        j                  d|d   ¬«      d¬«      }|}| j                  r&t	        j
                  | j                  |¬«      }	||	z  }| j                  j                  dkD  r&t	        j
                  | j                  |¬«      }
||
z  }| j                  | j                   k7  r| j#                  |«      }| j%                  |«      }|�§t'        t        |«      «      t'        t        |«      «      k7  ryt'        t        |«      «      d	k(  r,t	        j(                  t	        j(                  |d
¬«      d
¬«      }t	        j*                  t	        j                  |d¬«      | j,                  ¬«      }||z  }| j/                  ||¬«      }|S )z’
        Applies embedding based on inputs tensor.

        Returns:
            final_embeddings (`tf.Tensor`): output embedding tensor.
        Nz5Need to provide either `input_ids` or `input_embeds`.)Úparamsr4  rl   r   ©ÚdimsÚvalue)ÚstartÚlimitr  ry  r    rñ   r_   r9   )rK  r   r1   r�  rH   r3  rˆ   r   Úfillrø   rë   r‰  r�  r‘  r�  rˆ  rB   rŠ  rŸ   r÷   rù   rc   rf   r(   )r3   Ú	input_idsÚposition_idsÚtoken_type_idsÚinputs_embedsri   r:   rL   Úfinal_embeddingsÚposition_embedsÚtoken_type_embedss              r6   r?   zTFDebertaEmbeddings.callq  sÇ  € ð Ð Ð!6ÜÐTÓUÐUàÐ Ü*¨9°d·k±k×6LÑ6LÔMÜŸI™I¨T¯[©[À)ÔLˆMä  Ó/°°Ð4ˆàÐ!ÜŸW™W¨+¸QÔ?ˆNàÐÜŸ>™>¬"¯(©(¸À+ÈbÁ/Ô*RÐYZÔ[ˆLà(ÐØ×%Ò%Ü Ÿi™i¨t×/GÑ/GÐQ]Ô^ˆOØ Ñ/ÐØ�;‰;×&Ñ&¨Ò*Ü "§	¡	°×1KÑ1KÐUcÔ dÐØÐ 1Ñ1Ðà×Ñ $×"2Ñ"2Ò2Ø#Ÿ™Ð/?Ó@ÐàŸ>™>Ð*:Ó;ÐàÐÜ”:˜dÓ#Ó$¬¬JÐ7GÓ,HÓ(IÒIÜ”z $Ó'Ó(¨AÒ-ÜŸ:™:¤b§j¡j°¸AÔ&>ÀQÔG�DÜ—w‘wœrŸ~™~¨d¸Ô;À4×CUÑCUÔV�à/°$Ñ6ÐàŸ<™<Ð(8À8˜<ÓLÐàÐr7   rA   )NNNNNF)rš  r¿   r›  r¿   rœ  r¿   r�  r¿   ri   r¿   r:   rP   rQ   rm   r›   rX   s   @r6   r†  r†  2  sp   ø„ ÙQôZó&'IðV *.Ø,0Ø.2Ø-1Ø$(Øð5 à&ð5 ð *ð5 ð ,ð	5 ð
 +ð5 ð "ð5 ð ð5 ð 
÷5 r7   r†  c                  ó0   ‡ — e Zd Zdˆ fd„Zdd„Zdd„Zˆ xZS )Ú TFDebertaPredictionHeadTransformc                óÞ  •— t        ‰| �  di |¤Ž t        |d|j                  «      | _        t
        j                  j                  | j                  t        |j                  «      d¬«      | _
        t        |j                  t        «      rt        |j                  «      | _        n|j                  | _        t
        j                  j!                  |j"                  d¬«      | _        || _        y )Nrˆ  r%   rÄ   rŸ   r    r)   )r*   r+   rG   rB   rˆ  r   r,   r-   r   rÈ   r%   rÉ   rÊ   rË   r
   Útransform_act_fnr¢   r£   rŸ   r1   r2   s      €r6   r+   z)TFDebertaPredictionHeadTransform.__init__ª  s¼   ø€ Ü‰ÑÑ"˜6Ò"ä% fÐ.>À×@RÑ@RÓSˆÔä—\‘\×'Ñ'Ø×%Ñ%Ü.¨v×/GÑ/GÓHØð (ó 
ˆŒ
ô �f×'Ñ'¬Ô-Ü$5°f×6GÑ6GÓ$HˆDÕ!à$*×$5Ñ$5ˆDÔ!ÜŸ™×8Ñ8À×AVÑAVÐ]hÐ8ÓiˆŒØˆ�r7   c                ón   — | j                  |¬«      }| j                  |«      }| j                  |«      }|S rÎ   )r%   r¤  rŸ   rÐ   s     r6   r?   z%TFDebertaPredictionHeadTransform.call¼  s6   € ØŸ
™
¨-˜
Ó8ˆØ×-Ñ-¨mÓ<ˆØŸ™ }Ó5ˆàÐr7   c                ó  — | j                   ry d| _         t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   ŒrxY w# 1 sw Y   y xY w)NTr%   rŸ   )rF   rG   rH   rI   r%   r'   rJ   r1   rB   rŸ   rˆ  rK   s     r6   rJ   z&TFDebertaPredictionHeadTransform.buildÃ  sÚ   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ HØ—
‘
× Ñ  $¨¨d¯k©k×.EÑ.EÐ!FÔG÷Hä�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ HØ—‘×$Ñ$ d¨D°$×2EÑ2EÐ%FÔG÷Hð Hð 8÷Hð Hú÷Hð Hús   Á3C/Â<)C;Ã/C8Ã;DrM   rÒ   rA   r¬   rX   s   @r6   r¢  r¢  ©  s   ø„ õó$÷	Hr7   r¢  c                  óP   ‡ — e Zd Zdˆ fd„Zd	d„Zd
d„Zdd„Zdd„Zdd„Zdd„Z	ˆ xZ
S )ÚTFDebertaLMPredictionHeadc                óœ   •— t        ‰| �  di |¤Ž || _        t        |d|j                  «      | _        t        |d¬«      | _        || _        y )Nrˆ  Ú	transformr&   r)   )	r*   r+   r1   rG   rB   rˆ  r¢  rª  Úinput_embeddings©r3   r1   r«  r4   r5   s       €r6   r+   z"TFDebertaLMPredictionHead.__init__Ð  sJ   ø€ Ü‰ÑÑ"˜6Ò"àˆŒÜ% fÐ.>À×@RÑ@RÓSˆÔä9¸&À{ÔSˆŒð !1ˆÕr7   c                óX  — | j                  | j                  j                  fddd¬«      | _        | j                  ry d| _        t        | dd «      �Nt        j                  | j                  j                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NÚzerosTr‹   )r‰   rŠ   Ú	trainabler'   rª  )rŒ   r1   r�  r‹   rF   rG   rH   rI   rª  r'   rJ   rK   s     r6   rJ   zTFDebertaLMPredictionHead.buildÜ  s‘   € Ø—O‘O¨4¯;©;×+AÑ+AÐ*CÐQXÐdhÐou�OÓvˆŒ	à�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ð +ð 8÷+ð +ús   Á:B Â B)c                ó   — | j                   S rA   )r«  rC   s    r6   Úget_output_embeddingsz/TFDebertaLMPredictionHead.get_output_embeddingsæ  s   € Ø×$Ñ$Ð$r7   c                ó`   — || j                   _        t        |«      d   | j                   _        y ©Nr   )r«  rˆ   r   r�  ©r3   r–  s     r6   Úset_output_embeddingsz/TFDebertaLMPredictionHead.set_output_embeddingsé  s(   € Ø',ˆ×ÑÔ$Ü+5°eÓ+<¸QÑ+?ˆ×ÑÕ(r7   c                ó   — d| j                   iS )Nr‹   )r‹   rC   s    r6   Úget_biasz"TFDebertaLMPredictionHead.get_biasí  s   € Ø˜Ÿ	™	Ð"Ð"r7   c                óX   — |d   | _         t        |d   «      d   | j                  _        y )Nr‹   r   )r‹   r   r1   r�  r´  s     r6   Úset_biasz"TFDebertaLMPredictionHead.set_biasð  s'   € Ø˜&‘MˆŒ	Ü!+¨E°&©MÓ!:¸1Ñ!=ˆ�‰Õr7   c                ó–  — | j                  |¬«      }t        |«      d   }t        j                  |d| j                  g¬«      }t        j
                  || j                  j                  d¬«      }t        j                  |d|| j                  j                  g¬«      }t        j                  j                  || j                  ¬«      }|S )Nrß   r    rl   rW  T)Úara  r^  )r–  r‹   )rª  r   rH   r  rˆ  r_  r«  rˆ   r1   r�  ÚnnÚbias_addr‹   )r3   r<   Ú
seq_lengths      r6   r?   zTFDebertaLMPredictionHead.callô  sŸ   € ØŸ™°]˜ÓCˆÜ Ó.¨qÑ1ˆ
ÜŸ
™
¨-ÀÀD×DWÑDWÐ?XÔYˆÜŸ	™	 M°T×5JÑ5J×5QÑ5QÐ_cÔdˆÜŸ
™
¨-ÀÀJÐPT×P[ÑP[×PfÑPfÐ?gÔhˆÜŸ™Ÿ™¨]ÀÇÁ˜ÓKˆàÐr7   ©r1   r!   r«  úkeras.layers.LayerrA   ©rQ   rÀ  ©r–  ztf.Variable)rQ   zDict[str, tf.Variable]rÒ   )rS   rT   rU   r+   rJ   r±  rµ  r·  r¹  r?   rW   rX   s   @r6   r¨  r¨  Ï  s'   ø„ õ
1ó+ó%ó@ó#ó>÷r7   r¨  c                  ó0   ‡ — e Zd Zdˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFDebertaOnlyMLMHeadc                óJ   •— t        ‰| �  di |¤Ž t        ||d¬«      | _        y )NÚpredictionsr&   r)   )r*   r+   r¨  rÆ  r¬  s       €r6   r+   zTFDebertaOnlyMLMHead.__init__   s&   ø€ Ü‰ÑÑ"˜6Ò"Ü4°VÐ=MÐTaÔbˆÕr7   c                ó*   — | j                  |¬«      }|S )Nrß   )rÆ  )r3   Úsequence_outputÚprediction_scoress      r6   r?   zTFDebertaOnlyMLMHead.call  s   € Ø ×,Ñ,¸?Ð,ÓKÐà Ð r7   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTrÆ  )rF   rG   rH   rI   rÆ  r'   rJ   rK   s     r6   rJ   zTFDebertaOnlyMLMHead.build	  sm   € Ø�:Š:ØØˆŒ
Ü�4˜¨Ó-Ð9Ü—‘˜t×/Ñ/×4Ñ4Ó5ñ -Ø× Ñ ×&Ñ& tÔ,÷-ð -ð :÷-ð -úó   ÁA1Á1A:r¿  )rÈ  rm   rQ   rm   rA   r¬   rX   s   @r6   rÄ  rÄ  ÿ  s   ø„ õcó!÷
-r7   rÄ  c                  óŽ   ‡ — e Zd ZeZdˆ fd„Zdd„Zd	d„Zd„ Ze		 	 	 	 	 	 	 	 	 d
	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       Z
dd„Zˆ xZS )ÚTFDebertaMainLayerc                óz   •— t        ‰| �  di |¤Ž || _        t        |d¬«      | _        t        |d¬«      | _        y )NrŽ  r&   Úencoderr)   )r*   r+   r1   r†  rŽ  rç   rÏ  r2   s      €r6   r+   zTFDebertaMainLayer.__init__  s6   ø€ Ü‰ÑÑ"˜6Ò"àˆŒä-¨f¸<ÔHˆŒÜ'¨°YÔ?ˆ�r7   c                ó   — | j                   S rA   )rŽ  rC   s    r6   Úget_input_embeddingsz'TFDebertaMainLayer.get_input_embeddings  s   € Ø�‰Ðr7   c                ó`   — || j                   _        t        |«      d   | j                   _        y r³  )rŽ  rˆ   r   r�  r´  s     r6   Úset_input_embeddingsz'TFDebertaMainLayer.set_input_embeddings!  s$   € Ø!&ˆ�‰ÔÜ%/°Ó%6°qÑ%9ˆ�‰Õ"r7   c                ó   — t         ‚)z�
        Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base
        class PreTrainedModel
        )ÚNotImplementedError)r3   Úheads_to_prunes     r6   Ú_prune_headszTFDebertaMainLayer._prune_heads%  s
   € ô
 "Ð!r7   c
                ó˜  — |�|�t        d«      ‚|�t        |«      }
n|�t        |«      d d }
nt        d«      ‚|€t        j                  |
d¬«      }|€t        j                  |
d¬«      }| j	                  ||||||	¬«      }| j                  ||||||	¬«      }|d   }|s	|f|dd  z   S t        ||j                  |j                  ¬	«      S )
NzDYou cannot specify both input_ids and inputs_embeds at the same timerl   z5You have to specify either input_ids or inputs_embedsr    r”  r   )rš  r›  rœ  r�  ri   r:   )r<   r´   r¸   r  r  r:   r  )	rK  r   rH   r™  rŽ  rÏ  r   r<   r  )r3   rš  r´   rœ  r›  r�  r¸   r  r  r:   rL   Úembedding_outputÚencoder_outputsrÈ  s                 r6   r?   zTFDebertaMainLayer.call,  s  € ð Ð  ]Ð%>ÜÐcÓdÐdØÐ"Ü$ YÓ/‰KØÐ&Ü$ ]Ó3°C°RÐ8‰KäÐTÓUÐUàÐ!ÜŸW™W¨+¸QÔ?ˆNàÐ!ÜŸW™W¨+¸QÔ?ˆNàŸ?™?ØØ%Ø)Ø'ØØð +ó 
Ðð Ÿ,™,Ø*Ø)Ø/Ø!5Ø#Øð 'ó 
ˆð *¨!Ñ,ˆáØ#Ð%¨¸¸Ð(;Ñ;Ð;ä Ø-Ø)×7Ñ7Ø&×1Ñ1ô
ð 	
r7   c                óÆ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒexY w# 1 sw Y   y xY w)NTrŽ  rÏ  )rF   rG   rH   rI   rŽ  r'   rJ   rÏ  rK   s     r6   rJ   zTFDebertaMainLayer.builde  sµ   € Ø�:Š:ØØˆŒ
Ü�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ ,Ø—‘×%Ñ% dÔ+÷,ä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷,ð ,ú÷)ð )úr½   rM   rÁ  rÂ  ©	NNNNNNNNF)rš  úTFModelInputType | Noner´   únp.ndarray | tf.Tensor | Nonerœ  rÞ  r›  rÞ  r�  rÞ  r¸   úOptional[bool]r  rß  r  rß  r:   rP   rQ   r  rA   )rS   rT   rU   r!   Úconfig_classr+   rÑ  rÓ  r×  r   r?   rJ   rW   rX   s   @r6   rÍ  rÍ    s¶   ø„ Ø €Lõ@óó:ò"ð ð .2Ø8<Ø8<Ø6:Ø7;Ø,0Ø/3Ø&*Øð6
à*ð6
ð 6ð6
ð 6ð	6
ð
 4ð6
ð 5ð6
ð *ð6
ð -ð6
ð $ð6
ð ð6
ð 
4ò6
ó ð6
÷p	)r7   rÍ  c                  ó   — e Zd ZdZeZdZy)ÚTFDebertaPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚdebertaN)rS   rT   rU   rn   r!   rà  Úbase_model_prefixr)   r7   r6   râ  râ  q  s   „ ñð
 !€LØ!Ñr7   râ  a1
  
    The DeBERTa model was proposed in [DeBERTa: Decoding-enhanced BERT with Disentangled
    Attention](https://arxiv.org/abs/2006.03654) by Pengcheng He, Xiaodong Liu, Jianfeng Gao, Weizhu Chen. It's build
    on top of BERT/RoBERTa with two improvements, i.e. disentangled attention and enhanced mask decoder. With those two
    improvements, it out perform BERT/RoBERTa on a majority of tasks with 80GB pretraining data.

    This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it
    as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and
    behavior.

    <Tip>

    TensorFlow models and layers in `transformers` accept two formats as input:

    - having all inputs as keyword arguments (like PyTorch models), or
    - having all inputs as a list, tuple or dict in the first positional argument.

    The reason the second format is supported is that Keras methods prefer this format when passing inputs to models
    and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just
    pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second
    format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with
    the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first
    positional argument:

    - a single Tensor with `input_ids` only and nothing else: `model(input_ids)`
    - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring:
    `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])`
    - a dictionary with one or several input Tensors associated to the input names given in the docstring:
    `model({"input_ids": input_ids, "token_type_ids": token_type_ids})`

    Note that when creating models and layers with
    [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry
    about any of this, as you can just pass inputs like you would to any other Python function!

    </Tip>

    Parameters:
        config ([`DebertaConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a�	  
    Args:
        input_ids (`np.ndarray`, `tf.Tensor`, `List[tf.Tensor]` ``Dict[str, tf.Tensor]` or `Dict[str, np.ndarray]` and each example must have the shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`np.ndarray` or `tf.Tensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        inputs_embeds (`np.ndarray` or `tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput``] instead of a plain tuple.
zaThe bare DeBERTa Model transformer outputting raw hidden-states without any specific head on top.c                  óÄ   ‡ — e Zd Zdˆ fd„Ze eej                  d«      «       ee	e
e¬«      	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       «       «       Zd	d„Zˆ xZS )
ÚTFDebertaModelc                óP   •— t        ‰| �  |g|¢­i |¤Ž t        |d¬«      | _        y )Nrã  r&   )r*   r+   rÍ  rã  ©r3   r1   rh   r4   r5   s       €r6   r+   zTFDebertaModel.__init__Ö  s(   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3ä)¨&°yÔAˆ�r7   úbatch_size, sequence_length©Ú
checkpointÚoutput_typerà  c
                ó:   — | j                  |||||||||	¬«	      }
|
S )N©	rš  r´   rœ  r›  r�  r¸   r  r  r:   )rã  )r3   rš  r´   rœ  r›  r�  r¸   r  r  r:   rã   s              r6   r?   zTFDebertaModel.callÛ  s9   € ð& —,‘,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#Øð ó 

ˆð ˆr7   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTrã  )rF   rG   rH   rI   rã  r'   rJ   rK   s     r6   rJ   zTFDebertaModel.buildü  si   € Ø�:Š:ØØˆŒ
Ü�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷)ð )úrË  rM   rÜ  )rš  rÝ  r´   rÞ  rœ  rÞ  r›  rÞ  r�  rÞ  r¸   rß  r  rß  r  rß  r:   rß  rQ   r  rA   )rS   rT   rU   r+   r   r   ÚDEBERTA_INPUTS_DOCSTRINGÚformatr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCr?   rJ   rW   rX   s   @r6   ræ  ræ  Ñ  sÒ   ø„ õ
Bð
 Ù*Ð+C×+JÑ+JÐKhÓ+iÓjÙØ&Ø%Ø$ôð .2Ø8<Ø8<Ø6:Ø7;Ø,0Ø/3Ø&*Ø#(ðà*ðð 6ðð 6ð	ð
 4ðð 5ðð *ðð -ðð $ðð !ðð 
4òóó kó ð÷4)r7   ræ  z5DeBERTa Model with a `language modeling` head on top.c                  óÒ   ‡ — e Zd Zdˆ fd„Zdd„Ze eej                  d«      «       e	e
ee¬«      	 	 	 	 	 	 	 	 	 	 d		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d
d„«       «       «       Zdd„Zˆ xZS )ÚTFDebertaForMaskedLMc                óà   •— t        ‰| �  |g|¢­i |¤Ž |j                  rt        j	                  d«       t        |d¬«      | _        t        || j                  j                  d¬«      | _	        y )NzpIf you want to use `TFDebertaForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.rã  r&   Úcls)r«  r'   )
r*   r+   Ú
is_decoderÚloggerÚwarningrÍ  rã  rÄ  rŽ  Úmlmrè  s       €r6   r+   zTFDebertaForMaskedLM.__init__  s_   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3à×ÒÜ�N‰Nð1ôô
 *¨&°yÔAˆŒÜ'¨ÀÇÁ×AXÑAXÐ_dÔeˆ�r7   c                ó.   — | j                   j                  S rA   )rû  rÆ  rC   s    r6   Úget_lm_headz TFDebertaForMaskedLM.get_lm_head  s   € Ø�x‰x×#Ñ#Ð#r7   ré  rê  c                ó  — | j                  |||||||||
¬«	      }|d   }| j                  ||
¬«      }|	€dn| j                  |	|¬«      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j
                  ¬«      S )a«  
        labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        rî  r   )rÈ  r:   N©ÚlabelsÚlogitsrñ   ©Úlossr  r<   r  )rã  rû  Úhf_compute_lossr   r<   r  )r3   rš  r´   rœ  r›  r�  r¸   r  r  r   r:   rã   rÈ  rÉ  r  rk   s                   r6   r?   zTFDebertaForMaskedLM.call  s¾   € ð4 —,‘,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#Øð ó 

ˆð " !™*ˆØ ŸH™H°_Èx˜HÓXÐØ�~‰t¨4×+?Ñ+?ÀvÐVgÐ+?Ó+hˆáØ'Ð)¨G°A°B¨KÑ7ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r7   c                óÆ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒexY w# 1 sw Y   y xY w)NTrã  rû  )rF   rG   rH   rI   rã  r'   rJ   rû  rK   s     r6   rJ   zTFDebertaForMaskedLM.buildJ  s±   € Ø�:Š:ØØˆŒ
Ü�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ä�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ %Ø—‘—‘˜tÔ$÷%ð %ð 2÷)ð )ú÷%ð %úr½   rM   rÁ  ©
NNNNNNNNNF)rš  rÝ  r´   rÞ  rœ  rÞ  r›  rÞ  r�  rÞ  r¸   rß  r  rß  r  rß  r   rÞ  r:   rß  rQ   z)Union[TFMaskedLMOutput, Tuple[tf.Tensor]]rA   )rS   rT   rU   r+   rý  r   r   rð  rñ  r   rò  r   ró  r?   rJ   rW   rX   s   @r6   rõ  rõ    så   ø„ õ
fó$ð Ù*Ð+C×+JÑ+JÐKhÓ+iÓjÙØ&Ø$Ø$ôð .2Ø8<Ø8<Ø6:Ø7;Ø,0Ø/3Ø&*Ø04Ø#(ð+
à*ð+
ð 6ð+
ð 6ð	+
ð
 4ð+
ð 5ð+
ð *ð+
ð -ð+
ð $ð+
ð .ð+
ð !ð+
ð 
3ò+
óó kó ð+
÷Z	%r7   rõ  zŸ
    DeBERTa Model transformer with a sequence classification/regression head on top (a linear layer on top of the
    pooled output) e.g. for GLUE tasks.
    c                  óÊ   ‡ — e Zd Zdˆ fd„Ze eej                  d«      «       ee	e
e¬«      	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       «       «       Zd	d„Zˆ xZS )
Ú"TFDebertaForSequenceClassificationc                óÆ  •— t        ‰| �  |g|¢­i |¤Ž |j                  | _        t        |d¬«      | _        t        |d¬«      | _        t        |dd «      }|€| j                  j                  n|}t        |d¬«      | _        t        j                  j                  |j                  t        |j                   «      d¬«      | _        | j                  j$                  | _        y )Nrã  r&   ÚpoolerÚcls_dropoutÚ
classifierrÄ   )r*   r+   Ú
num_labelsrÍ  rã  r#   r
  rG   r1   r¤   r/   r(   r   r,   r-   r   rÈ   r  rD   )r3   r1   rh   r4   Údrop_outr5   s        €r6   r+   z+TFDebertaForSequenceClassification.__init__^  s½   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3à ×+Ñ+ˆŒä)¨&°yÔAˆŒÜ,¨V¸(ÔCˆŒä˜6 =°$Ó7ˆØ6>Ð6F�4—;‘;×2Ò2ÈHˆÜ-¨h¸]ÔKˆŒÜŸ,™,×,Ñ,Ø×#Ñ#Ü.¨v×/GÑ/GÓHØð -ó 
ˆŒð
 Ÿ+™+×0Ñ0ˆ�r7   ré  rê  c                óL  — | j                  |||||||||
¬«	      }|d   }| j                  ||
¬«      }| j                  ||
¬«      }| j                  |«      }|	€dn| j	                  |	|¬«      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a–  
        labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        rî  r   r9   Nrÿ  r    r  )rã  r
  r(   r  r  r   r<   r  )r3   rš  r´   rœ  r›  r�  r¸   r  r  r   r:   rã   rÈ  r>   r  r  rk   s                    r6   r?   z'TFDebertaForSequenceClassification.callp  sÜ   € ð4 —,‘,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#Øð ó 

ˆð " !™*ˆØŸ™ O¸h˜ÓGˆØŸ™ ]¸X˜ÓFˆØ—‘ Ó/ˆØ�~‰t¨4×+?Ñ+?ÀvÐV\Ð+?Ó+]ˆáØ�Y ¨¨ Ñ,ˆFà)-Ð)9�T�G˜fÑ$ÐE¸vÐEä)ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r7   c                óz  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �[t        j                  | j                  j
                  «      5  | j                  j                  d d | j                  g«       d d d «       y y # 1 sw Y   �Œ'xY w# 1 sw Y   ŒÙxY w# 1 sw Y   Œ‹xY w# 1 sw Y   y xY w)NTrã  r
  r(   r  )rF   rG   rH   rI   rã  r'   rJ   r
  r(   r  rD   rK   s     r6   rJ   z(TFDebertaForSequenceClassification.build§  s]  € Ø�:Š:ØØˆŒ
Ü�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ä�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ (Ø—‘×!Ñ! $Ô'÷(ä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ EØ—‘×%Ñ% t¨T°4·?±?Ð&CÔD÷Eð Eð 9÷)ñ )ú÷(ð (ú÷)ð )ú÷Eð Eús0   ÁFÂ%FÃ?F%Å)F1ÆFÆF"Æ%F.Æ1F:rM   r  )rš  rÝ  r´   rÞ  rœ  rÞ  r›  rÞ  r�  rÞ  r¸   rß  r  rß  r  rß  r   rÞ  r:   rß  rQ   z3Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]rA   )rS   rT   rU   r+   r   r   rð  rñ  r   rò  r   ró  r?   rJ   rW   rX   s   @r6   r  r  V  sà   ø„ õ1ð$ Ù*Ð+C×+JÑ+JÐKhÓ+iÓjÙØ&Ø.Ø$ôð .2Ø8<Ø8<Ø6:Ø7;Ø,0Ø/3Ø&*Ø04Ø#(ð.
à*ð.
ð 6ð.
ð 6ð	.
ð
 4ð.
ð 5ð.
ð *ð.
ð -ð.
ð $ð.
ð .ð.
ð !ð.
ð 
=ò.
óó kó ð.
÷`Er7   r  z¦
    DeBERTa Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                  óÊ   ‡ — e Zd Zdˆ fd„Ze eej                  d«      «       ee	e
e¬«      	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       «       «       Zd	d„Zˆ xZS )
ÚTFDebertaForTokenClassificationc                óf  •— t        ‰| �  |g|¢­i |¤Ž |j                  | _        t        |d¬«      | _        t
        j                  j                  |j                  ¬«      | _	        t
        j                  j                  |j                  t        |j                  «      d¬«      | _        || _        y )Nrã  r&   )Úrater  rÄ   )r*   r+   r  rÍ  rã  r   r,   ÚDropoutr¤   r(   r-   r   rÈ   r  r1   rè  s       €r6   r+   z(TFDebertaForTokenClassification.__init__Á  s‘   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3à ×+Ñ+ˆŒä)¨&°yÔAˆŒÜ—|‘|×+Ñ+°×1KÑ1KÐ+ÓLˆŒÜŸ,™,×,Ñ,Ø×#Ñ#¼È×H`ÑH`Ó8aÐhtð -ó 
ˆŒð ˆ�r7   ré  rê  c                ó(  — | j                  |||||||||
¬«	      }|d   }| j                  ||
¬«      }| j                  |¬«      }|	€dn| j                  |	|¬«      }|s|f|dd z   }|�|f|z   S |S t	        |||j
                  |j                  ¬«      S )	zä
        labels (`tf.Tensor` or `np.ndarray` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        rî  r   r9   rÏ   Nrÿ  r    r  )rã  r(   r  r  r   r<   r  )r3   rš  r´   rœ  r›  r�  r¸   r  r  r   r:   rã   rÈ  r  r  rk   s                   r6   r?   z$TFDebertaForTokenClassification.callÍ  sÌ   € ð0 —,‘,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#Øð ó 

ˆð " !™*ˆØŸ,™, À˜,ÓJˆØ—‘¨�Ó8ˆØ�~‰t¨4×+?Ñ+?ÀvÐV\Ð+?Ó+]ˆáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä&ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r7   c                óô  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   Œ|xY w# 1 sw Y   y xY w)NTrã  r  )
rF   rG   rH   rI   rã  r'   rJ   r  r1   rB   rK   s     r6   rJ   z%TFDebertaForTokenClassification.build   óË   € Ø�:Š:ØØˆŒ
Ü�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ MØ—‘×%Ñ% t¨T°4·;±;×3JÑ3JÐ&KÔL÷Mð Mð 9÷)ð )ú÷Mð Múó   ÁC"Â%3C.Ã"C+Ã.C7rM   r  )rš  rÝ  r´   rÞ  rœ  rÞ  r›  rÞ  r�  rÞ  r¸   rß  r  rß  r  rß  r   rÞ  r:   rß  rQ   z0Union[TFTokenClassifierOutput, Tuple[tf.Tensor]]rA   )rS   rT   rU   r+   r   r   rð  rñ  r   rò  r   ró  r?   rJ   rW   rX   s   @r6   r  r  ¹  sà   ø„ õ
ð Ù*Ð+C×+JÑ+JÐKhÓ+iÓjÙØ&Ø+Ø$ôð .2Ø8<Ø8<Ø6:Ø7;Ø,0Ø/3Ø&*Ø04Ø#(ð*
à*ð*
ð 6ð*
ð 6ð	*
ð
 4ð*
ð 5ð*
ð *ð*
ð -ð*
ð $ð*
ð .ð*
ð !ð*
ð 
:ò*
óó kó ð*
÷X	Mr7   r  zà
    DeBERTa Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear
    layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                  óÐ   ‡ — e Zd Zdˆ fd„Ze eej                  d«      «       ee	e
e¬«      	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       «       «       Zd	d„Zˆ xZS )
ÚTFDebertaForQuestionAnsweringc                ó  •— t        ‰| �  |g|¢­i |¤Ž |j                  | _        t        |d¬«      | _        t
        j                  j                  |j                  t        |j                  «      d¬«      | _
        || _        y )Nrã  r&   Ú
qa_outputsrÄ   )r*   r+   r  rÍ  rã  r   r,   r-   r   rÈ   r  r1   rè  s       €r6   r+   z&TFDebertaForQuestionAnswering.__init__  sr   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3à ×+Ñ+ˆŒä)¨&°yÔAˆŒÜŸ,™,×,Ñ,Ø×#Ñ#¼È×H`ÑH`Ó8aÐhtð -ó 
ˆŒð ˆ�r7   ré  rê  c                ó²  — | j                  |||||||||¬«	      }|d   }| j                  |¬«      }t        j                  |dd¬«      \  }}t        j                  |d¬«      }t        j                  |d¬«      }d}|	� |
�d	|	i}|
|d
<   | j                  |||f¬«      }|s||f|dd z   }|�|f|z   S |S t        ||||j                  |j                  ¬«      S )a  
        start_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`tf.Tensor` or `np.ndarray` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        rî  r   rÏ   rñ   rl   )r–  r\  r]   )Úinputr]   NÚstart_positionÚend_positionrÿ  )r  Ústart_logitsÚ
end_logitsr<   r  )	rã  r  rH   re  rù   r  r   r<   r  )r3   rš  r´   rœ  r›  r�  r¸   r  r  Ústart_positionsÚend_positionsr:   rã   rÈ  r  r"  r#  r  r   rk   s                       r6   r?   z"TFDebertaForQuestionAnswering.call  s  € ð> —,‘,ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#Øð ó 

ˆð " !™*ˆØ—‘¨�Ó8ˆÜ#%§8¡8°&ÈQÐUWÔ#XÑ ˆ�jÜ—z‘z¨¸2Ô>ˆÜ—Z‘Z j°rÔ:ˆ
ØˆàÐ&¨=Ð+DØ&¨Ð8ˆFØ%2ˆF�>Ñ"Ø×'Ñ'¨v¸|ÈZÐ>XÐ'ÓYˆDáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä-ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r7   c                óô  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   Œ|xY w# 1 sw Y   y xY w)NTrã  r  )
rF   rG   rH   rI   rã  r'   rJ   r  r1   rB   rK   s     r6   rJ   z#TFDebertaForQuestionAnswering.builda  r  r  rM   )NNNNNNNNNNF)rš  rÝ  r´   rÞ  rœ  rÞ  r›  rÞ  r�  rÞ  r¸   rß  r  rß  r  rß  r$  rÞ  r%  rÞ  r:   rß  rQ   z7Union[TFQuestionAnsweringModelOutput, Tuple[tf.Tensor]]rA   )rS   rT   rU   r+   r   r   rð  rñ  r   rò  r   ró  r?   rJ   rW   rX   s   @r6   r  r    sî   ø„ õ	ð Ù*Ð+C×+JÑ+JÐKhÓ+iÓjÙØ&Ø2Ø$ôð .2Ø8<Ø8<Ø6:Ø7;Ø,0Ø/3Ø&*Ø9=Ø7;Ø#(ð9
à*ð9
ð 6ð9
ð 6ð	9
ð
 4ð9
ð 5ð9
ð *ð9
ð -ð9
ð $ð9
ð 7ð9
ð 5ð9
ð !ð9
ð 
Aò9
óó kó ð9
÷v	Mr7   r  )rõ  r  r  r  ræ  râ  )Prn   Ú
__future__r   r•   Útypingr   r   r   r   r   ÚnumpyÚnpÚ
tensorflowrH   Úactivations_tfr
   Úmodeling_tf_outputsr   r   r   r   r   Úmodeling_tf_utilsr   r   r   r   r   r   r   r   r   Útf_utilsr   r   r   Úutilsr   r   r   r   Úconfiguration_debertar!   Ú
get_loggerrS   rù  ró  rò  r,   ÚLayerr#   rZ   r/   rƒ   r�   r®   rÂ   rÔ   rÙ   rç   rþ   r$  r'  r+  r;  r°   r†  r¢  r¨  rÄ  rÍ  râ  ÚDEBERTA_START_DOCSTRINGrð  ræ  rõ  r  r  r  Ú__all__r)   r7   r6   ú<module>r6     s  ðñ å "ã ß 9Õ 9ã Û å /÷õ ÷
÷ 
õ 
÷ SÑ Rß uÓ uÝ 0ð 
ˆ×	Ñ	˜HÓ	%€ð "€Ø.Ð ô)˜UŸ\™\×/Ñ/ô )ô@˜Ÿ™×*Ñ*ô ô,%˜UŸ\™\×/Ñ/ô %ôP9˜Ÿ™×+Ñ+ô 9ô()˜%Ÿ,™,×,Ñ,ô )ô:-.˜Ÿ™×+Ñ+ô -.ô`H˜EŸL™L×.Ñ.ô Hô:)�e—l‘l×(Ñ(ô )ôB0-�U—\‘\×'Ñ'ô 0-ôfi
�u—|‘|×)Ñ)ô i
òX*ò0,ò,ò.ò
ô0R¨¯©×);Ñ);ô Rôjt ˜%Ÿ,™,×,Ñ,ô t ôn#H u§|¡|×'9Ñ'9ô #HôL- §¡× 2Ñ 2ô -ô`-˜5Ÿ<™<×-Ñ-ô -ô([)˜Ÿ™×+Ñ+ô [)ô|"Ð0ô "ð(Ð ðT)Ð ñX ØgØóô-)Ð-ó -)ó	ð-)ñ` ÐQÐSjÓkôM%Ð3Ð5Qó M%ó lðM%ñ` ðð óôYEÐ)AÐC_ó YEóðYEñx ðð óôIMÐ&>Ð@Yó IMóðIMñX ðð óôWMÐ$<Ð>Uó WMóðWMòt�r7   