Ë
    S^(h ã                  óp  — d Z ddlmZ ddlZddlmZmZmZ ddlZ	ddl
ZddlmZmZmZ ddlmZmZmZmZmZ ddlmZmZmZmZmZmZmZmZmZ dd	l m!Z!m"Z" dd
l#m$Z$ ddl%m&Z&  e$jN                  e(«      Z)dZ*dZ+d„ Z,d„ Z-d„ Z.d„ Z/ G d„ dej`                  jb                  «      Z2 G d„ dej`                  jb                  «      Z3 G d„ dej`                  jb                  «      Z4 G d„ dej`                  jb                  «      Z5 G d„ dej`                  jb                  «      Z6 G d„ dej`                  jb                  «      Z7 G d„ d ej`                  jb                  «      Z8 G d!„ d"ej`                  jb                  «      Z9 G d#„ d$ej`                  jb                  «      Z: G d%„ d&ej`                  jb                  «      Z; G d'„ d(ej`                  jb                  «      Z< G d)„ d*e«      Z=d+Z>d,Z? ed-e>«       G d.„ d/ej`                  jb                  «      «       Z@ ed-e>«       G d0„ d1e=«      «       ZA ed2e>«       G d3„ d4e=e«      «       ZB G d5„ d6ej`                  jb                  «      ZC ed7e>«       G d8„ d9e=e«      «       ZD ed:e>«       G d;„ d<e=e«      «       ZE G d=„ d>ej`                  jb                  «      ZFdAd?„ZGg d@¢ZHy)BzPyTorch ESM model.é    )ÚannotationsN)ÚOptionalÚTupleÚUnioné   )Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forward)Ú+TFBaseModelOutputWithPastAndCrossAttentionsÚ.TFBaseModelOutputWithPoolingAndCrossAttentionsÚTFMaskedLMOutputÚTFSequenceClassifierOutputÚTFTokenClassifierOutput)	ÚTFMaskedLanguageModelingLossÚTFModelInputTypeÚTFPreTrainedModelÚTFSequenceClassificationLossÚTFTokenClassificationLossÚget_initializerÚkerasÚ
shape_listÚunpack_inputs)Úcheck_embeddings_within_boundsÚstable_softmax)Úloggingé   )Ú	EsmConfigzfacebook/esm2_t6_8M_UR50Dr   c                ól   — t        j                  | dd¬«      \  }}t        j                  | |fd¬«      S )Né   éÿÿÿÿ©Úaxis)ÚtfÚsplitÚconcat)ÚxÚx1Úx2s      úe/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/esm/modeling_tf_esm.pyÚrotate_halfr*   7   s/   € Ü�X‰X�a˜ Ô$�F€BˆÜ�9‰9�r�c˜2�Y RÔ(Ð(ó    c                óÆ   — |d d …d d …d t        j                  | «      d   …d d …f   }|d d …d d …d t        j                  | «      d   …d d …f   }| |z  t        | «      |z  z   S )Néþÿÿÿ)r#   Úshaper*   )r&   ÚcosÚsins      r)   Úapply_rotary_pos_embr1   <   sd   € Ø
Ša’Ð%”b—h‘h˜q“k "‘oÐ%¢qÐ(Ñ
)€CØ
Ša’Ð%”b—h‘h˜q“k "‘oÐ%¢qÐ(Ñ
)€Cà�‰Gœ A›¨Ñ,Ñ-Ð-r+   c                óF   — | t         j                  j                  | «      z   S )zJMake layer symmetric in final two dimensions, used for contact prediction.)r#   ÚlinalgÚmatrix_transpose)r&   s    r)   Ú
symmetrizer5   C   s   € àŒr�y‰y×)Ñ)¨!Ó,Ñ,Ð,r+   c                ó´   — t        j                  | dd¬«      }t        j                  | dd¬«      }t        j                  | dd¬«      }||z  }||z  }| |z
  }|S )z=Perform average product correct, used for contact prediction.r    T)Úkeepdimsr-   )r    r-   )r#   Ú
reduce_sum)r&   Úa1Úa2Úa12ÚavgÚ
normalizeds         r)   Úaverage_product_correctr>   H   sY   € ä	�‰�q˜" tÔ	,€BÜ	�‰�q˜" tÔ	,€BÜ
�-‰-˜˜8¨dÔ
3€Cà
ˆr‰'€CØ
�‰)€CØ�S‘€JØÐr+   c                  ó@   ‡ — e Zd ZdZddˆ fd„Zˆ fd„Zdd„Zd	d„Zˆ xZS )
ÚTFRotaryEmbeddingzå
    Rotary position embeddings based on those in
    [RoFormer](https://huggingface.co/docs/transformers/model_doc/roformer). Query and keys are transformed by rotation
    matrices which depend on their relative positions.
    c                ó4   •— t         ‰| �  |¬«       || _        y )N©Úname)ÚsuperÚ__init__Údim)ÚselfrF   rC   Ú	__class__s      €r)   rE   zTFRotaryEmbedding.__init__[   s   ø€ Ü‰Ñ˜dÐÔ#ð ˆ�r+   c           
     ó^  •— t         ‰| �  |«       | j                  d| j                  dz  ft        j
                  t        d«      d¬«      | _        | j                  j                  ddt	        j                  d| j                  dt        j
                  ¬«      | j                  z  z  z  «       y )	NÚinv_freqr   ç      ð?F)r.   ÚdtypeÚinitializerÚ	trainablei'  r   )ÚstartÚlimitÚdeltarL   )
rD   ÚbuildÚ
add_weightrF   r#   Úfloat32r   rJ   ÚassignÚrange)rG   Úinput_shaperH   s     €r)   rR   zTFRotaryEmbedding.builde   s‰   ø€ Ü‰‰�kÔ"ØŸ™Ø˜tŸx™x¨1™}Ð.´b·j±jÌoÐ^aÓNbÐnsð (ó 
ˆŒð 	�‰×ÑØ�5œRŸX™X¨A°T·X±XÀQÌbÏjÉjÔYÐ\`×\dÑ\dÑdÑeÑfõ	
r+   c                óf  — t        j                  |«      |   }t        j                  || j                  j                  ¬«      }t        j
                  d|| j                  «      }t        j                  ||fd¬«      d d d d …d d …f   }t        j                  |«      t        j                  |«      fS )N©rL   z
i, j -> ijr    r!   )	r#   r.   rV   rJ   rL   Úeinsumr%   r/   r0   )rG   r&   Úseq_dimensionÚseq_lenÚtÚfreqsÚembs          r)   Ú_compute_cos_sinz"TFRotaryEmbedding._compute_cos_sinn   s�   € Ü—(‘(˜1“+˜mÑ,ˆä�H‰H�W D§M¡M×$7Ñ$7Ô8ˆÜ—	‘	˜,¨¨4¯=©=Ó9ˆÜ�i‰i˜ ˜¨RÔ0°°tºQÂÐ1AÑBˆä�v‰v�c‹{œBŸF™F 3›KÐ'Ð'r+   c                ób   — | j                  |d¬«      \  }}t        |||«      t        |||«      fS )Nr-   )r[   )r`   r1   )rG   ÚqÚkÚcos_embÚsin_embs        r)   ÚcallzTFRotaryEmbedding.callw   s@   € Ø×0Ñ0°À"Ð0ÓEÑˆ�ô !  G¨WÓ5Ü   G¨WÓ5ð
ð 	
r+   ©N)rF   Úint)r   )rb   ú	tf.Tensorrc   ri   ÚreturnzTuple[tf.Tensor, tf.Tensor])	Ú__name__Ú
__module__Ú__qualname__Ú__doc__rE   rR   r`   rf   Ú__classcell__©rH   s   @r)   r@   r@   T   s   ø„ ñöô
ó(÷
r+   r@   c                  ó@   ‡ — e Zd ZdZ	 	 	 d	 	 	 dˆ fd„Zdd„Zd„ Zˆ xZS )ÚTFEsmContactPredictionHeadzWPerforms symmetrization, apc, and computes a logistic regression on the output featuresc                ó’   •— t         ‰| �  |¬«       || _        || _        t        j
                  j                  d|dd¬«      | _        y )NrB   r   ÚsigmoidÚ
regression)Úuse_biasÚ
activationrC   )rD   rE   Úeos_idxÚin_featuresr   ÚlayersÚDenseru   )rG   ry   Úbiasrx   rC   rH   s        €r)   rE   z#TFEsmContactPredictionHead.__init__ƒ   sD   ø€ ô 	‰Ñ˜dÐÔ#ØˆŒØ&ˆÔÜŸ,™,×,Ñ,¨Q¸È)ÐZfÐ,Ógˆ�r+   c                ó  — | j                   ry d| _         t        | dd «      �Zt        j                  | j                  j
                  «      5  | j                  j                  d | j                  f«       d d d «       y y # 1 sw Y   y xY w)NTru   )ÚbuiltÚgetattrr#   Ú
name_scoperu   rC   rR   ry   ©rG   rW   s     r)   rR   z TFEsmContactPredictionHead.build�   sy   € Ø�:Š:ØØˆŒ
Ü�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ @Ø—‘×%Ñ% t¨T×-=Ñ-=Ð&>Ô?÷@ð @ð 9÷@ð @ús   Á(A=Á=Bc                ó  — t        j                  || j                  k7  |j                  «      }t        j                  |d«      t        j                  |d«      z  }||d d …d d d d …d d …f   z  }|dd d…d d…f   }|ddd …dd …f   }t        |«      \  }}}}}t        j                  ||||z  ||f«      }t        t        |«      «      }t        j                  |d¬«      }t        j                  | j                  |«      d«      S )Nr   r   .r    )r   r   r   r   ©Úpermr   )r#   Úcastrx   rL   Úexpand_dimsr   Úreshaper>   r5   Ú	transposeÚsqueezeru   )	rG   ÚtokensÚ
attentionsÚeos_maskÚ
batch_sizerz   ÚheadsÚseqlenÚ_s	            r)   rf   zTFEsmContactPredictionHead.call—   sõ   € ä—7‘7˜6 T§\¡\Ñ1°:×3CÑ3CÓDˆÜ—>‘> (¨AÓ.´·±ÀÈ!Ó1LÑLˆØ (ª1¨d°Dº!ºQÐ+>Ñ"?Ñ?ˆ
Ø  S b S¨#¨2¨# Ñ.ˆ
à  Q¡R¨© Ñ,ˆ
Ü/9¸*Ó/EÑ,ˆ
�F˜E 6¨1Ü—Z‘Z 
¨Z¸À%¹ÈÐQWÐ,XÓYˆ
ô -¬Z¸
Ó-CÓDˆ
Ü—\‘\ *°<Ô@ˆ
Ü�z‰z˜$Ÿ/™/¨*Ó5°qÓ9Ð9r+   )Tr   N)ry   rh   rx   rh   rg   )rk   rl   rm   rn   rE   rR   rf   ro   rp   s   @r)   rr   rr   €   s6   ø„ Ùað
 ØØð
hàð
hð õ	
hó@ö:r+   rr   c                  ó<   ‡ — e Zd ZdZdˆ fd„	Z	 dd„Zd„ Zdd„Zˆ xZS )ÚTFEsmEmbeddingszV
    Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
    c                óØ  •— t         ‰| �  |¬«       t        j                  j	                  |j
                  |j                  t        |j                  «      d¬«      | _	        t        j                  j	                  |j                  |j                  t        |j                  «      d¬«      | _        |j                  r1t        j                  j                  |j                  d¬«      | _        nd | _        t!        |dd«      | _        t%        j&                  |j                  «      d d d …f   | _        |j*                  | _        |j.                  | _        |j0                  | _        || _        y )	NrB   Úword_embeddings)Úembeddings_initializerrC   Úposition_embeddingsÚ
layer_norm©ÚepsilonrC   Úposition_embedding_typeÚabsolute)rD   rE   r   rz   Ú	EmbeddingÚ
vocab_sizeÚhidden_sizer   Úinitializer_ranger”   Úmax_position_embeddingsr–   Úemb_layer_norm_beforeÚLayerNormalizationÚlayer_norm_epsr—   r   rš   r#   rV   Úposition_idsÚpad_token_idÚpadding_idxÚtoken_dropoutÚmask_token_idÚconfig©rG   r©   rC   rH   s      €r)   rE   zTFEsmEmbeddings.__init__­   s-  ø€ Ü‰Ñ˜dÐÔ#Ü$Ÿ|™|×5Ñ5Ø×ÑØ×ÑÜ#2°6×3KÑ3KÓ#LØ"ð	  6ó  
ˆÔô $)§<¡<×#9Ñ#9Ø×*Ñ*Ø×ÑÜ#2°6×3KÑ3KÓ#LØ&ð	 $:ó $
ˆÔ ð ×'Ò'Ü#Ÿl™l×=Ñ=Àf×F[ÑF[ÐbnÐ=ÓoˆD�Oà"ˆDŒOô (/¨vÐ7PÐR\Ó']ˆÔ$äŸH™H V×%CÑ%CÓDÀTÊ1ÀWÑMˆÔà!×.Ñ.ˆÔØ#×1Ñ1ˆÔØ#×1Ñ1ˆÔØˆ�r+   c                óZ  — |€+|�t        || j                  |«      }n| j                  |«      }|€1t        || j                  j
                  «       | j                  |«      }|}| j                  rÁt        j                  || j                  k(  d d …d d …d f   d|«      }d}t        j                  t        j                  |d¬«      t        j                  «      }|| j                  k(  }	t        j                  j                  |	t        j                  d¬«      |z  }
|d|z
  z  d|
z
  d d …d d f   z  }| j                   dk(  r| j#                  |«      }||z  }| j$                  �| j%                  |«      }|�7|t        j                  t        j&                  |d«      |j(                  «      z  }|S )Ng        g¸…ëQ¸¾?r    r!   )rL   r"   r   r›   )Ú"create_position_ids_from_input_idsr¦   Ú&create_position_ids_from_inputs_embedsr   r©   r�   r”   r§   r#   Úwherer¨   r…   r8   rT   ÚmathÚcount_nonzerorš   r–   r—   r†   rL   )rG   Ú	input_idsÚattention_maskr¤   Úinputs_embedsÚpast_key_values_lengthÚ
embeddingsÚmask_ratio_trainÚsrc_lengthsÚmasked_tokensÚmask_ratio_observedr–   s               r)   rf   zTFEsmEmbeddings.callË   s˜  € ð ÐØÐ$äAÀ)ÈT×M]ÑM]Ð_uÓv‘à#×JÑJÈ=ÓY�àÐ Ü*¨9°d·k±k×6LÑ6LÔMØ ×0Ñ0°Ó;ˆMð #ˆ
ð ×ÒÜŸ™ 9°×0BÑ0BÑ#BÂAÂqÈ$ÀJÑ"OÐQTÐV`ÓaˆJØ)ÐÜŸ'™'¤"§-¡-°ÀRÔ"HÌ"Ï*É*ÓUˆKØ%¨×);Ñ);Ñ;ˆMÜ"$§'¡'×"7Ñ"7¸ÌRÏZÉZÐ^`Ð"7Ó"aÐdoÑ"oÐØ# qÐ+;Ñ';Ñ<ÀÐDWÑ@WÒYZÐ\`ÐbfÐYfÑ?gÑgˆJà×'Ñ'¨:Ò5Ø"&×":Ñ":¸<Ó"HÐØÐ-Ñ-ˆJà�?‰?Ð&ØŸ™¨Ó4ˆJØÐ%Ø#¤b§g¡g¬b¯n©n¸^ÈRÓ.PÐR\×RbÑRbÓ&cÑcˆJð Ðr+   c                ó  — t        |«      dd }|d   }t        j                  | j                  dz   || j                  z   dz   t        j                  ¬«      }t        j
                  t        j                  |d«      |«      S )zÑ
        We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

        Args:
            inputs_embeds: tf.Tensor

        Returns: tf.Tensor
        Nr    r   )rO   rP   rL   r   )r   r#   rV   r¦   Úint64Úbroadcast_tor†   )rG   r³   rW   Úsequence_lengthr¤   s        r)   r­   z6TFEsmEmbeddings.create_position_ids_from_inputs_embedsø   st   € ô ! Ó/°°Ð4ˆØ% a™.ˆä—x‘xØ×"Ñ" QÑ&¨oÀ×@PÑ@PÑ.PÐSTÑ.TÔ\^×\dÑ\dô
ˆô �‰œrŸ~™~¨l¸AÓ>ÀÓLÐLr+   c                óÀ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   ŒÖxY w# 1 sw Y   ŒˆxY w# 1 sw Y   y xY w)NTr”   r–   r—   )r~   r   r#   r€   r”   rC   rR   r–   r—   r©   rž   r�   s     r)   rR   zTFEsmEmbeddings.build	  s!  € Ø�:Š:ØØˆŒ
Ü�4Ð*¨DÓ1Ð=Ü—‘˜t×3Ñ3×8Ñ8Ó9ñ 1Ø×$Ñ$×*Ñ*¨4Ô0÷1ä�4Ð.°Ó5ÐAÜ—‘˜t×7Ñ7×<Ñ<Ó=ñ 5Ø×(Ñ(×.Ñ.¨tÔ4÷5ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ MØ—‘×%Ñ% t¨T°4·;±;×3JÑ3JÐ&KÔL÷Mð Mð 9÷1ð 1ú÷5ð 5ú÷Mð Múó$   ÁD<Â%EÃ?3EÄ<EÅEÅErg   )NNNNr   )	rk   rl   rm   rn   rE   rf   r­   rR   ro   rp   s   @r)   r’   r’   ¨   s&   ø„ ñõð> rsó+òZM÷"Mr+   r’   c                  ój   ‡ — e Zd Zdˆ fd„	Zdd„Z	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zd	d„Zˆ xZS )
ÚTFEsmSelfAttentionc                ó*  •— t         ‰| �  |¬«       |j                  |j                  z  dk7  r2t	        |d«      s&t        d|j                  › d|j                  › d�«      ‚|j                  | _        t        |j                  |j                  z  «      | _        | j                  | j                  z  | _        t        j                  j                  | j                  t        |j                  «      d¬«      | _        t        j                  j                  | j                  t        |j                  «      d	¬«      | _        t        j                  j                  | j                  t        |j                  «      d
¬«      | _        t        j                  j#                  |j$                  «      | _        |xs t)        |dd«      | _        d | _        | j*                  dk(  s| j*                  dk(  rf|j.                  | _        t        j                  j1                  d|j.                  z  dz
  | j                  t        |j                  «      ¬«      | _        n+| j*                  dk(  rt5        | j                  d¬«      | _        |j6                  | _        || _        y )NrB   r   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (ú)Úquery©Úkernel_initializerrC   ÚkeyÚvaluerš   r›   Úrelative_keyÚrelative_key_queryr   r   )r•   ÚrotaryÚrotary_embeddings)rF   rC   )rD   rE   rž   Únum_attention_headsÚhasattrÚ
ValueErrorrh   Úattention_head_sizeÚall_head_sizer   rz   r{   r   rŸ   rÅ   rÈ   rÉ   ÚDropoutÚattention_probs_dropout_probÚdropoutr   rš   rÍ   r    rœ   Údistance_embeddingr@   Ú
is_decoderr©   )rG   r©   rš   rC   rH   s       €r)   rE   zTFEsmSelfAttention.__init__  s@  ø€ Ü‰Ñ˜dÐÔ#Ø×Ñ × :Ñ :Ñ:¸aÒ?ÌÐPVÐXhÔHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð
 $*×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔä—\‘\×'Ñ'Ø×Ñ´?À6×C[ÑC[Ó3\Ðcjð (ó 
ˆŒ
ô —<‘<×%Ñ%Ø×Ñ´?À6×C[ÑC[Ó3\Ðchð &ó 
ˆŒô —\‘\×'Ñ'Ø×Ñ´?À6×C[ÑC[Ó3\Ðcjð (ó 
ˆŒ
ô —|‘|×+Ñ+¨F×,OÑ,OÓPˆŒØ'>ò (
Ä'ØÐ-¨zóC
ˆÔ$ð "&ˆÔØ×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÒ=qØ+1×+IÑ+IˆDÔ(Ü&+§l¡l×&<Ñ&<Ø�F×2Ñ2Ñ2°QÑ6Ø×(Ñ(Ü'6°v×7OÑ7OÓ'Pð '=ó 'ˆDÕ#ð
 ×)Ñ)¨XÒ5Ü%6¸4×;SÑ;SÐZmÔ%nˆDÔ"à ×+Ñ+ˆŒØˆ�r+   c                óª   — t        |«      d d | j                  | j                  gz   }t        j                  ||«      }t        j
                  |d¬«      S )Nr    ©r   r   r   r   rƒ   )r   rÎ   rÑ   r#   r‡   rˆ   )rG   r&   Únew_x_shapes      r)   Útranspose_for_scoresz'TFEsmSelfAttention.transpose_for_scoresA  sI   € Ü  “m C RÐ(¨D×,DÑ,DÀd×F^ÑF^Ð+_Ñ_ˆÜ�J‰J�q˜+Ó&ˆÜ�|‰|˜A LÔ1Ð1r+   c	                ó  — | j                  |«      }	|d u}
|
r|�|d   }|d   }|}�n |
rC| j                  | j                  |«      «      }| j                  | j                  |«      «      }|}n»|�y| j                  | j                  |«      «      }| j                  | j                  |«      «      }t	        j
                  |d   |gd¬«      }t	        j
                  |d   |gd¬«      }n@| j                  | j                  |«      «      }| j                  | j                  |«      «      }| j                  |	«      }|| j                  dz  z  }| j                  r||f}| j                  dk(  r| j                  ||«      \  }}t	        j                  ||d¬«      }| j                  d	k(  s| j                  d
k(  �r7t        |«      d   }t	        j                  t	        j                  |t        j                  ¬«      d«      }t	        j                  t	        j                  |t        j                  ¬«      d«      }||z
  }| j                  || j                   z   dz
  «      }t	        j"                  ||j$                  «      }| j                  d	k(  rt	        j&                  d||«      }||z   }nE| j                  d
k(  r6t	        j&                  d||«      }t	        j&                  d||«      }||z   |z   }|�||z   }t)        |d¬«      }| j+                  ||¬«      }|�||z  }||z  }t	        j,                  |d¬«      }t        |«      d d | j.                  gz   }t	        j0                  ||«      }|r||fn|f}| j                  r||fz   }|S )Nr   r   r   r!   g      à¿rÌ   T©Útranspose_brÊ   rË   rY   r    zbhld,lrd->bhlrzbhrd,lrd->bhlr©ÚtrainingrÙ   rƒ   r-   )rÅ   rÛ   rÈ   rÉ   r#   r%   rÑ   r×   rš   rÍ   Úmatmulr   r†   rV   r»   rÖ   r    r…   rL   rZ   r   rÕ   rˆ   rÒ   r‡   )rG   Úhidden_statesr²   Ú	head_maskÚencoder_hidden_statesÚencoder_attention_maskÚpast_key_valueÚoutput_attentionsrà   Úmixed_query_layerÚis_cross_attentionÚ	key_layerÚvalue_layerÚquery_layerÚattention_scoresÚ
seq_lengthÚposition_ids_lÚposition_ids_rÚdistanceÚpositional_embeddingÚrelative_position_scoresÚrelative_position_scores_queryÚrelative_position_scores_keyÚattention_probsÚcontext_layerÚnew_context_layer_shapeÚoutputss                              r)   rf   zTFEsmSelfAttention.callF  st  € ð !ŸJ™J }Ó5Ðð
 3¸$Ð>Ðá .Ð"<à& qÑ)ˆIØ(¨Ñ+ˆKØ3ŠNÙØ×1Ñ1°$·(±(Ð;PÓ2QÓRˆIØ×3Ñ3°D·J±JÐ?TÓ4UÓVˆKØ3‰NØÐ'Ø×1Ñ1°$·(±(¸=Ó2IÓJˆIØ×3Ñ3°D·J±J¸}Ó4MÓNˆKÜŸ	™	 >°!Ñ#4°iÐ"@ÀqÔIˆIÜŸ)™) ^°AÑ%6¸Ð$DÈ1ÔM‰Kà×1Ñ1°$·(±(¸=Ó2IÓJˆIØ×3Ñ3°D·J±J¸}Ó4MÓNˆKà×/Ñ/Ð0AÓBˆð " D×$<Ñ$<¸dÑ$BÑBˆà�?Š?ð (¨Ð5ˆNà×'Ñ'¨8Ò3Ø%)×%;Ñ%;¸KÈÓ%SÑ"ˆK˜ô Ÿ9™9 [°)ÈÔNÐà×'Ñ'¨>Ò9¸T×=YÑ=YÐ]qÓ=qÜ# MÓ2°1Ñ5ˆJÜŸ^™^¬B¯H©H°ZÄrÇxÁxÔ,PÐRTÓUˆNÜŸ^™^¬B¯H©H°ZÄrÇxÁxÔ,PÐRSÓTˆNØ%¨Ñ6ˆHØ#'×#:Ñ#:¸8Àd×FbÑFbÑ;bÐefÑ;fÓ#gÐ Ü#%§7¡7Ð+?À×ARÑARÓ#SÐ à×+Ñ+¨~Ò=Ü+-¯9©9Ð5EÀ{ÐThÓ+iÐ(Ø#3Ð6NÑ#NÑ Ø×-Ñ-Ð1EÒEÜ13·±Ð;KÈ[ÐZnÓ1oÐ.Ü/1¯y©yÐ9IÈ9ÐVjÓ/kÐ,Ø#3Ð6TÑ#TÐWsÑ#sÐ àÐ%à/°.Ñ@Ðô )Ð)9ÀÔCˆð Ÿ,™, À˜,ÓJˆð Ð Ø-°	Ñ9ˆOà'¨+Ñ5ˆäŸ™ ]¸ÔFˆÜ",¨]Ó";¸C¸RÐ"@ÀD×DVÑDVÐCWÑ"WÐÜŸ
™
 =Ð2IÓJˆá6G�= /Ñ2ÈmÐM]ˆà�?Š?Ø Ð 1Ñ1ˆGØˆr+   c                óê  — | j                   ry d| _         t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   �ŒHxY w# 1 sw Y   ŒãxY w# 1 sw Y   Œ~xY w# 1 sw Y   y xY w)NTrÅ   rÈ   rÉ   rÍ   )r~   r   r#   r€   rÅ   rC   rR   r©   rž   rÈ   rÉ   rÍ   r�   s     r)   rR   zTFEsmSelfAttention.build­  sŠ  € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ HØ—
‘
× Ñ  $¨¨d¯k©k×.EÑ.EÐ!FÔG÷Hä�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ FØ—‘—‘  d¨D¯K©K×,CÑ,CÐDÔE÷Fä�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ HØ—
‘
× Ñ  $¨¨d¯k©k×.EÑ.EÐ!FÔG÷Hä�4Ð,¨dÓ3Ð?Ü—‘˜t×5Ñ5×:Ñ:Ó;ñ 3Ø×&Ñ&×,Ñ,¨TÔ2÷3ð 3ð @÷Hñ Hú÷Fð Fú÷Hð Hú÷3ð 3ús0   Á3GÂ<3GÄ-3GÆG)ÇGÇGÇG&Ç)G2©NN)r&   ri   rj   ri   ©NNNNNFF)râ   ri   r²   útf.Tensor | Nonerã   rý   rä   rý   rå   rý   ræ   zTuple[Tuple[tf.Tensor]] | Nonerç   úOptional[bool]rà   Úboolrj   zTuple[tf.Tensor]rg   )rk   rl   rm   rE   rÛ   rf   rR   ro   rp   s   @r)   rÁ   rÁ     s’   ø„ õ&óP2ð ,0Ø&*Ø26Ø37Ø9=Ø,1Øðeà ðeð )ðeð $ð	eð
  0ðeð !1ðeð 7ðeð *ðeð ðeð 
óe÷N3r+   rÁ   c                  ó0   ‡ — e Zd Zdˆ fd„	Zdd„Zdd„Zˆ xZS )ÚTFEsmSelfOutputc                ó  •— t         ‰| �  |¬«       t        j                  j	                  |j
                  t        |j                  «      d¬«      | _        t        j                  j                  |j                  «      | _        || _        y ©NrB   ÚdenserÆ   ©rD   rE   r   rz   r{   rž   r   rŸ   r  rÓ   Úhidden_dropout_probrÕ   r©   rª   s      €r)   rE   zTFEsmSelfOutput.__init__À  ól   ø€ Ü‰Ñ˜dÐÔ#Ü—\‘\×'Ñ'Ø×Ñ´?À6×C[ÑC[Ó3\Ðcjð (ó 
ˆŒ
ô —|‘|×+Ñ+¨F×,FÑ,FÓGˆŒØˆ�r+   c                óX   — | j                  |«      }| j                  ||¬«      }||z  }|S ©Nrß   ©r  rÕ   ©rG   râ   Úinput_tensorrà   s       r)   rf   zTFEsmSelfOutput.callÈ  ó2   € ØŸ
™
 =Ó1ˆØŸ™ ]¸X˜ÓFˆØ˜Ñ%ˆØÐr+   c                ó(  — | j                   ry d| _         t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   y xY w©NTr  ©	r~   r   r#   r€   r  rC   rR   r©   rž   r�   s     r)   rR   zTFEsmSelfOutput.buildÎ  ó}   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ HØ—
‘
× Ñ  $¨¨d¯k©k×.EÑ.EÐ!FÔG÷Hð Hð 4÷Hð Húó   Á3BÂBrg   ©F©rk   rl   rm   rE   rf   rR   ro   rp   s   @r)   r  r  ¿  s   ø„ õó÷Hr+   r  c                  óD   ‡ — e Zd Zdˆ fd„	Zd„ Z	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFEsmAttentionc                óú   •— t         ‰| �  |¬«       t        |d¬«      | _        t	        |d¬«      | _        t        «       | _        t        j                  j                  |j                  d¬«      | _        || _        y )NrB   rG   ÚoutputÚ	LayerNormr˜   )rD   rE   rÁ   rG   r  Úoutput_layerÚsetÚpruned_headsr   rz   r¢   r£   r  r©   rª   s      €r)   rE   zTFEsmAttention.__init__Ø  sc   ø€ Ü‰Ñ˜dÐÔ#Ü& v°FÔ;ˆŒ	Ü+¨F¸ÔBˆÔÜ›EˆÔÜŸ™×8Ñ8À×AVÑAVÐ]hÐ8ÓiˆŒØˆ�r+   c                ó   — t         ‚rg   ©ÚNotImplementedError)rG   rŽ   s     r)   Úprune_headszTFEsmAttention.prune_headsà  ó   € Ü!Ð!r+   c	           
     ó”   — | j                  |«      }	| j                  |	|||||||«      }
| j                  |
d   |«      }|f|
dd  z   }|S )Nr   r   )r  rG   r  )rG   râ   r²   rã   rä   rå   ræ   rç   rà   Úhidden_states_lnÚself_outputsÚattention_outputrù   s                r)   rf   zTFEsmAttention.callã  sk   € ð  Ÿ>™>¨-Ó8ÐØ—y‘yØØØØ!Ø"ØØØó	
ˆð  ×,Ñ,¨\¸!©_¸mÓLÐØ#Ð%¨°Q°RÐ(8Ñ8ˆØˆr+   c                óÀ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   ŒÖxY w# 1 sw Y   ŒˆxY w# 1 sw Y   y xY w)NTrG   r  r  )r~   r   r#   r€   rG   rC   rR   r  r  r©   rž   r�   s     r)   rR   zTFEsmAttention.buildý  s  € Ø�:Š:ØØˆŒ
Ü�4˜ Ó&Ð2Ü—‘˜tŸy™yŸ~™~Ó.ñ &Ø—	‘	—‘ Ô%÷&ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ LØ—‘×$Ñ$ d¨D°$·+±+×2IÑ2IÐ%JÔK÷Lð Lð 8÷&ð &ú÷.ð .ú÷Lð Lúr¿   rg   rü   )rk   rl   rm   rE   r   rf   rR   ro   rp   s   @r)   r  r  ×  s/   ø„ õò"ð ØØ"Ø#ØØØó÷4Lr+   r  c                  ó0   ‡ — e Zd Zdˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFEsmIntermediatec                óº   •— t        ‰| �  di |¤Ž t        j                  j	                  |j
                  t        |j                  «      d¬«      | _        || _	        y )Nr  )ÚunitsrÇ   rC   © )
rD   rE   r   rz   r{   Úintermediate_sizer   rŸ   r  r©   ©rG   r©   ÚkwargsrH   s      €r)   rE   zTFEsmIntermediate.__init__  sQ   ø€ Ü‰ÑÑ"˜6Ò"ä—\‘\×'Ñ'Ø×*Ñ*Ü.¨v×/GÑ/GÓHØð (ó 
ˆŒ
ð
 ˆ�r+   c                óh   — | j                  |¬«      }t        j                  j                  |«      }|S )N©Úinputs)r  r#   ÚnnÚgelu)rG   râ   s     r)   rf   zTFEsmIntermediate.call  s*   € ØŸ
™
¨-˜
Ó8ˆÜŸ™Ÿ
™
 =Ó1ˆØÐr+   c                ó(  — | j                   ry d| _         t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   y xY wr  r  r�   s     r)   rR   zTFEsmIntermediate.build  r  r  ©r©   r   ©râ   ri   rj   ri   rg   r  rp   s   @r)   r(  r(    s   ø„ õó÷
Hr+   r(  c                  ó0   ‡ — e Zd Zdˆ fd„	Zdd„Zdd„Zˆ xZS )ÚTFEsmOutputc                ó  •— t         ‰| �  |¬«       t        j                  j	                  |j
                  t        |j                  «      d¬«      | _        t        j                  j                  |j                  «      | _        || _        y r  r  rª   s      €r)   rE   zTFEsmOutput.__init__&  r  r+   c                óX   — | j                  |«      }| j                  ||¬«      }||z  }|S r	  r
  r  s       r)   rf   zTFEsmOutput.call.  r  r+   c                ó(  — | j                   ry d| _         t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   y xY wr  )	r~   r   r#   r€   r  rC   rR   r©   r,  r�   s     r)   rR   zTFEsmOutput.build4  s}   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ NØ—
‘
× Ñ  $¨¨d¯k©k×.KÑ.KÐ!LÔM÷Nð Nð 4÷Nð Núr  rg   r  r  rp   s   @r)   r8  r8  %  s   ø„ õó÷Nr+   r8  c                  ó>   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )Ú
TFEsmLayerc                óà  •— t         ‰| �  |¬«       |j                  | _        d| _        t	        |d¬«      | _        |j                  | _        |j                  | _        | j                  r*| j                  st        | › d�«      ‚t	        |«      | _	        t        |d¬«      | _        t        |d¬«      | _        t        j                  j!                  |j"                  d¬«      | _        || _        y )	NrB   r   Ú	attentionz> should be used as a decoder model if cross attention is addedÚintermediater  r  r˜   )rD   rE   Úchunk_size_feed_forwardÚseq_len_dimr  r?  r×   Úadd_cross_attentionÚRuntimeErrorÚcrossattentionr(  r@  r8  r  r   rz   r¢   r£   r  r©   rª   s      €r)   rE   zTFEsmLayer.__init__>  sÉ   ø€ Ü‰Ñ˜dÐÔ#Ø'-×'EÑ'EˆÔ$ØˆÔÜ'¨°[ÔAˆŒØ ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ Ø×#Ò#Ø—?’?Ü" d VÐ+iÐ#jÓkÐkÜ"0°Ó"8ˆDÔÜ-¨f¸>ÔJˆÔÜ'¨°XÔ>ˆÔÜŸ™×8Ñ8À×AVÑAVÐ]hÐ8ÓiˆŒØˆ�r+   c	           
     óî  — |�|d d nd }	| j                  |||||	|¬«      }
|
d   }| j                  r|
dd }|
d   }n|
dd  }d }| j                  rV|�Tt        | d«      st        d| › d�«      ‚|�|d	d  nd }| j	                  ||||||||¬
«      }|d   }||dd z   }|d   }|z   }| j                  |«      }| j                  |¬«      }| j                  |||¬«      }|f|z   }| j                  r|fz   }|S )Nr   )rç   ræ   rà   r   r   r    rE  z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r-   rß   ©râ   )râ   r  rà   )r?  r×   rÏ   ÚAttributeErrorrE  r  r@  r  )rG   râ   r²   rã   rä   rå   ræ   rç   rà   Úself_attn_past_key_valueÚself_attention_outputsr%  rù   Úpresent_key_valueÚcross_attn_present_key_valueÚcross_attn_past_key_valueÚcross_attention_outputsÚlayernorm_outputÚintermediate_outputÚlayer_outputs                       r)   rf   zTFEsmLayer.callN  s¥  € ð :HÐ9S >°"°1Ñ#5ÐY]Ð Ø!%§¡ØØØØ/Ø3Øð "0ó "
Ðð 2°!Ñ4Ðð �?Š?Ø,¨Q¨rÐ2ˆGØ 6°rÑ :Ñà,¨Q¨RÐ0ˆGà'+Ð$Ø�?Š?Ð4Ð@Ü˜4Ð!1Ô2Ü$Ø=¸d¸Vð D`ð `óð ð @NÐ?Y¨°r°sÑ(;Ð_cÐ%Ø&*×&9Ñ&9Ø ØØØ%Ø&Ø)Ø!Ø!ð ':ó 	'Ð#ð  7°qÑ9ÐØÐ 7¸¸"Ð =Ñ=ˆGð ,CÀ2Ñ+FÐ(Ø 1Ð4PÑ PÐàŸ>™>Ð*:Ó;ÐØ"×/Ñ/Ð>NÐ/ÓOÐØ×(Ñ(Ø-Ð<LÐW_ð )ó 
ˆð  �/ GÑ+ˆð �?Š?ØÐ!2Ð 4Ñ4ˆGàˆr+   c                óŽ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   �Œ1xY w# 1 sw Y   ŒãxY w# 1 sw Y   Œ•xY w# 1 sw Y   y xY w)NTr?  r@  r  r  )r~   r   r#   r€   r?  rC   rR   r@  r  r  r©   rž   r�   s     r)   rR   zTFEsmLayer.build”  sk  € Ø�:Š:ØØˆŒ
Ü�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ +Ø—‘×$Ñ$ TÔ*÷+ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ä�4˜ dÓ+Ð7Ü—‘˜tŸ~™~×2Ñ2Ó3ñ LØ—‘×$Ñ$ d¨D°$·+±+×2IÑ2IÐ%JÔK÷Lð Lð 8÷+ñ +ú÷.ð .ú÷.ð .ú÷Lð Lús0   ÁFÂ%F#Ã?F/Å3F;ÆF Æ#F,Æ/F8Æ;Grg   rü   r  rp   s   @r)   r=  r=  =  s,   ø„ õð& ØØ"Ø#ØØØóD÷LLr+   r=  c                  óD   ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zˆ xZS )ÚTFEsmEncoderc                ó
  •— t         ‰| �  |¬«       || _        t        |j                  «      D �cg c]  }t        |d|› �¬«      ‘Œ c}| _        t        j                  j                  |j                  d¬«      | _        y c c}w )NrB   zlayer_._Úemb_layer_norm_afterr˜   )rD   rE   r©   rV   Únum_hidden_layersr=  Úlayerr   rz   r¢   r£   rV  )rG   r©   rC   ÚirH   s       €r)   rE   zTFEsmEncoder.__init__§  ss   ø€ Ü‰Ñ˜dÐÔ#ØˆŒÜGLÈV×MeÑMeÓGfÖgÀ!”j °¸¸¨nÖ=ÒgˆŒ
Ü$)§L¡L×$CÑ$CØ×)Ñ)Ð0Fð %Dó %
ˆÕ!ùò hs   °B c                óþ  — |	rdnd }|rdnd }|r| j                   j                  rdnd }|rdnd }t        | j                  «      D ]j  \  }}|	r||fz   }|�||   nd }|�||   nd } |||||||||«      }|d   }|r	||d   fz  }|sŒB||d   fz   }| j                   j                  sŒb||d   fz   }Œl | j                  r| j	                  |«      }|	r||fz   }|
st        d„ |||||fD «       «      S t        |||||¬«      S )Nr+  r   r    r   r   c              3  ó$   K  — | ]  }|�|–— Œ
 y ­wrg   r+  )Ú.0Úvs     r)   ú	<genexpr>z$TFEsmEncoder.call.<locals>.<genexpr>ã  s   è ø€ ò 
àð �=ô ñ
ùs   ‚)Úlast_hidden_stateÚpast_key_valuesrâ   r‹   Úcross_attentions)r©   rC  Ú	enumeraterX  rV  Útupler   )rG   râ   r²   rã   rä   rå   r`  Ú	use_cacherç   Úoutput_hidden_statesÚreturn_dictrà   Úall_hidden_statesÚall_self_attentionsÚall_cross_attentionsÚnext_decoder_cacherY  Úlayer_moduleÚlayer_head_maskræ   Úlayer_outputss                        r)   rf   zTFEsmEncoder.call¯  s  € ñ #7™B¸DÐÙ$5™b¸4ÐÙ%6¸4¿;¹;×;ZÒ;Z™rÐ`dÐá#,™R°$ÐÜ(¨¯©Ó4ò 	V‰OˆAˆ|Ù#Ø$5¸Ð8HÑ$HÐ!à.7Ð.C˜i¨šlÈˆOØ3BÐ3N˜_¨QÒ/ÐTXˆNá(ØØØØ%Ø&ØØ!Øó	ˆMð *¨!Ñ,ˆMÙØ" }°RÑ'8Ð&:Ñ:Ð"Ú Ø&9¸]È1Ñ=MÐ<OÑ&OÐ#Ø—;‘;×2Ó2Ø+?À=ÐQRÑCSÐBUÑ+UÑ(ð1	Vð4 ×$Ò$Ø ×5Ñ5°mÓDˆMáØ 1°]Ð4DÑ DÐáÜñ 
ð "Ø&Ø%Ø'Ø(ðô
ó 
ð 
ô ;Ø+Ø.Ø+Ø*Ø1ô
ð 	
r+   c                óî  — | j                   ry d| _         t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �K| j                  D ];  }t        j                  |j
                  «      5  |j                  d «       d d d «       Œ= y y # 1 sw Y   ŒbxY w# 1 sw Y   ŒUxY w)NTrV  rX  )
r~   r   r#   r€   rV  rC   rR   r©   rž   rX  )rG   rW   rX  s      r)   rR   zTFEsmEncoder.buildö  sÖ   € Ø�:Š:ØØˆŒ
Ü�4Ð/°Ó6ÐBÜ—‘˜t×8Ñ8×=Ñ=Ó>ñ WØ×)Ñ)×/Ñ/°°t¸T¿[¹[×=TÑ=TÐ0UÔV÷Wä�4˜ $Ó'Ð3ØŸ™ò &�Ü—]‘] 5§:¡:Ó.ñ &Ø—K‘K Ô%÷&ð &ñ&ð 4÷Wð Wú÷&ð &ús   Á3CÃC+ÃC(Ã+C4	rg   )
NNNNNNFFTFr  rp   s   @r)   rT  rT  ¦  s4   ø„ õ
ð ØØ"Ø#ØØØØ"ØØóE
÷N
&r+   rT  c                  ó0   ‡ — e Zd Zdˆ fd„Zdd„Zdd„Zˆ xZS )ÚTFEsmPoolerc                ó¼   •— t        ‰| �  di |¤Ž t        j                  j	                  |j
                  t        |j                  «      dd¬«      | _        || _	        y )NÚtanhr  )r*  rÇ   rw   rC   r+  )
rD   rE   r   rz   r{   rž   r   rŸ   r  r©   r-  s      €r)   rE   zTFEsmPooler.__init__  sT   ø€ Ü‰ÑÑ"˜6Ò"ä—\‘\×'Ñ'Ø×$Ñ$Ü.¨v×/GÑ/GÓHØØð	 (ó 
ˆŒ
ð ˆ�r+   c                ó<   — |d d …df   }| j                  |¬«      }|S )Nr   r0  )r  )rG   râ   Úfirst_token_tensorÚpooled_outputs       r)   rf   zTFEsmPooler.call  s*   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð*<˜
Ó=ˆàÐr+   c                ó(  — | j                   ry d| _         t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   y xY wr  r  r�   s     r)   rR   zTFEsmPooler.build  r  r  r5  r6  rg   r  rp   s   @r)   rp  rp    s   ø„ õ	ó÷Hr+   rp  c                  ó   — e Zd ZdZeZdZy)ÚTFEsmPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚesmN)rk   rl   rm   rn   r   Úconfig_classÚbase_model_prefixr+  r+   r)   rx  rx  !  s   „ ñð
 €LØÑr+   rx  a2  

    This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a Keras [Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a
    regular Keras model and refer to the TF/Keras documentation for all matters related to general usage and behavior.

    Parameters:
        config ([`EsmConfig`]): Model configuration class with all the parameters of the
            model. Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights.
a–  
    Args:
        input_ids (`tf.Tensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`tf.Tensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`tf.Tensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~file_utils.ModelOutput`] instead of a plain tuple.
z]The bare ESM Model transformer outputting raw hidden-states without any specific head on top.c                  ó¦   ‡ — e Zd ZdZdgZd
ˆ fd„	Zdd„Zd„ Zdd„Zd„ Z		 	 	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z
d	„ Zˆ xZS )ÚTFEsmMainLayera  

    The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
    cross-attention is added between the self-attention layers, following the architecture described in [Attention is
    all you need](https://arxiv.org/abs/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
    Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

    To behave as an decoder the model needs to be initialized with the `is_decoder` argument of the configuration set
    to `True`. To be used in a Seq2Seq model, the model needs to initialized with both `is_decoder` argument and
    `add_cross_attention` set to `True`; an `encoder_hidden_states` is then expected as an input to the forward pass.
    r¤   c                óH  •— t        ‰| �  d	d|i|¤Ž || _        |j                  | _        t	        |d¬«      | _        t        |d¬«      | _        |rt        |d¬«      nd | _	        t        | j                  j                  | j                  j                  z  dd¬«      | _        y )
NrC   rµ   rB   ÚencoderÚpoolerTÚcontact_head)ry   r|   rC   r+  )rD   rE   r©   r×   r’   rµ   rT  r  rp  r€  rr   rW  rÎ   r�  )rG   r©   Úadd_pooling_layerrC   r.  rH   s        €r)   rE   zTFEsmMainLayer.__init__w  sŠ   ø€ Ü‰ÑÑ-˜dÐ- fÒ-àˆŒØ ×+Ñ+ˆŒä)¨&°|ÔDˆŒÜ# F°Ô;ˆŒÙ<M”k &¨xÕ8ÐSWˆŒä6ØŸ™×5Ñ5¸¿¹×8WÑ8WÑWÐ^bÐiwô
ˆÕr+   c                ó`  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   �ŒxY w# 1 sw Y   ŒÌxY w# 1 sw Y   Œ~xY w# 1 sw Y   y xY w)NTrµ   r  r€  r�  )
r~   r   r#   r€   rµ   rC   rR   r  r€  r�  r�   s     r)   rR   zTFEsmMainLayer.build…  sQ  € Ø�:Š:ØØˆŒ
Ü�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ ,Ø—‘×%Ñ% dÔ+÷,ä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ä�4˜ 4Ó(Ð4Ü—‘˜tŸ{™{×/Ñ/Ó0ñ (Ø—‘×!Ñ! $Ô'÷(ä�4˜¨Ó.Ð:Ü—‘˜t×0Ñ0×5Ñ5Ó6ñ .Ø×!Ñ!×'Ñ'¨Ô-÷.ð .ð ;÷,ñ ,ú÷)ð )ú÷(ð (ú÷.ð .ús0   ÁE?Â%FÃ?FÅF$Å?F	ÆFÆF!Æ$F-c                ó.   — | j                   j                  S rg   )rµ   r”   ©rG   s    r)   Úget_input_embeddingsz#TFEsmMainLayer.get_input_embeddings–  s   € Ø�‰×.Ñ.Ð.r+   c                ót   — || j                   j                  _        t        |«      d   | j                   _        y )Nr   )rµ   r”   Úweightr   r�   )rG   rÉ   s     r)   Úset_input_embeddingsz#TFEsmMainLayer.set_input_embeddings™  s*   € Ø16ˆ�‰×'Ñ'Ô.Ü%/°Ó%6°qÑ%9ˆ�‰Õ"r+   c                ó   — t         ‚rg   r  )rG   Úheads_to_prunes     r)   Ú_prune_headszTFEsmMainLayer._prune_heads�  r!  r+   c                óê  — | j                   j                  sd}	|�|�t        d«      ‚|�t        |«      }n|�t        |«      d d }nt        d«      ‚|\  }}|€&d}d gt	        | j
                  j                  «      z  }nt        |d   d   «      d   }|€t        j                  |||z   fd¬«      }| j                  ||||||¬	«      }t        |«      }||z   }| j                  rÈt        j                  |«      }t        j                  t        j                  |d d d d …f   ||df«      |d d d …d f   «      }t        j                  ||j                  ¬
«      }||d d …d d d …f   z  }t        |«      }t        j                  ||d   d|d   |d   f«      }|d   �3|d d …d d …| d …d d …f   }n t        j                  ||d   dd|d   f«      }t        j                  ||j                  ¬
«      }t        j                   d|j                  ¬
«      }t        j                   d|j                  ¬
«      }t        j"                  t        j$                  ||«      |«      }| j                  rf|�dt        j                  ||j                  ¬
«      }t	        t        |«      «      }|dk(  r|d d …d d d …d d …f   }|dk(  r|d d …d d d d …f   }dz
  dz  }nd }|�t&        ‚d g| j                   j(                  z  }| j                  |||||||	|
|||¬«      }|d   }| j*                  �| j+                  |¬«      nd }|s
||f|dd  z   S t-        |||j.                  |j0                  |j2                  |j4                  ¬«      S )NFzDYou cannot specify both input_ids and inputs_embeds at the same timer    z5You have to specify either input_ids or inputs_embedsr   r-   r   )ÚdimsrÉ   )r±   r²   r¤   r³   r´   rà   rY   r   rK   g     ˆÃÀr   )râ   r²   rã   rä   rå   r`  rd  rç   re  rf  rà   rG  )r_  Úpooler_outputr`  râ   r‹   ra  )r©   r×   rÐ   r   Úlenr  rX  r#   Úfillrµ   rV   Ú
less_equalÚtiler…   rL   r‡   ÚconstantÚmultiplyÚsubtractr  rW  r€  r   r`  râ   r‹   ra  )rG   r±   r²   r¤   rã   r³   rä   rå   r`  rd  rç   re  rf  rà   rW   r�   rî   r´   Úembedding_outputÚattention_mask_shapeÚmask_seq_lengthÚseq_idsÚcausal_maskÚextended_attention_maskÚone_cstÚten_thousand_cstÚnum_dims_encoder_attention_maskÚencoder_extended_attention_maskÚencoder_outputsÚsequence_outputru  s                                  r)   rf   zTFEsmMainLayer.call   sî  € ð  �{‰{×%Ò%ØˆIàÐ  ]Ð%>ÜÐcÓdÐdØÐ"Ü$ YÓ/‰KØÐ&Ü$ ]Ó3°C°RÐ8‰KäÐTÓUÐUà!,Ñˆ
�JàÐ"Ø%&Ð"Ø#˜f¤s¨4¯<©<×+=Ñ+=Ó'>Ñ>‰Oä%/°ÀÑ0BÀ1Ñ0EÓ%FÀrÑ%JÐ"àÐ!ÜŸW™W¨:°zÐDZÑ7ZÐ*[ÐcdÔeˆNàŸ?™?ØØ)Ø%Ø'Ø#9Øð +ó 
Ðô  *¨.Ó9Ðà$Ð'=Ñ=ˆð
 �?Š?Ü—h‘h˜Ó/ˆGÜŸ-™-Ü—‘˜  dªA Ñ.°¸_ÈaÐ0PÓQØ˜ša ˜Ñ&óˆKô Ÿ'™' +°^×5IÑ5IÔJˆKØ&1°NÂ1ÀdÊAÀ:Ñ4NÑ&NÐ#Ü#-Ð.EÓ#FÐ Ü&(§j¡jØ'Ð*>¸qÑ*AÀ1ÐFZÐ[\ÑF]Ð_sÐtuÑ_vÐ)wó'Ð#ð ˜qÑ!Ð-à*AÂ!ÂQÈÈÉÒVWÐBWÑ*XÑ'ä&(§j¡jØÐ!5°aÑ!8¸!¸QÐ@TÐUVÑ@WÐ Xó'Ð#ô #%§'¡'Ð*AÐIY×I_ÑI_Ô"`ÐÜ—+‘+˜cÐ)9×)?Ñ)?Ô@ˆÜŸ;™; xÐ7G×7MÑ7MÔNÐÜ"$§+¡+¬b¯k©k¸'ÐCZÓ.[Ð]mÓ"nÐð �?Š?Ð5ÐAô &(§W¡WÐ-CÐKb×KhÑKhÔ%iÐ"Ü.1´*Ð=SÓ2TÓ.UÐ+Ø.°!Ò3Ø2HÊÈDÒRSÒUVÈÑ2WÐ/Ø.°!Ò3Ø2HÊÈDÐRVÒXYÐIYÑ2ZÐ/ð 03Ð5TÑ/TÐX`Ñ.`Ñ+à.2Ð+ð Ð Ü%Ð%à˜ §¡×!>Ñ!>Ñ>ˆIàŸ,™,Ø*Ø2ØØ"7Ø#BØ+ØØ/Ø!5Ø#Øð 'ó 
ˆð *¨!Ñ,ˆØFJÇkÁkÐF]˜Ÿ™°/˜ÔBÐcgˆáàØðð    Ð#ñ$ð $ô
 >Ø-Ø'Ø+×;Ñ;Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ô
ð 	
r+   c                óþ   —  | ||dd¬«      j                   }t        j                  |d¬«      }t        j                  ||j                  «      }||d d …d d d f   z  }||d d …d d d d …d f   z  }| j                  ||«      S )NT)r²   rf  rç   r   r!   )r‹   r#   Ústackr…   rL   r�  )rG   rŠ   r²   Úattnss       r)   Úpredict_contactszTFEsmMainLayer.predict_contacts9  s„   € Ù�V¨NÈÐ`dÔe×pÑpˆÜ—‘˜ QÔ'ˆô
 Ÿ™ °·±Ó=ˆØ�¢ 4¨¨tÐ 3Ñ4Ñ4ˆØ�¢ 4¨ªq°$Ð 6Ñ7Ñ7ˆØ× Ñ  ¨Ó/Ð/r+   )TNrg   )rÉ   ztf.Variable©NNNNNNNNNNNNF)r±   úTFModelInputType | Noner²   únp.ndarray | tf.Tensor | Noner¤   r©  rã   r©  r³   r©  rä   r©  rå   r©  r`  ú4Optional[Tuple[Tuple[Union[np.ndarray, tf.Tensor]]]]rd  rþ   rç   rþ   re  rþ   rf  rþ   rà   rÿ   rj   úGUnion[TFBaseModelOutputWithPoolingAndCrossAttentions, Tuple[tf.Tensor]])rk   rl   rm   rn   Ú_keys_to_ignore_on_load_missingrE   rR   r†  r‰  rŒ  rf   r¦  ro   rp   s   @r)   r}  r}  d  sû   ø„ ñ

ð (7Ð&7Ð#õ
ó.ò"/ó:ò"ð
 .2Ø8<Ø6:Ø37Ø7;Ø?CØ@DØPTØ$(Ø,0Ø/3Ø&*ØðW
à*ðW
ð 6ðW
ð 4ð	W
ð
 1ðW
ð 5ðW
ð  =ðW
ð !>ðW
ð NðW
ð "ðW
ð *ðW
ð -ðW
ð $ðW
ð ðW
ð 
QóW
ör
0r+   r}  c                  óä   ‡ — e Zd Zddˆ fd„Ze eej                  d«      «       ee	e
e¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 d		 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d
d„«       «       «       Zd„ Zdd„Zˆ xZS )Ú
TFEsmModelc                óR   •— t        ‰| �  |g|¢­i |¤Ž t        ||d¬«      | _        y )Nry  ©r‚  rC   )rD   rE   r}  ry  )rG   r©   r‚  r1  r.  rH   s        €r)   rE   zTFEsmModel.__init__K  s,   ø€ Ü‰Ñ˜Ð3 &Ò3¨FÒ3ä! &Ð<MÐTYÔZˆ�r+   úbatch_size, sequence_length©Ú
checkpointÚoutput_typerz  c                óB   — | j                  |||||||||	|
|||¬«      }|S )aÓ  
        encoder_hidden_states  (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention if
            the model is configured as a decoder.
        encoder_attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on the padding token indices of the encoder input. This mask is used in
            the cross-attention if the model is configured as a decoder. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

        past_key_values (`Tuple[Tuple[tf.Tensor]]` of length `config.n_layers`)
            contains precomputed key and value hidden states of the attention blocks. Can be used to speed up decoding.
            If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `decoder_input_ids` of shape `(batch_size, sequence_length)`.
        use_cache (`bool`, *optional*, defaults to `True`):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`). Set to `False` during training, `True` during generation
        )r±   r²   r¤   rã   r³   rä   rå   r`  rd  rç   re  rf  rà   )ry  )rG   r±   r²   r¤   rã   r³   rä   rå   r`  rd  rç   re  rf  rà   rù   s                  r)   rf   zTFEsmModel.callP  sF   € ðV —(‘(ØØ)Ø%ØØ'Ø"7Ø#9Ø+ØØ/Ø!5Ø#Øð ó 
ˆð ˆr+   c                ó:   — | j                   j                  ||«      S rg   ©ry  r¦  ©rG   rŠ   r²   s      r)   r¦  zTFEsmModel.predict_contactsŒ  ó   € Ø�x‰x×(Ñ(¨°Ó@Ð@r+   c                óú   — | j                   ry d| _         t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   y xY w)NTry  )r~   r   r#   r€   ry  rC   rR   r�   s     r)   rR   zTFEsmModel.build�  se   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ %Ø—‘—‘˜tÔ$÷%ð %ð 2÷%ð %ús   ÁA1Á1A:)Tr5  r§  )r±   r¨  r²   r©  r¤   r©  rã   r©  r³   r©  rä   r©  rå   r©  r`  rª  rd  rþ   rç   rþ   re  rþ   rf  rþ   rà   rþ   rj   r«  rg   )rk   rl   rm   rE   r   r
   ÚESM_INPUTS_DOCSTRINGÚformatr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCrf   r¦  rR   ro   rp   s   @r)   r®  r®  F  s  ø„ ö
[ð
 Ù*Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&ØBØ$ôð .2Ø8<Ø6:Ø37Ø7;Ø?CØ@DØPTØ$(Ø,0Ø/3Ø&*Ø#(ð3à*ð3ð 6ð3ð 4ð	3ð
 1ð3ð 5ð3ð  =ð3ð !>ð3ð Nð3ð "ð3ð *ð3ð -ð3ð $ð3ð !ð3ð 
Qò3óó gó ð3òjA÷%r+   r®  z1ESM Model with a `language modeling` head on top.c                  óú   ‡ — e Zd ZdgZdgZˆ fd„Zd„ Zd„ Zd„ Ze	 e
ej                  d«      «       eeeed¬	«      	 	 	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd
„«       «       «       Zd„ Zdd„Zˆ xZS )ÚTFEsmForMaskedLMr¤   r€  c                óP  •— t         ‰| �  |«       |j                  rt        j	                  d«       t        |dd¬«      | _        t        |d¬«      | _        |j                  r¸t        j                  t        j                  j                  | j                  «       ddd«      «      5  | j                  j                   j"                  j%                  d	«       d d d «       | j                  j                   j"                  j&                  d
   | j                  _        y y # 1 sw Y   ŒGxY w)NzjIf you want to use `EsmForMaskedLM` make sure `config.is_decoder=False` for bi-directional self-attention.Fry  r°  Úlm_headrB   rµ   r”   rû   r   )rD   rE   r×   ÚloggerÚwarningr}  ry  ÚTFEsmLMHeadrÂ  Útie_word_embeddingsr#   r€   ÚosÚpathÚjoinÚ_name_scoperµ   r”   rR   ÚweightsÚdecoder©rG   r©   rH   s     €r)   rE   zTFEsmForMaskedLM.__init__�  sß   ø€ Ü‰Ñ˜Ô à×ÒÜ�N‰Nð1ôô
 " &¸EÈÔNˆŒÜ" 6°	Ô:ˆŒØ×%Ò%ä—‘œrŸw™wŸ|™|¨D×,<Ñ,<Ó,>ÀÀ|ÐUfÓgÓhñ HØ—‘×#Ñ#×3Ñ3×9Ñ9¸,ÔG÷Hà#'§8¡8×#6Ñ#6×#FÑ#F×#NÑ#NÈqÑ#QˆD�L‰LÕ ð	 &÷Hð Hús   Â&0DÄD%c                ó.   — | j                   j                  S rg   ©rÂ  rÌ  r…  s    r)   Úget_output_embeddingsz&TFEsmForMaskedLM.get_output_embeddings®  s   € Ø�|‰|×#Ñ#Ð#r+   c                ó&   — || j                   _        y rg   rÏ  )rG   Únew_embeddingss     r)   Úset_output_embeddingsz&TFEsmForMaskedLM.set_output_embeddings±  s   € Ø-ˆ�‰Õr+   c                ó   — | j                   S rg   )rÂ  r…  s    r)   Úget_lm_headzTFEsmForMaskedLM.get_lm_head´  s   € Ø�|‰|Ðr+   r±  z<mask>)r³  r´  rz  Úmaskc                ó8  — |�|n| j                   j                  }| j                  ||||||||	|
||¬«      }|d   }| j                  |«      }d}|�| j	                  ||¬«      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a!  
        labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
            config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
            loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
        kwargs (`Dict[str, any]`, *optional*, defaults to `{}`):
            Used to hide legacy arguments that have been deprecated.
        N)
r²   r¤   rã   r³   rä   rå   rç   re  rf  rà   r   )ÚlabelsÚlogitsr   ©ÚlossrÙ  râ   r‹   )r©   Úuse_return_dictry  rÂ  Úhf_compute_lossr   râ   r‹   )rG   r±   r²   r¤   rã   r³   rä   rå   rØ  rç   re  rf  rà   rù   r¢  Úprediction_scoresÚmasked_lm_lossr  s                     r)   rf   zTFEsmForMaskedLM.call·  sÝ   € ð> &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—(‘(ØØ)Ø%ØØ'Ø"7Ø#9Ø/Ø!5Ø#Øð ó 
ˆð " !™*ˆØ ŸL™L¨Ó9ÐàˆØÐØ!×1Ñ1¸ÐHYÐ1ÓZˆNáØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÐ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   c                ó:   — | j                   j                  ||«      S rg   r·  r¸  s      r)   r¦  z!TFEsmForMaskedLM.predict_contacts÷  r¹  r+   c                óÆ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒexY w# 1 sw Y   y xY w)NTry  rÂ  )r~   r   r#   r€   ry  rC   rR   rÂ  r�   s     r)   rR   zTFEsmForMaskedLM.buildú  s±   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ %Ø—‘—‘˜tÔ$÷%ä�4˜ DÓ)Ð5Ü—‘˜tŸ|™|×0Ñ0Ó1ñ )Ø—‘×"Ñ" 4Ô(÷)ð )ð 6÷%ð %ú÷)ð )úó   ÁCÂ%CÃCÃC )NNNNNNNNNNNF)r±   r¨  r²   r©  r¤   r©  rã   r©  r³   r©  rä   r©  rå   r©  rØ  r©  rç   rþ   re  rþ   rf  rþ   rà   rÿ   rj   z)Union[TFMaskedLMOutput, Tuple[tf.Tensor]]rg   )rk   rl   rm   r¬  Ú"_keys_to_ignore_on_load_unexpectedrE   rÐ  rÓ  rÕ  r   r
   r»  r¼  r   r½  r   r¾  rf   r¦  rR   ro   rp   s   @r)   rÀ  rÀ  ˜  s#  ø„ à'6Ð&7Ð#Ø*3¨Ð&ôRò"$ò.òð Ù*Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø$Ø$Øô	ð .2Ø8<Ø6:Ø37Ø7;Ø?CØ@DØ04Ø,0Ø/3Ø&*Øð6
à*ð6
ð 6ð6
ð 4ð	6
ð
 1ð6
ð 5ð6
ð  =ð6
ð !>ð6
ð .ð6
ð *ð6
ð -ð6
ð $ð6
ð ð6
ð 
3ò6
óó gó ð6
òpA÷	)r+   rÀ  c                  ó8   ‡ — e Zd ZdZdˆ fd„	Zdd„Zd„ Zd„ Zˆ xZS )rÅ  z&ESM Head for masked language modeling.c                óÜ  •— t         ‰| �  |¬«       t        j                  j	                  |j
                  t        |j                  «      d¬«      | _        t        j                  j                  |j                  d¬«      | _        |j                  rd | _        || _        y t        j                  j	                  |j                  t        |j                  «      dd¬«      | _        || _        y )	NrB   r  rÆ   r—   r˜   rÌ  F)rÇ   rC   rv   )rD   rE   r   rz   r{   rž   r   rŸ   r  r¢   r£   r—   rÆ  rÌ  r�   r©   rª   s      €r)   rE   zTFEsmLMHead.__init__	  sÇ   ø€ Ü‰Ñ˜dÐÔ#Ü—\‘\×'Ñ'Ø×Ñ´?À6×C[ÑC[Ó3\Ðcjð (ó 
ˆŒ
ô  Ÿ,™,×9Ñ9À&×BWÑBWÐ^jÐ9ÓkˆŒØ×%Ò%ØˆDŒLð ˆ�ô !Ÿ<™<×-Ñ-Ø×!Ñ!Ü#2°6×3KÑ3KÓ#LØØð	 .ó ˆDŒLð ˆ�r+   c                óª  — | j                   ry d| _         | j                  d| j                  j                  fdd¬«      | _        t        | dd «      �dt        j                  | j                  j                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �dt        j                  | j                  j                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �|| j                  j                  set        j                  | j                  j                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y y # 1 sw Y   �ŒxY w# 1 sw Y   Œ xY w# 1 sw Y   y xY w)NTr|   Úzeros)r.   rM   rN   r  r—   rÌ  )r~   rS   r©   r�   r|   r   r#   r€   r  rC   rR   rž   r—   rÆ  rÌ  r�   s     r)   rR   zTFEsmLMHead.build  sx  € ð �:Š:ØØˆŒ
Ø—O‘O F°4·;±;×3IÑ3IÐ2KÐY`Ðlp�OÓqˆŒ	Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ HØ—
‘
× Ñ  $¨¨d¯k©k×.EÑ.EÐ!FÔG÷Hä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ MØ—‘×%Ñ% t¨T°4·;±;×3JÑ3JÐ&KÔL÷Mä�4˜ DÓ)Ð5¸d¿k¹k×>]Ò>]Ü—‘˜tŸ|™|×0Ñ0Ó1ñ JØ—‘×"Ñ" D¨$°·±×0GÑ0GÐ#HÔI÷Jð Jð ?^Ð5÷Hñ Hú÷Mð Mú÷Jð Jús$   Á:3F0Ã+3F=Å23G	Æ0F:Æ=GÇ	Gc                ó   — d| j                   iS )Nr|   )r|   r…  s    r)   Úget_biaszTFEsmLMHead.get_bias,  s   € Ø˜Ÿ	™	Ð"Ð"r+   c                óR  — | j                  |«      }t        j                  j                  |«      }| j	                  |«      }| j
                  j                  r1t        j                  || j                  d¬«      | j                  z   }|S | j                  |«      | j                  z   }|S )NTrÝ   )
r  r#   r2  r3  r—   r©   rÆ  rá   rÌ  r|   )rG   Úfeaturesr&   s      r)   rf   zTFEsmLMHead.call/  s�   € Ø�J‰J�xÓ ˆÜ�E‰E�J‰J�q‹MˆØ�O‰O˜AÓˆð �;‰;×*Ò*Ü—	‘	˜!˜TŸ\™\°tÔ<¸t¿y¹yÑHˆAð ˆð —‘˜Q“ $§)¡)Ñ+ˆAØˆr+   rg   )	rk   rl   rm   rn   rE   rR   ré  rf   ro   rp   s   @r)   rÅ  rÅ    s   ø„ Ù0õó$Jò"#ö
r+   rÅ  z›
    ESM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    c                  óÎ   ‡ — e Zd ZdgZˆ fd„Ze eej                  d«      «       e	e
ee¬«      	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„«       «       «       Zd	d„Zˆ xZS )
ÚTFEsmForSequenceClassificationr¤   c                óž   •— t         ‰| �  |«       |j                  | _        || _        t	        |dd¬«      | _        t        |d¬«      | _        y ©NFry  r°  Ú
classifierrB   )rD   rE   Ú
num_labelsr©   r}  ry  ÚTFEsmClassificationHeadrð  rÍ  s     €r)   rE   z'TFEsmForSequenceClassification.__init__F  sB   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒØˆŒä! &¸EÈÔNˆŒÜ1°&¸|ÔLˆ�r+   r±  r²  c                ó2  — |	�|	n| j                   j                  }	| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }|€dn| j	                  ||«      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a†  
        labels (`tf.Tensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N©r²   r¤   rã   r³   rç   re  rf  rà   r   r   rÚ  )r©   rÜ  ry  rð  rÝ  r   râ   r‹   ©rG   r±   r²   r¤   rã   r³   rØ  rç   re  rf  rà   rù   r¢  rÙ  rÛ  r  s                   r)   rf   z#TFEsmForSequenceClassification.callN  sÊ   € ð4 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—(‘(ØØ)Ø%ØØ'Ø/Ø!5Ø#Øð ó 

ˆð " !™*ˆØ—‘ Ó1ˆà�~‰t¨4×+?Ñ+?ÀÈÓ+OˆáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä)ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   c                óÆ  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �Nt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       y y # 1 sw Y   ŒexY w# 1 sw Y   y xY w©NTry  rð  )r~   r   r#   r€   ry  rC   rR   rð  r�   s     r)   rR   z$TFEsmForSequenceClassification.build…  s±   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ %Ø—‘—‘˜tÔ$÷%ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ ,Ø—‘×%Ñ% dÔ+÷,ð ,ð 9÷%ð %ú÷,ð ,úrâ  ©
NNNNNNNNNF)r±   r¨  r²   r©  r¤   r©  rã   r©  r³   r©  rØ  r©  rç   rþ   re  rþ   rf  rþ   rà   rÿ   rj   z3Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]rg   )rk   rl   rm   r¬  rE   r   r
   r»  r¼  r   r½  r   r¾  rf   rR   ro   rp   s   @r)   rí  rí  <  së   ø„ ð (7Ð&7Ð#ôMð Ù*Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø.Ø$ôð .2Ø8<Ø6:Ø37Ø7;Ø04Ø,0Ø/3Ø&*Øð.
à*ð.
ð 6ð.
ð 4ð	.
ð
 1ð.
ð 5ð.
ð .ð.
ð *ð.
ð -ð.
ð $ð.
ð ð.
ð 
=ò.
óó gó ð.
÷`	,r+   rí  z¢
    ESM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for
    Named-Entity-Recognition (NER) tasks.
    c                  óÔ   ‡ — e Zd ZdgZdgZˆ fd„Ze eej                  d«      «       e
eee¬«      	 	 	 	 	 	 	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d	d„«       «       «       Zd
d„Zˆ xZS )ÚTFEsmForTokenClassificationr€  r¤   c                ó6  •— t         ‰| �  |«       |j                  | _        t        |dd¬«      | _        t
        j                  j                  |j                  «      | _	        t
        j                  j                  |j                  d¬«      | _        || _        y rï  )rD   rE   rñ  r}  ry  r   rz   rÓ   r  rÕ   r{   rð  r©   rÍ  s     €r)   rE   z$TFEsmForTokenClassification.__init__œ  sq   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä! &¸EÈÔNˆŒÜ—|‘|×+Ñ+¨F×,FÑ,FÓGˆŒÜŸ,™,×,Ñ,¨V×->Ñ->À\Ð,ÓRˆŒØˆ�r+   r±  r²  c                óX  — |	�|	n| j                   j                  }	| j                  ||||||||	|
¬«	      }|d   }| j                  ||
¬«      }| j	                  |«      }|€dn| j                  ||«      }|	s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )zÔ
        labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
        Nrô  r   rß   r   rÚ  )	r©   rÜ  ry  rÕ   rð  rÝ  r   râ   r‹   rõ  s                   r)   rf   z TFEsmForTokenClassification.call¥  sÜ   € ð0 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—(‘(ØØ)Ø%ØØ'Ø/Ø!5Ø#Øð ó 

ˆð " !™*ˆàŸ,™, À˜,ÓJˆØ—‘ Ó1ˆà�~‰t¨4×+?Ñ+?ÀÈÓ+OˆáØ�Y ¨¨ Ñ,ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä&ØØØ!×/Ñ/Ø×)Ñ)ô	
ð 	
r+   c                óô  — | j                   ry d| _         t        | dd «      �Mt        j                  | j                  j
                  «      5  | j                  j                  d «       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   Œ|xY w# 1 sw Y   y xY wr÷  )
r~   r   r#   r€   ry  rC   rR   rð  r©   rž   r�   s     r)   rR   z!TFEsmForTokenClassification.buildÝ  sÇ   € Ø�:Š:ØØˆŒ
Ü�4˜ Ó%Ð1Ü—‘˜tŸx™xŸ}™}Ó-ñ %Ø—‘—‘˜tÔ$÷%ä�4˜ tÓ,Ð8Ü—‘˜tŸ™×3Ñ3Ó4ñ MØ—‘×%Ñ% t¨T°4·;±;×3JÑ3JÐ&KÔL÷Mð Mð 9÷%ð %ú÷Mð Mús   ÁC"Â%3C.Ã"C+Ã.C7rø  )r±   r¨  r²   r©  r¤   r©  rã   r©  r³   r©  rØ  r©  rç   rþ   re  rþ   rf  rþ   rà   rÿ   rj   z0Union[TFTokenClassifierOutput, Tuple[tf.Tensor]]rg   )rk   rl   rm   rã  r¬  rE   r   r
   r»  r¼  r   r½  r   r¾  rf   rR   ro   rp   s   @r)   rú  rú  ‘  só   ø„ ð +4¨Ð&Ø'6Ð&7Ð#ôð Ù*Ð+?×+FÑ+FÐGdÓ+eÓfÙØ&Ø+Ø$ôð .2Ø8<Ø6:Ø37Ø7;Ø04Ø,0Ø/3Ø&*Øð/
à*ð/
ð 6ð/
ð 4ð	/
ð
 1ð/
ð 5ð/
ð .ð/
ð *ð/
ð -ð/
ð $ð/
ð ð/
ð 
:ò/
óó gó ð/
÷b	Mr+   rú  c                  ó4   ‡ — e Zd ZdZdˆ fd„	Zdd„Zdd„Zˆ xZS )rò  z-Head for sentence-level classification tasks.c                ó¤  •— t         ‰| �  |¬«       t        j                  j	                  |j
                  t        |j                  «      dd¬«      | _        t        j                  j                  |j                  «      | _        t        j                  j	                  |j                  t        |j                  «      dd¬«      | _        || _        y )NrB   rr  r  )rÇ   rw   rC   ÚlinearÚout_proj)rD   rE   r   rz   r{   rž   r   rŸ   r  rÓ   r  rÕ   rñ  r  r©   rª   s      €r)   rE   z TFEsmClassificationHead.__init__ì  sª   ø€ Ü‰Ñ˜dÐÔ#Ü—\‘\×'Ñ'Ø×ÑÜ.¨v×/GÑ/GÓHØØð	 (ó 
ˆŒ
ô —|‘|×+Ñ+¨F×,FÑ,FÓGˆŒÜŸ™×*Ñ*Ø×ÑÜ.¨v×/GÑ/GÓHØØð	 +ó 
ˆŒð ˆ�r+   c                ó®   — |d d …dd d …f   }| j                  ||¬«      }| j                  |«      }| j                  ||¬«      }| j                  |«      }|S )Nr   rß   )rÕ   r  r  )rG   rë  rà   r&   s       r)   rf   zTFEsmClassificationHead.callý  sV   € Ø’Q˜š1�WÑˆØ�L‰L˜ XˆLÓ.ˆØ�J‰J�q‹MˆØ�L‰L˜ XˆLÓ.ˆØ�M‰M˜!ÓˆØˆr+   c                ó"  — | j                   ry d| _         t        | dd «      �dt        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       t        | dd «      �et        j                  | j                  j
                  «      5  | j                  j                  d d | j                  j                  g«       d d d «       y y # 1 sw Y   Œ|xY w# 1 sw Y   y xY w)NTr  r  )
r~   r   r#   r€   r  rC   rR   r©   rž   r  r�   s     r)   rR   zTFEsmClassificationHead.build  sÞ   € Ø�:Š:ØØˆŒ
Ü�4˜ $Ó'Ð3Ü—‘˜tŸz™zŸ™Ó/ñ HØ—
‘
× Ñ  $¨¨d¯k©k×.EÑ.EÐ!FÔG÷Hä�4˜ TÓ*Ð6Ü—‘˜tŸ}™}×1Ñ1Ó2ñ KØ—‘×#Ñ# T¨4°·±×1HÑ1HÐ$IÔJ÷Kð Kð 7÷Hð Hú÷Kð Kús   Á3C9Â<3DÃ9DÄDrg   r  )rk   rl   rm   rn   rE   rf   rR   ro   rp   s   @r)   rò  rò  é  s   ø„ Ù7õó"÷	Kr+   rò  c                ó”   — t        j                  | |k7  t         j                  «      }t        j                  |d¬«      |z   |z  }||z   S )zÿ
    Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
    are ignored. This is modified from fairseq's `utils.make_positions`.

    Args:
        x: tf.Tensor x:

    Returns: tf.Tensor
    r   r!   )r#   r…   r»   Úcumsum)r±   r¦   r´   rÖ  Úincremental_indicess        r)   r¬   r¬     sD   € ô �7‰7�9 Ñ+¬R¯X©XÓ6€DÜŸ9™9 T°Ô2Ð5KÑKÈtÑSÐØ Ñ,Ð,r+   )rÀ  rí  rú  r®  rx  )r   )Irn   Ú
__future__r   rÇ  Útypingr   r   r   ÚnumpyÚnpÚ
tensorflowr#   Ú
file_utilsr   r	   r
   Úmodeling_tf_outputsr   r   r   r   r   Úmodeling_tf_utilsr   r   r   r   r   r   r   r   r   Útf_utilsr   r   Úutilsr   Úconfiguration_esmr   Ú
get_loggerrk   rÃ  r½  r¾  r*   r1   r5   r>   rz   ÚLayerr@   rr   r’   rÁ   r  r  r(  r8  r=  rT  rp  rx  ÚESM_START_DOCSTRINGr»  r}  r®  rÀ  rÅ  rí  rú  rò  r¬   Ú__all__r+  r+   r)   ú<module>r     sÖ  ðñ å "ã 	ß )Ñ )ã Û ç qÑ q÷õ ÷
÷ 
õ 
÷ GÝ Ý (ð 
ˆ×	Ñ	˜HÓ	%€à1Ð Ø€ò)ò
.ò-ò
	ô)
˜Ÿ™×*Ñ*ô )
ôX%: §¡×!3Ñ!3ô %:ôPmM�e—l‘l×(Ñ(ô mMô`d3˜Ÿ™×+Ñ+ô d3ôNH�e—l‘l×(Ñ(ô Hô02L�U—\‘\×'Ñ'ô 2LôjH˜Ÿ™×*Ñ*ô Hô2N�%—,‘,×$Ñ$ô Nô0fL�—‘×#Ñ#ô fLôRZ&�5—<‘<×%Ñ%ô Z&ô|H�%—,‘,×$Ñ$ô Hô:Ð,ô ðÐ ð'Ð ñT ØcØóô[0�U—\‘\×'Ñ'ó [0ó	ð[0ñ| ØcØóôK%Ð%ó K%ó	ðK%ñ\ ÐMÐObÓcôj)Ð+Ð-Ió j)ó dðj)ôZ3�%—,‘,×$Ñ$ô 3ñl ðð óôK,Ð%9Ð;Wó K,óðK,ñ\ ðð óôNMÐ"6Ð8Qó NMóðNMôb%K˜eŸl™l×0Ñ0ô %KóP-ò �r+   