Ë
    S^(h¡T ã                   ó0  — d dl Z d dlZd dlmZ d dlmZ d dlmZmZm	Z	m
Z
mZmZmZ d dlZd dlZd dlmZ d dlmZ ddlmZ ddlmZ dd	lmZmZmZmZmZmZ d
dlm Z  d
dl!m"Z"m#Z#m$Z$ d
dl%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1  ejd                  e3«      Z4dZ5dZ6e G d„ de«      «       Z7dZ8d„ Z9d„ Z:dUde	ejv                     de<dejv                  fd„Z=dejv                  de>fd„Z?dejv                  de	e>   fd„Z@d„ ZAdVd „ZBd!„ ZC G d"„ d#ejˆ                  «      ZE G d$„ d%ejŒ                  «      ZGej�                  j’                  dWdejv                  d&e>dejv                  fd'„«       ZJ G d(„ d)ejŒ                  «      ZK G d*„ d+ejŒ                  «      ZL G d,„ d-ejŒ                  «      ZM G d.„ d/e$«      ZN G d0„ d1ejŒ                  «      ZO G d2„ d3ejŒ                  «      ZP G d4„ d5ejŒ                  «      ZQ G d6„ d7ejŒ                  «      ZR G d8„ d9ejŒ                  «      ZS G d:„ d;ejŒ                  «      ZT G d<„ d=«      ZUdXd>„ZVd?„ ZW G d@„ dAejŒ                  «      ZX G dB„ dCejŒ                  «      ZY G dD„ dEejŒ                  «      ZZ G dF„ dGejŒ                  «      Z[ G dH„ dIejŒ                  «      Z\ G dJ„ dKejŒ                  «      Z] G dL„ dMejŒ                  «      Z^ G dN„ dOejŒ                  «      Z_ G dP„ dQejŒ                  «      Z` edRe"«       G dS„ dTe$«      «       ZadTd/gZby)Yé    N)Ú	dataclass)Úpartial)ÚCallableÚDictÚListÚOptionalÚSequenceÚTupleÚUnion)Ú	LayerNormé   )Úis_deepspeed_available)ÚModelOutput)ÚContextManagersÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚis_scipy_availableÚloggingÚreplace_return_docstringsé   )Ú	EsmConfig)ÚESM_START_DOCSTRINGÚEsmModelÚEsmPreTrainedModel)Ú	OFProteinÚRigidÚRotationÚatom14_to_atom37Úchunk_layerÚcompute_predicted_aligned_errorÚ
compute_tmÚ-frames_and_literature_positions_to_atom14_posÚmake_atom14_masksÚresidue_constantsÚto_pdbÚtorsion_angles_to_frameszfacebook/esmfold_v1r   c                   óª  — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed	<   dZeej                     ed
<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   dZeej                     ed<   y)ÚEsmForProteinFoldingOutputa!  
    Output type of [`EsmForProteinFoldingOutput`].

    Args:
        frames (`torch.FloatTensor`):
            Output frames.
        sidechain_frames (`torch.FloatTensor`):
            Output sidechain frames.
        unnormalized_angles (`torch.FloatTensor`):
            Predicted unnormalized backbone and side chain torsion angles.
        angles (`torch.FloatTensor`):
            Predicted backbone and side chain torsion angles.
        positions (`torch.FloatTensor`):
            Predicted positions of the backbone and side chain atoms.
        states (`torch.FloatTensor`):
            Hidden states from the protein folding trunk.
        s_s (`torch.FloatTensor`):
            Per-residue embeddings derived by concatenating the hidden states of each layer of the ESM-2 LM stem.
        s_z (`torch.FloatTensor`):
            Pairwise residue embeddings.
        distogram_logits (`torch.FloatTensor`):
            Input logits to the distogram used to compute residue distances.
        lm_logits (`torch.FloatTensor`):
            Logits output by the ESM-2 protein language model stem.
        aatype (`torch.FloatTensor`):
            Input amino acids (AlphaFold2 indices).
        atom14_atom_exists (`torch.FloatTensor`):
            Whether each atom exists in the atom14 representation.
        residx_atom14_to_atom37 (`torch.FloatTensor`):
            Mapping between atoms in the atom14 and atom37 representations.
        residx_atom37_to_atom14 (`torch.FloatTensor`):
            Mapping between atoms in the atom37 and atom14 representations.
        atom37_atom_exists (`torch.FloatTensor`):
            Whether each atom exists in the atom37 representation.
        residue_index (`torch.FloatTensor`):
            The index of each residue in the protein chain. Unless internal padding tokens are used, this will just be
            a sequence of integers from 0 to `sequence_length`.
        lddt_head (`torch.FloatTensor`):
            Raw outputs from the lddt head used to compute plddt.
        plddt (`torch.FloatTensor`):
            Per-residue confidence scores. Regions of low confidence may indicate areas where the model's prediction is
            uncertain, or where the protein structure is disordered.
        ptm_logits (`torch.FloatTensor`):
            Raw logits used for computing ptm.
        ptm (`torch.FloatTensor`):
            TM-score output representing the model's high-level confidence in the overall structure.
        aligned_confidence_probs (`torch.FloatTensor`):
            Per-residue confidence scores for the aligned structure.
        predicted_aligned_error (`torch.FloatTensor`):
            Predicted error between the model's prediction and the ground truth.
        max_predicted_aligned_error (`torch.FloatTensor`):
            Per-sample maximum predicted error.
    NÚframesÚsidechain_framesÚunnormalized_anglesÚanglesÚ	positionsÚstatesÚs_sÚs_zÚdistogram_logitsÚ	lm_logitsÚaatypeÚatom14_atom_existsÚresidx_atom14_to_atom37Úresidx_atom37_to_atom14Úatom37_atom_existsÚresidue_indexÚ	lddt_headÚplddtÚ
ptm_logitsÚptmÚaligned_confidence_probsÚpredicted_aligned_errorÚmax_predicted_aligned_error)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r)   r   ÚtorchÚFloatTensorÚ__annotations__r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   © ó    úf/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/esm/modeling_esmfold.pyr(   r(   ;   s÷  … ñ4ðl +/€FˆH�U×&Ñ&Ñ'Ó.Ø48Ð�h˜u×0Ñ0Ñ1Ó8Ø7;Ð˜ %×"3Ñ"3Ñ4Ó;Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø-1€Iˆx˜×)Ñ)Ñ*Ó1Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø'+€Cˆ�%×#Ñ#Ñ	$Ó+Ø'+€Cˆ�%×#Ñ#Ñ	$Ó+Ø48Ð�h˜u×0Ñ0Ñ1Ó8Ø-1€Iˆx˜×)Ñ)Ñ*Ó1Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø6:Ð˜ ×!2Ñ!2Ñ3Ó:Ø;?Ð˜X e×&7Ñ&7Ñ8Ó?Ø;?Ð˜X e×&7Ñ&7Ñ8Ó?Ø6:Ð˜ ×!2Ñ!2Ñ3Ó:Ø15€M�8˜E×-Ñ-Ñ.Ó5Ø-1€Iˆx˜×)Ñ)Ñ*Ó1Ø)-€Eˆ8�E×%Ñ%Ñ&Ó-Ø.2€J�˜×*Ñ*Ñ+Ó2Ø'+€Cˆ�%×#Ñ#Ñ	$Ó+Ø<@Ð˜h u×'8Ñ'8Ñ9Ó@Ø;?Ð˜X e×&7Ñ&7Ñ8Ó?Ø?CÐ ¨%×*;Ñ*;Ñ!<ÔCrH   r(   aJ  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.max_position_embeddings - 1]`.

            [What are position IDs?](../glossary#position-ids)
        masking_pattern (`torch.LongTensor` of shape `({0})`, *optional*):
            Locations of tokens to mask during training as a form of regularization. Mask values selected in `[0, 1]`.
        num_recycles (`int`, *optional*, defaults to `None`):
            Number of times to recycle the input sequence. If `None`, defaults to `config.num_recycles`. "Recycling"
            consists of passing the output of the folding trunk back in as input to the trunk. During training, the
            number of recycles should vary with each batch, to ensure that the model learns to output valid predictions
            after each recycle. During inference, num_recycles should be set to the highest value that the model was
            trained with for maximum accuracy. Accordingly, when this value is set to `None`, config.max_recycles is
            used.
c                  ó€   — t        j                  «       t         j                  k(  } | xr t        j                  «       } | S ©N)rD   Úget_autocast_gpu_dtypeÚfloat16Úis_autocast_enabled)Úfp16_enableds    rI   Úis_fp16_enabledrP   ­   s2   € ä×/Ñ/Ó1´U·]±]ÑB€LØÒ?¤E×$=Ñ$=Ó$?€LàÐrH   c                  ót   — t        «       ry	 dd l} | j                  j                  «       S # t        $ r Y yw xY w)NFr   )r   Ú	deepspeedÚutilsÚis_initializedÚ	Exception)rR   s    rI   Úis_deepspeed_initializedrV   µ   s:   € ÜÔØð	Ûð —?‘?×1Ñ1Ó3Ð3øÜò 	Ùð	ús   �+ «	7¶7ÚsamplesÚpad_vÚreturnc                 óà  — t        | «      dk(  rt        j                  «       S t        | D �ch c]  }|j                  «       ’Œ c}«      dk7  r*t	        d| D �cg c]  }|j                  «       ‘Œ c}› �«      ‚t        | D �ch c]  }|j                  ’Œ c}«      \  }t        | D �cg c]  }|j                  ‘Œ c}Ž D �cg c]  }t        |«      ‘Œ }}t        j                  t        | «      g|¢­| d   j                  |dœŽ}|j                  |«       t        t        | «      «      D ]+  }||   }| |   }	|	|t        d„ |	j                  D «       «      <   Œ- |S c c}w c c}w c c}w c c}w c c}w )zõ
    Takes a list of tensors with the following dimensions:
        [(d_11, ..., d_1K),
         (d_21, ..., d_2K), ..., (d_N1, ..., d_NK)]
    and stack + pads them into a single tensor of:
    (N, max_i=1,N { d_i1 }, ..., max_i=1,N {diK})
    r   r   z Samples has varying dimensions: )ÚdtypeÚdevicec              3   ó4   K  — | ]  }t        d |«      –— Œ y­w)r   N©Úslice)Ú.0Úks     rI   ú	<genexpr>z(collate_dense_tensors.<locals>.<genexpr>Õ   s   è ø€ Ò4 q”u˜Q —{Ñ4ùs   ‚)ÚlenrD   ÚTensorÚdimÚRuntimeErrorÚtupler\   ÚzipÚshapeÚmaxÚemptyr[   Úfill_Úrange)
rW   rX   Úxr\   ÚlstÚ	max_shapeÚresultÚiÚresult_iÚts
             rI   Úcollate_dense_tensorsru   Â   s6  € ô ˆ7ƒ|�qÒÜ�|‰|‹~ÐÜ
˜WÖ%˜ˆA�E‰E�GÒ%Ó&¨!Ò+ÜÐ=ÐPWÖ>XÈ1¸q¿u¹u½wÒ>XÐ=YÐZÓ[Ð[Ü¨Ö1 A�q—x“xÒ1Ó2�I€VÜ%(¸GÖ*D°q¨1¯7«7Ò*DÐ%EÖF˜c”�S•ÐF€IÐFÜ�[‰[œ˜W›ÐY¨	ÑY¸À¹×9IÑ9IÐRXÒY€FØ
‡L�L�ÔÜ”3�w“<Ó ò :ˆØ˜!‘9ˆØ�A‰JˆØ89ˆ”Ñ4¨A¯G©GÔ4Ó4Ò5ð:ð €Mùò &ùÚ>XùÚ1ùÚ*DùÒFs   ¬EÁE
Â E!Â%E&Â>E+rt   Úno_dimsc                 óF   — | j                  | j                  d |  dz   «      S )N©éÿÿÿÿ)Úreshaperi   )rt   rv   s     rI   Úflatten_final_dimsr{   Ù   s#   € Ø�9‰9�Q—W‘W˜Y˜w˜hÐ'¨%Ñ/Ó0Ð0rH   ÚtensorÚindsc           	      óÆ   — dt        |«      z  }t        t        t        | j                  d | «      «      «      }| j	                  ||D �cg c]  }||z   ‘Œ	 c}z   «      S c c}w )Nry   )rc   Úlistrm   ri   Úpermute)r|   r}   Ú
zero_indexÚ
first_indsrr   s        rI   Úpermute_final_dimsrƒ   Ý   sT   € Ø”c˜$“i‘€JÜ”eœC §¡¨[¨jÐ 9Ó:Ó;Ó<€JØ�>‰>˜*ÀÖ'E¸1¨
°Q«Ò'EÑEÓFÐFùÒ'Es   Á	A
c                 óÈ   — |d   }i }|j                  «       D ]B  \  }}|D �cg c]  }||   ‘Œ	 }}t        |t        «      rt        | |«      ||<   Œ8 | |«      ||<   ŒD |S c c}w )Nr   )ÚitemsÚ
isinstanceÚdictÚdict_multimap)ÚfnÚdictsÚfirstÚnew_dictra   ÚvÚdÚall_vs           rI   rˆ   rˆ   ã   ss   € Ø�!‰H€EØ€HØ—‘“ò $‰ˆˆ1Ø$Ö%˜!��1“Ð%ˆÐ%Ü�aœÔÜ'¨¨EÓ2ˆH�QŠKá˜U›)ˆH�QŠKð$ð €Oùò &s   ¢Ac                 ó^  — | j                   }|t        d|d   «      z  }t        «       sjt        j	                  d«       t        j                  |«      }t        j                  j                  j                  | |¬«      j                  dd|z  ¬«       y ddlm} t        j                  |«      |j                  d	d
dd¬«      z  }|j                  d	d
d|| j!                  «       ¬«      }t#        j$                  ||«      }| j'                  t        j(                  || j*                  ¬«      «       y )Nr   zlThis init requires scipy, but scipy was not found, default to an approximation that might not be equivalent.)Ústdç        ç       @)Úminrj   r   )Ú	truncnorméþÿÿÿé   )ÚaÚbÚlocÚscale)r˜   r™   rš   r›   Úsize©r\   )ri   rj   r   ÚloggerÚwarningÚmathÚsqrtrD   ÚnnÚinitÚnormal_ÚclampÚscipy.statsr•   r‘   ÚrvsÚnumelÚnprz   Úcopy_r|   r\   )Úweightsr›   Úfanri   r‘   r•   rW   s          rI   Útrunc_normal_init_r­   ð   sè   € Ø�M‰M€EØ”C˜˜5 ™8Ó$Ñ$€EäÔÜ�‰ðô	
ô �i‰i˜ÓˆÜ�‰�‰×Ñ˜g¨3ÐÓ/×5Ñ5¸#À3ÈÁ9Ð5ÕMõ 	*ä�i‰i˜Ó §¡°°q¸aÀq Ó!IÑIˆØ—-‘- "¨¨q¸À'Ç-Á-Ã/�-ÓRˆÜ—*‘*˜W eÓ,ˆØ�‰”e—l‘l 7°7·>±>ÔBÕCrH   c                 ó|   — t        j                  «       5  d}| j                  |«       d d d «       y # 1 sw Y   y xY w)NgËab€ˆRá?)rD   Úno_gradrl   )r«   Úsoftplus_inverse_1s     rI   Úipa_point_weights_init_r±     s3   € Ü	�‰‹ñ *Ø.ÐØ�‰Ð(Ô)÷*÷ *ñ *ús   •2²;c                   ó|   ‡ — e Zd ZdZ	 	 	 d	dededededeee	j                  e	j                  gdf      f
ˆ fd„Zˆ xZS )
ÚEsmFoldLinearz¾
    A Linear layer with built-in nonstandard initializations. Called just like torch.nn.Linear.

    Implements the initializers in 1.11.4, plus some additional ones found in the code.
    NÚin_dimÚout_dimÚbiasr£   Úinit_fnc                 óð   •— t         ‰| �  |||¬«       |r8t        j                  «       5  | j                  j                  d«       ddd«       || _        || _        |dvrt        d«      ‚y# 1 sw Y   Œ'xY w)aM  
        Args:
            in_dim:
                The final dimension of inputs to the layer
            out_dim:
                The final dimension of layer outputs
            bias:
                Whether to learn an additive bias. True by default
            init:
                The initializer to use. Choose from:

                "default": LeCun fan-in truncated normal initialization "relu": He initialization w/ truncated normal
                distribution "glorot": Fan-average Glorot uniform initialization "gating": Weights=0, Bias=1 "normal":
                Normal initialization with std=1/sqrt(fan_in) "final": Weights=0, Bias=0

                Overridden by init_fn if the latter is not None.
            init_fn:
                A custom initializer taking weight and bias as inputs. Overrides init if not None.
        ©r¶   r   N)ÚdefaultÚreluÚglorotÚgatingÚnormalÚfinalzInvalid init string.)	ÚsuperÚ__init__rD   r¯   r¶   rl   r£   r·   Ú
ValueError)Úselfr´   rµ   r¶   r£   r·   Ú	__class__s         €rI   rÁ   zEsmFoldLinear.__init__  ss   ø€ ô6 	‰Ñ˜ ¨tÐÔ4áÜ—‘“ñ #Ø—	‘	—‘ Ô"÷#àˆŒ	ØˆŒàÐQÑQÜÐ3Ó4Ð4ð R÷#ð #ús   ªA,Á,A5)Trº   N)r@   rA   rB   rC   ÚintÚboolÚstrr   r   rD   rd   rÁ   Ú__classcell__©rÄ   s   @rI   r³   r³     si   ø„ ñð ØØJNñ$5àð$5ð ð$5ð ð	$5ð
 ð$5ð ˜( E§L¡L°%·,±,Ð#?ÀÐ#EÑFÑG÷$5ñ $5rH   r³   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEsmFoldLayerNormc                 óô   •— t         ‰| �  «        |f| _        || _        t	        j
                  t        j                  |«      «      | _        t	        j
                  t        j                  |«      «      | _
        y rK   )rÀ   rÁ   Úc_inÚepsr¢   Ú	ParameterrD   ÚonesÚweightÚzerosr¶   )rÃ   rÍ   rÎ   rÄ   s      €rI   rÁ   zEsmFoldLayerNorm.__init__:  sP   ø€ Ü‰ÑÔà�GˆŒ	ØˆŒä—l‘l¤5§:¡:¨dÓ#3Ó4ˆŒÜ—L‘L¤§¡¨TÓ!2Ó3ˆ�	rH   c           	      óF  — |j                   }|t        j                  u rªt        «       s t        j                  j
                  j                  d¬«      5  t        j                  j                  || j                  | j                  j                  |¬«      | j                  j                  |¬«      | j                  «      }d d d «       |S t        j                  j                  || j                  | j                  | j                  | j                  «      }|S # 1 sw Y   S xY w)NF©Úenabled©r[   )r[   rD   Úbfloat16rV   ÚcudaÚampÚautocastr¢   Ú
functionalÚ
layer_normrÍ   rÑ   Útor¶   rÎ   )rÃ   rn   rŽ   Úouts       rI   ÚforwardzEsmFoldLayerNorm.forwardC  sÚ   € Ø�G‰GˆØ”—‘ÑÔ'?Ô'AÜ—‘—‘×(Ñ(°Ð(Ó7ñ wÜ—m‘m×.Ñ.¨q°$·)±)¸T¿[¹[¿^¹^ÐRS¸^Ó=TÐVZ×V_ÑV_×VbÑVbÐijÐVbÓVkÐmq×muÑmuÓv�÷wð
 ˆ
ô —-‘-×*Ñ*¨1¨d¯i©i¸¿¹ÀdÇiÁiÐQU×QYÑQYÓZˆCàˆ
÷wð
 ˆ
ús   ÁA,DÄD )gñhãˆµøä>©r@   rA   rB   rÁ   rß   rÈ   rÉ   s   @rI   rË   rË   9  s   ø„ õ4örH   rË   re   c                 ó†  — | j                   }|t        j                  u rjt        «       s`t        j                  j
                  j                  d¬«      5  t        j                  j                  j                  | |¬«      }ddd«       |S t        j                  j                  j                  | |¬«      }|S # 1 sw Y   S xY w)z[
    Softmax, but without automatic casting to fp32 when the input is of type bfloat16
    FrÔ   ©re   N)
r[   rD   r×   rV   rØ   rÙ   rÚ   r¢   rÛ   Úsoftmax)rt   re   rŽ   Úss       rI   Úsoftmax_no_castrå   N  s›   € ð
 	
�‰€AØŒE�N‰NÑÔ#;Ô#=Ü�Z‰Z�^‰^×$Ñ$¨UÐ$Ó3ñ 	8Ü—‘×#Ñ#×+Ñ+¨A°3Ð+Ó7ˆA÷	8ð
 €Hô �H‰H×Ñ×'Ñ'¨¨sÐ'Ó3ˆà€H÷	8ð
 €Hús   Á,B6Â6C c                   óÊ  ‡ — e Zd ZdZ	 ddedededededefˆ fd„Zd	ej                  d
ej                  de	ej                  ej                  ej                  f   fd„Z
dej                  d	ej                  dej                  fd„Z	 	 	 	 	 	 	 dd	ej                  d
ej                  deeej                        dedededededeej                     dej                  fd„Zˆ xZS )ÚEsmFoldAttentionzu
    Standard multi-head attention using AlphaFold's default layer initialization. Allows multiple bias vectors.
    Úc_qÚc_kÚc_vÚc_hiddenÚno_headsr½   c                 óÜ  •— t         ‰| �  «        || _        || _        || _        || _        || _        || _        t        | j                  | j
                  | j                  z  dd¬«      | _	        t        | j                  | j
                  | j                  z  dd¬«      | _
        t        | j                  | j
                  | j                  z  dd¬«      | _        t        | j
                  | j                  z  | j                  d¬«      | _        d| _        | j                  r4t        | j                  | j
                  | j                  z  d¬«      | _        t        j                  «       | _        y)aª  
        Args:
            c_q:
                Input dimension of query data
            c_k:
                Input dimension of key data
            c_v:
                Input dimension of value data
            c_hidden:
                Per-head hidden dimension
            no_heads:
                Number of attention heads
            gating:
                Whether the output should be gated using query data
        Fr¼   ©r¶   r£   r¿   ©r£   Nr½   )rÀ   rÁ   rè   ré   rê   rë   rì   r½   r³   Úlinear_qÚlinear_kÚlinear_vÚlinear_oÚlinear_gr¢   ÚSigmoidÚsigmoid)rÃ   rè   ré   rê   rë   rì   r½   rÄ   s          €rI   rÁ   zEsmFoldAttention.__init__b  s
  ø€ ô0 	‰ÑÔàˆŒØˆŒØˆŒØ ˆŒØ ˆŒØˆŒô
 & d§h¡h°·±ÀÇÁÑ0MÐTYÐ`hÔiˆŒÜ% d§h¡h°·±ÀÇÁÑ0MÐTYÐ`hÔiˆŒÜ% d§h¡h°·±ÀÇÁÑ0MÐTYÐ`hÔiˆŒÜ% d§m¡m°d·m±mÑ&CÀTÇXÁXÐT[Ô\ˆŒàˆŒØ�;Š;Ü)¨$¯(©(°D·M±MÀDÇMÁMÑ4QÐX`ÔaˆDŒMä—z‘z“|ˆ�rH   Úq_xÚkv_xrY   c                 ó0  — | j                  |«      }| j                  |«      }| j                  |«      }|j                  |j                  d d | j
                  dfz   «      }|j                  |j                  d d | j
                  dfz   «      }|j                  |j                  d d | j
                  dfz   «      }|j                  dd«      }|j                  dd«      }|j                  dd«      }|t        j                  | j                  «      z  }|||fS )Nry   r–   éýÿÿÿ)
rð   rñ   rò   Úviewri   rì   Ú	transposer    r¡   rë   )rÃ   r÷   rø   Úqra   r�   s         rI   Ú	_prep_qkvzEsmFoldAttention._prep_qkv‘  sô   € à�M‰M˜#ÓˆØ�M‰M˜$ÓˆØ�M‰M˜$Óˆð �F‰F�1—7‘7˜3˜B�< 4§=¡=°"Ð"5Ñ5Ó6ˆØ�F‰F�1—7‘7˜3˜B�< 4§=¡=°"Ð"5Ñ5Ó6ˆØ�F‰F�1—7‘7˜3˜B�< 4§=¡=°"Ð"5Ñ5Ó6ˆð �K‰K˜˜BÓˆØ�K‰K˜˜BÓˆØ�K‰K˜˜BÓˆà	ŒT�Y‰Y�t—}‘}Ó%Ñ%ˆà�!�QˆwˆrH   Úoc                 óü   — | j                   �R| j                  | j                  |«      «      }|j                  |j                  d d | j                  dfz   «      }||z  }t        |d«      }| j                  |«      }|S )Nry   r—   )rô   rö   rû   ri   rì   r{   ró   )rÃ   rÿ   r÷   Úgs       rI   Ú_wrap_upzEsmFoldAttention._wrap_up¥  sw   € Ø�=‰=Ð$Ø—‘˜TŸ]™]¨3Ó/Ó0ˆAð —‘�q—w‘w˜s �| t§}¡}°bÐ&9Ñ9Ó:ˆAØ�A‘ˆAô ˜q !Ó$ˆð �M‰M˜!ÓˆàˆrH   ÚbiasesÚuse_memory_efficient_kernelÚuse_lmaÚlma_q_chunk_sizeÚlma_kv_chunk_sizeÚ	use_flashÚ
flash_maskc
                 óž  — |r|�|€t        d«      ‚|r|�t        d«      ‚|||g}
t        |
«      dkD  rt        d«      ‚|€g }| j                  ||«      \  }}}t        |d«      }t	        j
                  ||«      }|D ]  }||z  }Œ	 t        |d«      }t	        j
                  ||«      }|j                  dd«      }| j                  ||«      }|S )	a{  
        Args:
            q_x:
                [*, Q, C_q] query data
            kv_x:
                [*, K, C_k] key data
            biases:
                List of biases that broadcast to [*, H, Q, K]
            use_memory_efficient_kernel:
                Whether to use a custom memory-efficient attention kernel. This should be the default choice for most.
                If none of the "use_<...>" flags are True, a stock PyTorch implementation is used instead
            use_lma:
                Whether to use low-memory attention (Staats & Rabe 2021). If none of the "use_<...>" flags are True, a
                stock PyTorch implementation is used instead
            lma_q_chunk_size:
                Query chunk size (for LMA)
            lma_kv_chunk_size:
                Key/Value chunk size (for LMA)
        Returns
            [*, Q, C_q] attention update
        zPIf use_lma is specified, lma_q_chunk_size and lma_kv_chunk_size must be providedzSuse_flash is incompatible with the bias option. For masking, use flash_mask insteadr   z2Choose at most one alternative attention algorithm)r   r   ry   r–   rú   )	rÂ   Úsumrþ   rƒ   rD   Úmatmulrå   rü   r  )rÃ   r÷   rø   r  r  r  r  r  r  r	  Úattn_optionsÚqueryÚkeyÚvalueÚoutputr™   s                   rI   rß   zEsmFoldAttention.forwardµ  s÷   € ñB Ð(Ð0Ð4EÐ4MÜÐoÓpÐpá˜Ð+ÜÐrÓsÐsà3°W¸iÐHˆÜˆ|Ó˜qÒ ÜÐQÓRÐRàˆ>ØˆFð !ŸN™N¨3°Ó5Ñˆˆs�EÜ   fÓ-ˆô —‘˜e SÓ)ˆØò 	ˆAØ�a‰K‰Fð	ä  ¨Ó,ˆô —‘˜f eÓ,ˆØ×!Ñ! " bÓ)ˆØ—‘˜v sÓ+ˆàˆrH   ©T)NFFi   i   FN)r@   rA   rB   rC   rÅ   rÆ   rÁ   rD   rd   r
   rþ   r  r   r   rß   rÈ   rÉ   s   @rI   rç   rç   ]  sT  ø„ ñð ñ-$àð-$ð ð-$ð ð	-$ð
 ð-$ð ð-$ð õ-$ð^˜UŸ\™\ð °·±ð À%ÈÏÉÐV[×VbÑVbÐdi×dpÑdpÐHpÑBqó ð(˜%Ÿ,™,ð ¨U¯\©\ð ¸e¿l¹ló ð( 04Ø,1ØØ $Ø!%ØØ-1ñ=à�\‰\ð=ð �l‰lð=ð ˜˜eŸl™lÑ+Ñ,ð	=ð
 &*ð=ð ð=ð ð=ð ð=ð ð=ð ˜UŸ\™\Ñ*ð=ð 
�‰÷=rH   rç   c                   ó8  ‡ — e Zd Zdˆ fd„	Zej
                  j                  	 	 	 ddej                  deej                     de	de
de
de
dej                  fd	„«       Z	 	 	 	 	 ddej                  d
eej                     dee	   de
de
de
dej                  fd„Zˆ xZS )ÚEsmFoldTriangleAttentionc                 ód  •— t         ‰| �  «        || _        || _        || _        || _        || _        t        | j                  «      | _        t        || j                  dd¬«      | _
        t        | j                  | j                  | j                  | j                  | j                  «      | _        y)zç
        Args:
            c_in:
                Input channel dimension
            c_hidden:
                Overall hidden channel dimension (not per-head)
            no_heads:
                Number of attention heads
        Fr¾   rî   N)rÀ   rÁ   rÍ   rë   rì   ÚstartingÚinfr   rÜ   r³   Úlinearrç   Úmha)rÃ   rÍ   rë   rì   r  r  rÄ   s         €rI   rÁ   z!EsmFoldTriangleAttention.__init__ö  s„   ø€ ô 	‰ÑÔàˆŒ	Ø ˆŒØ ˆŒØ ˆŒØˆŒä# D§I¡IÓ.ˆŒä# D¨$¯-©-¸eÈ(ÔSˆŒä# D§I¡I¨t¯y©y¸$¿)¹)ÀTÇ]Á]ÐTX×TaÑTaÓbˆ�rH   rn   r  Ú
chunk_sizer  r  Úinplace_saferY   c           
      ó˜   — |||dœ}t        t        | j                  ||¬«      ||t        |j                  dd «      |r|¬«      S d¬«      S )ztriangle! triangle!)r÷   rø   r  )r  r  Nr–   )r  Úno_batch_dimsÚ_out)r   r   r  rc   ri   )rÃ   rn   r  r  r  r  r  Ú
mha_inputss           rI   Ú_chunkzEsmFoldTriangleAttention._chunk  sa   € ð ØØñ
ˆ
ô Ü�D—H‘HÐ:UÐ_fÔgØØ!Ü˜aŸg™g c r˜lÓ+Ù"�ô
ð 	
ð
 )-ô
ð 	
rH   Úmaskc                 ó   — |€|j                  |j                  dd «      }| j                  s$|j                  dd«      }|j                  dd«      }| j	                  |«      }| j
                  |dz
  z  ddd…dddd…f   }t        | j                  |«      d«      }|j                  d«      }||g}	|�| j                  ||	||||¬	«      }n| j                  |||	||¬
«      }| j                  s|j                  dd«      }|S )z­
        Args:
            x:
                [*, I, J, C_in] input tensor (e.g. the pair representation)
        Returns:
            [*, I, J, C_in] output tensor
        Nry   r–   rú   r   .©r—   r   r   éüÿÿÿ)r  r  r  )r÷   rø   r  r  r  )Únew_onesri   r  rü   rÜ   r  rƒ   r  Ú	unsqueezer   r  )
rÃ   rn   r!  r  r  r  r  Ú	mask_biasÚtriangle_biasr  s
             rI   rß   z EsmFoldTriangleAttention.forward'  s  € ð  ˆ<à—:‘:Ø—‘˜˜�óˆDð �}Š}Ø—‘˜B Ó#ˆAØ—>‘> " bÓ)ˆDð �O‰O˜AÓˆð —X‘X ¨¡Ñ*¨C²°D¸$ÂÐ,AÑBˆ	ô +¨4¯;©;°q«>¸9ÓEˆð &×/Ñ/°Ó3ˆà˜]Ð+ˆàÐ!Ø—‘ØØØØ,GØØ)ð ó ‰Að —‘Ø˜A fÐJeÐovð ó ˆAð �}Š}Ø—‘˜B Ó#ˆAàˆrH   )Tç    eÍÍA)FFF)NNFFF)r@   rA   rB   rÁ   rD   ÚjitÚignorerd   r   rÅ   rÆ   r   r   rß   rÈ   rÉ   s   @rI   r  r  õ  sö   ø„ õcð0 ‡Y�Y×Ñð -2ØØ"ñ
à�<‰<ð
ð �U—\‘\Ñ"ð
ð ð	
ð
 &*ð
ð ð
ð ð
ð 
�‰ò
ó ð
ð6 (,Ø$(Ø,1ØØ"ñ9à�<‰<ð9ð �u—|‘|Ñ$ð9ð ˜S‘Mð	9ð
 &*ð9ð ð9ð ð9ð 
�‰÷9rH   r  c                   óH  ‡ — e Zd ZdZdˆ fd„	Z	 ddej                  dej                  dee   dej                  fd„Z		 	 	 ddej                  d	eej                     d
ee   de
fd„Z	 	 	 	 ddej                  d	eej                     de
de
dee   dej                  fd„Zˆ xZS )Ú#EsmFoldTriangleMultiplicativeUpdatez*
    Implements Algorithms 11 and 12.
    c                 ó–  •— t         ‰| �  «        |j                  }|| _        t	        ||«      | _        t	        ||d¬«      | _        t	        ||«      | _        t	        ||d¬«      | _        t	        ||d¬«      | _	        t	        ||d¬«      | _
        t        |«      | _        t        |«      | _        t        j                  «       | _        y )Nr½   rï   r¿   )rÀ   rÁ   Úpairwise_state_dimÚ	_outgoingr³   Ú
linear_a_pÚ
linear_a_gÚ
linear_b_pÚ
linear_b_grô   Úlinear_zr   Úlayer_norm_inÚlayer_norm_outr¢   rõ   rö   )rÃ   Úconfigr0  rë   rÄ   s       €rI   rÁ   z,EsmFoldTriangleMultiplicativeUpdate.__init__h  s£   ø€ Ü‰ÑÔØ×,Ñ,ˆØ"ˆŒä'¨°(Ó;ˆŒÜ'¨°(ÀÔJˆŒÜ'¨°(Ó;ˆŒÜ'¨°(ÀÔJˆŒÜ% h°¸xÔHˆŒÜ% h°¸wÔGˆŒä& xÓ0ˆÔÜ'¨Ó1ˆÔä—z‘z“|ˆ�rH   r˜   r™   Ú_inplace_chunk_sizerY   c                 óœ  — | j                   rt        |d«      }t        |d«      }nt        |d«      }t        |d«      }|�lt        d|j                  d   |«      D ]L  }|d|||z   …d d …d d …f   }|d|||z   …d d …d d …f   }t	        j
                  ||«      |d|||z   …d d …d d …f<   ŒN |}nt	        j
                  ||«      }t        |d«      S )Nr#  )r—   r   r   r   rú   .©r   r—   r   )r0  rƒ   rm   ri   rD   r  )rÃ   r˜   r™   r9  rr   Úa_chunkÚb_chunkÚps           rI   Ú_combine_projectionsz8EsmFoldTriangleMultiplicativeUpdate._combine_projectionsy  sï   € ð �>Š>Ü" 1 iÓ0ˆAÜ" 1 iÓ0‰Aä" 1 iÓ0ˆAÜ" 1 iÓ0ˆAàÐ*ä˜1˜aŸg™g b™kÐ+>Ó?ò �Ø˜C  QÐ)<Ñ%<Ð!<ºaÂÐBÑC�Ø˜C  QÐ)<Ñ%<Ð!<ºaÂÐBÑC�Ü<A¿L¹LØØó=��#�q˜1Ð2Ñ2Ð2²A²qÐ8Ò9ðð ‰Aä—‘˜Q Ó"ˆAä! ! YÓ/Ð/rH   Úzr!  Úinplace_chunk_sizeÚwith_addc                 ó’  ‡ ‡‡#‡$‡%‡&‡'‡(‡)— |€|j                  |j                  dd «      }|j                  d«      }dˆ fd„	Š$dˆ$ˆˆ fd„	} |||dd¬«      }‰��p|j                  d   Š'‰'dz  ‰'dz  z   Š&dŠ(d	Š#‰ j                  r‰(n‰#}d
„ Š%ˆ%fd„Š)ˆ#ˆ%ˆ&ˆ'ˆ(ˆ)fd„}t	        |j                  «      }	‰&|	‰#<   |j                  |	«      }
 ‰%|
«      }t        d‰&«      |‰#<   |
j                  ||   «       d}t	        t        d‰&‰«      «      }t        ||dd ‰&gz   «      D ��cg c]
  \  }}||z
  ‘Œ }}}t	        t        ‰&‰'‰«      «      }|D �cg c]  }‰‘Œ }}t        ||z   ||z   «      }|D �][  \  }}|s|‰&k\  r ||
|«      }
d} ‰)||||z   |«      } ‰)||||z   |«      }|j                  «       }|‰#k(  r ‰)||||z   ‰#«      }n>|s) ‰%|«      }t        d‰&«      |‰#<    ‰)|
|||z   ‰(«      ||<   n|‰&z
  } ‰)|
|||z   ‰(«      } |||dd¬«      }~t        j                  ||«      }t        |d«      }‰ j                  |«      }‰ j                  |«      } ‰)||||z   ‰#«      }‰ j!                  ‰ j#                  |«      «      }|j%                  «        ~||z  } ‰%|«      }t        |||z   «      |‰#<   |r||xx   |z  cc<   �ŒW|||<   �Œ^ |S  |||dd«      } t        j                  || «      }!‰ j                  |!«      }!‰ j                  |!«      }!‰ j!                  |«      }"|"j%                  «        |!|"z  }!|r||!z  }|S |!}|S c c}}w c c}w )a™  
        Args:
            z:
                A [*, N, N, C_z] pair representation
            mask:
                A [*, N, N] pair mask
            inplace_chunk_size:
                Size of chunks used in the main computation. Increase to trade memory for speed.
            with_add:
                If True, z is overwritten with (z + update). Otherwise, it is overwritten with (update).
        Returns:
            A reference to the overwritten z

        More memory-efficient, inference-only version of the forward function. Uses in-place operations, fusion of the
        addition that happens after this module in the Evoformer, a smidge of recomputation, and a cache of overwritten
        values to lower peak memory consumption of this module from 5x the size of the input tensor z to 2.5x its size.
        Useful for inference on extremely long sequences.

        It works as follows. We will make reference to variables used in the default forward implementation below.
        Naively, triangle multiplication attention requires the manifestation of 5 tensors the size of z: 1) z, the
        "square" input tensor, 2) a, the first projection of z, 3) b, the second projection of b, 4) g, a z-sized mask,
        and 5) a z-sized tensor for intermediate computations. For large N, this is prohibitively expensive; for
        N=4000, for example, z is more than 8GB alone. To avoid this problem, we compute b, g, and all intermediate
        tensors in small chunks, noting that the chunks required to compute a chunk of the output depend only on the
        tensor a and corresponding vertical and horizontal chunks of z. This suggests an algorithm that loops over
        pairs of chunks of z: hereafter "columns" and "rows" of z, even though each "column" and "row" in fact contains
        inplace_chunk_size contiguous true columns and rows of z. Writing output chunks to a new tensor would bring
        total memory consumption down to 3x the size of z. However, more memory can be saved by writing output chunks
        directly to z in-place. WLOG, we choose to write output chunks vertically, overwriting the ith "column" of z at
        the end of the ith iteration of the main loop. Despite this overwriting, the ith column is always one column
        ahead of previously overwritten columns and can be recovered directly from z. After the first iteration,
        however, the ith row of z is always at least partially overwritten. For this reason, we introduce the z-cache,
        a tensor one-half the size of z. The z-cache initially contains the left half (2nd and 3rd quadrants) of z. For
        0 < i < N/2, the missing left part of the ith row of z is recovered from this cache at the beginning of the ith
        iteration. Once i exceeds n/2, the cache is "reoriented" to encompass the 3rd and 4th quadrants of z instead.
        Though the 3rd quadrant of the original z is entirely overwritten at this point, it can be recovered from the
        z-cache itself. Thereafter, the ith row of z can be recovered in its entirety from the reoriented z-cache.
        After the final iteration, z has been completely overwritten and contains the triangular multiplicative update.
        If with_add is True, it instead contains the sum of z and the triangular multiplicative update. In either case,
        peak memory consumption is just 2.5x the size of z, disregarding memory used for chunks and other small
        variables.
        Nry   Tc                 óø   •— |r‰j                   }‰j                  }n‰j                  }‰j                  }‰j	                  | «      }  || «      }|j                  «        | || «      z  }||z  }t        |d«      }|S )Nr#  )r2  r1  r4  r3  r6  Úsigmoid_rƒ   )Úpairr!  r˜   rô   Úlinear_pr>  rÃ   s         €rI   Úcompute_projection_helperzYEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.compute_projection_helperÉ  st   ø€ ÙØŸ?™?�ØŸ?™?‘àŸ?™?�ØŸ?™?�à×%Ñ% dÓ+ˆDÙ˜“ˆAØ�J‰JŒLØ‘˜$“ÑˆAØ�‰IˆAÜ" 1 iÓ0ˆAØˆHrH   c           
      óP  •— ‰j                   |z  }|s  ‰| ||«      }|r|j                  dd«      }|S |r‰j                  n‰j                  }|j                  j
                  d   }| j
                  d d |fz   | j
                  dd z   }| j                  |«      }t        d| j
                  d   ‰«      D ]o  }	| d|	|	‰z   …d d …d d …f   }
 ‰| d|	|	‰z   …d d …d d …f   |d|	|	‰z   …d d …d d …f   |«      }
|r|
j                  dd«      }
|
|d|	|	‰z   …f<   n|
|d|	|	‰z   …d d …f<   ~
Œq |S )Nry   r–   rú   r   .)r0  rü   r2  r4  r¶   ri   Ú	new_zerosrm   )rF  r!  r˜   ÚchunkedÚneed_transposer>  rô   ÚcÚ	out_shaperr   Ú
pair_chunkrH  rA  rÃ   s              €€€rI   Úcompute_projectionzREsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.compute_projectionÙ  sr  ø€ Ø!Ÿ^™^¨aÑ/ˆNÙÙ-¨d°D¸!Ó<�Ù!ØŸ™ B¨Ó+�Að. ˆHñ' /0˜4Ÿ?š?°T·_±_�Ø—M‘M×'Ñ'¨Ñ+�Ø ŸJ™J s¨˜O¨q¨dÑ2°T·Z±ZÀÀ2Ð5FÑF�	Ø—N‘N 9Ó-�Ü˜q $§*¡*¨R¡.Ð2DÓEò #�AØ!% c¨1¨qÐ3EÑ/EÐ+EÂqÊ!Ð&KÑ!L�JÙ!:Ø˜S ! aÐ*<Ñ&<Ð"<ºaÂÐBÑCØ˜S ! aÐ*<Ñ&<Ð"<ºaÂÐBÑCØó"�Jñ
 &Ø%/×%9Ñ%9¸"¸bÓ%A˜
Ø=G˜˜#˜q 1Ð'9Ñ#9Ð9Ð9Ò:à@J˜˜#˜q 1Ð'9Ñ#9Ð9º1Ð<Ñ=á"ð#ð ˆHrH   )rK  r—   rú   r–   c                 óR   — | j                   D �cg c]  }t        d «      ‘Œ c}S c c}w rK   )ri   r_   )rt   Ú_s     rI   Úempty_slicerzLEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.empty_slicer  s   € Ø-.¯W©WÖ5¨œ˜d�Ò5Ð5ùÒ5s   �$c                 ó<   •—  ‰| «      }t        ||«      ||<   | |   S rK   r^   )rt   ÚstartÚendre   rä   rS  s        €rI   Úslice_tensorzLEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.slice_tensor  s%   ø€ á  “O�Ü˜u cÓ*��#‘Ø˜‘t�rH   c                 ó  •—  ‰| ‰d ‰
«      }| j                  ‰
‰«      } | dd ‰	dz  …d d …d d …f   }  ‰| «      }t        d‰«      |‰<   || |<    ‰|‰d ‰
«      } ‰|‰d ‰«      } ‰| «      }t        ‰d «      |‰<   || |<   | S )N.r—   r   )rü   r_   )Úz_cacher@  Ú
quadrant_3Úfirst_half_slicerÚ
quadrant_4Úquadrant_3_slicerÚcol_dimrS  Úhalf_nÚnÚrow_dimrW  s         €€€€€€rI   Úflip_z_cache_zMEsmFoldTriangleMultiplicativeUpdate._inference_forward.<locals>.flip_z_cache_  s¿   ø€ ñ *¨'°6¸4ÀÓI�
Ø!×+Ñ+¨G°WÓ=�ð " # z¨!¨q©& z²1²aÐ"7Ñ8�ñ %1°Ó$9Ð!Ü-2°1°fÓ-=Ð! 'Ñ*Ø-7�Ð)Ñ*ñ *¨!¨V°T¸7ÓC�
Ù)¨*°f¸dÀGÓL�
ñ %1°Ó$9Ð!Ü-2°6¸4Ó-@Ð! 'Ñ*à-7�Ð)Ñ*à�rH   r   Fr   )r˜   rK  r;  r  )TT)r%  ri   r&  r0  r   rJ  r_   rª   rm   rh   ÚclonerD   r  rƒ   r7  r5  rô   r6  rE  )*rÃ   r@  r!  rA  rB  rP  r˜   Úb_chunk_dimrb  Úz_cache_shaperY  Úz_cache_slicerÚz_cache_rotatedÚi_rangeÚi_1Úi_2Úinitial_offsetsÚ
after_halfrR  Úafter_half_offsetsÚcombined_range_with_offsetsrr   ÚoffsetÚ	z_chunk_bÚ
mask_chunkÚz_chunk_slicerÚz_cache_offsetr=  Úx_chunkÚ	z_chunk_gÚg_chunkÚz_slicerr™   rn   r  r^  rH  rS  r_  r`  ra  rW  s*   `  `                               @@@@@@@rI   Ú_inference_forwardz6EsmFoldTriangleMultiplicativeUpdate._inference_forward“  s¸  ÿø€ ðb ˆ<Ø—:‘:˜aŸg™g c r˜lÓ+ˆDà�~‰~˜bÓ!ˆõ	÷ 	ñB ˜q $¨°dÔ;ˆàÑ)Ø—‘˜‘ˆAØ˜!‘V˜a !™e‘^ˆFØˆGØˆGØ%)§^¢^™'¸ˆKò6ô÷ñ ô8 ! §¡›MˆMØ%+ˆM˜'Ñ"Ø—k‘k -Ó0ˆGÙ)¨'Ó2ˆNÜ&+¨A¨vÓ&6ˆN˜7Ñ#Ø�M‰M˜!˜NÑ+Ô,Ø#ˆOô
 œ5  FÐ,>Ó?Ó@ˆGÜ9<¸WÀgÈaÈbÀkÐU[ÐT\ÑF\Ó9]×^©X¨S°#˜s S›yÐ^ˆOÑ^Üœe F¨AÐ/AÓBÓCˆJØ>HÖ!I¸Ò"4Ð!IÐÐ!IÜ*-¨g¸
Ñ.BÀOÐVhÑDhÓ*iÐ'Ø8ó .*‘	��6Ù&¨1°ª;Ù+¨G°QÓ7�GØ&*�Oá(¨¨A¨q°6©z¸;ÓG�	Ù)¨$°°1°v±:¸{ÓK�
à%ŸO™OÓ-�	Ø 'Ò)Ù ,¨Q°°1°v±:¸wÓ G‘Iñ
 +Ù)5°iÓ)@˜Ü27¸¸6Ó2B˜ wÑ/Ù4@ÀÈ!ÈQÐQWÉZÐY`Ó4a˜	 .Ò1à)*¨V©˜Ù$0°¸.È.Ð[aÑJaÐcjÓ$k˜	á,¨Y¸
ÀeÐUZÔ[�ØäŸ,™, q¨'Ó2�Ü,¨W°iÓ@�Ø×-Ñ-¨gÓ6�ØŸ-™-¨Ó0�ñ )¨¨A¨q°6©z¸7ÓC�	ØŸ-™-¨×(:Ñ(:¸9Ó(EÓF�Ø× Ñ Ô"Øà˜7Ñ"�ñ (¨›?�Ü$)¨!¨Q°©ZÓ$8�˜Ñ!ÙØ�h“K 7Ñ*•Kà")�A�h“Kð].*ðx ˆñ # 1 d¨E°5Ó9ˆAÜ—‘˜Q Ó"ˆAØ×#Ñ# AÓ&ˆAØ—‘˜aÓ ˆAØ—‘˜aÓ ˆAØ�J‰JŒLØ�‰FˆAÙØ�Q‘�ð ˆð �àˆùóA _ùâ!Is   ÄL>Å	Mr  Ú_add_with_inplacec                 óF  — |r| j                  ||||¬«      }|S |€|j                  |j                  dd «      }|j                  d«      }| j	                  |«      }|}|| j                  | j                  |«      «      z  }|| j                  |«      z  }|}|| j                  | j                  |«      «      z  }|| j                  |«      z  }t        «       rbt        j                  j                  j                  d¬«      5  | j                  |j!                  «       |j!                  «       «      }ddd«       n| j                  ||«      }~~| j#                  «      }| j%                  |«      }| j                  | j'                  |«      «      }	||	z  }|S # 1 sw Y   ŒTxY w)zÛ
        Args:
            x:
                [*, N_res, N_res, C_z] input tensor
            mask:
                [*, N_res, N_res] input mask
        Returns:
            [*, N_res, N_res, C_z] output tensor
        )rA  rB  Nry   FrÔ   )rx  r%  ri   r&  r6  rö   r2  r1  r4  r3  rP   rD   rØ   rÙ   rÚ   r?  Úfloatr7  r5  rô   )
rÃ   r@  r!  r  ry  r9  rn   r˜   r™   r  s
             rI   rß   z+EsmFoldTriangleMultiplicativeUpdate.forwardv  sŠ  € ñ" Ø×'Ñ'ØØØ#6Ø*ð	 (ó ˆAð ˆHàˆ<Ø—:‘:˜aŸg™g c r˜lÓ+ˆDà�~‰~˜bÓ!ˆà×Ñ˜qÓ!ˆØˆØ�—‘˜TŸ_™_¨QÓ/Ó0Ñ0ˆØ�—‘ Ó"Ñ"ˆØˆØ�—‘˜TŸ_™_¨QÓ/Ó0Ñ0ˆØ�—‘ Ó"Ñ"ˆäÔÜ—‘—‘×(Ñ(°Ð(Ó7ñ DØ×-Ñ-¨a¯g©g«i¸¿¹»ÓC�÷Dð Dð ×)Ñ)¨!¨QÓ/ˆAàˆqØ×Ñ Ó"ˆØ�M‰M˜!ÓˆØ�L‰L˜Ÿ™ qÓ)Ó*ˆØ�‰Eˆàˆ÷Dð Dús   Ä/FÆF r  rK   )NNT)NFFé   )r@   rA   rB   rC   rÁ   rD   rd   r   rÅ   r?  rÆ   rx  rß   rÈ   rÉ   s   @rI   r-  r-  c  sø   ø„ ñõ$ð$ VZñ0Ø—‘ð0Ø"'§,¡,ð0ØEMÈcÁ]ð0à	�‰ó0ð: (,Ø,0Øñaà�<‰<ðað �u—|‘|Ñ$ðað % S™Mð	að
 óaðL (,Ø"Ø"'Ø-0ñ3à�<‰<ð3ð �u—|‘|Ñ$ð3ð ð	3ð
  ð3ð & c™]ð3ð 
�‰÷3rH   r-  c                   ó"   ‡ — e Zd ZdZˆ fd„Zˆ xZS )ÚEsmFoldPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    c                 óÞ  •— t        |t        «      �r�t        j                  «       5  |j                  �(|j	                  |j
                  |j                  «       �nK|j                  dk(  rt        |j
                  d¬«       �n#|j                  dk(  rt        |j
                  d¬«       nü|j                  dk(  r,t        j                  j                  |j
                  d¬	«       nÁ|j                  d
k(  rC|j
                  j                  d«       |j                  r‹|j                  j                  d«       no|j                  dk(  r6t        j                  j                  j                  |j
                  d¬«       n*|j                  dk(  r|j
                  j                  d«       ddd«       yt        |t        «      rt        |j                  «       yt        |t         «      �rìt        j                  j                  j#                  |j$                  j&                  j
                  «       t        j                  j                  j#                  |j$                  j&                  j                  «       t        j                  j                  j#                  |j(                  j&                  j
                  «       t        j                  j                  j#                  |j(                  j&                  j                  «       t        j                  j                  j#                  |j*                  j,                  j.                  j
                  «       t        j                  j                  j#                  |j*                  j,                  j.                  j                  «       t        j                  j                  j#                  |j0                  j,                  j.                  j
                  «       t        j                  j                  j#                  |j0                  j,                  j.                  j                  «       t        j                  j                  j#                  |j2                  j4                  j
                  «       t        j                  j                  j#                  |j2                  j4                  j                  «       t        j                  j                  j#                  |j6                  j8                  j
                  «       t        j                  j                  j#                  |j:                  j4                  j
                  «       t        j                  j                  j#                  |j:                  j4                  j                  «       t        j                  j                  j#                  |j<                  j>                  d   j
                  «       t        j                  j                  j#                  |j<                  j>                  d   j                  «       t        j                  j                  j#                  |j@                  j>                  d   j
                  «       t        j                  j                  j#                  |j@                  j>                  d   j                  «       ytB        ‰| �‰  |«       y# 1 sw Y   yxY w)zInitialize the weightsNrº   ç      ð?)r›   r»   r“   r¼   r   )Úgainr½   r’   r¾   r  )Únonlinearityr¿   r–   )#r†   r³   rD   r¯   r·   rÑ   r¶   r£   r­   r¢   Úxavier_uniform_rl   Úkaiming_normal_ÚEsmFoldInvariantPointAttentionr±   Úhead_weightsÚ#EsmFoldTriangularSelfAttentionBlockÚzeros_Ú
tri_mul_inr5  Útri_mul_outÚtri_att_startr  ró   Útri_att_endÚsequence_to_pairÚo_projÚpair_to_sequencer  Úseq_attentionÚmlp_seqÚmlpÚmlp_pairrÀ   Ú_init_weights)rÃ   ÚmodulerÄ   s     €rI   r”  z$EsmFoldPreTrainedModel._init_weights³  s+  ø€ ä�fœmÕ,Ü—‘“ñ -Ø—>‘>Ð-Ø—N‘N 6§=¡=°&·+±+Ö>Ø—[‘[ IÒ-Ü& v§}¡}¸C×@Ø—[‘[ FÒ*Ü& v§}¡}¸CÖ@Ø—[‘[ HÒ,Ü—G‘G×+Ñ+¨F¯M©MÀÐ+ÕBØ—[‘[ HÒ,Ø—M‘M×'Ñ'¨Ô,Ø—{’{ØŸ™×)Ñ)¨#Õ.Ø—[‘[ HÒ,Ü—H‘H—M‘M×1Ñ1°&·-±-ÈhÐ1ÕWØ—[‘[ GÒ+Ø—M‘M×'Ñ'¨Ô,÷!-ð -ô" ˜Ô >Ô?Ü# F×$7Ñ$7Õ8Ü˜Ô CÕDÜ�H‰H�M‰M× Ñ  ×!2Ñ!2×!;Ñ!;×!BÑ!BÔCÜ�H‰H�M‰M× Ñ  ×!2Ñ!2×!;Ñ!;×!@Ñ!@ÔAÜ�H‰H�M‰M× Ñ  ×!3Ñ!3×!<Ñ!<×!CÑ!CÔDÜ�H‰H�M‰M× Ñ  ×!3Ñ!3×!<Ñ!<×!AÑ!AÔBÜ�H‰H�M‰M× Ñ  ×!5Ñ!5×!9Ñ!9×!BÑ!B×!IÑ!IÔJÜ�H‰H�M‰M× Ñ  ×!5Ñ!5×!9Ñ!9×!BÑ!B×!GÑ!GÔHÜ�H‰H�M‰M× Ñ  ×!3Ñ!3×!7Ñ!7×!@Ñ!@×!GÑ!GÔHÜ�H‰H�M‰M× Ñ  ×!3Ñ!3×!7Ñ!7×!@Ñ!@×!EÑ!EÔFä�H‰H�M‰M× Ñ  ×!8Ñ!8×!?Ñ!?×!FÑ!FÔGÜ�H‰H�M‰M× Ñ  ×!8Ñ!8×!?Ñ!?×!DÑ!DÔEÜ�H‰H�M‰M× Ñ  ×!8Ñ!8×!?Ñ!?×!FÑ!FÔGÜ�H‰H�M‰M× Ñ  ×!5Ñ!5×!<Ñ!<×!CÑ!CÔDÜ�H‰H�M‰M× Ñ  ×!5Ñ!5×!<Ñ!<×!AÑ!AÔBÜ�H‰H�M‰M× Ñ  §¡×!3Ñ!3°BÑ!7×!>Ñ!>Ô?Ü�H‰H�M‰M× Ñ  §¡×!3Ñ!3°BÑ!7×!<Ñ!<Ô=Ü�H‰H�M‰M× Ñ  §¡×!4Ñ!4°RÑ!8×!?Ñ!?Ô@Ü�H‰H�M‰M× Ñ  §¡×!4Ñ!4°RÑ!8×!=Ñ!=Õ>ä‰GÑ! &Õ)÷O-ð -ús   §F [#Û#[,)r@   rA   rB   rC   r”  rÈ   rÉ   s   @rI   r~  r~  ¬  s   ø„ ñ÷**ð **rH   r~  c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚEsmFoldSelfAttentionc                 ó¸  •— t         ‰| �  «        |||z  k(  sJ ‚|| _        || _        || _        t        j                  ||dz  d¬«      | _        t        j                  ||d¬«      | _        || _	        |r•t        j                  ||«      | _
        t        j
                  j                  j                  | j                  j                  «       t        j
                  j                  j                  | j                  j                   «       | j                  dz  | _        t        j
                  j                  j                  | j                  j                   «       y )Nr   Fr¹   Tç      à¿)rÀ   rÁ   Ú	embed_dimÚ	num_headsÚ
head_widthr¢   ÚLinearÚprojrŽ  ÚgatedÚg_projrD   r£   rˆ  rÑ   Úones_r¶   Úrescale_factor)rÃ   rš  r›  rœ  rŸ  rÄ   s        €rI   rÁ   zEsmFoldSelfAttention.__init__á  sï   ø€ Ü‰ÑÔØ˜I¨
Ñ2Ò2Ð2Ð2à"ˆŒØ"ˆŒØ$ˆŒä—I‘I˜i¨°Q©¸UÔCˆŒ	Ü—i‘i 	¨9¸4Ô@ˆŒØˆŒ
ÙÜŸ)™) I¨yÓ9ˆDŒKÜ�H‰H�M‰M× Ñ  §¡×!3Ñ!3Ô4Ü�H‰H�M‰M×Ñ §¡× 0Ñ 0Ô1à"Ÿo™o¨tÑ3ˆÔä�‰�‰×Ñ˜TŸ[™[×-Ñ-Õ.rH   c                 ó
  —  | j                  |«      j                  g |j                  dd ¢| j                  ‘d‘­Ž }|j	                  dddd«      }|j                  dd¬«      \  }}}| j                  |z  }t        j                  d||«      }	|�|	|j	                  dddd«      z   }	|�.|dd…ddf   }|	j                  |d	k(  t        j                   «      }	t        j                  j                  |	d¬«      }	t        j                  d
|	|«      }
 |
j                  g |
j                  dd ¢d‘­Ž }
| j                   r"| j#                  |«      j%                  «       |
z  }
| j'                  |
«      }
|
|	j	                  dddd«      fS )aä  
        Basic self attention with optional mask and external pairwise bias. To handle sequences of different lengths,
        use mask.

        Inputs:
            x: batch of input sequences (.. x L x C) mask: batch of boolean masks where 1=valid, 0=padding position (..
            x L_k) bias: batch of scalar pairwise attention biases (.. x Lq x Lk x num_heads)

        Outputs:
          sequence projection (B x L x embed_dim), attention maps (B x L x L x num_heads)
        Nr—   ry   r   r   r   râ   z...qc,...kc->...qkFz...hqk,...hkc->...qhc)rž  rû   ri   r›  r€   Úchunkr¢  rD   ÚeinsumÚmasked_fillr©   r  r¢   rÛ   rã   rz   rŸ  r   rö   rŽ  )rÃ   rn   r!  r¶   Úindicesrt   rý   ra   r�   r˜   Úys              rI   rß   zEsmFoldSelfAttention.forwardõ  sr  € ð ˆD�I‰I�a‹L×ÑÐ?˜qŸw™w r¨˜{Ð?¨D¯N©NÐ?¸BÒ?ˆØ�I‰I�a˜˜A˜qÓ!ˆØ—'‘'˜! �'Ó$‰ˆˆ1ˆaà×Ñ !Ñ#ˆÜ�L‰LÐ-¨q°!Ó4ˆð ÐØ�D—L‘L  A q¨!Ó,Ñ,ˆAð ÐØš˜4 ˜Ñ&ˆDØ—‘˜d e™m¬b¯f©f¨WÓ5ˆAä�M‰M×!Ñ! !¨Ð!Ó,ˆä�L‰LÐ0°!°QÓ7ˆØˆA�I‰IÐ'�q—w‘w˜r �{Ð' BÒ'ˆà�:Š:Ø—‘˜A“×&Ñ&Ó(¨1Ñ,ˆAØ�K‰K˜‹Nˆà�!—)‘)˜A˜q ! QÓ'Ð'Ð'rH   )F)NNNrà   rÉ   s   @rI   r—  r—  à  s   ø„ õ/÷(&(rH   r—  c                   ót   ‡ — e Zd ZdZdedeeee   f   fˆ fd„Zde	j                  de	j                  fd„Zˆ xZS )ÚEsmFoldDropoutzl
    Implementation of dropout with the ability to share the dropout mask along a particular dimension.
    ÚrÚ	batch_dimc                 ó¬   •— t         ‰| �  «        || _        t        |t        «      r|g}|| _        t        j                  | j                  «      | _        y rK   )	rÀ   rÁ   r«  r†   rÅ   r¬  r¢   ÚDropoutÚdropout)rÃ   r«  r¬  rÄ   s      €rI   rÁ   zEsmFoldDropout.__init__#  sA   ø€ Ü‰ÑÔàˆŒÜ�i¤Ô%Ø"˜ˆIØ"ˆŒÜ—z‘z $§&¡&Ó)ˆ�rH   rn   rY   c                 ó¶   — t        |j                  «      }| j                  �| j                  D ]  }d||<   Œ	 || j                  |j	                  |«      «      z  S )Nr   )r   ri   r¬  r¯  r%  )rÃ   rn   ri   Úbds       rI   rß   zEsmFoldDropout.forward,  sR   € Ü�Q—W‘W“ˆØ�>‰>Ð%Ø—n‘nò �Ø��b’	ðà�4—<‘< §
¡
¨5Ó 1Ó2Ñ2Ð2rH   )r@   rA   rB   rC   r{  r   rÅ   r   rÁ   rD   rd   rß   rÈ   rÉ   s   @rI   rª  rª    sC   ø„ ñð*˜%ð *¨E°#°t¸C±y°.Ñ,Aõ *ð3˜Ÿ™ð 3¨%¯,©,÷ 3rH   rª  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEsmFoldSequenceToPairc                 óÊ  •— t         ‰| �  «        t        j                  |«      | _        t        j
                  ||dz  d¬«      | _        t        j
                  d|z  |d¬«      | _        t        j                  j                  j                  | j                  j                  «       t        j                  j                  j                  | j                  j                  «       y )Nr—   Tr¹   )rÀ   rÁ   r¢   r   Ú	layernormr�  rž  rŽ  rD   r£   rˆ  r¶   )rÃ   Úsequence_state_dimÚ	inner_dimr/  rÄ   s       €rI   rÁ   zEsmFoldSequenceToPair.__init__5  s�   ø€ Ü‰ÑÔäŸ™Ð&8Ó9ˆŒÜ—I‘IÐ0°)¸a±-ÀdÔKˆŒ	Ü—i‘i  I¡Ð/AÈÔMˆŒä�‰�‰×Ñ˜TŸY™YŸ^™^Ô,Ü�‰�‰×Ñ˜TŸ[™[×-Ñ-Õ.rH   c                 óz  — t        |j                  «      dk(  sJ ‚| j                  |«      }| j                  |«      }|j	                  dd¬«      \  }}|dd…ddd…dd…f   |dd…dd…ddd…f   z  }|dd…ddd…dd…f   |dd…dd…ddd…f   z
  }t        j                  ||gd¬«      }| j                  |«      }|S )z×
        Inputs:
          sequence_state: B x L x sequence_state_dim

        Output:
          pairwise_state: B x L x L x pairwise_state_dim

        Intermediate state:
          B x L x L x 2*inner_dim
        r   r—   ry   râ   N)rc   ri   rµ  rž  r¤  rD   ÚcatrŽ  )rÃ   Úsequence_staterä   rý   ra   ÚprodÚdiffrn   s           rI   rß   zEsmFoldSequenceToPair.forward?  s¼   € ô �>×'Ñ'Ó(¨AÒ-Ð-Ð-à�N‰N˜>Ó*ˆØ�I‰I�a‹LˆØ�w‰w�q˜bˆwÓ!‰ˆˆ1à’�Dš!šQ�Ñ !¢A¢q¨$² MÑ"2Ñ2ˆØ’�Dš!šQ�Ñ !¢A¢q¨$² MÑ"2Ñ2ˆä�I‰I�t˜T�l¨Ô+ˆØ�K‰K˜‹NˆàˆrH   rà   rÉ   s   @rI   r³  r³  4  s   ø„ ô/örH   r³  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚEsmFoldPairToSequencec                 ó�   •— t         ‰| �  «        t        j                  |«      | _        t        j
                  ||d¬«      | _        y )NFr¹   )rÀ   rÁ   r¢   r   rµ  r�  r  )rÃ   r/  r›  rÄ   s      €rI   rÁ   zEsmFoldPairToSequence.__init__[  s4   ø€ Ü‰ÑÔäŸ™Ð&8Ó9ˆŒÜ—i‘iÐ 2°IÀEÔJˆ�rH   c                 ó~   — t        |j                  «      dk(  sJ ‚| j                  |«      }| j                  |«      }|S )z’
        Inputs:
          pairwise_state: B x L x L x pairwise_state_dim

        Output:
          pairwise_bias: B x L x L x num_heads
        é   )rc   ri   rµ  r  )rÃ   Úpairwise_stater@  Úpairwise_biass       rI   rß   zEsmFoldPairToSequence.forwarda  s>   € ô �>×'Ñ'Ó(¨AÒ-Ð-Ð-Ø�N‰N˜>Ó*ˆØŸ™ A›ˆØÐrH   rà   rÉ   s   @rI   r¾  r¾  Z  s   ø„ ôKörH   r¾  c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚEsmFoldResidueMLPc           	      ó  •— t         ‰| �  «        t        j                  t        j                  |«      t        j
                  ||«      t        j                  «       t        j
                  ||«      t        j                  |«      «      | _        y rK   )	rÀ   rÁ   r¢   Ú
Sequentialr   r�  ÚReLUr®  r’  )rÃ   rš  r·  r¯  rÄ   s       €rI   rÁ   zEsmFoldResidueMLP.__init__p  s[   ø€ Ü‰ÑÔä—=‘=Ü�L‰L˜Ó#Ü�I‰I�i Ó+Ü�G‰G‹IÜ�I‰I�i Ó+Ü�J‰J�wÓó
ˆ�rH   c                 ó*   — || j                  |«      z   S rK   )r’  )rÃ   rn   s     rI   rß   zEsmFoldResidueMLP.forward{  s   € Ø�4—8‘8˜A“;‰ÐrH   ©r   rà   rÉ   s   @rI   rÅ  rÅ  o  s   ø„ õ	
örH   rÅ  c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )r‡  c                 ó\  •— t         ‰| �  «        || _        |j                  }|j                  }||j
                  z  }||j                  z  }t        j                  |«      | _	        t        ||dz  |«      | _        t        ||«      | _        t        |||j
                  d¬«      | _        t!        |d¬«      | _        t!        |d¬«      | _        t'        ||j                  |dd¬«      | _        t'        ||j                  |dd¬«      | _        t-        |d|z  |j.                  ¬	«      | _        t-        |d|z  |j.                  ¬	«      | _        t        j4                  |j.                  «      | _        t9        |j.                  dz  d«      | _        t9        |j.                  dz  d
«      | _        y )Nr—   T)rŸ  )r0  Fr)  )r  r  rÁ  )r¯  r   )rÀ   rÁ   r8  r¶  r/  Úsequence_head_widthÚpairwise_head_widthr¢   r   Úlayernorm_1r³  r�  r¾  r�  r—  r�  r-  rŠ  r‰  r  r‹  rŒ  rÅ  r¯  r‘  r“  r®  Údroprª  Úrow_dropÚcol_drop)rÃ   r8  r¶  r/  Úsequence_num_headsÚpairwise_num_headsrÄ   s         €rI   rÁ   z,EsmFoldTriangularSelfAttentionBlock.__init__€  s�  ø€ Ü‰ÑÔØˆŒà#×6Ñ6ÐØ#×6Ñ6ÐØ/°6×3MÑ3MÑMÐØ/°6×3MÑ3MÑMÐäŸ<™<Ð(:Ó;ˆÔä 5Ð6HÐJ\Ð`aÑJaÐcuÓ vˆÔÜ 5Ð6HÐJ\Ó ]ˆÔä1ØÐ 2°F×4NÑ4NÐVZô
ˆÔô ?¸vÐQUÔVˆÔÜ=¸fÐPUÔVˆŒä5Ø × :Ñ :Ð<NÐTWÐbfô
ˆÔô 4Ø × :Ñ :Ð<NÐTWÐbgô
ˆÔô )Ð);¸QÐASÑ=SÐ]c×]kÑ]kÔlˆŒÜ)Ð*<¸aÐBTÑ>TÐ^d×^lÑ^lÔmˆŒä—J‘J˜vŸ~™~Ó.ˆŒ	Ü& v§~¡~¸Ñ'9¸1Ó=ˆŒÜ& v§~¡~¸Ñ'9¸1Ó=ˆ�rH   c           	      ó,  — t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚|�:t        |j                  «      dk7  r"t        dt        |j                  «      › d�«      ‚|j                  \  }}}|j                  d   }	|| j                  j                  k7  r&t        d	|› d
| j                  j                  › d�«      ‚|	| j                  j
                  k7  r&t        d|	› d
| j                  j
                  › d�«      ‚||j                  d   k7  rt        d|› d
|j                  d   › d�«      ‚||j                  d   k7  s||j                  d   k7  r/t        d|› d
|j                  d   › d|j                  d   › d�«      ‚| j                  |«      }
| j                  |«      }| j                  |||
¬«      \  }}|| j                  |«      z   }| j                  |«      }|| j                  |«      z   }|�#|j                  d«      |j                  d«      z  nd}|| j                  | j                  ||¬«      «      z   }|| j                  | j!                  ||¬«      «      z   }|| j                  | j#                  |||¬«      «      z   }|| j                  | j%                  |||¬«      «      z   }| j'                  |«      }||fS )a*  
        Inputs:
          sequence_state: B x L x sequence_state_dim pairwise_state: B x L x L x pairwise_state_dim mask: B x L boolean
          tensor of valid positions

        Output:
          sequence_state: B x L x sequence_state_dim pairwise_state: B x L x L x pairwise_state_dim
        r   z,`sequence_state` should be a 3d-tensor, got ú dims.rÁ  z,`pairwise_state` should be a 4d-tensor, got Nr—   ú"`mask` should be a 2d-tensor, got zR`sequence_state` last dimension should be equal to `self.sequence_state_dim`. Got ú != ú.zR`pairwise_state` last dimension should be equal to `self.pairwise_state_dim`. Got r   zD`sequence_state` and `pairwise_state` have inconsistent batch size: r   zI`sequence_state` and `pairwise_state` have inconsistent sequence length: z or )r!  r¶   ©r!  )r!  r  )rc   ri   rÂ   r8  r¶  r/  r�  rÏ  r�  rÐ  r‘  r�  r&  rÑ  rŠ  rÒ  r‰  r‹  rŒ  r“  )rÃ   rº  rÂ  r!  r  Ú,_EsmFoldTriangularSelfAttentionBlock__kwargsr¬  Úseq_dimr¶  r/  r¶   r¨  rR  Útri_masks                 rI   rß   z+EsmFoldTriangularSelfAttentionBlock.forward¢  sQ  € ô ˆ~×#Ñ#Ó$¨Ò)ÜÐKÌCÐP^×PdÑPdÓLeÐKfÐflÐmÓnÐnÜˆ~×#Ñ#Ó$¨Ò)ÜÐKÌCÐP^×PdÑPdÓLeÐKfÐflÐmÓnÐnØÐ¤ D§J¡J£°1Ò 4ÜÐAÄ#ÀdÇjÁjÃ/ÐARÐRXÐYÓZÐZà1?×1EÑ1EÑ.ˆ	�7Ð.Ø+×1Ñ1°!Ñ4Ðà §¡×!?Ñ!?Ò?ÜØdØ%Ð& d¨4¯;©;×+IÑ+IÐ*JÈ!ðMóð ð  §¡×!?Ñ!?Ò?ÜØdØ%Ð& d¨4¯;©;×+IÑ+IÐ*JÈ!ðMóð ð ˜×,Ñ,¨QÑ/Ò/ÜØVÐW`ÐVaÐaeØ!×'Ñ'¨Ñ*Ð+¨1ð.óð ð �n×*Ñ*¨1Ñ-Ò-°¸N×<PÑ<PÐQRÑ<SÒ1SÜØ[Ð\cÐ[dÐdhØ!×'Ñ'¨Ñ*Ð+¨4°×0DÑ0DÀQÑ0GÐ/HÈðKóð ð ×$Ñ$ ^Ó4ˆð ×Ñ˜^Ó,ˆØ×!Ñ! !¨$°TÐ!Ó:‰ˆˆ1Ø'¨$¯)©)°A«,Ñ6ˆØŸ™ nÓ5ˆð (¨$×*?Ñ*?ÀÓ*OÑOˆð =AÐ<L�4—>‘> !Ó$ t§~¡~°aÓ'8Ò8ÐRVˆØ'¨$¯-©-¸×8HÑ8HÈÐ^fÐ8HÓ8gÓ*hÑhˆØ'¨$¯-©-¸¿¹ÈÐ]e¸Ó8fÓ*gÑgˆØ'¨$¯-©-Ø×Ñ˜~°HÈÐÓTó+
ñ 
ˆð (¨$¯-©-Ø×Ñ˜^°(ÀzÐÓRó+
ñ 
ˆð
 Ÿ™ ~Ó6ˆà˜~Ð-Ð-rH   )NNrà   rÉ   s   @rI   r‡  r‡    s   ø„ ô >÷DB.rH   r‡  c                   ó    — e Zd Zdd„Zd„ Zd„ Zy)ÚEsmCategoricalMixturec                 óÂ   — || _         t        j                  |||dz   | j                   j                  | j                   j                  ¬«      }|d d |dd  z   dz  | _        y )Nr   ©r\   r[   ry   r—   )ÚlogitsrD   Úlinspacer\   r[   Úv_bins)rÃ   ÚparamÚbinsrU  rV  s        rI   rÁ   zEsmCategoricalMixture.__init__è  sW   € àˆŒÜ�~‰~˜e S¨$°©(¸4¿;¹;×;MÑ;MÐUY×U`ÑU`×UfÑUfÔgˆØ˜C˜R�y 4¨¨ 8Ñ+¨qÑ0ˆ�rH   c                 ó<  — |j                  d«      | j                  d g|j                  z     z
  j                  «       j	                  d«      }| j
                  j                  d«      }t        j                  ||j                  d«      d¬«      j                  d«      S )Nry   râ   )
r&  rä  ÚndimÚabsÚargminrâ  Úlog_softmaxrD   Útake_along_dimÚsqueeze)rÃ   ÚtrueÚ
true_indexÚnlls       rI   Úlog_probzEsmCategoricalMixture.log_probî  s€   € ð —n‘n RÓ(¨4¯;©;¸°vÀÇ	Á	Ñ7IÑ+JÑJ×OÑOÓQ×XÑXÐY[Ó\ˆ
Ø�k‰k×%Ñ% bÓ)ˆÜ×#Ñ# C¨×)=Ñ)=¸bÓ)AÀrÔJ×RÑRÐSUÓVÐVrH   c                 óŽ   — | j                   j                  d«      | j                  j                  d«      z  j	                  d«      S )Nry   r   )râ  rã   rä  r&  rí  )rÃ   s    rI   ÚmeanzEsmCategoricalMixture.meanö  s6   € Ø—‘×#Ñ# BÓ'¨$¯+©+×*?Ñ*?ÀÓ*BÑB×KÑKÈBÓOÐOrH   N)é2   r   r   )r@   rA   rB   rÁ   rñ  ró  rG   rH   rI   rß  rß  ç  s   „ ó1òWóPrH   rß  c                 ó8   — t        | |¬«      j                  «       S )N©ræ  )rß  ró  )râ  ræ  s     rI   Úcategorical_lddtr÷  ú  s   € ä  ¨dÔ3×8Ñ8Ó:Ð:rH   c                 ó  — | €yt        | j                  «      dk7  r"t        dt        | j                  «      › d�«      ‚| j                  \  }}| j                  d«      j	                  |||«      }|j                  ||z  |«      }|S )zÍ
    Helper to convert B x L mask of valid positions to axial mask used in row column attentions.

    Input:
      mask: B x L tensor of booleans

    Output:
      mask: B x L x L tensor of booleans
    Nr—   r×  rÖ  r   )rc   ri   rÂ   r&  Úexpandrz   )r!  r¬  rÜ  Úms       rI   Úget_axial_maskrû  ÿ  s   € ð €|Øä
ˆ4�:‰:ƒ˜!ÒÜÐ=¼cÀ$Ç*Á*»oÐ=NÈfÐUÓVÐVØŸ™Ñ€IˆwØ�‰�qÓ× Ñ  ¨G°WÓ=€AØ	�	‰	�)˜gÑ% wÓ/€AØ€HrH   c                   ó&   ‡ — e Zd Zˆ fd„Zdd„Zˆ xZS )ÚEsmFoldRelativePositionc                 óÂ   •— t         ‰| �  «        |j                  | _        t        j
                  j                  d| j                  z  dz   |j                  «      | _        y ©Nr—   )	rÀ   rÁ   Úposition_binsræ  rD   r¢   Ú	Embeddingr/  Ú	embedding©rÃ   r8  rÄ   s     €rI   rÁ   z EsmFoldRelativePosition.__init__  sH   ø€ Ü‰ÑÔØ×(Ñ(ˆŒ	ô Ÿ™×+Ñ+¨A°·	±	©M¸AÑ,=¸v×?XÑ?XÓYˆ�rH   c                 ó   — |j                   t        j                  k7  rt        d|j                   › d�«      ‚|�?|j                  |j                  k7  r&t        d|j                  › d|j                  › d�«      ‚|dd…ddd…f   |dd…dd…df   z
  }|j                  | j                   | j                  «      }|| j                  z   dz   }|�!|dd…ddd…f   |dd…dd…df   z  }d||d	k(  <   | j                  |«      }|S )
zÚ
        Input:
          residue_index: B x L tensor of indices (dtype=torch.long) mask: B x L tensor of booleans

        Output:
          pairwise_state: B x L x L x pairwise_state_dim tensor of embeddings
        z`residue_index` has dtype z, it should be `torch.long`.Nz5`residue_index` and `mask` have inconsistent shapes: rØ  rÙ  r   r   F)r[   rD   ÚlongrÂ   ri   r¥   ræ  r  )rÃ   r8   r!  r¼  r  s        rI   rß   zEsmFoldRelativePosition.forward  s  € ð ×Ñ¤%§*¡*Ò,ÜÐ9¸-×:MÑ:MÐ9NÐNjÐkÓlÐlØÐ × 3Ñ 3°t·z±zÒ AÜØGÈ×H[ÑH[ÐG\Ð\`Ðae×akÑakÐ`lÐlmÐnóð ð šQ ¢a˜ZÑ(¨=ººA¸t¸Ñ+DÑDˆØ�z‰z˜4Ÿ9™9˜* d§i¡iÓ0ˆØ�d—i‘iÑ !Ñ#ˆàÐØš˜4¢˜
Ñ# dª1ªa°¨:Ñ&6Ñ6ˆDØ"#ˆD�˜‘Ñà—‘ Ó%ˆØˆrH   rK   rà   rÉ   s   @rI   rý  rý    s   ø„ ôZ÷rH   rý  c                   óV   ‡ — e Zd Zˆ fd„Zdej
                  dej
                  fd„Zˆ xZS )ÚEsmFoldAngleResnetBlockc                 óð   •— t         ‰| �  «        t        |j                  |j                  d¬«      | _        t        |j                  |j                  d¬«      | _        t        j                  «       | _        y ©Nr»   rï   r¿   )	rÀ   rÁ   r³   Ú
resnet_dimÚlinear_1Úlinear_2r¢   rÈ  r»   r  s     €rI   rÁ   z EsmFoldAngleResnetBlock.__init__:  sU   ø€ Ü‰ÑÔä% f×&7Ñ&7¸×9JÑ9JÐQWÔXˆŒÜ% f×&7Ñ&7¸×9JÑ9JÐQXÔYˆŒä—G‘G“Iˆ�	rH   r˜   rY   c                 ó˜   — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }||z   S rK   )r»   r  r  )rÃ   r˜   Ú	s_initials      rI   rß   zEsmFoldAngleResnetBlock.forwardB  sG   € Øˆ	à�I‰I�a‹LˆØ�M‰M˜!ÓˆØ�I‰I�a‹LˆØ�M‰M˜!Óˆà�9‰}ÐrH   )r@   rA   rB   rÁ   rD   rd   rß   rÈ   rÉ   s   @rI   r  r  9  s#   ø„ ôð˜Ÿ™ð ¨%¯,©,÷ rH   r  c                   ó�   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  deej                  ej                  f   fd„Zˆ xZ	S )ÚEsmFoldAngleResnetz.
    Implements Algorithm 20, lines 11-14
    c                 óø  •— t         ‰| �  «        || _        t        |j                  |j
                  «      | _        t        |j                  |j
                  «      | _        t        j                  «       | _
        t        |j                  «      D ](  }t        |«      }| j                  j                  |«       Œ* t        |j
                  |j                  dz  «      | _        t        j"                  «       | _        y rÿ  )rÀ   rÁ   r8  r³   Úsequence_dimr
  Ú	linear_inÚlinear_initialr¢   Ú
ModuleListÚlayersrm   Únum_resnet_blocksr  ÚappendÚ
num_anglesÚ
linear_outrÈ  r»   )rÃ   r8  rR  ÚlayerrÄ   s       €rI   rÁ   zEsmFoldAngleResnet.__init__R  s»   ø€ Ü‰ÑÔØˆŒä& v×':Ñ':¸F×<MÑ<MÓNˆŒÜ+¨F×,?Ñ,?À×ARÑARÓSˆÔä—m‘m“oˆŒÜ�v×/Ñ/Ó0ò 	&ˆAÜ+¨FÓ3ˆEØ�K‰K×Ñ˜uÕ%ð	&ô (¨×(9Ñ(9¸6×;LÑ;LÈqÑ;PÓQˆŒä—G‘G“Iˆ�	rH   rä   r  rY   c           	      ó  — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }||z   }| j                  D ]
  } ||«      }Œ | j                  |«      }| j	                  |«      }|j                  |j                  dd dz   «      }|}t        j                  t        j                  t        j                  |dz  dd¬«      | j                  j                  ¬«      «      }||z  }||fS )a  
        Args:
            s:
                [*, C_hidden] single embedding
            s_initial:
                [*, C_hidden] single embedding as of the start of the StructureModule
        Returns:
            [*, no_angles, 2] predicted angles
        Nry   )ry   r—   r—   T)re   Úkeepdim)r”   )r»   r  r  r  r  rû   ri   rD   r¡   r¥   r  r8  Úepsilon)rÃ   rä   r  ÚlÚunnormalized_sÚ
norm_denoms         rI   rß   zEsmFoldAngleResnet.forwardb  sö   € ð —I‘I˜iÓ(ˆ	Ø×'Ñ'¨	Ó2ˆ	Ø�I‰I�a‹LˆØ�N‰N˜1ÓˆØ�	‰Mˆà—‘ò 	ˆAÙ�!“‰Að	ð �I‰I�a‹Lˆð �O‰O˜AÓˆð �F‰F�1—7‘7˜3˜B�< 'Ñ)Ó*ˆàˆÜ—Z‘ZÜ�K‰KÜ—	‘	˜!˜Q™$ B°Ô5Ø—K‘K×'Ñ'ôó
ˆ
ð �
‰Nˆà˜qÐ Ð rH   ©
r@   rA   rB   rC   rÁ   rD   rd   r
   rß   rÈ   rÉ   s   @rI   r  r  M  sD   ø„ ñôð )!˜Ÿ™ð )!°%·,±,ð )!À5ÈÏÉÐW\×WcÑWcÐIcÑCd÷ )!rH   r  c                   óÂ   ‡ — e Zd ZdZˆ fd„Z	 	 ddej                  deej                     dedej                  de	dee
ej                        d	ej                  fd
„Zˆ xZS )r…  z"
    Implements Algorithm 22.
    c                 ó¼  •— t         ‰| �  «        || _        |j                  }|j                  }|j
                  | _        |j                  | _        |j                  | _	        |j                  | _
        |j
                  |j                  z  }t        ||«      | _        t        |d|z  «      | _        |j                  |j                  z  dz  }t        ||«      | _        |j                  |j                  |j                  z   z  dz  }t        ||«      | _        t        ||j                  «      | _        t#        j$                  t'        j(                  |j                  «      «      | _        |j                  ||j
                  z   |j                  dz  z   z  }t        ||d¬«      | _        t#        j.                  d¬«      | _        t#        j2                  «       | _        y )Nr—   r   rÁ  r¿   rï   ry   râ   )rÀ   rÁ   r8  r  Úpairwise_dimÚipa_dimÚ
hidden_dimÚnum_heads_ipar›  Únum_qk_pointsÚnum_v_pointsr³   rð   Ú	linear_kvÚlinear_q_pointsÚlinear_kv_pointsÚlinear_br¢   rÏ   rD   rÒ   r†  r  ÚSoftmaxrã   ÚSoftplusÚsoftplus)	rÃ   r8  Úc_sÚc_zÚhcÚhpqÚhpkvÚconcat_out_dimrÄ   s	           €rI   rÁ   z'EsmFoldInvariantPointAttention.__init__“  s€  ø€ Ü‰ÑÔØˆŒà×!Ñ!ˆØ×!Ñ!ˆØ Ÿ.™.ˆŒØ×-Ñ-ˆŒØ#×1Ñ1ˆÔØ"×/Ñ/ˆÔð �^‰^˜f×2Ñ2Ñ2ˆÜ% c¨2Ó.ˆŒÜ& s¨A°©FÓ3ˆŒà×"Ñ" V×%9Ñ%9Ñ9¸AÑ=ˆÜ,¨S°#Ó6ˆÔà×#Ñ# v×';Ñ';¸f×>QÑ>QÑ'QÑRÐUVÑVˆÜ -¨c°4Ó 8ˆÔä% c¨6×+?Ñ+?Ó@ˆŒäŸL™L¬¯©°f×6JÑ6JÓ)LÓMˆÔà×-Ñ-°°v·~±~Ñ1EÈ×H[ÑH[Ð^_ÑH_Ñ1_Ñ`ˆÜ'¨¸À'ÔJˆŒä—z‘z bÔ)ˆŒÜŸ™›ˆ�rH   rä   r@  r«  r!  Ú_offload_inferenceÚ_z_reference_listrY   c           	      ó>  — |g}| j                  |«      }| j                  |«      }|j                  |j                  dd | j                  dfz   «      }|j                  |j                  dd | j                  dfz   «      }t        j                  || j                  d¬«      \  }	}
| j                  |«      }t        j                  ||j                  d   dz  d¬«      }t        j                  |d¬«      }|d   j                  |«      }|j                  |j                  dd | j                  | j                  dfz   «      }| j                  |«      }t        j                  ||j                  d   dz  d¬«      }t        j                  |d¬«      }|d   j                  |«      }|j                  |j                  dd | j                  ddfz   «      }t        j                  || j                  | j                  gd¬«      \  }}| j                  |d   «      }|r3t        j                   |d   «      dk(  sJ ‚|d   j#                  «       |d<   t%        «       rzt
        j&                  j(                  j+                  d	¬
«      5  t        j,                  t/        |j1                  «       d«      t/        |	j1                  «       d«      «      }ddd«       n*t        j,                  t/        |d«      t/        |	d«      «      }t3        j4                  dd| j                  z  z  «      z  }|t3        j4                  d«      t/        |d«      z  z  }|j7                  d«      |j7                  d«      z
  }|dz  }t9        t        j:                  |d¬«      «      } | j=                  | j>                  «      j                  dtA        |j                  dd «      z  dz   Ž }|t3        j4                  dd| j                  dz  dz  z  z  «      z  }||z  }t        j8                  |d¬«      dz  }|j7                  d«      |j7                  d«      z  }| jB                  jD                  |dz
  z  }t/        |d«      }||z   }||j7                  d«      z   }| jG                  |«      }t        j,                  ||
jI                  dd«      jK                  |jL                  ¬«      «      jI                  dd«      }tO        |d«      }t        j8                  |dddd…dd…df   t/        |d«      dddd…dd…f   z  d¬«      }t/        |d«      }|d   jQ                  |«      }tO        t        j4                  t        j8                  |dz  d¬«      | jB                  jR                  z   «      d«      } |jT                  g |j                  dd ¢d‘d‘­Ž }|r!|d   jK                  |jV                  «      |d<   t        j,                  |jI                  dd«      |d   jK                  |jL                  ¬«      «      }tO        |d«      }| jY                  t        jZ                  |gt        j:                  |d¬«      ¢|‘|‘­d¬«      jK                  |d   jL                  ¬«      «      }|S # 1 sw Y   �Œ¡xY w)ab  
        Args:
            s:
                [*, N_res, C_s] single representation
            z:
                [*, N_res, N_res, C_z] pair representation
            r:
                [*, N_res] transformation object
            mask:
                [*, N_res] mask
        Returns:
            [*, N_res, C_s] single representation update
        Nry   râ   r   ).Nr–   r   r—   FrÔ   )r   r   r—   r;  r€  gUUUUUUÕ?r#  r$  éûÿÿÿ)r   )ry   r   g      "@r™  r   rú   rÖ   .)r   r   r   r—   )r—   r   r   r   ).NN).rð   r+  rû   ri   r›  rD   Úsplitr'  r,  ÚstackÚapplyr)  r-  r*  r.  ÚsysÚgetrefcountÚcpurP   rØ   rÙ   rÚ   r  rƒ   r{  r    r¡   r&  r  Úunbindr1  r†  rc   r8  r  rã   rü   rÝ   r[   r{   Úinvert_applyr  rz   r\   r  r¹  )rÃ   rä   r@  r«  r!  r8  r9  rý   Úkvra   r�   Úq_ptsÚkv_ptsÚk_ptsÚv_ptsr™   r˜   Úpt_attr†  Úsquare_maskrÿ   Úo_ptÚ	o_pt_normÚo_pairs                           rI   rß   z&EsmFoldInvariantPointAttention.forward¶  sâ  € ð, ˆCˆð �M‰M˜!ÓˆØ�^‰^˜AÓˆð �F‰F�1—7‘7˜3˜B�< 4§>¡>°2Ð"6Ñ6Ó7ˆð �W‰W�R—X‘X˜c˜r�] d§n¡n°bÐ%9Ñ9Ó:ˆô �{‰{˜2˜tŸ™°BÔ7‰ˆˆ1ð ×$Ñ$ QÓ'ˆô —‘˜E 5§;¡;¨r¡?°aÑ#7¸RÔ@ˆÜ—‘˜E rÔ*ˆØ�)‘×"Ñ" 5Ó)ˆð —
‘
˜5Ÿ;™; s¨Ð+¨t¯~©~¸t×?QÑ?QÐSTÐ.UÑUÓVˆð ×&Ñ& qÓ)ˆô —‘˜V V§\¡\°"Ñ%5¸Ñ%:ÀÔCˆÜ—‘˜V¨Ô,ˆØ�9‘×#Ñ# FÓ+ˆð —‘˜VŸ\™\¨#¨2Ð.°$·.±.À"ÀaÐ1HÑHÓIˆô —{‘{ 6¨D×,>Ñ,>À×@QÑ@QÐ+RÐXZÔ[‰ˆˆuð �M‰M˜!˜A™$ÓˆáÜ—?‘? 1 Q¡4Ó(¨AÒ-Ð-Ð-Ø�Q‘4—8‘8“:ˆAˆa‰Dô ÔÜ—‘—‘×(Ñ(°Ð(Ó7ñ Ü—L‘LÜ& q§w¡w£y°)Ó<Ü& q§w¡w£y°)Ó<ó�÷ð ô —‘Ü" 1 iÓ0Ü" 1 iÓ0óˆAð
 	
ŒT�Y‰Y�s˜a $§/¡/Ñ1Ñ2Ó3Ñ3ˆØ	ŒT�Y‰Y�wÓÔ"4°Q¸	Ó"BÑBÑBˆð —‘ Ó$ u§¡°rÓ':Ñ:ˆØ˜‘ˆô ”U—\‘\ &¨bÔ1Ó2ˆØ<�t—}‘} T×%6Ñ%6Ó7×<Ñ<¸tÄcÈ&Ï,É,ÐWZÐXZÐJ[ÓF\Ñ?\Ð_fÑ?fÐhˆØ#¤d§i¡i°°q¸D×<NÑ<NÐQTÑ<TÐWXÑ<XÑ7YÑ0ZÓ&[Ñ[ˆØ˜,Ñ&ˆô —‘˜6 rÔ*¨dÑ3ˆà—n‘n RÓ(¨4¯>©>¸"Ó+=Ñ=ˆØ—k‘k—o‘o¨°q©Ñ9ˆô $ F¨IÓ6ˆà�‰JˆØ�×%Ñ% bÓ)Ñ)ˆØ�L‰L˜‹Oˆô �L‰L˜˜AŸK™K¨¨BÓ/×2Ñ2¸¿¹Ð2ÓAÓB×LÑLÈRÐQSÓTˆô ˜q !Ó$ˆô �y‰yØˆs�Dš!šQ Ð$Ñ%Ô(:¸5À,Ó(OÐPSÐUYÒ[\Ò^_ÐP_Ñ(`Ñ`Øô
ˆô " $¨Ó5ˆØ�Ñ!×.Ñ.¨tÓ4ˆô '¤u§z¡z´%·)±)¸DÀ!¹GÈÔ2LÈtÏ{É{×ObÑObÑ2bÓ'cÐefÓgˆ	ð ˆt�|‰|Ð4˜TŸZ™Z¨¨˜_Ð4¨bÐ4°!Ò4ˆáØ�Q‘4—7‘7˜4Ÿ;™;Ó'ˆAˆa‰Dô —‘˜aŸk™k¨"¨bÓ1°1°Q±4·7±7ÀÇÁ°7Ó3IÓJˆô $ F¨AÓ.ˆð �O‰OÜ�I‰I�qÐIœ5Ÿ<™<¨°"Ô5ÐI°yÐIÀ&ÑIÈrÔR×UÑUÐ\]Ð^_Ñ\`×\fÑ\fÐUÓgó
ˆð ˆ÷Yñ ús   ÊAZÚZ)FN)r@   rA   rB   rC   rÁ   rD   rd   r   r   rÆ   r	   rß   rÈ   rÉ   s   @rI   r…  r…  Ž  sŠ   ø„ ñô!&ðR $)Ø>BñZà�<‰<ðZð �E—L‘LÑ!ðZð ð	Zð
 �l‰lðZð !ðZð $ H¨U¯\©\Ñ$:Ñ;ðZð 
�‰÷ZrH   r…  c                   óx   ‡ — e Zd ZdZˆ fd„Zdej                  deej                  ej                  f   fd„Zˆ xZ	S )ÚEsmFoldBackboneUpdatez*
    Implements part of Algorithm 23.
    c                 ó\   •— t         ‰| �  «        t        |j                  dd¬«      | _        y )Né   r¿   rï   )rÀ   rÁ   r³   r  r  r  s     €rI   rÁ   zEsmFoldBackboneUpdate.__init__X  s$   ø€ Ü‰ÑÔä# F×$7Ñ$7¸ÀÔIˆ�rH   rä   rY   c                 ó(   — | j                  |«      }|S )z‚
        Args:
            [*, N_res, C_s] single representation
        Returns:
            [*, N_res, 6] update vector
        )r  )rÃ   rä   Úupdates      rI   rß   zEsmFoldBackboneUpdate.forward]  s   € ð —‘˜Q“ˆàˆrH   r"  rÉ   s   @rI   rO  rO  S  s7   ø„ ñôJð

˜Ÿ™ð 
¨%°·±¸e¿l¹lÐ0JÑ*K÷ 
rH   rO  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú%EsmFoldStructureModuleTransitionLayerc                 ó>  •— t         ‰| �  «        t        |j                  |j                  d¬«      | _        t        |j                  |j                  d¬«      | _        t        |j                  |j                  d¬«      | _        t        j                  «       | _	        y r	  )
rÀ   rÁ   r³   r  r  r  Úlinear_3r¢   rÈ  r»   r  s     €rI   rÁ   z.EsmFoldStructureModuleTransitionLayer.__init__k  sr   ø€ Ü‰ÑÔä% f×&9Ñ&9¸6×;NÑ;NÐU[Ô\ˆŒÜ% f×&9Ñ&9¸6×;NÑ;NÐU[Ô\ˆŒÜ% f×&9Ñ&9¸6×;NÑ;NÐU\Ô]ˆŒä—G‘G“Iˆ�	rH   c                 ó¾   — |}| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }||z   }|S rK   )r  r»   r  rW  )rÃ   rä   r  s      rI   rß   z-EsmFoldStructureModuleTransitionLayer.forwardt  sY   € Øˆ	Ø�M‰M˜!ÓˆØ�I‰I�a‹LˆØ�M‰M˜!ÓˆØ�I‰I�a‹LˆØ�M‰M˜!Óˆà�	‰MˆàˆrH   rà   rÉ   s   @rI   rU  rU  j  s   ø„ ôö
rH   rU  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú EsmFoldStructureModuleTransitionc                 ó^  •— t         ‰| �  «        || _        t        j                  «       | _        t        |j                  «      D ](  }t        |«      }| j
                  j                  |«       Œ* t        j                  |j                  «      | _        t        |j                  «      | _        y rK   )rÀ   rÁ   r8  r¢   r  r  rm   Únum_transition_layersrU  r  r®  Údropout_rater¯  r   r  rÜ   )rÃ   r8  rR  r  rÄ   s       €rI   rÁ   z)EsmFoldStructureModuleTransition.__init__‚  s‚   ø€ Ü‰ÑÔØˆŒä—m‘m“oˆŒÜ�v×3Ñ3Ó4ò 	"ˆAÜ5°fÓ=ˆAØ�K‰K×Ñ˜qÕ!ð	"ô —z‘z &×"5Ñ"5Ó6ˆŒÜ# F×$7Ñ$7Ó8ˆ�rH   c                 ó|   — | j                   D ]
  } ||«      }Œ | j                  |«      }| j                  |«      }|S rK   )r  r¯  rÜ   )rÃ   rä   r  s      rI   rß   z(EsmFoldStructureModuleTransition.forwardŽ  s@   € Ø—‘ò 	ˆAÙ�!“‰Að	ð �L‰L˜‹OˆØ�O‰O˜AÓˆàˆrH   rà   rÉ   s   @rI   rZ  rZ  �  s   ø„ ô
9örH   rZ  c                   ó<   ‡ — e Zd Zˆ fd„Z	 	 dd„Zd„ Zd„ Zd„ Zˆ xZS )ÚEsmFoldStructureModulec                 óÞ  •— t         ‰| �  «        || _        t        |j                  «      | _        t        |j                  «      | _        t        |j                  |j                  «      | _	        t        |«      | _        t        j                  |j                  «      | _        t        |j                  «      | _        t#        |«      | _        t'        |«      | _        t+        |«      | _        y rK   )rÀ   rÁ   r8  r   r  Úlayer_norm_sr%  Úlayer_norm_zr³   r  r…  Úipar¢   r®  r]  Úipa_dropoutÚlayer_norm_iparZ  Ú
transitionrO  Ú	bb_updater  Úangle_resnetr  s     €rI   rÁ   zEsmFoldStructureModule.__init__™  s²   ø€ Ü‰ÑÔØˆŒô & f×&9Ñ&9Ó:ˆÔÜ% f×&9Ñ&9Ó:ˆÔä& v×':Ñ':¸F×<OÑ<OÓPˆŒä1°&Ó9ˆŒäŸ:™: f×&9Ñ&9Ó:ˆÔÜ'¨×(;Ñ(;Ó<ˆÔä:¸6ÓBˆŒÜ.¨vÓ6ˆŒÜ.¨vÓ6ˆÕrH   c           
      óŠ  — |d   }|€|j                  |j                  dd «      }| j                  |«      }| j                  |d   «      }d}|r8t	        j
                  |d   «      dk(  sJ ‚|d   j                  «       |d<   |g}d}|}| j                  |«      }t        j                  |j                  dd |j                  |j                  | j                  d¬«      }	g }
t        | j                  j                  «      D �]{  }|| j!                  |||	|||¬«      z   }| j#                  |«      }| j%                  |«      }| j'                  |«      }|	j)                  | j+                  |«      «      }	t        t-        |	j/                  «       j1                  «       d¬	«      |	j3                  «       «      }|j5                  | j                  j6                  «      }| j9                  ||«      \  }}| j;                  |||«      }| j=                  ||«      }|	j5                  | j                  j6                  «      }|j?                  «       |jA                  «       ||||d
œ}|
jC                  |«       |	jE                  «       }	�Œ~ ~~|r!|d   jG                  |j                  «      |d<   tI        tJ        jL                  |
«      }
||
d<   |
S )a×  
        Args:
            evoformer_output_dict:
                Dictionary containing:
                    "single":
                        [*, N_res, C_s] single representation
                    "pair":
                        [*, N_res, N_res, C_z] pair representation
            aatype:
                [*, N_res] amino acid indices
            mask:
                Optional [*, N_res] sequence mask
        Returns:
            A dictionary of outputs
        ÚsingleNry   rF  r—   Úquat)Úfmt)r8  r9  )Úrot_matsÚquats)r)   r*   r+   r,   r-   r.   )'r%  ri   rb  rc  r?  r@  rA  r  r   Úidentityr[   r\   Útrainingrm   r8  Ú
num_blocksrd  re  rf  rg  Úcompose_q_update_vecrh  r   Úget_rotsÚget_rot_matsÚ	get_transÚscale_translationÚtrans_scale_factorri  r&   r"   Úto_tensor_7Úto_tensor_4x4r  Ústop_rot_gradientrÝ   rˆ   rD   r=  )rÃ   Úevoformer_output_dictr3   r!  r8  rä   r@  Úz_reference_listr  ÚrigidsÚoutputsrr   Úbackb_to_globalr+   r,   Úall_frames_to_globalÚpred_xyzÚscaled_rigidsÚpredss                      rI   rß   zEsmFoldStructureModule.forward±  sÈ  € ð, " (Ñ+ˆàˆ<à—:‘:˜aŸg™g c r˜lÓ+ˆDð ×Ñ˜aÓ ˆð ×ÑÐ3°FÑ;Ó<ˆàÐÙÜ—?‘?Ð#8¸Ñ#@ÓAÀQÒFÐFÐFØ,AÀ&Ñ,I×,MÑ,MÓ,OÐ! &Ñ)Ø !˜sÐØˆAð ˆ	Ø�N‰N˜1Óˆô —‘Ø�G‰G�C�RˆLØ�G‰GØ�H‰HØ�M‰MØô
ˆð ˆÜ�t—{‘{×-Ñ-Ó.ó /	0ˆAà�D—H‘HØØØØØ#5Ø"2ð ó ñ ˆAð × Ñ  Ó#ˆAØ×#Ñ# AÓ&ˆAØ—‘ Ó"ˆAð ×0Ñ0°·±ÀÓ1BÓCˆFô
 $Ü &§/¡/Ó"3×"@Ñ"@Ó"BÈ$ÔOØ× Ñ Ó"óˆOð
 .×?Ñ?ÀÇÁ×@^Ñ@^Ó_ˆOð +/×*;Ñ*;¸A¸yÓ*IÑ'Ð à#'×#@Ñ#@ÀÐRXÐZ`Ó#aÐ à×IÑIÐJ^Ð`fÓgˆHà"×4Ñ4°T·[±[×5SÑ5SÓTˆMð (×3Ñ3Ó5Ø$8×$FÑ$FÓ$HØ':Ø Ø%ØñˆEð �N‰N˜5Ô!à×-Ñ-Ó/ŠFð_/	0ðb ÐáØ,AÀ&Ñ,I×,LÑ,LÈQÏXÉXÓ,VÐ! &Ñ)ä¤§¡¨WÓ5ˆØˆ�ÑàˆrH   c           	      ó,  — t        | d«      s9| j                  dt        j                  t        j
                  ||d¬«      d¬«       t        | d«      s8| j                  dt        j                  t        j                  |d¬«      d¬«       t        | d«      s9| j                  dt        j                  t        j                  ||d¬«      d¬«       t        | d«      s:| j                  dt        j                  t        j                  ||d¬«      d¬«       y y )	NÚdefault_framesF)r[   r\   Úrequires_grad)Ú
persistentÚ	group_idx)r\   r‡  Ú	atom_maskÚlit_positions)	ÚhasattrÚregister_bufferrD   r|   r$   Ú!restype_rigid_group_default_frameÚrestype_atom14_to_rigid_groupÚrestype_atom14_maskÚ$restype_atom14_rigid_group_positions)rÃ   Úfloat_dtyper\   s      rI   Ú_init_residue_constantsz.EsmFoldStructureModule._init_residue_constants"  s  € Ü�tÐ-Ô.Ø× Ñ Ø Ü—‘Ü%×GÑGØ%Ø!Ø"'ô	ð !ð !ô 	ô �t˜[Ô)Ø× Ñ ØÜ—‘Ü%×CÑCØ!Ø"'ôð
 !ð !ô ô �t˜[Ô)Ø× Ñ ØÜ—‘Ü%×9Ñ9Ø%Ø!Ø"'ô	ð !ð !ô 	ô �t˜_Ô-Ø× Ñ ØÜ—‘Ü%×JÑJØ%Ø!Ø"'ô	ð !ð !õ 	ð .rH   c                 ó~   — | j                  |j                  |j                  «       t        |||| j                  «      S rK   )r“  r[   r\   r&   r†  )rÃ   r«  ÚalphaÚfs       rI   r&   z/EsmFoldStructureModule.torsion_angles_to_framesO  s1   € à×$Ñ$ U§[¡[°%·,±,Ô?ä'¨¨5°!°T×5HÑ5HÓIÐIrH   c                 óö   — | j                  |j                  «       j                  |j                  «       j                  «       t	        ||| j
                  | j                  | j                  | j                  «      S rK   )	r“  rt  r[   r\   r"   r†  r‰  rŠ  r‹  )rÃ   r«  r–  s      rI   r"   zDEsmFoldStructureModule.frames_and_literature_positions_to_atom14_posU  s]   € à×$Ñ$ Q§Z¡Z£\×%7Ñ%7¸¿¹»×9LÑ9LÔMÜ<ØØØ×ÑØ�N‰NØ�N‰NØ×Ñó
ð 	
rH   )NF)	r@   rA   rB   rÁ   rß   r“  r&   r"   rÈ   rÉ   s   @rI   r`  r`  ˜  s(   ø„ ô7ð8 Ø óoòb+òZJö

rH   r`  c                   ó:   ‡ — e Zd Zˆ fd„Zd„ Zd„ Zed„ «       Zˆ xZS )ÚEsmFoldingTrunkc                 óN  •— t         ‰| �  «        || _        |j                  }|j                  }t        |«      | _        t        j                  t        |j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        d| _        t        j                  |«      | _        t        j                  |«      | _        t        j"                  | j                  |«      | _        | j$                  j&                  d   j)                  «       j+                  «        t-        |j.                  «      | _        t        j0                  ||j.                  j2                  «      | _        t        j0                  ||j.                  j6                  «      | _        |j:                  | _        y c c}w )Né   r   )rÀ   rÁ   r8  r¶  r/  rý  Úpairwise_positional_embeddingr¢   r  rm   rr  r‡  ÚblocksÚrecycle_binsr   Úrecycle_s_normÚrecycle_z_normr  Úrecycle_distorÑ   ÚdetachÚzero_r`  Ústructure_moduler�  r  Ú
trunk2sm_sr%  Ú
trunk2sm_zr  )rÃ   r8  r2  r3  rR  rÄ   s        €rI   rÁ   zEsmFoldingTrunk.__init__c  s2  ø€ Ü‰ÑÔØˆŒà×'Ñ'ˆØ×'Ñ'ˆä-DÀVÓ-LˆÔ*ä—m‘mÔZ_Ð`f×`qÑ`qÓZrÖ$sÐUVÔ%HÈÕ%PÒ$sÓtˆŒàˆÔÜ Ÿl™l¨3Ó/ˆÔÜ Ÿl™l¨3Ó/ˆÔÜŸ\™\¨$×*;Ñ*;¸SÓAˆÔØ×Ñ×!Ñ! !Ñ$×+Ñ+Ó-×3Ñ3Ô5ä 6°v×7NÑ7NÓ OˆÔÜŸ)™) C¨×)@Ñ)@×)MÑ)MÓNˆŒÜŸ)™) C¨×)@Ñ)@×)MÑ)MÓNˆŒà ×+Ñ+ˆ�ùò %ts   Á%F"c                 ó   — || _         y rK   )r  )rÃ   r  s     rI   Úset_chunk_sizezEsmFoldingTrunk.set_chunk_sizez  s   € ð
 %ˆ�rH   c           	      ó6  ‡ — |j                   }|}|}	|€‰ j                  j                  }n|dk  rt        d«      ‚|dz  }ˆ fd„}
|}|	}t	        j
                  |«      }t	        j
                  |«      }t	        j                  |j                  dd |t        j                  dœŽ}t        |«      D �]O  }t        ||dz
  k(  rg nt	        j                  «       g«      5  ‰ j                  |j                  «       «      j                  |«      }‰ j                  |j                  «       «      j                  |«      }|‰ j!                  |j                  «       «      j                  |«      z  } |
||z   |	|z   ||«      \  }}‰ j#                  ‰ j%                  |«      ‰ j'                  |«      dœ||j)                  «       «      }|}|}t*        j-                  |d	   d   dd…dd…dd
…f   dd‰ j.                  «      }ddd«       �ŒR |d<   ||d<   |S # 1 sw Y   �ŒixY w)a~  
        Inputs:
          seq_feats: B x L x C tensor of sequence features pair_feats: B x L x L x C tensor of pair features residx: B
          x L long tensor giving the position in the sequence mask: B x L boolean tensor indicating valid residues

        Output:
          predicted_structure: B x L x (num_atoms_per_residue * 3) tensor wrapped in a Coordinates object
        Nr   z(Number of recycles must not be negative.r   c                 óŽ   •— |‰j                  ||¬«      z   }‰j                  D ]  } || |||‰j                  ¬«      \  } }Œ | |fS )NrÚ  )r!  r8   r  )rœ  r�  r  )rä   r@  Úresidxr!  ÚblockrÃ   s        €rI   Ú
trunk_iterz+EsmFoldingTrunk.forward.<locals>.trunk_iter–  sU   ø€ Ø�D×6Ñ6°vÀDÐ6ÓIÑIˆAàŸ™ò `�Ù˜Q ¨¸FÈtÏÉÔ_‘�‘1ð`à�a�4ˆKrH   ry   rá  )rk  rF  r-   r   g      @g     `5@r/   r0   )r\   r8  Úmax_recyclesrÂ   rD   Ú
zeros_likerÒ   ri   Úint64rm   r   r¯   rŸ  r¢  rÝ   r   r¡  r¤  r¥  r¦  r{  r™  Ú	distogramrž  )rÃ   Ú	seq_featsÚ
pair_featsÚtrue_aar«  r!  Úno_recyclesr\   Ús_s_0Ús_z_0r­  r/   r0   Ú	recycle_sÚ	recycle_zrž  Úrecycle_idxÚ	structures   `                 rI   rß   zEsmFoldingTrunk.forward�  s  ø€ ð ×!Ñ!ˆØˆØˆàÐØŸ+™+×2Ñ2‰Kà˜QŠÜ Ð!KÓLÐLØ˜1ÑˆKô	ð ˆØˆÜ×$Ñ$ SÓ)ˆ	Ü×$Ñ$ SÓ)ˆ	Ü—{‘{ C§I¡I¨c¨r N¸6ÌÏÉÒUˆä  Ó-ó 	ˆKÜ  {°kÀA±oÒ'E¡ÌEÏMÉMËOÐK\Ó]ñ à ×/Ñ/°	×0@Ñ0@Ó0BÓC×FÑFÀvÓN�	Ø ×/Ñ/°	×0@Ñ0@Ó0BÓC×FÑFÀvÓN�	Ø˜T×/Ñ/°×0CÑ0CÓ0EÓF×IÑIÈ&ÓQÑQ�	á% e¨iÑ&7¸ÀÑ9JÈFÐTXÓY‘��Sð !×1Ñ1Ø#Ÿ™¨sÓ3¸T¿_¹_ÈSÓ=QÑRØØ—J‘J“Ló�	ð  �	Ø�	ä.×8Ñ8Ø˜kÑ*¨2Ñ.ªq²!°R°a°R¨xÑ8ØØØ×%Ñ%ó	 �÷%ñ ð	ð4 ˆ	�%ÑØˆ	�%ÑàÐ÷9ñ ús   ÃDHÈH	c                 óÚ  — t        j                  |||dz
  | j                  ¬«      }|dz  }| j                  dd¬«      D �cg c]  }|j	                  d«      ‘Œ c}\  }}}||z
  }	||z
  }
|	j                  |
d¬«      }d|z  d	|	z  z   d
|
z  z
  |z   }|dd d d …d d …f   |dd d …d d d …f   z
  j                  d«      j                  dd¬«      }t        j                  ||kD  d¬«      }|S c c}w )Nr   r�   r—   r   r–   râ   ry   gÆ vlÂ¥â¿g‡O[ŸI-â?g»:Ïñ4Má?.T)re   Úkeepdims)rD   rã  r\   r¤  rí  ÚcrossÚpowr  )ÚcoordsÚmin_binÚmax_binÚnum_binsÚ
boundariesrn   ÚNÚCAÚCr™   rM  r˜   ÚCBÚdistsræ  s                  rI   r±  zEsmFoldingTrunk.distogramÂ  s   € ô —^‘^ØØØ�q‰LØ—=‘=ô	
ˆ
ð   ‘]ˆ
Ø+1¯<©<¸¸r¨<Ó+BÖC a�A—I‘I˜b•MÒC‰ˆˆ2ˆqà�‰FˆØ�‰FˆØ�G‰G�A˜2ˆGÓˆØ˜1‰_˜z¨A™~Ñ-°
¸Q±Ñ>ÀÑCˆØ�C˜šq¢!�OÑ$ r¨#ªq°$º¨/Ñ':Ñ:×?Ñ?ÀÓB×FÑFÈ2ÐX\ÐFÓ]ˆÜ�y‰y˜ Ñ+°Ô4ˆØˆùò Ds   ÁC()	r@   rA   rB   rÁ   r¨  rß   Ústaticmethodr±  rÈ   rÉ   s   @rI   r™  r™  b  s'   ø„ ô,ò.%ò?ðB ñó ôrH   r™  a}  
    ESMForProteinFolding is the HuggingFace port of the original ESMFold model. It consists of an ESM-2 "stem" followed
    by a protein folding "head", although unlike most other output heads, this "head" is similar in size and runtime to
    the rest of the model combined! It outputs a dictionary containing predicted structural information about the input
    protein(s).
    c                   ó  ‡ — e Zd ZddgZˆ fd„Zedee   dej                  fd„«       Z
 eej                  d«      «       eee¬«      	 	 	 	 dd	ej                  d
eej                     deej                     deej                     dee   defd„«       «       Zd„ Zdej                  dej                  fd„Zd„ Z ej.                  «       	 ddeeee   f   fd„«       Zededee   fd„«       Zdefd„Zdee   dee   fd„Zˆ xZS )ÚEsmForProteinFoldingr`  r‡  c           
      óš	  •— t         ‰| �  |«       || _        d| _        t	        |d¬«      | _        | j
                  j                  d«       | j                  j                  j                  r| j
                  j                  «        | j                  j                  | _        | j                  j                  | j                  j                  z  | _        | j                  j                  | _        | j!                  d| j#                  |j$                  «      «       t'        j(                  t+        j,                  | j                  dz   «      «      | _        | j                  j                  j0                  }|j2                  }|j4                  }t'        j6                  t9        | j                  «      t'        j:                  | j                  |«      t'        j<                  «       t'        j:                  ||«      «      | _        t@        jB                  dz   | _"        d| _#        | jD                  dz
  | _$        | jD                  dz
  | _%        | j                  j$                  jM                  d	«      | _'        | j                  j$                  jM                  d
«      | _(        | j                  j$                  jM                  d«      | _)        | j                  j$                  jM                  d«      | _*        | j                  j                  jV                  r't'        jX                  | jD                  |d¬«      | _-        t]        |«      | _        t'        j:                  || j                  «      | _/        t'        j:                  || j                  «      | _0        t'        j:                  || jD                  «      | _1        d| _2        |jf                  }t'        j6                  t'        j8                  |jh                  «      t'        j:                  |jh                  | j                  j                  jj                  «      t'        j:                  | j                  j                  jj                  | j                  j                  jj                  «      t'        j:                  | j                  j                  jj                  d| jd                  z  «      «      | _6        y )Né@   F)Úadd_pooling_layerÚ
af2_to_esmr   r   r   r—   z<cls>z<mask>z<eos>ú<pad>)Úpadding_idxrô  é%   )7rÀ   rÁ   r8  Údistogram_binsr   ÚesmÚrequires_grad_Úesmfold_configÚfp16_esmÚhalfÚhidden_sizeÚ	esm_featsÚnum_hidden_layersÚnum_attention_headsÚ	esm_attnsÚ
esm_layersr�  Ú_af2_to_esm_from_vocab_listÚ
vocab_listr¢   rÏ   rD   rÒ   Úesm_s_combineÚtrunkr¶  r/  rÇ  r   r�  rÈ  Ú	esm_s_mlpr$   Úrestype_numÚn_tokens_embedÚpad_idxÚunk_idxÚmask_idxÚindexÚesm_dict_cls_idxÚesm_dict_mask_idxÚesm_dict_eos_idxÚesm_dict_padding_idxÚembed_aar  r  r™  Údistogram_headÚptm_headÚlm_headÚ	lddt_binsr¤  r  Úlddt_head_hid_dimr9   )rÃ   r8  Útrunk_configr2  r3  Ústructure_module_configrÄ   s         €rI   rÁ   zEsmForProteinFolding.__init__ç  s<  ø€ Ü‰Ñ˜Ô àˆŒà ˆÔä˜F°eÔ<ˆŒà�‰×Ñ Ô&Ø�;‰;×%Ñ%×.Ò.Ø�H‰H�M‰MŒOàŸ™×0Ñ0ˆŒØŸ™×6Ñ6¸¿¹×9XÑ9XÑXˆŒØŸ+™+×7Ñ7ˆŒØ×Ñ˜\¨4×+KÑ+KÈF×L]ÑL]Ó+^Ô_ÜŸ\™\¬%¯+©+°d·o±oÈÑ6IÓ*JÓKˆÔà—{‘{×1Ñ1×7Ñ7ˆØ×-Ñ-ˆØ×-Ñ-ˆÜŸ™Ü�d—n‘nÓ%Ü�I‰I�d—n‘n cÓ*Ü�G‰G‹IÜ�I‰I�c˜3Óó	
ˆŒô 0×;Ñ;¸aÑ?ˆÔØˆŒØ×*Ñ*¨QÑ.ˆŒØ×+Ñ+¨aÑ/ˆŒØ $§¡× 6Ñ 6× <Ñ <¸WÓ EˆÔØ!%§¡×!7Ñ!7×!=Ñ!=¸hÓ!GˆÔØ $§¡× 6Ñ 6× <Ñ <¸WÓ EˆÔØ$(§K¡K×$:Ñ$:×$@Ñ$@ÀÓ$IˆÔ!Ø�;‰;×%Ñ%×.Ò.ÜŸ\™\¨$×*=Ñ*=¸sÐPQÔRˆDŒNä$ \Ó2ˆŒ
ä Ÿi™i¨¨T×-@Ñ-@ÓAˆÔÜŸ	™	 # t×':Ñ':Ó;ˆŒÜ—y‘y  d×&9Ñ&9Ó:ˆŒØˆŒØ".×"?Ñ"?ÐÜŸ™Ü�L‰LÐ0×=Ñ=Ó>Ü�I‰IÐ-×:Ñ:¸D¿K¹K×<VÑ<V×<hÑ<hÓiÜ�I‰I�d—k‘k×0Ñ0×BÑBÀDÇKÁK×D^ÑD^×DpÑDpÓqÜ�I‰I�d—k‘k×0Ñ0×BÑBÀBÈÏÉÑDWÓXó	
ˆ�rH   rá  rY   c                 ó¶   — | j                  d«      gt        j                  D �cg c]  }| j                  |«      ‘Œ c}z   }t        j                  |«      S c c}w )NrÑ  )rê  r$   Úrestypes_with_xrD   r|   )rá  r�   Úesm_reorders      rI   rà  z0EsmForProteinFolding._af2_to_esm_from_vocab_list  sN   € ð "×'Ñ'¨Ó0Ð1ÔRc×RsÑRsÖ4tÈQ°Z×5EÑ5EÀaÕ5HÒ4tÑtˆÜ�|‰|˜KÓ(Ð(ùò 5us   ¤Azbatch_size, sequence_length)Úoutput_typeÚconfig_classÚ	input_idsÚattention_maskÚposition_idsÚmasking_patternÚnum_recyclesc                 óš  — | j                   j                  }|}|j                  d   }|j                  d   }	|j                  }
|€t	        j
                  ||
¬«      }|€&t	        j                  |	|
¬«      j                  |«      }| j                  ||«      }|�| j                  ||||«      \  }}}n|}d}| j                  |«      }|j                  | j                  j                  «      }|j                  r|dz  }|j                  «       }| j                  j!                  d«      j#                  d«      |z  j%                  d«      }| j'                  |«      }|j)                  ||	|	|j*                  j,                  «      }| j                   j                  j.                  r|| j1                  |«      z  }| j+                  ||||||¬«      }|j3                  «       D ��ci c]  \  }}|dv r||“Œ }}}|r||d<   | j5                  |d	   «      }||j7                  dd«      z   dz  }||d
<   | j9                  |d   «      }||d<   ||d<   t;        |«       dD ]  }||xx   |j#                  d«      z  cc<   Œ  ||d<   | j=                  |d   «      j?                  |d   j                  d   ||	d| j@                  «      }||d<   tC        |d   | j@                  ¬«      }||d<   | jE                  |d	   «      }||d<   tG        |d| jH                  ¬«      |d<   |jK                  tM        |d| jH                  ¬«      «       tO        di |¤ŽS c c}}w )a  
        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, EsmForProteinFolding

        >>> model = EsmForProteinFolding.from_pretrained("facebook/esmfold_v1")
        >>> tokenizer = AutoTokenizer.from_pretrained("facebook/esmfold_v1")
        >>> inputs = tokenizer(["MLKNVQVQLV"], return_tensors="pt", add_special_tokens=False)  # A tiny random peptide
        >>> outputs = model(**inputs)
        >>> folded_positions = outputs.positions
        ```

        r   r   Nr�   r—   )rµ  )r0   r/   r)   r*   r+   r,   r-   r.   Úmlm_targetsr0   r1   r/   r2   r3   )r4   r7   ry   r8   r.   r9   rö  r:   r;   é   )rÂ  Úno_binsr<   rG   )(r8  r×  ri   r\   rD   Ú	ones_likeÚarangeÚ	expand_asÚaf2_idx_to_esm_idxÚ	bert_maskÚ&compute_language_model_representationsrÝ   râ  r[   Úesm_ablate_sequencer¢  rã   r&  rí  rä  rJ  rã  r/  rï  r  r…   rð  rü   rò  r#   r9   rz   ró  r÷  rñ  r!   rÔ  rS  r    r(   )rÃ   rü  rý  rþ  rÿ  r   ÚcfgÚaaÚBÚLr\   ÚesmaaÚ	masked_aar  Úesm_sr¶  r·  r»  ra   r�   Údisto_logitsr2   r9   r:   r;   s                            rI   rß   zEsmForProteinFolding.forward$  s^  € ð4 �k‰k×(Ñ(ˆàˆØ�H‰H�Q‰KˆØ�H‰H�Q‰KˆØ×!Ñ!ˆØÐ!Ü"Ÿ_™_¨R¸Ô?ˆNØÐÜ Ÿ<™<¨°&Ô9×CÑCÀIÓNˆLð ×'Ñ'¨¨NÓ;ˆàÐ&Ø,0¯N©N¸2¸uÀnÐVeÓ,fÑ)ˆI�u™kàˆIØˆKð ×;Ñ;¸EÓBˆð
 —‘˜×+Ñ+×1Ñ1Ó2ˆà×"Ò"Ø˜A‘IˆEà—‘“ˆð ×#Ñ#×+Ñ+¨AÓ.×8Ñ8¸Ó;¸eÑC×LÑLÈQÓOˆØ—‘˜uÓ%ˆà—‘  1 a¨¯©×)EÑ)EÓFˆà�;‰;×%Ñ%×.Ò.Ø�T—^‘^ IÓ.Ñ.ˆEàŸ*™* U¨E°2°|À^Ðam˜*Ónˆ	ð "Ÿ™Ó)÷
á��1Øð	ñ
ð ˆq‰Dð
ˆ	ñ 
ñ" Ø'2ˆI�mÑ$à×*Ñ*¨9°UÑ+;Ó<ˆØ$ |×'=Ñ'=¸aÀÓ'CÑCÀqÑHˆØ(4ˆ	Ð$Ñ%à—L‘L ¨5Ñ!1Ó2ˆ	Ø!*ˆ	�+Ñà ˆ	�(ÑÜ˜)Ô$ð

ò 	9ˆAð �a‹L˜N×4Ñ4°RÓ8Ñ8ŒLð		9ð
 &2ˆ	�/Ñ"à—N‘N 9¨XÑ#6Ó7×?Ñ?À	È(Ñ@S×@YÑ@YÐZ[Ñ@\Ð^_ÐabÐdfÐhl×hvÑhvÓwˆ	Ø!*ˆ	�+ÑÜ  ¨2¡°T·^±^ÔDˆØ"ˆ	�'Ñà—]‘] 9¨UÑ#3Ó4ˆ
Ø",ˆ	�,ÑÜ% j¸"Àd×FYÑFYÔZˆ	�%ÑØ×ÑÔ8¸ÈRÐY]×YlÑYlÔmÔnä)Ñ6¨IÑ6Ð6ùóe
s   ÇMc                 óê   — | j                   j                  |j                  k7  r*| j                   j                  |j                  «      | _         |dz   j                  |dk7  d«      }| j                   |   S ©Nr   r   )rÐ  r\   rÝ   r¦  )rÃ   r  r!  s      rI   r  z'EsmForProteinFolding.af2_idx_to_esm_idx¢  s[   € à�?‰?×!Ñ! R§Y¡YÒ.Ø"Ÿo™o×0Ñ0°·±Ó;ˆDŒOØ�1‰f×!Ñ! $¨!¡)¨QÓ/ˆØ�‰˜rÑ"Ð"rH   r  c                 óª  — t        | j                  «       «      j                  }|j                  \  }}| j                  j
                  j                  r>t        j                  ||| j                  j                  d   d| j                  |¬«      }|S | j                  | j                  }}|j                  |df|«      }|j                  |df| j                  «      }	t        j                   |||	gd¬«      }||t#        |«      |dk7  j%                  d«      f<   | j'                  ||dk7  d¬«      d   }
t        j(                  |
d	¬«      }|d d …dd…f   }|S )
Nr   ry   r�   r   râ   T)rý  Úoutput_hidden_statesÚhidden_statesr—   )ÚnextÚ
parametersr\   ri   r8  r×  Ú	bypass_lmrD   rÒ   râ  rœ   rÛ  rë  rí  Únew_fullrî  r¹  rm   r  rÕ  r=  )rÃ   r  r\   r  r  r  ÚbosiÚeosiÚbosÚeosÚesm_hidden_statess              rI   r
  z;EsmForProteinFolding.compute_language_model_representations©  s1  € Ü�d—o‘oÓ'Ó(×/Ñ/ˆØ�{‰{‰ˆˆ1à�;‰;×%Ñ%×/Ò/Ü—K‘K  1 d×&8Ñ&8×&=Ñ&=¸aÑ&@À"ÀdÇnÁnÐ]cÔdˆEØˆLà×*Ñ*¨D×,AÑ,AˆdˆØ�n‰n˜a ˜V TÓ*ˆØ�n‰n˜a ˜V T×%>Ñ%>Ó?ˆÜ—	‘	˜3  sÐ+°Ô3ˆà/3ˆŒe�A‹h˜ !™×(Ñ(¨Ó+Ð+Ñ,ð
 !ŸH™H U¸5ÀA¹:Ð\`˜HÓaÐbqÑrÐÜ—‘Ð-°1Ô5ˆà’a˜˜2˜�g‘ˆàˆrH   c                 óÄ   — |j                  «       }|j                  «       }|j                  «       }| j                  ||dk(  <   d||dk7  <   | j                  ||dk(  <   |||fS r  )rc  ré  rì  )rÃ   r  r  r!  ÚpatternÚnew_aaÚtargetÚ	new_esmaas           rI   r	  zEsmForProteinFolding.bert_maskÂ  sd   € Ø—‘“ˆØ—‘“ˆØ—K‘K“Mˆ	Ø#Ÿ}™}ˆˆw˜!‰|ÑØ ˆˆw˜!‰|ÑØ"&×"8Ñ"8ˆ	�'˜Q‘,ÑØ�y &Ð(Ð(rH   Úseqsc                 óÐ  — t        |t        «      r|g}n|}t        | j                  «       «      j                  }t        |D �cg c]Z  }t        j                  t        j                  |t        j                  d¬«      «      j                  |«      j                  d¬«      ‘Œ\ c}«      }t        |D �cg c]  }|j                  t        |«      «      ‘Œ c}«      }|€=t        j                  |j                   d   |¬«      j#                  t        |«      d«      n|j                  |«      }|j$                  dk(  r|j'                  d«      }| j)                  |||¬«      S c c}w c c}w )	NT)ÚsequenceÚmappingÚmap_unknown_to_xr   râ   r�   ry   r   )rþ  )r†   rÇ   r  r  r\   ru   rD   Ú
from_numpyr$   Úsequence_to_onehotÚrestype_order_with_xrÝ   Úargmaxr%  rc   r  ri   rù  rè  r&  rß   )rÃ   r'  rþ  ro   r\   Úseqr3   r!  s           rI   ÚinferzEsmForProteinFolding.inferË  sH  € ô �dœCÔ Ø�&‰CàˆCä�d—o‘oÓ'Ó(×/Ñ/ˆÜ&ð öð ô × Ñ Ü%×8Ñ8Ø!$Ü 1× FÑ FØ)-ôó÷ ‘�F“ß‘˜A�“ñòó
ˆô %È3Ö%OÀC f§o¡o´c¸#³hÕ&?Ò%OÓPˆð Ð#ô �L‰L˜Ÿ™ a™°Ô8×?Ñ?ÄÀCÃÈ"ÔMà—‘ Ó(ð 	ð
 ×Ñ Ò!Ø'×1Ñ1°!Ó4ˆLØ�|‰|ØØØ%ð ó 
ð 	
ùò+ùò &Ps   ÁAEÂ2!E#r  c           	      óž  — | j                  «       D ��ci c]%  \  }}||j                  d«      j                  «       “Œ' } }}g }t        | d   d   | «      }| d   }t	        | d   j
                  d   «      D ]O  }| d   |   }||   }||   }	| d   |   dz   }
t        |||	|
| d	   |   ¬
«      }|j                  t        |«      «       ŒQ |S c c}}w )zDReturns the pbd (file) string from the model given the model output.rA  r-   ry   r7   r3   r   r8   r   r:   )r3   Úatom_positionsrŠ  r8   Ú	b_factors)	r…   rÝ   Únumpyr   rm   ri   r   r  r%   )r  ra   r�   ÚpdbsÚfinal_atom_positionsÚfinal_atom_maskrr   r  Úpred_posr!  ÚresidÚpreds               rI   Úoutput_to_pdbz"EsmForProteinFolding.output_to_pdbó  sú   € ð 6<·\±\³^×D©T¨Q°�!�Q—T‘T˜%“[×&Ñ&Ó(Ñ(ÐDˆÑDØˆÜ/°°{Ñ0CÀBÑ0GÈÓPÐØ Ð!5Ñ6ˆÜ�v˜hÑ'×-Ñ-¨aÑ0Ó1ò 	&ˆAØ˜Ñ! !Ñ$ˆBØ+¨AÑ.ˆHØ" 1Ñ%ˆDØ˜?Ñ+¨AÑ.°Ñ2ˆEÜØØ'ØØ#Ø  ™/¨!Ñ,ôˆDð �K‰Kœ˜t›Õ%ð	&ð ˆùó# Es   ”*C	c                 óz   — t        |t        «      sJ ‚ | j                  |g|¢­i |¤Ž}| j                  |«      d   S )úEReturns the pdb (file) string from the model given an input sequence.r   )r†   rÇ   r1  r<  ©rÃ   r'  ÚargsÚkwargsr  s        rI   Ú	infer_pdbzEsmForProteinFolding.infer_pdb		  sB   € ä˜$¤Ô$Ð$Ð$Ø�—‘˜DÐ2 4Ò2¨6Ñ2ˆØ×!Ñ! &Ó)¨!Ñ,Ð,rH   c                 óP   —  | j                   |g|¢­i |¤Ž}| j                  |«      S )r>  )r1  r<  r?  s        rI   Ú
infer_pdbszEsmForProteinFolding.infer_pdbs	  s-   € à�—‘˜DÐ2 4Ò2¨6Ñ2ˆØ×!Ñ! &Ó)Ð)rH   )NNNNrK   )r@   rA   rB   Ú_no_split_modulesrÁ   rÊ  r   rÇ   rD   rd   rà  r   ÚESMFOLD_INPUTS_DOCSTRINGÚformatr   r(   r   r   rÅ   rß   r  r
  r	  r¯   r   r1  r   r<  rB  rD  rÈ   rÉ   s   @rI   rÌ  rÌ  Û  s—  ø„ ð 2Ð3XÐYÐô5
ðn ð)°°S±	ð )¸e¿l¹lò )ó ð)ñ
 +Ð+C×+JÑ+JÐKhÓ+iÓjÙÐ+EÐT]Ô^ð 26Ø/3Ø26Ø&*ñz7à—<‘<ðz7ð ! §¡Ñ.ðz7ð ˜uŸ|™|Ñ,ð	z7ð
 " %§,¡,Ñ/ðz7ð ˜s‘mðz7ð 
$òz7ó _ó kðz7òx#ð¸E¿L¹Lð ÈUÏ\É\ó ò2)ð €U‡]�]ƒ_ð ñ%
à�C˜˜c™�NÑ#ò%
ó ð%
ðN ð˜dð  t¨C¡yò ó ðð*-°#ó -ð*˜t C™yð *¸dÀ3¹i÷ *rH   rÌ  rÊ  )r€  Úfan_inrx   )rô  )cr    r?  Údataclassesr   Ú	functoolsr   Útypingr   r   r   r   r	   r
   r   r5  r©   rD   Útorch.nnr¢   r   Úintegrations.deepspeedr   Úmodeling_outputsr   rS   r   r   r   r   r   r   Úconfiguration_esmr   Úmodeling_esmr   r   r   Úopenfold_utilsr   r   r   r   r   r    r!   r"   r#   r$   r%   r&   Ú
get_loggerr@   rž   Ú_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCr(   rF  rP   rV   rd   r{  ru   rÅ   r{   rƒ   rˆ   r­   r±   r�  r³   ÚModulerË   r*  r+  rå   rç   r  r-  r~  r—  rª  r³  r¾  rÅ  r‡  rß  r÷  rû  rý  r  r  r…  rO  rU  rZ  r`  r™  rÌ  Ú__all__rG   rH   rI   ú<module>rW     s  ðó Û 
Ý !Ý ß I× IÑ Iã Û Ý Ý å <Ý +÷÷ õ )ß KÑ K÷÷ ÷ ó ð  
ˆ×	Ñ	˜HÓ	%€Ø+Ð Ø€ð ôMD ó MDó ðMDð`Ð òBò
ñ 4¨¯©Ñ#5ð ¸eð ÈEÏLÉLó ð.1˜%Ÿ,™,ð 1°ó 1ðG˜uŸ|™|ð G°4¸±9ó Gò
óDò**ô+5�B—I‘Iô +5ô\�r—y‘yô ð* ‡�×Ññ�u—|‘|ð ¨#ð °u·|±|ò ó ðôU�r—y‘yô Uôpk˜rŸy™yô kô\F¨"¯)©)ô FôR
1*Ð/ô 1*ôh;(˜2Ÿ9™9ô ;(ô|3�R—Y‘Yô 3ô,#˜BŸI™Iô #ôL˜BŸI™Iô ô*˜Ÿ	™	ô ô e.¨"¯)©)ô e.÷PPñ Pó&;ò
ô,!˜bŸi™iô !ôH˜bŸi™iô ô(>!˜Ÿ™ô >!ôBB R§Y¡Yô BôJ˜BŸI™Iô ô.¨B¯I©Iô ô. r§y¡yô ô.G
˜RŸY™Yô G
ôTr�b—i‘iô rñr ðð óôn*Ð-ó n*óðn*ðb	 "Ð#;Ð
<�rH   