Ë
    T^(ho ã                   ó  — d dl Z d dlZd dlmZ d dlmZmZmZmZm	Z	m
Z
mZ d dlZd dlmZ d dlmZmZmZ ddlmZ ddlmZmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZm Z  ddl!m"Z"m#Z# ddl$m%Z%m&Z& ddl'm(Z( ddl)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/ ddl0m1Z1m2Z2 ddl3m4Z4  e2«       rd dl5m6Z6 d dl7m8Z8m9Z9 nd\  Z6Z8Z9 e1«       r	d dl:m;Z;m<Z< nd\  Z<Z; e,jz                  e>«      Z?dZ@ G d„ dej                  j‚                  «      ZB G d„ dej‚                  «      ZC G d„ d e«      ZD G d!„ d"ej‚                  «      ZEd#ejŒ                  d$eGd%ejŒ                  fd&„ZH	 dOd'ej‚                  d(ejŒ                  d)ejŒ                  d*ejŒ                  d+e	ejŒ                     d,eId-eIfd.„ZJd/„ ZKdPd0„ZL G d1„ d2ej‚                  «      ZMd3ejŒ                  d4eGfd5„ZNd6„ ZOd7„ ZP eQe6e;e<f«      ZR G d8„ d9ej‚                  «      ZS G d:„ d;ej‚                  «      ZT G d<„ d=ej‚                  «      ZU G d>„ d?ej‚                  «      ZV G d@„ dAej‚                  «      ZW G dB„ dCe&«      ZXdDZYdEZZ e*dFeY«       G dG„ dHeX«      «       Z[ G dI„ dJeXe«      Z\ e*dKeY«       G dL„ dMeX«      «       Z]g dN¢Z^y)Qé    N)Úcycle)ÚAnyÚCallableÚDictÚListÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)ÚAttentionMaskConverter)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚ SequenceClassifierOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstrings)Údeprecate_kwarg)Úis_causal_conv1d_availableÚis_mamba_ssm_availableé   )ÚZamba2Config)Úselective_state_update)Úmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combined©NNN)Úcausal_conv1d_fnÚcausal_conv1d_update©NNzZyphra/Zamba2-2.7Bc                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚZamba2RMSNormGatedc                 ó˜   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        || _        y ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilonÚ
group_size)ÚselfÚhidden_sizer9   ÚepsÚ	__class__s       €úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/zamba2/modeling_zamba2.pyr3   zZamba2RMSNormGated.__init__F   s6   ø€ Ü‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÔØ$ˆ�ó    c                 ób  — |j                   }|j                  t        j                  «      }|�?|t        j
                  j                  |j                  t        j                  «      «      z  }|j                  �^ }}|| j                  z  } |j                  g |¢|‘| j                  ‘­Ž }|j                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  } |j                  g |¢|| j                  z  ‘­Ž }| j                  |j                  |«      z  S ©Né   éÿÿÿÿT)Úkeepdim)ÚdtypeÚtor5   Úfloat32r   Ú
functionalÚsiluÚshaper9   ÚviewÚpowÚmeanÚrsqrtr8   r7   )	r:   Úhidden_statesÚgateÚinput_dtypeÚprefix_dimsÚlast_dimÚgroup_countÚhidden_states_groupÚvariances	            r>   ÚforwardzZamba2RMSNormGated.forwardL   s  € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØÐØ)¬B¯M©M×,>Ñ,>¸t¿w¹wÄuÇ}Á}Ó?UÓ,VÑVˆMØ!.×!4Ñ!4Ñˆ�hØ $§/¡/Ñ1ˆØ0˜m×0Ñ0Ð\°+Ð\¸{Ð\ÈDÏOÉOÒ\ÐØ&×*Ñ*¨1Ó-×2Ñ2°2¸tÐ2ÓDˆØ1´E·K±KÀÈ4×K`ÑK`Ñ@`Ó4aÑaÐØ0Ð+×0Ñ0Ð]°+Ð]¸{ÈTÏ_É_Ñ?\Ò]ˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r?   ©g�íµ ÷Æ°>r1   )Ú__name__Ú
__module__Ú__qualname__r3   rW   Ú__classcell__©r=   s   @r>   r/   r/   E   s   ø„ õ%÷;r?   r/   c                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚZamba2RMSNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)z<
        Zamba2RMSNorm is equivalent to T5LayerNorm
        N)r2   r3   r   r4   r5   r6   r7   r8   )r:   r;   r<   r=   s      €r>   r3   zZamba2RMSNorm.__init__[   s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr?   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S rA   )	rE   rF   r5   rG   rL   rM   rN   r8   r7   )r:   rO   rQ   rV   s       r>   rW   zZamba2RMSNorm.forwardc   sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r?   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler7   rJ   r8   ©r:   s    r>   Ú
extra_reprzZamba2RMSNorm.extra_reprj   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr?   rX   )rY   rZ   r[   r3   rW   re   r\   r]   s   @r>   r_   r_   Z   s   ø„ õ$ò;öJr?   r_   c                   ó  — e Zd ZdZej
                  dfdededej                  de	e
   fd„Z	 ddej                  d	ej                  d
ede	ee
ef      deej                  ej                  f   f
d„Zdej"                  fd„Zdd
e	e   defd„Zdeeej                     eej                     f   fd„Zedde	eeej,                           ddfd„«       Zd
edej                  dej"                  dej                  fd„Zd„ Zy)ÚZamba2HybridDynamicCachea¤  
    A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache
    (which has a constant shape regardless of seq_len).

    This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states`
    and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor
    For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`,
    while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors).
    For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors),
    while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`,
    and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`.
    NÚconfigÚ
batch_sizerE   Údevicec           	      ó.  — || _         |j                  | _        d| _        t        |j                  |j
                  z  «      | _        |j                  | _        |j                  | _
        |j                  | _        g | _        i | _        i | _        i | _        i | _        i | _        t%        |j&                  «      D ]Î  }t)        j*                  || j                  d|j,                  z  |j                  z  z   | j                  ||¬«      | j                   |<   t)        j*                  || j                  |j.                  | j                  ||¬«      | j"                  |<   | j                  |   dk(  sŒ´| j                  j1                  |«       ŒÐ t%        |j&                  «      D �cg c]  }t)        j2                  g g|z  |¬«      ‘Œ c}| _        t%        |j&                  «      D �cg c]  }t)        j2                  g g|z  |¬«      ‘Œ c}| _        y c c}w c c}w )NFrB   ©rj   rE   Úhybrid©rj   )rE   Úlayers_block_typeÚhas_previous_stateÚintÚmamba_expandr;   Úintermediate_sizeÚmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizeÚn_mamba_headsÚtransformer_layersÚ_modulesÚ_parametersÚ_buffersÚconv_statesÚ
ssm_statesÚrangeÚnum_hidden_layersr5   ÚzerosÚmamba_ngroupsÚmamba_headdimÚappendÚtensorÚ	key_cacheÚvalue_cache)r:   rh   ri   rE   rj   ÚiÚ_s          r>   r3   z!Zamba2HybridDynamicCache.__init__|   sÐ  € ð ˆŒ
Ø!'×!9Ñ!9ˆÔØ"'ˆÔÜ!$ V×%8Ñ%8¸6×;MÑ;MÑ%MÓ!NˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔØ#×1Ñ1ˆÔØ"$ˆÔØˆŒØˆÔØˆŒØˆÔØˆŒÜ�v×/Ñ/Ó0ò 	2ˆAÜ"'§+¡+ØØ×&Ñ&¨¨V×-AÑ-AÑ)AÀF×DXÑDXÑ)XÑXØ×%Ñ%ØØô#ˆD×Ñ˜QÑô "'§¡Ø˜D×.Ñ.°×0DÑ0DÀd×FYÑFYÐbhÐpuô"ˆD�O‰O˜AÑð ×%Ñ% aÑ(¨HÓ4Ø×'Ñ'×.Ñ.¨qÕ1ð	2ô SXÐX^×XpÑXpÓRqÖrÈQœ%Ÿ,™,¨ t¨jÑ'8ÀÖHÒrˆŒÜTYÐZ`×ZrÑZrÓTsÖtÈqœEŸL™L¨"¨°
Ñ):À6ÖJÒtˆÕùò sùÚts   Æ!"HÇ""HÚ
key_statesÚvalue_statesÚ	layer_idxÚcache_kwargsÚreturnc                 ó†  — | j                   |   j                  d   dk(  r|| j                   |<   || j                  |<   nft        j                  | j                   |   |gd¬«      | j                   |<   t        j                  | j                  |   |gd¬«      | j                  |<   | j                   |   | j                  |   fS )NrC   r   rB   ©Údim)r†   rJ   r‡   r5   Úcat)r:   rŠ   r‹   rŒ   r�   s        r>   ÚupdatezZamba2HybridDynamicCache.updateœ   s²   € ð �>‰>˜)Ñ$×*Ñ*¨2Ñ.°!Ò3Ø(2ˆD�N‰N˜9Ñ%Ø*6ˆD×Ñ˜YÒ'ä(-¯	©	°4·>±>À)Ñ3LÈjÐ2YÐ_`Ô(aˆD�N‰N˜9Ñ%Ü*/¯)©)°T×5EÑ5EÀiÑ5PÐR^Ð4_ÐefÔ*gˆD×Ñ˜YÑ'à�~‰~˜iÑ(¨$×*:Ñ*:¸9Ñ*EÐEÐEr?   Úbeam_idxc                 óî  — t        t        | j                  «      «      D �]S  }| j                  |   j                  }| j                  |   j	                  d|j                  |«      «      | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |«      «      | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |«      «      | j                  |<   | j                  |   j                  }| j                  |   j	                  d|j                  |«      «      | j                  |<   �ŒV y)zDReorders the cache for beam search, given the selected beam indices.r   N)	r   Úlenr†   rj   Úindex_selectrF   r‡   r}   r~   )r:   r”   rŒ   rj   s       r>   Úreorder_cachez&Zamba2HybridDynamicCache.reorder_cache­   sD  € äœs 4§>¡>Ó2Ó3ó 		iˆIØ—^‘^ IÑ.×5Ñ5ˆFØ(,¯©°yÑ(A×(NÑ(NÈqÐRZ×R]ÑR]Ð^dÓReÓ(fˆD�N‰N˜9Ñ%Ø×%Ñ% iÑ0×7Ñ7ˆFØ*.×*:Ñ*:¸9Ñ*E×*RÑ*RÐSTÐV^×VaÑVaÐbhÓViÓ*jˆD×Ñ˜YÑ'à×%Ñ% iÑ0×7Ñ7ˆFØ*.×*:Ñ*:¸9Ñ*E×*RÑ*RÐSTÐV^×VaÑVaÐbhÓViÓ*jˆD×Ñ˜YÑ'Ø—_‘_ YÑ/×6Ñ6ˆFØ)-¯©¸Ñ)C×)PÑ)PÐQRÐT\×T_ÑT_Ð`fÓTgÓ)hˆD�O‰O˜IÓ&ñ		ir?   c                 óê   — || j                   vr| j                   d   n|}t        | j                  «      |k  s | j                  |   j                  «       dk(  ry| j                  |   j                  d   S )zYReturns the sequence length of the cached states. A layer index can be optionally passed.r   éþÿÿÿ)ry   r–   r†   ÚnumelrJ   )r:   rŒ   s     r>   Úget_seq_lengthz'Zamba2HybridDynamicCache.get_seq_lengthº   sl   € ð 3<À4×CZÑCZÑ2Z�D×+Ñ+¨AÒ.Ð`iˆ	Üˆt�~‰~Ó )Ò+¨t¯~©~¸iÑ/H×/NÑ/NÓ/PÐTUÒ/UØØ�~‰~˜iÑ(×.Ñ.¨rÑ2Ð2r?   c                 ó   — t        d«      ‚©NzAZamba2HybridDynamicCache does not have a legacy cache equivalent.©ÚNotImplementedErrorrd   s    r>   Úto_legacy_cachez(Zamba2HybridDynamicCache.to_legacy_cacheÂ   s   € Ü!Ð"eÓfÐfr?   Úpast_key_valuesr   c                 ó   — t        d«      ‚rž   rŸ   )Úclsr¢   s     r>   Úfrom_legacy_cachez*Zamba2HybridDynamicCache.from_legacy_cacheÅ   s   € ä!Ð"eÓfÐfr?   Únew_conv_stateÚcache_positionc                 óT  — | j                   |   }|j                  d| j                  dz
  «      }|j                  dd¬«      }|j	                  |j
                  «      |d d …d d …|f<   | j                   |   j                  «        | j                   |xx   |z  cc<   | j                   |   S )Nr   r%   rC   ©ÚshiftsÚdims)r}   Úclamprw   ÚrollrF   rj   Úzero_)r:   rŒ   r¦   r§   Ú
conv_states        r>   Úupdate_conv_statez*Zamba2HybridDynamicCache.update_conv_stateÉ   s£   € ð ×%Ñ% iÑ0ˆ
Ø'×-Ñ-¨a°×1FÑ1FÈÑ1JÓKˆà—_‘_¨B°R�_Ó8ˆ
Ø+9×+<Ñ+<¸Z×=NÑ=NÓ+Oˆ
’1’a˜Ð'Ñ(Ø×Ñ˜Ñ#×)Ñ)Ô+Ø×Ñ˜Ó# zÑ1Ó#Ø×Ñ 	Ñ*Ð*r?   c                 ól   — | j                   j                  «        | j                  j                  «        y r1   )r}   r®   r~   rd   s    r>   ÚresetzZamba2HybridDynamicCache.resetÕ   s$   € Ø×Ñ×ÑÔ Ø�‰×ÑÕr?   r1   )r   )rY   rZ   r[   Ú__doc__r5   Úfloat16r&   rq   rE   r   Ústrr3   ÚTensorr   r   r	   r“   Ú
LongTensorr˜   rœ   r¡   ÚclassmethodÚFloatTensorr¥   r°   r²   © r?   r>   rg   rg   n   sz  „ ñð KPÏ-É-ÐquñuØ"ðuØ03ðuØ<A¿K¹KðuØaiÐjmÑanóuðJ 26ñFà—L‘LðFð —l‘lðFð ð	Fð
 ˜t C¨ H™~Ñ.ðFð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*óFð"i e×&6Ñ&6ó iñ3¨°©ð 3¸có 3ðg  u¨U¯\©\Ñ':¸EÀ%Ç,Á,Ñ<OÐ'OÑ!Pó gð ñg°¸¸uÀU×EVÑEVÑ?WÑ9XÑ0Yð gÐesò gó ðgð
+Øð
+Ø.3¯l©lð
+ØLQ×L\ÑL\ð
+à	�‰ó
+ó r?   rg   c                   ó`   ‡ — e Zd Z	 ddefˆ fd„Z ej                  «       ed„ «       «       Zˆ xZ	S )ÚZamba2RotaryEmbeddingrh   c                 ó  •— t         ‰| �  «        t        |d«      rG|j                  �;|j                  j	                  d|j                  j	                  d«      «      | _        nd| _        |j                  | _        |j                  | _        || _	        t        | j
                     | _        | j                  ||j                  |j                  ¬«      \  }| _        | j                  d|d¬«       | j                   | _        y )	NÚrope_scalingÚ	rope_typeÚtypeÚdefault)rj   Úbaser‘   Úinv_freqF)Ú
persistent)r2   r3   Úhasattrr¾   Úgetr¿   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrh   r   Úrope_init_fnÚ
rope_thetaÚattention_head_dimÚattention_scalingÚregister_bufferrÃ   Úoriginal_inv_freq)r:   rh   rj   rÃ   r=   s       €r>   r3   zZamba2RotaryEmbedding.__init__Û   sß   ø€ ô
 	‰ÑÔä�6˜>Ô*¨v×/BÑ/BÐ/NØ#×0Ñ0×4Ñ4°[À&×BUÑBU×BYÑBYÐZ`ÓBaÓbˆD�Nà&ˆDŒNØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àˆŒÜ/°·±Ñ?ˆÔà+/×+<Ñ+<Ø × 1Ñ 1°v×7PÑ7Pð ,=ó ,
Ñ(ˆ�$Ô(ð 	×Ñ˜Z¨¸eÐÔDØ!%§¡ˆÕr?   c                 ób  — | j                   d d d …d f   j                  «       j                  |j                  d   dd«      j	                  |j
                  «      }|d d …d d d …f   j                  «       }t        |j
                  j                  t        «      r/|j
                  j                  dk7  r|j
                  j                  nd}t        j                  |d¬«      5  |j                  «       |j                  «       z  j                  dd«      }t        j                  ||fd¬	«      }|j                  «       | j                  z  }|j                  «       | j                  z  }	d d d «       j	                  |j                   ¬
«      	j	                  |j                   ¬
«      fS # 1 sw Y   ŒAxY w)Nr   rC   r%   ÚmpsÚcpuF)Údevice_typeÚenabledrB   r�   ©rE   )rÃ   ÚfloatÚexpandrJ   rF   rj   Ú
isinstancerÀ   rµ   r5   ÚautocastÚ	transposer’   ÚcosrÍ   ÚsinrE   )
r:   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrÓ   ÚfreqsÚembrÛ   rÜ   s
             r>   rW   zZamba2RotaryEmbedding.forwardò   sV  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓe×hÑhÐij×iqÑiqÓrÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-ÄÔ'EÈ!Ï(É(Ï-É-Ð[`ÒJ`�a—h‘h—m’mÐfkˆÜ�^‰^¨¸UÔCñ 	5Ø&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ—)‘)˜U E˜N°Ô3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷		5ð �v‰v˜AŸG™GˆvÓ$ c§f¡f°1·7±7 fÓ&;Ð;Ð;÷	5ð 	5ús   Ã BF%Æ%F.r1   )
rY   rZ   r[   r&   r3   r5   Úno_gradr   rW   r\   r]   s   @r>   r¼   r¼   Ú   s9   ø„ ð ñ/àõ/ð. €U‡]�]ƒ_Øñ<ó ó ô<r?   r¼   rO   Ún_reprŽ   c                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    r%   N)rJ   r×   Úreshape)rO   rä   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         r>   Ú	repeat_kvrë     so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTr?   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutc                 óT  — t        || j                  «      }t        || j                  «      }	t        j                  ||j	                  dd«      «      |z  }
|�#|d d …d d …d d …d |j
                  d   …f   }|
|z   }
t        j                  j                  |
dt        j                  ¬«      j                  |j                  «      }
t        j                  j                  |
|| j                  ¬«      }
t        j                  |
|	«      }|j	                  dd«      j                  «       }||
fS )NrB   r   rš   rC   )r‘   rE   )ÚpÚtrainingr%   )rë   Únum_key_value_groupsr5   ÚmatmulrÚ   rJ   r   rH   ÚsoftmaxrG   rF   rE   rò   rõ   Ú
contiguous)rì   rí   rî   rï   rð   rñ   rò   ÚkwargsrŠ   r‹   Úattn_weightsÚcausal_maskÚattn_outputs                r>   Úeager_attention_forwardrþ     s  € ô ˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÐ!Ø$¢Qªª1Ð.D°
×0@Ñ0@ÀÑ0DÐ.DÐ%DÑEˆØ# kÑ1ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜Ð$Ð$r?   c                 óš   — | dd| j                   d   dz  …f   }| d| j                   d   dz  d…f   }t        j                  | |fd¬«      S )z*Rotates half the hidden dims of the input..NrC   rB   r�   )rJ   r5   r’   )rÝ   Úx1Úx2s      r>   Úrotate_halfr  (  sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9‰9�r�c˜2�Y BÔ'Ð'r?   c                 óž   — |j                  |«      }|j                  |«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )aÛ  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    )Ú	unsqueezer  )ÚqÚkrÛ   rÜ   rÞ   Úunsqueeze_dimÚq_embedÚk_embeds           r>   Úapply_rotary_pos_embr
  /  sY   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€GØ�GÐÐr?   c                   óF  ‡ — e Zd ZdZ	 	 	 ddedee   dee   dee   fˆ fd„Z	 	 	 ddej                  dedeej                     d	ee
   d
eeej                  ej                  f      dee   deej                  eej                     eeej                        f   fd„Zˆ xZS )ÚZamba2Attentiona  
    Multi-headed attention from 'Attention Is All You Need' paper. Modified to use sliding window attention: Longformer
    and "Generating Long Sequences with Sparse Transformers".

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://arxiv.org/pdf/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)

    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://arxiv.org/pdf/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://arxiv.org/pdf/2411.15242).
    rh   rŒ   Únum_fwd_mem_blocksÚblock_idc           	      óø  •— t         ‰| �  «        || _        || _        |j                  | _        |j
                  | _        |j                  |j                  z  | _	        |j                  | _
        | j                  dz  dz  | _        d| _        |j                  | _        t        j                  |j                  |j                  | j                  z  d¬«      | _        t        j                  |j                  |j                  | j                  z  d¬«      | _        t        j                  |j                  |j                  | j                  z  d¬«      | _        t        j                  |j                  | j                  z  |j&                  d¬«      | _        || _        |j,                  | _        || _        |j2                  �r…t        j4                  g «      | _        t        j4                  g «      | _        t        j4                  g «      | _        t=        | j*                  «      D �]  }||j>                  z  |k(  �r{t        j@                  t        j                  | j                  | j                  jB                  d¬«      t        j                  | j                  jB                  | j                  d¬«      «      }t        j@                  t        j                  | j                  | j                  jB                  d¬«      t        j                  | j                  jB                  | j                  d¬«      «      }t        j@                  t        j                  | j                  | j                  jB                  d¬«      t        j                  | j                  jB                  | j                  d¬«      «      }n<t        jD                  «       }t        jD                  «       }t        jD                  «       }| j6                  jG                  |«       | j8                  jG                  |«       | j:                  jG                  |«       �Œ! tI        | j.                  «      D �	�
ci c]  \  }	}
|
|	“Œ
 c}
}	| _%        y c c}
}	w )NrB   g      à¿TF©Úbias)&r2   r3   rh   rŒ   Úattention_hidden_sizerÌ   rê   Únum_attention_headsrè   rö   rÇ   rñ   Ú	is_causalÚattention_dropoutr   ÚLinearÚq_projÚk_projÚv_projr;   Úo_projr  Úhybrid_layer_idsÚlayer_block_mapr  Úuse_shared_attention_adapterÚ
ModuleListÚlinear_q_adapter_listÚlinear_k_adapter_listÚlinear_v_adapter_listr   Únum_mem_blocksÚ
SequentialÚadapter_rankÚIdentityr„   Ú	enumerateÚ	layer_dic)r:   rh   rŒ   r  r  rˆ   Úlinear_q_adapterÚlinear_k_adapterÚlinear_v_adapterÚindexrï   r=   s              €r>   r3   zZamba2Attention.__init__e  sJ  ø€ ô 	‰ÑÔØˆŒØ"ˆŒà%+×%AÑ%AˆÔ"Ø×1Ñ1ˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø'-×'EÑ'EˆÔ$ØŸ™¨Ñ)¨dÑ2ˆŒØˆŒØ!'×!9Ñ!9ˆÔä—i‘i × <Ñ <¸f×>XÑ>XÐ[_×[hÑ[hÑ>hÐotÔuˆŒÜ—i‘i × <Ñ <¸f×>XÑ>XÐ[_×[hÑ[hÑ>hÐotÔuˆŒÜ—i‘i × <Ñ <¸f×>XÑ>XÐ[_×[hÑ[hÑ>hÐotÔuˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆŒØ"4ˆÔØ%×6Ñ6ˆÔØ ˆŒà×.Ó.Ü)+¯©°rÓ):ˆDÔ&Ü)+¯©°rÓ):ˆDÔ&Ü)+¯©°rÓ):ˆDÔ&ä˜4×2Ñ2Ó3ó D�Ø�v×,Ñ,Ñ,°Ó8Ü')§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ð$ô (*§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ð$ô (*§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ñ$ô
 (*§{¡{£}Ð$Ü')§{¡{£}Ð$Ü')§{¡{£}Ð$Ø×*Ñ*×1Ñ1Ð2BÔCØ×*Ñ*×1Ñ1Ð2BÔCØ×*Ñ*×1Ñ1Ð2BÖCð)Dô, <EÀT×EYÑEYÓ;Z×[©<¨5°%˜% ™,Ó[ˆ�ùÓ[s   ÑQ6rO   rð   Úpast_key_valueÚposition_embeddingsrú   rŽ   c                 ó¦  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      }	| j                  |«      }
| j	                  |«      }| j
                  j                  rW| j                  |   }|	 | j                  |   |«      z   }	|
 | j                  |   |«      z   }
| | j                  |   |«      z   }|	j                  |«      j                  dd«      }	|
j                  |«      j                  dd«      }
|j                  |«      j                  dd«      }| j
                  j                  r|\  }}t        |	|
||«      \  }	}
|�|j                  |
||«      \  }
}t         }| j
                  j"                  dk7  r^| j
                  j"                  dk(  r(|j%                  dd«      rt&        j)                  d«       nt*        | j
                  j"                     } || |	|
||f| j,                  sd	n| j.                  | j0                  d
œ|¤Ž\  }} |j2                  g |¢d‘­Ž j5                  «       }| j7                  |«      }||fS )NrC   r%   rB   ÚeagerÚsdpaÚoutput_attentionsFzã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.ç        )rò   rñ   )rJ   rê   r  r  r  rh   r  r'  r  r   r!  rK   rÚ   Úuse_mem_roper
  r“   rþ   Ú_attn_implementationrÆ   ÚloggerÚwarning_oncer   rõ   r  rñ   ræ   rù   r  )r:   rO   rŒ   rð   r,  r-  rú   Úinput_shapeÚhidden_shapeÚquery_statesrŠ   r‹   Úadapter_layer_idxrÛ   rÜ   Úattention_interfacerý   rû   s                     r>   rW   zZamba2Attention.forward�  s\  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆØ�;‰;×3Ò3Ø $§¡¨yÑ 9ÐØ'Ð*W¨$×*DÑ*DÐEVÑ*WÐXeÓ*fÑfˆLØ#Ð&S d×&@Ñ&@ÐARÑ&SÐTaÓ&bÑbˆJØ'Ð*W¨$×*DÑ*DÐEVÑ*WÐXeÓ*fÑfˆLà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà�;‰;×#Ò#Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ%Ø'5×'<Ñ'<¸ZÈÐW`Ó'aÑ$ˆJ˜ä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9¸f¿j¹jÐI\Ð^cÔ>dÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r?   r*   )rY   rZ   r[   r³   r&   r   rq   r3   r5   r¶   rg   r	   r   r   rW   r\   r]   s   @r>   r  r  J  sû   ø„ ñð: $(Ø,0Ø"&ñ6\àð6\ð ˜C‘=ð6\ð % S™Mð	6\ð
 ˜3‘-õ6\ðx 26Ø=AØKOñ7)à—|‘|ð7)ð ð7)ð ! §¡Ñ.ð	7)ð
 !Ð!9Ñ:ð7)ð & e¨E¯L©L¸%¿,¹,Ð,FÑ&GÑHð7)ð Ð-Ñ.ð7)ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷7)r?   r  Úinput_tensorÚpad_sizec                 ó°   — t        | j                  «      dk(  r
ddddd|ddfnddd|ddf}t        j                  j                  j                  | |dd¬«      S )z‚
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
    é   r   Úconstant)Úmoderï   )r–   rJ   r5   r   rH   Úpad)r<  r=  Ú	pad_shapes      r>   Úpad_tensor_by_sizerD  Ú  sf   € ô 47°|×7IÑ7IÓ3JÈaÒ3O��A�q˜!˜Q ¨!¨QÑ/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€Iä�8‰8×Ñ×"Ñ" <°ÀÐSTÐ"ÓUÐUr?   c                 ó  — t        | |«      } t        | j                  «      dk(  r.| j                  | j                  d   d|| j                  d   «      S | j                  | j                  d   d|| j                  d   | j                  d   «      S )zÀ
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r   r   rC   rB   )rD  r–   rJ   ræ   )r<  r=  Ú
chunk_sizes      r>   Úreshape_into_chunksrG  å  s“   € ô & l°HÓ=€Lä
ˆ<×ÑÓ !Ò#à×#Ñ# L×$6Ñ$6°qÑ$9¸2¸zÈ<×K]ÑK]Ð^_ÑK`ÓaÐað ×#Ñ#Ø×Ñ˜qÑ! 2 z°<×3EÑ3EÀaÑ3HÈ,×J\ÑJ\Ð]^ÑJ_ó
ð 	
r?   c                 ó"  — | j                  d«      } | d   j                  g | j                  «       ¢|‘­Ž } t        j                  t        j                  ||| j
                  t        j                  ¬«      d¬«      }| j                  | d«      } t        j                  | d¬«      }t        j                  t        j                  ||| j
                  t        j                  ¬«      d¬«      }|j                  | t        j                   «      }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    rC   ©.Nrl   ©Údiagonalr   rš   r�   )
Úsizer×   r5   Útrilr6   rj   ÚboolÚmasked_fillÚcumsumÚinf)r<  rF  ÚmaskÚtensor_segsums       r>   Úsegment_sumrT  ù  sÞ   € ð ×"Ñ" 2Ó&€Jð 2�< 	Ñ*×1Ñ1ÐS°<×3DÑ3DÓ3FÐSÈ
ÒS€Lä�:‰:”e—j‘j ¨ZÀ×@SÑ@SÔ[`×[eÑ[eÔfÐqsÔt€DØ×+Ñ+¨T¨E°1Ó5€Lä—L‘L °2Ô6€Mô �:‰:”e—j‘j ¨ZÀ×@SÑ@SÔ[`×[eÑ[eÔfÐqrÔs€DØ!×-Ñ-¨t¨e´e·i±i°ZÓ@€MØÐr?   c                   óê   ‡ — e Zd ZdZddedee   fˆ fd„Z	 	 ddej                  dee
   deej                     fd„Zddee
   deej                     fd	„Z	 	 ddee
   deej                     fd
„Zˆ xZS )ÚZamba2MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    rh   rŒ   c           	      ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j                  | _        t        |j                  | j                  z  «      | _
        || _        |j                  | _        d| _        t        j                  «       | _        |j"                  | _        |j$                  | _        |j(                  | _        | j                  j,                  | _        |j0                  | _        |j2                  | _        |j4                  | _        |j6                  | _        | j                  d| j&                  z  | j
                  z  z   | _        t        j:                  | j8                  | j8                  d|j                  | j8                  |j                  dz
  ¬«      | _        | j                  | j8                  z   | j.                  z   }t        j>                  | j                  ||j@                  ¬«      | _!        t        jD                  tG        jH                  | j.                  «      «      | _%        tG        jL                  d| j.                  dz   «      }t        jD                  tG        jN                  |«      «      | _(        d| jP                  _)        tU        | j                  | j                  | j&                  z  d¬«      | _+        t        jD                  tG        jH                  | j.                  «      «      | _,        d| jX                  _)        t        j>                  | j                  | j                  |j@                  ¬«      | _-        t\        st^        ja                  d	«       y y )
NrI   rB   Tr%   )Úin_channelsÚout_channelsr  Úkernel_sizeÚgroupsÚpaddingr  gñhãˆµøä>)r9   r<   a  The fast path is not available because on of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)1r2   r3   rh   r;   rt   ru   rv   rw   rq   rr   rs   rŒ   Úuse_conv_biasÚ
activationr   ÚSiLUÚactÚuse_mem_eff_pathr‚   Ún_groupsrƒ   rê   rx   Ú	num_headsrF  Útime_step_limitÚtime_step_minÚtime_step_maxÚconv_dimÚConv1dÚconv1dr  Úadd_bias_linearÚin_projr4   r5   r6   Údt_biasÚarangeÚlogÚA_logÚ_no_weight_decayr/   ÚnormÚDÚout_projÚis_fast_path_availabler5  r6  )r:   rh   rŒ   Úprojection_sizeÚAr=   s        €r>   r3   zZamba2MambaMixer.__init__  s“  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔÜ!$ V×%8Ñ%8¸4×;KÑ;KÑ%KÓ!LˆÔØ"ˆŒØ#×1Ñ1ˆÔØ ˆŒÜ—7‘7“9ˆŒØ &× 7Ñ 7ˆÔà×,Ñ,ˆŒØ×,Ñ,ˆŒØŸ™×2Ñ2ˆŒØ ×+Ñ+ˆŒà%×5Ñ5ˆÔØ#×1Ñ1ˆÔØ#×1Ñ1ˆÔà×.Ñ.°°T·]±]Ñ1BÀT×EXÑEXÑ1XÑXˆŒÜ—i‘iØŸ™ØŸ™ØØ×+Ñ+Ø—=‘=Ø×'Ñ'¨!Ñ+ô
ˆŒð ×0Ñ0°4·=±=Ñ@À4Ç>Á>ÑQˆÜ—y‘yØ×ÑØØ×'Ñ'ô
ˆŒô —|‘|¤E§J¡J¨t¯~©~Ó$>Ó?ˆŒô �L‰L˜˜DŸN™N¨QÑ.Ó/ˆÜ—\‘\¤%§)¡)¨A£,Ó/ˆŒ
Ø&*ˆ�
‰
Ô#Ü&Ø×"Ñ"¨t×/EÑ/EÈÏÉÑ/VÐ\`ô
ˆŒ	ô —‘œeŸj™j¨¯©Ó8Ó9ˆŒØ"&ˆ�‰ÔäŸ	™	 $×"8Ñ"8¸$×:JÑ:JÐQW×QgÑQgÔhˆŒå%Ü×Ñð>õð &r?   rO   Úcache_paramsrð   c                 ó‚  — |j                   \  }}}| j                  | j                  z  }d| j                  z  d| j                  z  | j                  z  z   | j                  z   }|��4|j
                  �r'| j                  |j                  d«      «      }	|	j                   d   |z
  dz  }
|
|
| j                  | j                  | j                  g}t        j                  |	|d¬«      \  }}}}}t        ||j                  | j                     | j                  j                  j                  d«      | j                  j                   | j"                  «      }t        j                  || j                  ||gd¬«      \  }}}t        j$                  | j&                  j)                  «       «       }|d d …d df   d d …d d …d f   j+                  d| j,                  | j                  «      j/                  t        j0                  ¬«      }|d d …d d …d f   j+                  dd| j,                  «      }| j2                  d d …d df   j+                  d| j,                  «      }| j4                  d d …d df   j+                  d| j,                  «      }|j7                  || j                  |j                   d   | j                  z  «      }|j7                  || j                  |j                   d   | j                  z  «      }|j7                  || j                  | j,                  «      }t9        |j:                  | j                     ||||||d |d¬«
      }|j7                  || j                  | j,                  z  «      }| j=                  ||«      }| j?                  |«      d d …d df   }|S |�Bt        j@                  |dk(  «      s*|jB                  }||d d …d d …d f   z  j/                  |«      }| j                  |«      }t        j$                  | j&                  j)                  «       «       }| jD                  €i nd	| jD                  i}|�t        j@                  |dk(  «      }nd}| jF                  rõ| jH                  ré|€ç|råtK        || j                  j                  j                  d«      | j                  j                   | j2                  |f| j4                  | jL                  d | j"                  | j<                  j                  | j<                  jN                  | j>                  j                  | j>                  j                   | j,                  | j                  d
ddœ|¤Ž\  }}|S t        j                  || j                  | j                  | j                  gd¬«      \  }}}|�v|jQ                  dd«      }tR        jT                  jW                  || jX                  |j                   d   z
  df«      }|j                  | j                     j[                  |«       t\        �| j"                  dvrJ| j_                  | j                  |jQ                  dd«      «      jQ                  dd«      d d …d |…f   «      }nyt]        |jQ                  dd«      | j                  j                  j                  d«      | j                  j                   | j"                  ¬«      jQ                  dd«      d d …d |…f   }t        j                  || j                  ||gd¬«      \  }}}|�Bt        j@                  |dk(  «      s*|jB                  }||d d …d d …d f   z  j/                  |«      }ta        |j7                  ||d| j,                  «      |||j7                  ||| j                  d«      |j7                  ||| j                  d«      f| jL                  | j4                  d d d| j2                  ddœ|¤Ž\  }}|�*|�(|j:                  | j                     j[                  |«       |j7                  ||d«      }| j=                  ||«      }| j?                  |«      }|S )NrB   r%   rC   r�   .rÕ   T)Úzrl  Údt_softplusÚdt_limitF)rr  rF  Úseq_idxr^  Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_statesr   )rI   Úswish)rÝ   r7   r  r^  )rF  rr  ry  r|  r„  rl  rz  )1rJ   rb  ru   rs   rc  rp   rk  Úsqueezerg  r5   Úsplitr,   r}   rŒ   ri  r7   r  r^  Úexpro  rÖ   r×   rê   rF   rG   rl  rr  rK   r'   r~   rq  rs  ÚallrE   rd  ra  rõ   r)   rF  r8   rÚ   r   rH   rB  rw   Úcopy_r+   r`  r(   )r:   rO   rw  rð   ri   Úseq_lenr‰   Úgroups_time_state_sizeÚd_to_removeÚin_projected_statesÚd_mlpÚsplit_projection_dimrP   Úhidden_states_B_CÚdtÚBÚCrv  rl  rr  Úhidden_states_reshapedÚoutrE   Úprojected_statesÚdt_limit_kwargsÚinput_not_maskedÚ	ssm_stateÚ	time_stepÚhidden_states_B_C_tr¯   Úscan_outputs                                  r>   Úcuda_kernels_forwardz%Zamba2MambaMixer.cuda_kernels_forwardY  sv  € ð "/×!4Ñ!4Ñˆ
�G˜QØ!%§¡°×1DÑ1DÑ!DÐØ˜$×0Ñ0Ñ0°1°t·}±}Ñ3DÀt×GZÑGZÑ3ZÑZÐ]a×]kÑ]kÑkˆð Ñ#¨×(GÓ(GØ"&§,¡,¨}×/DÑ/DÀQÓ/GÓ"HÐØ(×.Ñ.¨rÑ2°[Ñ@ÀQÑFˆEØ$)¨5°$×2HÑ2HÈ$Ï-É-ÐY]×YgÑYgÐ#hÐ Ü05·±Ð<OÐQeÐkmÔ0nÑ-ˆAˆq�$Ð)¨2ä 4Ø!Ø×(Ñ(¨¯©Ñ8Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ Ø—‘ó!Ðô #(§+¡+Ø!Ø×'Ñ'Ð)?ÐAWÐXØô#ÑˆM˜1˜aô
 —‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAà’!�T˜3�,‘¢¢1 d 
Ñ+×2Ñ2°2°t·}±}Àd×FYÑFYÓZ×]Ñ]Ôdi×dqÑdqÐ]ÓrˆAØ’A’q˜$�J‘×&Ñ& r¨2¨t¯}©}Ó=ˆBØ—l‘l¢1 d¨C <Ñ0×7Ñ7¸¸D¿M¹MÓJˆGØ—‘’q˜$ �|Ñ$×+Ñ+¨B°·±Ó>ˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ%2×%7Ñ%7¸
ÀDÇNÁNÐTX×TaÑTaÓ%bÐ"Ü2Ø×'Ñ'¨¯©Ñ7Ø&ØØØØØØØØ ôˆMð *×.Ñ.¨z¸4¿>¹>ÈDÏMÉMÑ;YÓZˆMØ ŸI™I m°TÓ:ˆMØ—-‘- Ó.ªq°$¸¨|Ñ<ˆCðz ˆ
ðu Ð)´%·)±)¸NÈaÑ<OÔ2Pà%×+Ñ+�Ø!.°ÂÂ1ÀdÀ
Ñ1KÑ!K× OÑ OÐPUÓ V�à#Ÿ|™|¨MÓ:ÐÜ—‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAØ$(×$8Ñ$8Ð$@™bÀzÐSW×SgÑSgÐFhˆOØÐ)Ü#(§9¡9¨^¸qÑ-@Ó#AÑ à#'Ð à×$Ò$¨¯ª¸<Ð;OÑTdÜ!AØ$Ø—K‘K×&Ñ&×.Ñ.¨qÓ1Ø—K‘K×$Ñ$Ø—L‘LØð"ð —f‘fØ#Ÿ™Ø Ø#Ÿ™Ø#'§9¡9×#3Ñ#3Ø $§	¡	× :Ñ :Ø#'§=¡=×#7Ñ#7Ø!%§¡×!3Ñ!3Ø ŸM™MØ ŸM™MØ%*Ø(,ñ#"ð$ &ñ%"‘��YðX ˆ
ôm 6;·[±[Ø$Ø×+Ñ+¨T¯]©]¸D¿N¹NÐKØô6Ñ2�Ð'¨ð  Ð+Ø*;×*EÑ*EÀaÈÓ*KÐ'Ü!#§¡×!2Ñ!2Ø+¨d×.CÑ.CÐFY×F_ÑF_Ð`bÑFcÑ.cÐefÐ-gó"�Jð !×,Ñ,¨T¯^©^Ñ<×BÑBÀ:ÔNÜ#Ð+¨t¯©ÐFWÑ/WØ(,¯©ØŸ™Ð$5×$?Ñ$?ÀÀ1Ó$EÓF×PÑPÐQRÐTUÓVÒWXÐZbÐ[bÐZbÐWbÑcó)Ñ%ô )9Ø+×5Ñ5°a¸Ó;Ø#Ÿ{™{×1Ñ1×9Ñ9¸!Ó<Ø!Ÿ[™[×-Ñ-Ø#'§?¡?ô	)÷
  ‘i  1“o¢a¨¨'¨ kñ)3Ð%ô ',§k¡kØ%Ø×+Ñ+Ð-CÐE[Ð\Øô'Ñ#�˜q !ð
 "Ð-´e·i±iÀÐRSÑ@SÔ6Tà)×/Ñ/�EØ%2°^ÂAÂqÈ$ÀJÑ5OÑ%O×$SÑ$SÐTYÓ$Z�MÜ)BØ!×&Ñ& z°7¸BÀÇÁÓNØØØ—F‘F˜: w°·±¸rÓBØ—F‘F˜: w°·±¸rÓBð*ð  $Ÿ™Ø—f‘fØØ Ø(,Ø ŸL™LØ $ñ*ð &ñ*Ñ&�˜Yð Ð(¨\Ð-EØ ×+Ñ+¨D¯N©NÑ;×AÑAÀ)ÔLØ)×.Ñ.¨z¸7ÀBÓG�à"Ÿi™i¨°TÓ:�Ø—m‘m KÓ0�Øˆ
r?   c                 óî  — |j                   \  }}}|j                  }|�-|j                  r!| j                  |j	                  d«      «      }nI|�6t        j                  |dk(  «      s||d d …d d …d f   z  j                  |«      }| j                  |«      }|j                   d   d| j                  z  z
  d| j                  z  | j                  z  z
  | j                  z
  dz  }	|j                  |	|	| j                  | j                  | j                  gd¬«      \  }}}
}}|��w|j                  | j                     j!                  «       }|j                  |j"                  «      }|j                  �r1|
j%                  d«      }
|j&                  | j                     }t        j(                  |dd¬«      }|j*                  dk(  r|d d …dd d …f   n||d d …d d …df<   |j&                  | j                     j-                  |«       t        j.                  |j                  |j"                  «      | j0                  j2                  d d …dd d …f   z  d¬«      }| j4                  r|| j0                  j6                  z  }| j9                  |«      j                  |«      d d …d df   }�n‚|j;                  dd«      }t<        j>                  jA                  || jB                  |j                   d   z
  df«      }|j&                  | j                     j-                  |«       | j9                  | j1                  |«      j;                  dd«      «      d d …d |…d d …f   }|�Ît        j                  |dk(  «      s¶|j                  }||d d …d d …d f   z  j                  |«      }n‹t        jD                  || j                  | jF                  | j                  f|j"                  |¬	«      }| j9                  | j1                  |j;                  dd«      «      dd |…f   j;                  dd«      «      }t        j                  || j                  | j                  | j                  z  | j                  | j                  z  gd¬«      \  }}}t        jH                  | jJ                  jM                  «       «       }|��t|j                  �rg|j*                  dk(  r
|d d …d df   n|d d …dd d …f   d d …d df   }|j;                  dd«      jO                  ||j                   d   | jF                  «      }| jP                  d
   jO                  | jP                  j                   d   | jF                  «      }t
        j<                  j>                  jS                  ||j                  |j                  «      z   «      }t        jT                  || jV                  «      }|d   jO                  | j                  | jF                  | j                  «      j                  t
        jX                  ¬«      }t        jH                  |d
   |z  «      }|j[                  || j                  d«      dd d d …f   }|jO                  || j                  | j                  | j                  z  |j                   d   «      j]                  «       }|j[                  |d|j                   d   «      }|d
   |dd d d …f   z  }|j[                  |d| jF                  «      }||d
   z  }|j                  | j                     j-                  |j                  | j                     |z  |z   «       |j[                  || j                  d«      dd d d …f   }|jO                  || j                  | j                  | j                  z  |j                   d   «      j]                  «       }|j[                  |d|j                   d   «      }|j                  | j                     j                  |j                  «      }|j_                  || j                  z  | jF                  | j                  «      }|j_                  || j                  z  | j                  d«      }t        j`                  ||«      }|j_                  || j                  | jF                  «      }| jb                  d
   jO                  | jb                  j                   d   | jF                  «      }|||z  z   j                  |j                  «      }|j[                  |d«      d d …d df   }�n t<        j>                  jS                  || jP                  z   «      }t        jT                  || jV                  «      }|j[                  ||d| jF                  «      jM                  «       }|j[                  ||d| j                  «      jM                  «       }|j[                  ||d| j                  «      jM                  «       }|je                  dd| j                  | j                  z  d«      }|je                  dd| j                  | j                  z  d«      }| jf                  || jf                  z  z
  | jf                  z  }| jb                  d
   ti        ||«      z  }||d
   z  }|j                  |j                  «      |z  }||||fD �cg c]  }tk        ||| jf                  «      ‘Œ c}\  }}}}|jm                  dddd«      }t        jn                  |d¬«      }t        jH                  tq        |«      «      }|d d …d d …d d …d d d …d d …f   |d d …d d …d d d …d d …d d …f   z  } | j/                  d¬«      }!|!d
   |jm                  ddddd«      d
   z  }"|"j/                  d¬«      }#|#d
   |d d …d d …d f   z  j/                  d«      }$t        jH                  |d d …d d …d d …dd …f   |z
  «      }%||%jm                  dddd«      d
   z  }&|&jm                  ddddd«      d
   |jm                  ddddd«      dd d d …f   z  j/                  d¬«      jm                  ddddd«      }'|�.|j                  r"|j                  | j                     d d …d df   }(nt        jr                  |'d d …d d…f   «      }(t        jt                  |(|'gd¬«      }'t        jH                  tq        t<        j>                  jA                  |d d …d d …d d …df   d«      «      «      })|'jm                  ddddd«      }*|)d   |*d d …d d …d df   z  j/                  d¬«      }+|+jm                  ddddd«      },|,d d …d d…f   |,d d …df   }}'t        jH                  |«      }-|dd d d …f   |'d d …d d …d df   z  }.|-jm                  dddd«      }/|.j/                  d«      |/d
   z  }0|$|0z   }|j[                  |d| j                  | jF                  «      }||z   }|dkD  r|d d …d |…d d …d d …f   }|j[                  ||d«      }|�*|�(|j                  | j                     j-                  |«       | jw                  ||
«      }1| jy                  |1j                  |«      «      }2|2S c c}w )Nr%   rC   rB   r�   r©   r   r   .rl   rI  ).NNrÕ   r?  )r%   r   )=rJ   rE   rp   rk  r†  r5   r‰  rF   rs   rb  ru   rc  r‡  rg  r~   rŒ   Úclonerj   r  r}   r­   ÚndimrŠ  Úsumri  r7   r]  r  r`  rÚ   r   rH   rB  rw   r�   rê   rˆ  ro  rÖ   r×   rl  Úsoftplusr¬   re  rG   ræ   rù   rK   Úbmmrr  ÚrepeatrF  rD  rG  ÚpermuterP  rT  Ú
zeros_liker’   rq  rs  )3r:   Úinput_statesrw  rð   ri   r‹  r‰   rE   r—  r�  rP   rO   r’  rš  r¯   r“  r”  rv  rl  ÚdAÚdBÚdBxr~   Ússm_states_reshapedÚ
C_reshapedÚyrr  r=  Ú
D_residualÚtÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decay_contractionÚstatesÚprevious_statesÚdecay_chunkÚstates_permutedÚresultÚ
new_statesÚstate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offr�  Úcontextualized_statess3                                                      r>   Útorch_forwardzZamba2MambaMixer.torch_forwardð  sj  € Ø!-×!3Ñ!3Ñˆ
�G˜QØ×"Ñ"ˆàÐ#¨×(GÒ(GØ $§¡¨\×-AÑ-AÀ!Ó-DÓ EÑàÐ)´%·)±)¸NÈAÑ<MÔ2Nà$0°>Â!ÂQÈÀ*Ñ3MÑ$M×#QÑ#QÐRWÓ#X�LØ $§¡¨\Ó :ÐØ!×'Ñ'¨Ñ+¨a°$×2HÑ2HÑ.HÑHÈAÐPT×P]ÑP]ÑL]Ð`d×`sÑ`sÑLsÑsÐuy÷  vDñ  vDñ  Dð  IJñ  JˆØ(8×(>Ñ(>Ø˜˜t×5Ñ5¸¿¹ÀtÇ~Á~ÐVÐ\^ð )?ó )
Ñ%ˆˆ1ˆd�M 2ð
 Ñ#Ø$×/Ñ/°·±Ñ?×EÑEÓGˆIØ!Ÿ™ ]×%9Ñ%9Ó:ˆIØ×.Ó.Ø—~‘~ aÓ(�Ø)×5Ñ5°d·n±nÑE�
Ü"ŸZ™Z¨
¸2ÀBÔG�
àAN×ASÑASÐWXÒAX }²Q¸º1°WÒ'=Ð^k�
š1ša ˜8Ñ$Ø×(Ñ(¨¯©Ñ8×>Ñ>¸zÔJÜ %§	¡	¨*¯-©-Ð8H×8OÑ8OÓ*PÐSW×S^ÑS^×SeÑSeÒfgÐijÒlmÐfmÑSnÑ*nÐtvÔ w�Ø×%Ò%Ø! T§[¡[×%5Ñ%5Ñ5�MØ $§¡¨Ó 7× :Ñ :¸5Ó AÂ!ÀTÈ3À,Ñ O’à -× 7Ñ 7¸¸!Ó <�ÜŸ]™]×.Ñ.Ø!Ø×*Ñ*¨]×-@Ñ-@ÀÑ-DÑDÀaÐHó�
ð ×(Ñ(¨¯©Ñ8×>Ñ>¸zÔJØ $§¡¨¯©°]Ó)C×)MÑ)MÈaÐPQÓ)RÓ SÒTUÐW_ÐX_ÐW_ÒabÐTbÑ c�Ø!Ð-´e·i±iÀÐPQÑ@QÔ6RØ)×/Ñ/�Eà%2°^ÂAÂqÈ$ÀJÑ5OÑ%O×$SÑ$SÐTYÓ$Z‘MäŸ™Ø˜TŸ^™^¨T¯]©]¸D×<OÑ<OÐPØ$×+Ñ+°5ôˆIð !ŸH™H T§[¡[°×1HÑ1HÈÈAÓ1NÓ%OÐPSÐU]ÐV]ÐU]ÐP]Ñ%^×%hÑ%hÐijÐlmÓ%nÓoˆMÜ#Ÿk™k¨-¸$×:PÑ:PÐRV×R_ÑR_Ðbf×buÑbuÑRuÐw{÷  xEñ  xEð  HL÷  H[ñ  H[ñ  x[ð  :\ð  bdô  eÑˆ�q˜!Ü�Y‰Y�t—z‘z×'Ñ'Ó)Ó*Ð*ˆØÑ#¨×(GÓ(Gð &(§W¡W°¢\�’A�t˜S�LÒ!°rº!¸QÂ¸'±{Â1ÀdÈCÀ<Ñ7PˆBØ—‘˜a Ó#×*Ñ*¨:°r·x±xÀ±|ÀTÇ]Á]ÓSˆBà—l‘l 9Ñ-×4Ñ4°T·\±\×5GÑ5GÈÑ5JÈDÏMÉMÓZˆGä—‘×$Ñ$×-Ñ-¨b°7·:±:¸b¿h¹hÓ3GÑ.GÓHˆBÜ—‘˜R ×!3Ñ!3Ó4ˆBØ�/Ñ"×)Ñ)¨$¯.©.¸$¿-¹-È×I\ÑI\Ó]×`Ñ`Ôgl×gtÑgtÐ`ÓuˆAä—‘˜2˜i™=¨1Ñ,Ó-ˆBð
 —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAà�I‘  3¨ªa <¡Ñ0ˆBð *×1Ñ1°*¸bÀ$Ç-Á-ÓPˆMØ�} YÑ/Ñ/ˆCð ×#Ñ# D§N¡NÑ3×9Ñ9Ø×'Ñ'¨¯©Ñ7¸"Ñ<¸sÑBôð —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAð &×0Ñ0°·±Ñ@×CÑCÀAÇGÁGÓLˆJà",§/¡/°*¸t¿~¹~Ñ2MÈtÏ}É}Ð^b×^qÑ^qÓ"rÐØŸ™ 
¨T¯^©^Ñ ;¸T×=PÑ=PÐRSÓTˆJÜ—	‘	Ð-¨zÓ:ˆAØ—‘�z 4§>¡>°4·=±=ÓAˆAð —‘�yÑ!×(Ñ(¨¯©¯©°a©¸$¿-¹-ÓHˆAØ�] QÑ&Ñ&×*Ñ*¨1¯7©7Ó3ˆAð —	‘	˜* bÓ)ª!¨T°3¨,Ñ7ŠAô —‘×'Ñ'¨¨T¯\©\Ñ(9Ó:ˆBÜ—‘˜R ×!3Ñ!3Ó4ˆBØ)×1Ñ1°*¸gÀrÈ4Ï=É=ÓY×_Ñ_ÓaˆMØ—	‘	˜* g°°D×4GÑ4GÓH×NÑNÓPˆAØ—	‘	˜* g¨r°4×3FÑ3FÓG×MÑMÓOˆAØ—‘˜˜A˜tŸ~™~°·±Ñ>ÀÓBˆAØ—‘˜˜A˜tŸ~™~°·±Ñ>ÀÓBˆAØŸ™¨'°D·O±OÑ*CÑCÀtÇÁÑVˆHàŸ™ 	Ñ*Ô-?ÀÈxÓ-XÑXˆJð *¨B¨y©MÑ9ˆMØ—‘�]×(Ñ(Ó)¨BÑ.ˆAð cpÐqrÐtuÐwxÐayÖ%zÐ\]Ô&9¸!¸XÀtÇÁÕ&WÒ%zÑ"ˆM˜1˜a ð —	‘	˜!˜Q  1Ó%ˆAÜ—|‘| A¨2Ô.ˆHô —	‘	œ+ a›.Ó)ˆAð šq¢!¢Q¨ªa²Ð2Ñ3°aºº1¸dÂAÂqÊ!Ð8KÑ6LÑLˆNØ×"Ñ" rÐ"Ó*ˆAð ˜y™\¨A¯I©I°a¸¸A¸qÀ!Ó,DÀYÑ,OÑOˆNØ×"Ñ" rÐ"Ó*ˆAð ˜	‘l ]²1²a¸°:Ñ%>Ñ>×CÑCÀAÓFˆFô !Ÿ9™9 hªq²!²Q¸¹¨|Ñ&<¸xÑ&GÓIˆLØ"# l×&:Ñ&:¸1¸aÀÀAÓ&FÀyÑ&QÑ"QÐà)×1Ñ1°!°Q¸¸1¸aÓ@ÀÑKÈ}×OdÑOdÐefÐhiÐklÐnoÐqrÓOsÐtwÐy}ò  @Að  uAñ  PBñ  B÷  Gñ  Gð  LMð  Gó  N÷  Vñ  Vð  WXð  Z[ð  ]^ð  `að  cdó  eˆFØÐ'¨L×,KÒ,KØ".×"9Ñ"9¸$¿.¹.Ñ"IÊ!ÈTÐSVÈ,Ñ"W‘ä"'×"2Ñ"2°6º!¸R¸a¸R¸%±=Ó"A�Ü—Y‘Y °Ð8¸aÔ@ˆFÜŸ)™)¤K´·±×0AÑ0AÀ(Ê1ÊaÒQRÐTVÈ;ÑBWÐY_Ó0`Ó$aÓbˆKà$Ÿn™n¨Q°°1°a¸Ó;ˆOØ! /Ñ2°_ÂQÊÈ4ÐQTÀ_Ñ5UÑU×ZÑZÐ_`ÐZÓaˆFØŸ™¨¨1¨a°°AÓ6ˆJØ *ª1¨c¨r¨c¨6Ñ 2°JºqÀ"¸uÑ4E�IˆFô $Ÿi™i¨Ó1ˆOà  Tª1 ™o°²qº!¸TÀ3°Ñ0GÑGˆNØ'6×'>Ñ'>¸qÀ!ÀQÈÓ'JÐ$Ø#×'Ñ'¨Ó+Ð.FÀyÑ.QÑQˆEð ˜‘ˆAà—	‘	˜* b¨$¯.©.¸$¿-¹-ÓHˆAà�J‘ˆAà˜!Š|Ø’a˜˜'˜¢1¢aÐ'Ñ(�Ø—	‘	˜* g¨rÓ2ˆAØÐ$¨Ð)AØ×'Ñ'¨¯©Ñ7×=Ñ=¸iÔHà—i‘i  4Ó(ˆð
 !%§¡¨k¯n©n¸UÓ.CÓ DÐØ$Ð$ùòI &{s   ê7y2c                 ó²   — t         r?d| j                  j                  j                  j                  v r| j                  |||«      S | j                  |||«      S )NÚcuda)rt  rk  r7   rj   rÀ   rž  rÅ  )r:   rO   rw  rð   s       r>   rW   zZamba2MambaMixer.forward²  sN   € õ " f°·±×0CÑ0C×0JÑ0J×0OÑ0OÑ&OØ×,Ñ,¨]¸LÈ.ÓYÐYà×!Ñ! -°¸~ÓNÐNr?   r1   r-   )rY   rZ   r[   r³   r&   r   rq   r3   r5   r¶   rg   rž  rÅ  rW   r\   r]   s   @r>   rV  rV    sÌ   ø„ ññ?˜|ð ?¸À¹õ ?ðH <@Ø15ñ	Tà—|‘|ðTð Ð7Ñ8ðTð ! §¡Ñ.ó	Tñn%¸ÐAYÑ8Zð %ÐqyÐz÷  {Gñ  {Gñ  rHó %ðJ <@Ø15ñ		Oð Ð7Ñ8ð	Oð ! §¡Ñ.÷		Or?   rV  c                   ó8   ‡ — e Zd Zddedee   fˆ fd„Zdd„Zˆ xZS )Ú	Zamba2MLPrh   r  c           	      ó   •— t         ‰	| �  «        || _        |j                  | _        |j                  | _        || _        || _        t        j                  | j                  d| j                  z  |j                  ¬«      | _
        t        j                  | j                  | j                  |j                  ¬«      | _        t        |j                     | _        t        j                  g «      | _        t#        | j
                  «      D ]Ï  }||j$                  z  |k(  rŒt        j&                  t        j                  | j                  j                  | j                  j(                  d¬«      t        j                  | j                  j(                  d| j                  z  d¬«      «      }nt        j*                  «       }| j                   j-                  |«       ŒÑ |j.                  }t1        |«      D ��ci c]  \  }}||“Œ
 c}}| _        yc c}}w )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        rB   r  FN)r2   r3   rh   r;   rs   r  r  r   r  rj  Úgate_up_projÚ	down_projr   Ú
hidden_actÚact_fnr  Úgate_up_proj_adapter_listr   r"  r#  r$  r%  r„   r  r&  r'  )
r:   rh   r  r  rˆ   Úgate_up_proj_adapterr  r+  rï   r=   s
            €r>   r3   zZamba2MLP.__init__¿  s”  ø€ ô
 	‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØ"4ˆÔØ ˆŒäŸI™I d×&6Ñ&6¸¸D×<RÑ<RÑ8RÐY_×YoÑYoÔpˆÔÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RhÑRhÔiˆŒÜ˜V×.Ñ.Ñ/ˆŒä)+¯©°rÓ):ˆÔ&Ü�t×.Ñ.Ó/ò 	HˆAØ�6×(Ñ(Ñ(¨HÒ4Ü')§}¡}Ü—I‘I˜dŸk™k×5Ñ5°t·{±{×7OÑ7OÐV[Ô\Ü—I‘I˜dŸk™k×6Ñ6¸¸D×<RÑ<RÑ8RÐY^Ô_ó(Ñ$ô
 (*§{¡{£}Ð$Ø×*Ñ*×1Ñ1Ð2FÕGð	Hð !×1Ñ1ˆÜ;DÀ_Ó;U×V©<¨5°%˜% ™,ÓVˆ�ùÓVs   Ç3H
c                 óü   — | j                  |«      }| j                  |   }| | j                  |   |«      z   }t        j                  |dd¬«      }| j                  |d   «      |d   z  }| j                  |«      }|S )NrB   rC   r�   r   r%   )rË  r'  rÏ  r5   ÚchunkrÎ  rÌ  )r:   Úhidden_staterŒ   Úgate_up_stateÚoutputs        r>   rW   zZamba2MLP.forwardÝ  s€   € Ø×)Ñ)¨,Ó7ˆØ—N‘N 9Ñ-ˆ	Ø%Ð(Q¨×(FÑ(FÀyÑ(QÐR^Ó(_Ñ_ˆäŸ™ M°1¸"Ô=ˆØ—{‘{ =°Ñ#3Ó4°}ÀQÑ7GÑGˆØ—‘ Ó-ˆØˆr?   r-   r1   )	rY   rZ   r[   r&   r   rq   r3   rW   r\   r]   s   @r>   rÉ  rÉ  ¾  s%   ø„ ñW˜|ð WÐPXÐY\ÑP]õ W÷<r?   rÉ  c                   ó4  ‡ — e Zd Zddedee   dee   fˆ fd„Z	 	 	 	 ddej                  dej                  dedeej                     dee	   d	ee
   d
eej                     dee   deej                  eeej                  ej                  f      f   fd„Zˆ xZS )ÚZamba2AttentionDecoderLayerrh   r  rŒ   c                 ó@  •— t         ‰| �  «        || _        t        |j                  «      }t        |d||¬«      | _        t        |||¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        y )NrC   )rŒ   r  r  )r  r  ©r<   )r2   r3   r  r–   r  r  Ú	self_attnrÉ  Úfeed_forwardr_   r  Úrms_norm_epsÚinput_layernormr;   Úpre_ff_layernorm)r:   rh   r  rŒ   Únum_gsr=   s        €r>   r3   z$Zamba2AttentionDecoderLayer.__init__é  s�   ø€ Ü‰ÑÔØ ˆŒÜ�V×,Ñ,Ó-ˆÜ(¨¸2ÐRXÐckÔlˆŒÜ% fÀÐRZÔ[ˆÔÜ,¨V×-IÑ-IÈv×ObÑObÔcˆÔÜ -¨f×.@Ñ.@Àf×FYÑFYÔ ZˆÕr?   rO   Úoriginal_hidden_statesrð   r,  r1  r-  rú   rŽ   c           
      óî   — t        j                  ||gd¬«      }| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	| j	                  |«      }| j                  ||«      }|f}
|r|
|	fz  }
|
S )aú  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://arxiv.org/pdf/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        rC   r�   )rO   rŒ   rð   r,  r1  r-  rº   )r5   ÚconcatenaterÝ  rÚ  rÞ  rÛ  )r:   rO   rà  rŒ   rð   r,  r1  r-  rú   Úself_attn_weightsÚoutputss              r>   rW   z#Zamba2AttentionDecoderLayer.forwardò  s¦   € ô> ×)Ñ)¨=Ð:PÐ*QÐWYÔZˆØ×,Ñ,¨]Ó;ˆØ+9¨4¯>©>ð ,
Ø'ØØ)Ø)Ø/Ø 3ñ,
ð ñ,
Ñ(ˆÐ(ð ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-¸ÓCˆà Ð"ˆáØÐ)Ð+Ñ+ˆGàˆr?   r-   )NNFN)rY   rZ   r[   r&   r   rq   r3   r5   r¶   rg   rN  r·   r   r   r	   r¹   rW   r\   r]   s   @r>   r×  r×  è  sî   ø„ ñ[˜|ð [°xÀ±}ð [ÐX`ÐadÑXeõ [ð 26Ø=AØ,1Ø:>ñ3à—|‘|ð3ð !&§¡ð3ð ð	3ð
 ! §¡Ñ.ð3ð !Ð!9Ñ:ð3ð $ D™>ð3ð & e×&6Ñ&6Ñ7ð3ð Ð-Ñ.ð3ð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷3r?   r×  c                   ót  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 	 	 ddej                  deej                     dee   deej                     deej                     dee	   d	ee
   d
ee
   deej                     deej                     deej                  eeej                  ej                  f      f   fd„Zˆ xZS )ÚZamba2MambaDecoderLayerrh   rŒ   c                 ó    •— t         ‰| �  «        t        ||¬«      | _        t	        |j
                  |j                  ¬«      | _        || _        y )N)rh   rŒ   rÙ  )	r2   r3   rV  Úmambar_   r;   rÜ  rÝ  rŒ   )r:   rh   rŒ   r=   s      €r>   r3   z Zamba2MambaDecoderLayer.__init__)  s>   ø€ Ü‰ÑÔÜ%¨V¸yÔIˆŒ
Ü,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔØ"ˆ�r?   rO   rà  rð   rü   r,  r1  Ú	use_cacher§   Útransformer_hidden_statesrŽ   c                 óš   — |}|
�||
z   n|}| j                  |«      }| j                  |||¬«      }d}||z   }|f}|r||fz  }|r||fz  }|S )a÷  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence.
        N)rO   rw  rð   )rÝ  rè  )r:   rO   rà  rŒ   rð   rü   r,  r1  ré  r§   rê  rú   Úresidualrã  rä  s                  r>   rW   zZamba2MambaDecoderLayer.forward/  s’   € ð< !ˆð
 :SÐ9^ˆMÐ5Ò5Ðdqð 	ð ×,Ñ,¨]Ó;ˆàŸ
™
Ø'Ø'Ø)ð #ó 
ˆð !Ðð ! =Ñ0ˆà Ð"ˆáØÐ)Ð+Ñ+ˆGáØ˜Ð(Ñ(ˆGàˆr?   )	NNNNNFFNN)rY   rZ   r[   r&   rq   r3   r5   r¶   r   rg   rN  r·   r	   r¹   rW   r\   r]   s   @r>   ræ  ræ  (  s  ø„ ð#˜|ð #¸õ #ð :>Ø#'Ø15Ø.2Ø=AØ,1Ø$)Ø59Ø<@ñ:à—|‘|ð:ð !)¨¯©Ñ 6ð:ð ˜C‘=ð	:ð
 ! §¡Ñ.ð:ð ˜eŸl™lÑ+ð:ð !Ð!9Ñ:ð:ð $ D™>ð:ð ˜D‘>ð:ð ! ×!1Ñ!1Ñ2ð:ð $,¨E¯L©LÑ#9ð:ð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷:r?   ræ  c                   ól  ‡ — e Zd Zdedej
                  defˆ fd„Z	 	 	 	 	 	 	 	 ddej                  de
ej                     de
e   de
ej                     d	e
ej                     d
e
e   de
e   de
e   de
ej                     deej                   e
eej                   ej                   f      f   fd„Zˆ xZS )ÚZamba2HybridLayerÚshared_transformerÚlinearrè  c                 óL   •— t         ‰| �  «        || _        || _        || _        y r1   )r2   r3   rð  Úmamba_decoderrï  )r:   rï  rð  rè  r=   s       €r>   r3   zZamba2HybridLayer.__init__m  s'   ø€ ô 	‰ÑÔØˆŒØ"ˆÔØ"4ˆÕr?   rO   rà  rŒ   rð   rü   r,  r1  ré  r-  rŽ   c
           	      ó¾   — | j                  |||||||	¬«      }
|
d   }|r|
d   }| j                  |«      }| j                  |||||||	¬«      }
|r|
d   f|
dd z   }
|
S )aX  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )rà  rŒ   rð   r,  r1  r-  r   r%   )rê  rð   r,  r1  ré  r-  rB   N)rï  rð  rò  )r:   rO   rà  rŒ   rð   rü   r,  r1  ré  r-  Úlayer_outputsrê  rã  s                r>   rW   zZamba2HybridLayer.forwardu  s­   € ð@ ×/Ñ/ØØ#9ØØ&Ø)Ø/Ø 3ð 0ó 
ˆð %2°!Ñ$4Ð!áØ -¨aÑ 0Ðà$(§K¡KÐ0IÓ$JÐ!à×*Ñ*ØØ&?Ø)Ø)Ø/ØØ 3ð +ó 
ˆñ Ø*¨1Ñ-Ð/@ÐAÀMÐRSÐRTÐDUÑUˆMàÐr?   )NNNNNFFN)rY   rZ   r[   r×  r   r  ræ  r3   r5   r¶   r   rq   rg   rN  r·   r	   r¹   rW   r\   r]   s   @r>   rî  rî  l  s  ø„ ð5Ø"=ð5ØGIÇyÁyð5ØYpõ5ð :>Ø#'Ø15Ø.2Ø=AØ,1Ø$)Ø:>ñ>à—|‘|ð>ð !)¨¯©Ñ 6ð>ð ˜C‘=ð	>ð
 ! §¡Ñ.ð>ð ˜eŸl™lÑ+ð>ð !Ð!9Ñ:ð>ð $ D™>ð>ð ˜D‘>ð>ð & e×&6Ñ&6Ñ7ð>ð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷>r?   rî  c                   ó>   — e Zd ZeZdZdZddgZdZdZ	dZ
dZdZdZd„ Zy)ÚZamba2PreTrainedModelÚmodelTr×  ræ  r¢   c                 ó^  — | j                   j                  }t        |t        j                  t        j
                  f«      rY|j                  j                  j                  d|¬«       |j                  �%|j                  j                  j                  «        y y t        |t        j                  «      rf|j                  j                  j                  d|¬«       |j                  �2|j                  j                  |j                     j                  «        y y t        |t        «      �rwd|j                  _        d|j                   _        t#        j$                  t#        j&                  | j                   j(                  «      t+        j,                  | j                   j.                  «      t+        j,                  | j                   j0                  «      z
  z  t+        j,                  | j                   j0                  «      z   «      j3                  | j                   j4                  ¬«      }|t#        j,                  t#        j6                  | «       «      z   }t#        j8                  «       5  |j:                  j=                  |«       d d d «       d|j:                  _        y y # 1 sw Y   ŒxY w)Nr2  )rM   ÚstdT)Úmin) rh   Úinitializer_rangerØ   r   r  rh  r7   ÚdataÚnormal_r  r®   Ú	EmbeddingÚpadding_idxrV  ro  rp  rr  r5   rˆ  Úrandrx   Úmathrn  rf  re  r¬   Útime_step_floorÚexpm1rã   rl  rŠ  Ú
_no_reinit)r:   rì   rù  r’  Úinv_dts        r>   Ú_init_weightsz#Zamba2PreTrainedModel._init_weightsÂ  sÜ  € Ø�k‰k×+Ñ+ˆÜ�fœrŸy™y¬"¯)©)Ð4Ô5Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ä˜Ô 0Õ1Ø,0ˆF�L‰LÔ)Ø(,ˆF�H‰HÔ%ä—‘Ü—
‘
˜4Ÿ;™;×4Ñ4Ó5Ü—8‘8˜DŸK™K×5Ñ5Ó6¼¿¹À$Ç+Á+×B[ÑB[Ó9\Ñ\ñ^ä—(‘(˜4Ÿ;™;×4Ñ4Ó5ñ6ó÷ ‰e˜Ÿ™×3Ñ3ˆeÓ4ð	 ð œ%Ÿ)™)¤U§[¡[°"°Ó%5Ð$5Ó6Ñ6ˆFä—‘“ñ -Ø—‘×$Ñ$ VÔ,÷-à(,ˆF�N‰NÕ%ð 2÷-ð -ús   É,J#Ê#J,N)rY   rZ   r[   r&   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attn_2Ú_supports_flex_attnÚ_supports_sdpaÚ_supports_cache_classÚ_is_statefulr  rº   r?   r>   rö  rö  ¶  sF   „ Ø€LØÐØ&*Ð#Ø6Ð8QÐRÐØ"3ÐØ!ÐØÐØ€NØ ÐØ€Ló-r?   rö  aK  
    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`Zamba2Config`]):
            Model configuration class with all the parameters of the model. Initializing with a config file does not
            load the weights associated with the model, only the configuration. Check out the
            [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aí  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
            Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide
            it.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            If `past_key_values` is used, optionally only the last `input_ids` have to be input (see
            `past_key_values`).

            If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`]
            and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more
            information on the default strategy.

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.
        position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`.

            [What are position IDs?](../glossary#position-ids)
        past_key_values (`Zamba2HybridDynamicCache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`):
            A Zamba2HybridDynamicCache object containing pre-computed hidden-states (keys and values in the
            self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see
            `past_key_values` input) to speed up sequential decoding.
            Key and value cache tensors have shape `(batch_size, num_heads, seq_len, head_dim)`.
            Convolution and ssm states tensors have shape `(batch_size, d_inner, d_conv)` and
            `(batch_size, d_inner, d_state)` respectively.
            See the `Zamba2HybridDynamicCache` class for more details.

            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that
            don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all
            `input_ids` of shape `(batch_size, sequence_length)`.
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        use_cache (`bool`, *optional*):
            If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see
            `past_key_values`).
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
            Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
            this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
            the complete sequence length.
zTThe bare Zamba2 Model outputting raw hidden-states without any specific head on top.c                   óJ  ‡ — e Zd ZdZdefˆ fd„Zd„ Zd„ Z ee	«      	 	 	 	 	 	 	 	 	 	 dde
ej                     de
ej                     de
ej                     d	e
e   d
e
ej                     de
e   de
e   de
e   de
e   de
ej                     deeef   fd„«       Zd„ Zd„ Zˆ xZS )ÚZamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    rh   c                 ó¸  •— t         ‰| �  |«       || _        |j                  | _        |j
                  | _        t        j                  |j
                  |j                  | j                  «      | _	        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ }}g }g }|j                  | _        t        |j                  «      D ]¯  }|j                  |   dk(  r|j                  t!        ||¬«      «       Œ2|j                  |   dk(  sŒE|j                  t        j"                  | j                  j                  | j                  j                  d¬«      «       |j                  t!        ||¬«      «       Œ± t%        |«      }t%        |«      }t'        |«      }| j)                  |||«      }t        j*                  |«      | _        |j.                  | _        t1        |j                  |j2                  ¬«      | _        |j6                  r1|j8                  rt:        j=                  d«       t?        |«      | _         d| _!        | jE                  «        y c c}w )	N)r  rè  ©rŒ   rm   Fr  rÙ  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.)#r2   r3   rh   Úpad_token_idrÿ  Ú
vocab_sizer   rþ  r;   Úembed_tokensr   r"  r×  ro   r€   r„   ræ  r  Úiterr   Ú
get_layersr  Úlayersr4  r_   rÜ  Úfinal_layernormr3  Úuse_long_contextr5  r6  r¼   Ú
rotary_embÚgradient_checkpointingÚ	post_init)	r:   rh   r  ÚblocksÚmamba_layersÚlinear_layersrˆ   r  r=   s	           €r>   r3   zZamba2Model.__init__?  sì  ø€ Ü‰Ñ˜Ô ØˆŒØ!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜKPÐQW×QfÑQfÓKgÖhÀaÔ-¨f¸qÖAÐhˆÐhØˆØˆØ!'×!9Ñ!9ˆÔÜ�v×/Ñ/Ó0ò 	RˆAØ×'Ñ'¨Ñ*¨gÒ5Ø×#Ñ#Ô$;¸FÈaÔ$PÕQØ×)Ñ)¨!Ñ,°Ó8Ø×$Ñ$¤R§Y¡Y¨t¯{©{×/FÑ/FÈÏÉ×H_ÑH_ÐfkÔ%lÔmØ×#Ñ#Ô$;¸FÈaÔ$PÕQð	Rô ˜LÓ)ˆÜ˜]Ó+ˆÜ�v“ˆØ—‘ ¨¸ÓEˆÜ—m‘m FÓ+ˆŒà$*×$?Ñ$?ˆÔ!Ü,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔØ×ÒØ×&Ò&Ü×#Ñ#Ø{ôô 4°FÓ;ˆDŒOØ&+ˆÔ#ð 	�‰Õùò7 is   ÂIc                 ó   — | j                   S r1   ©r  rd   s    r>   Úget_input_embeddingsz Zamba2Model.get_input_embeddingsc  s   € Ø× Ñ Ð r?   c                 ó   — || _         y r1   r$  ©r:   rï   s     r>   Úset_input_embeddingsz Zamba2Model.set_input_embeddingsf  s
   € Ø!ˆÕr?   Ú	input_idsrð   rÞ   r¢   Úinputs_embedsré  r1  Úoutput_hidden_statesÚreturn_dictr§   rŽ   c                 ór  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|d u |d uz  rt        d«      ‚| j                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|}t        j                  |«      }|rO|€M|�|j                  d   n|j                  d   }t        | j                   || j                  | j                   ¬«      }|
€R|�|j#                  | j$                  ¬«      nd}t        j&                  |||j                  d   z   |j                   ¬«      }
|€|
j)                  d«      }| j+                  |||
«      }| j                   j,                  r| j/                  ||«      }nd }|rd	nd }|rd	nd }t1        | j2                  «      D ]r  \  }}|r||fz  }| j                  r1| j                  r%| j5                  |j6                  |||||||||«
      }n ||||||||||¬
«	      }|d   }|sŒd|d   €Œj||d   fz  }Œt | j9                  |«      }|r||fz  }|r|j:                  sd|_        t=        ||r|nd ||¬«      }|	r|S |j?                  «       S )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either onezX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fr   ©rE   rj   r  r%   rn   rº   )rà  rŒ   rð   rü   r,  r1  ré  r-  T)Úlast_hidden_stater¢   rO   Ú
attentions) rh   r1  r+  ré  Úuse_return_dictÚ
ValueErrorr  rõ   r5  r6  r  r5   r   rJ   rg   rE   rj   rœ   Úfirst_transformer_layer_idrm  r  Ú_update_causal_maskr3  r  r&  r  Ú_gradient_checkpointing_funcÚ__call__r  rp   r   Úto_tuple)r:   r)  rð   rÞ   r¢   r*  ré  r1  r+  r,  r§   rO   rà  ri   Úpast_seen_tokensrü   r-  Úall_hidden_statesÚall_self_attnsrŒ   Úlayerrô  rÕ  s                          r>   rW   zZamba2Model.forwardi  s  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	à%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà˜Ð -°tÐ";Ò<ÜØsóð ð ×&Ò&¨4¯=ª=¹YÜ×ÑØjôð ˆIàÐ Ø ×-Ñ-¨iÓ8ˆMà%ˆä!&§¡¨]Ó!;Ðñ ˜Ð0Ø/8Ð/D˜Ÿ™¨Ò+È-×J]ÑJ]Ð^_ÑJ`ˆJÜ6°t·{±{ÀJÐVZ×V`ÑV`Ðim×itÑitÔuˆOàÐ!ð #Ð.ð  ×.Ñ.¸×9XÑ9XÐ.ÔYàð ô
 #Ÿ\™\Ø Ð"2°]×5HÑ5HÈÑ5KÑ"KÐTa×ThÑThôˆNð ÐØ)×3Ñ3°AÓ6ˆLà×.Ñ.¨~¸}ÈnÓ]ˆð �;‰;×#Ò#Ø"&§/¡/°-ÀÓ"NÑà"&Ðá"6™B¸DÐÙ0™°dˆä )¨$¯+©+Ó 6ò "	:ÑˆI�uÙ#Ø! mÐ%5Ñ5Ð!à×*Ò*¨t¯}ª}Ø $× AÑ AØ—N‘NØ!Ø*ØØ"ØØ#Ø%ØØ'ó!‘ñ !&Ø!Ø+AØ'Ø#1Ø +Ø#2Ø&7Ø'Ø(;ô
!�ð *¨!Ñ,ˆMâ Ø  Ñ#Ñ/à" }°QÑ'7Ð&9Ñ9‘NðE"	:ðH ×,Ñ,¨]Ó;ˆñ  Ø -Ð!1Ñ1Ðá ?×#EÒ#EØ15ˆOÔ.ä(Ø+Ù/8™O¸dØ+Ø%ô	
ˆñ %ˆvÐ;¨&¯/©/Ó*;Ð;r?   c                 óª  — | j                   j                  dk(  r	|�d|v r|S y |j                  |j                  }}t	        j
                  |«      j                  }|j                  d   }|d   dz   }t	        j                  ||f|||¬«      }	|dk7  rt	        j                  |	d¬«      }	|	t	        j                  ||¬«      |j                  dd«      kD  z  }	|	d d d d …d d …f   j                  |j                  d   ddd«      }	|�‡|	j                  «       }	|j                  «       d	k(  rd|j                  d   }
|	d
d |
…f   j                  d«      |d d …d d d d …f   j                  d«      z  }|	d
d |
…f   j!                  ||«      |	d
d |
…f<   | j                   j                  dk(  r0|�.|j                  j"                  dv rt%        j&                  |	|«      }	|	S )NÚflash_attention_2r2  r%   rC   )Ú
fill_valuerE   rj   rJ  rn   r   rB   .r0  )rÇ  Úxpu)rh   r4  rE   rj   r5   Úfinforú  rJ   ÚfullÚtriurm  ræ   r×   r   r‘   ÚeqrO  rÀ   r   Ú_unmask_unattended)r:   rð   r<  r§   rE   rj   Ú	min_dtypeÚsequence_lengthÚtarget_lengthrü   Úmask_lengthÚpadding_masks               r>   r4  zZamba2Model._update_causal_maskâ  sð  € Ø�;‰;×+Ñ+Ð/BÒBØÐ)¨c°^Ñ.CØ%Ð%Øà$×*Ñ*¨L×,?Ñ,?ˆvˆÜ—K‘K Ó&×*Ñ*ˆ	Ø&×,Ñ,¨QÑ/ˆØ& rÑ*¨QÑ.ˆä—j‘j /°=Ð!AÈiÐ_dÐmsÔtˆØ˜aÒÜŸ*™* [¸1Ô=ˆKØ”u—|‘| M¸&ÔAÀN×DZÑDZÐ[]Ð_`ÓDaÑaÑaˆØ! $¨ªa²Ð"2Ñ3×:Ñ:¸<×;MÑ;MÈaÑ;PÐRSÐUWÐY[Ó\ˆØÐ%Ø%×+Ñ+Ó-ˆKØ×!Ñ!Ó# qÒ(Ø,×2Ñ2°2Ñ6�Ø*¨3°°°Ð+<Ñ=×@Ñ@ÀÓEÈÒWXÐZ^Ð`dÒfgÐWgÑHh×HkÑHkÐloÓHpÑp�Ø1<¸SÀ,À;À,Ð=NÑ1O×1[Ñ1[Ð\hÐjsÓ1t�˜C  + Ð-Ñ.ð �K‰K×,Ñ,°Ò6ØÐ*Ø×%Ñ%×*Ñ*¨oÑ=ô
 1×CÑCÀKÐQZÓ[ˆKàÐr?   c           
      óx  — g }g | _         d| _        t        | j                  «      D �]  \  }}|dk(  �rê| j                  dk(  r|| _        t	        |«      }| j
                  j                  t        | j
                  j                  «      z  dkD  �r_d|› d�}t        j                  |dz   dz   dz   d	z   d
z   «      }	| j                   j                  |	«       d}
| j                  D ]q  }|dk(  re|
| j
                  j                  z  |j                  k(  r?t        j                  dt        |
«      z   dz   «      }| j                   j                  |«       |
dz  }
Œs | j
                  j                  r‚d}
| j                  D ]q  }|dk(  re|
| j
                  j                  z  |j                  k(  r?t        j                  dt        |
«      z   dz   «      }| j                   j                  |«       |
dz  }
Œs |j                  t        |t	        |«      t	        |«      «      «       �Œ÷|j                  t	        |«      «       �Œ |S )Nr   rm   r%   z	^layers\.z\.shared_transformer\.z(?:z3self_attn\.(?:q_proj|k_proj|v_proj|o_proj)\.weight|z1feed_forward\.(?:gate_up_proj|down_proj)\.weight|z,(?:input_layernorm|pre_ff_layernorm)\.weightz)$z>^shared_transformer\.feed_forward\.gate_up_proj_adapter_list\.z\.(?:0|1)\.weight$zg^shared_transformer\.self_attn\.(?:linear_q_adapter_list|linear_k_adapter_list|linear_v_adapter_list)\.)Ú_tied_weights_keysr3  r&  ro   Únextrh   r"  r–   r  ÚreÚcompiler„   r  rµ   r  rî  )r:   r   r"  r!  r  Úlayer_idÚ
layer_typeÚblockÚprefix_patternÚmain_keys_patternÚ
adapter_idÚ_layer_typeÚadapter_patternÚattn_adapter_patterns                 r>   r  zZamba2Model.get_layers  sH  € ØˆØ"$ˆÔØ*+ˆÔ'Ü$-¨d×.DÑ.DÓ$Eó )	2Ñ ˆH�jØ˜XÓ%Ø×2Ñ2°aÒ7Ø6>�DÔ3Ü˜V›�Ø—;‘;×-Ñ-´°D·K±K×4PÑ4PÓ0QÑQÐTUÓUØ(1°(°Ð;QÐ%R�NÜ(*¯
©
Ø&Ø ñ!àPñQð OñOð Jñ	Jð
  ñ ó)Ð%ð ×+Ñ+×2Ñ2Ð3DÔEà!"�JØ'+×'=Ñ'=ò (˜Ø&¨(Ò2°zÀDÇKÁK×D^ÑD^Ñ7^Ðbg×bpÑbpÒ7pÜ.0¯j©jØ aÜ"% j£/ñ!2à"7ñ!8ó/˜Oð
 !×3Ñ3×:Ñ:¸?ÔKØ" a™™
ð(ð —{‘{×?Ò?Ø%&˜
Ø+/×+AÑ+Aò 	,˜KØ*¨hÒ6¸:ÈÏÉ×HbÑHbÑ;bÐfk×ftÑftÒ;tÜ79·z±zð%qä&)¨*£oñ%6ð '<ñ%<ó8"Ð 4ð !%× 7Ñ 7× >Ñ >Ð?SÔ TØ&¨!™O™Jð	,ð —‘Ô/°´t¸MÓ7JÌDÐQ]ÓL^Ó_Ö`à—‘œd <Ó0Ö1ðS)	2ðT ˆr?   ©
NNNNNNNNNN)rY   rZ   r[   r³   r&   r3   r%  r(  r   ÚZAMBA2_INPUTS_DOCSTRINGr   r5   r·   r¶   rg   r¹   rN  r
   r	   r   rW   r4  r  r\   r]   s   @r>   r  r  3  s5  ø„ ñ
ð"˜|õ "òH!ò"ñ +Ð+BÓCð 15Ø15Ø37Ø>BØ59Ø$(Ø,0Ø/3Ø&*Ø59ñv<à˜E×,Ñ,Ñ-ðv<ð ! §¡Ñ.ðv<ð ˜u×/Ñ/Ñ0ð	v<ð
 "Ð":Ñ;ðv<ð   × 1Ñ 1Ñ2ðv<ð ˜D‘>ðv<ð $ D™>ðv<ð ' t™nðv<ð ˜d‘^ðv<ð ! ×!1Ñ!1Ñ2ðv<ð 
ˆuÐ-Ð-Ñ	.òv<ó Dðv<òp!öF.r?   r  c                    óà  ‡ — e Zd Zdefˆ fd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d„ Z
 ed	d
d¬«       ee«       eee¬«      	 	 	 	 	 	 	 	 	 	 	 	 ddeej&                     deej(                     deej&                     dee   deej,                     deej&                     dee   dee   dee   dee   deej&                     deeej(                  f   deeef   fd„«       «       «       Z	 	 	 	 	 	 dd„Zˆ xZS )ÚZamba2ForCausalLMrh   c                 ó$  •— t         ‰| �  |«       t        |«      | _        dg| j                  j                  ¢| _        |j
                  | _        t        j                  |j                  |j
                  d¬«      | _	        | j                  «        y )Nzlm_head.weightFr  )r2   r3   r  r÷  rK  r  r   r  r;   Úlm_headr  ©r:   rh   r=   s     €r>   r3   zZamba2ForCausalLM.__init__8  so   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø#3Ð"T°d·j±j×6SÑ6SÐ"TˆÔØ ×+Ñ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒð 	�‰Õr?   c                 ó.   — | j                   j                  S r1   ©r÷  r  rd   s    r>   r%  z&Zamba2ForCausalLM.get_input_embeddingsB  ó   € Ø�z‰z×&Ñ&Ð&r?   c                 ó&   — || j                   _        y r1   r`  r'  s     r>   r(  z&Zamba2ForCausalLM.set_input_embeddingsE  ó   € Ø"'ˆ�
‰
Õr?   c                 ó   — | j                   S r1   ©r]  rd   s    r>   Úget_output_embeddingsz'Zamba2ForCausalLM.get_output_embeddingsH  s   € Ø�|‰|Ðr?   c                 ó   — || _         y r1   re  )r:   Únew_embeddingss     r>   Úset_output_embeddingsz'Zamba2ForCausalLM.set_output_embeddingsK  s	   € Ø%ˆ�r?   c                 ó   — || _         y r1   ©r÷  )r:   Údecoders     r>   Úset_decoderzZamba2ForCausalLM.set_decoderN  s	   € Øˆ�
r?   c                 ó   — | j                   S r1   rk  rd   s    r>   Úget_decoderzZamba2ForCausalLM.get_decoderQ  s   € Ø�z‰zÐr?   Únum_logits_to_keepz4.50Úlogits_to_keep)ÚversionÚnew_name)Úoutput_typer  r)  rð   rÞ   r¢   r*  Úlabelsré  r1  r+  r,  r§   rŽ   c                 ó  — |�|n| j                   j                  }|	�|	n| j                   j                  }	|
�|
n| j                   j                  }
| j	                  ||||||||	||
¬«
      }|d   }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }d}|� | j                  ||| j                  fi |¤Ž}|
s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  |j                  ¬«      S )a"  
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

            logits_to_keep (`int` or `torch.Tensor`, *optional*):
                If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all
                `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
                token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
                If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.
                This is useful when using packed tensor format (single dimension for batch and sequence length).

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Zamba2ForCausalLM

        >>> model = Zamba2ForCausalLM.from_pretrained("Zyphra/Zamba2-7B-v1")
        >>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba2-7B-v1")

        >>> prompt = "Hey, are you conscious? Can you talk to me?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
        ```N)
r)  rð   rÞ   r¢   r*  ré  r1  r+  r§   r,  r   r%   ©ÚlossÚlogitsr¢   rO   r0  )rh   r1  r+  r1  r÷  rØ   rq   Úslicer]  Úloss_functionr  r   r¢   rO   r0  )r:   r)  rð   rÞ   r¢   r*  ru  ré  r1  r+  r,  r§   rq  Úloss_kwargsrä  rO   Úslice_indicesry  rx  rÕ  s                       r>   rW   zZamba2ForCausalLM.forwardT  sL  € ðf 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐð %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆð —*‘*ØØ)Ø%Ø+Ø'ØØ/Ø!5Ø)Ø#ð ó 
ˆð   ™
ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑUÈÑUˆDáØ�Y ¨¨ Ñ,ˆFØ'+Ð'7�D�7˜VÑ#ÐC¸VÐCä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r?   c           	      ót  — |d u }	|	sZ|€|d   |j                   d   k\  r|d d …|j                   d    d …f   }nc|j                   d   |j                   d   k7  rD|d d …|f   }n:t        | j                  |j                   d   | j                  | j                  ¬«      }|�T|€R|j                  «       j                  d«      dz
  }|j                  |dk(  d«       |	s|d d …|j                   d    d …f   }|�|	rd|i}
nd|j                  «       i}
|
j                  ||||| j                  j                  |dœ«       |
S )NrC   r%   r   r.  r*  r)  )rÞ   r¢   ré  rð   rq  r§   )rJ   rg   rh   rE   rj   ÚlongrP  Úmasked_fill_rù   r“   rp  )r:   r)  r¢   rð   r*  r§   rÞ   ré  rú   Úempty_past_kvÚmodel_inputss              r>   Úprepare_inputs_for_generationz/Zamba2ForCausalLM.prepare_inputs_for_generation±  sc  € ð (¨4Ð/ˆñ ð Ð)Ø! "Ñ%¨¯©¸Ñ);Ò;à%¢a¨.×*>Ñ*>¸qÑ*AÐ)AÑ)CÐ&CÑD‘	Ø—‘ Ñ# ~×';Ñ';¸AÑ'>Ò>Ø%¢a¨Ð&7Ñ8‘	ä6Ø—‘˜YŸ_™_¨QÑ/°t·z±zÈ$Ï+É+ôˆOð Ð%¨,Ð*>à)×.Ñ.Ó0×7Ñ7¸Ó;¸aÑ?ˆLØ×%Ñ% n¸Ñ&9¸1Ô=Ù Ø+ªA°	·±ÀÑ0BÐ/BÑ/DÐ,DÑE�ð Ð$©Ø+¨]Ð;‰Là'¨×)=Ñ)=Ó)?Ð@ˆLà×Ñà ,Ø#2Ø&Ø"0Ø"&§+¡+×"@Ñ"@Ø"0ñô		
ð Ðr?   )NNNNNNNNNNNr   )NNNNNT)rY   rZ   r[   r&   r3   r%  r(  rf  ri  rm  ro  r"   r   rY  r!   r   Ú_CONFIG_FOR_DOCr   r5   r·   r¶   rg   r¹   rN  r
   rq   r	   rW   rƒ  r\   r]   s   @r>   r[  r[  7  s¬  ø„ ð˜|õ ò'ò(òò&òòñ Ð)°6ÐDTÔUÙ*Ð+BÓCÙÐ+AÐP_Ô`ð 15Ø15Ø37Ø>BØ59Ø-1Ø$(Ø,0Ø/3Ø&*Ø59Ø34ñX
à˜E×,Ñ,Ñ-ðX
ð ! §¡Ñ.ðX
ð ˜u×/Ñ/Ñ0ð	X
ð
 "Ð":Ñ;ðX
ð   × 1Ñ 1Ñ2ðX
ð ˜×)Ñ)Ñ*ðX
ð ˜D‘>ðX
ð $ D™>ðX
ð ' t™nðX
ð ˜d‘^ðX
ð ! ×!1Ñ!1Ñ2ðX
ð ˜c 5§<¡<Ð/Ñ0ðX
ð 
ˆuÐ,Ð,Ñ	-òX
ó aó Dó VðX
ðz ØØØØØ÷9r?   r[  aÌ  
    The Zamba2 Model with a sequence classification head on top (linear layer).

    [`Zamba2ForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT-2) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   óX  ‡ — e Zd Zˆ fd„Zd„ Zd„ Z ee«      	 	 	 	 	 	 	 	 	 	 ddee	j                     dee	j                     dee	j                     deeeee	j                     f      dee	j                     d	ee	j                     d
ee   dee   dee   dee   deeef   fd„«       Zˆ xZS )ÚZamba2ForSequenceClassificationc                 ó  •— t         ‰| �  |«       |j                  | _        t        |«      | _        | j                  j
                  | _        t        j                  |j                  | j                  d¬«      | _	        | j                  «        y )NFr  )r2   r3   Ú
num_labelsr  r÷  rK  r   r  r;   Úscorer  r^  s     €r>   r3   z(Zamba2ForSequenceClassification.__init__ý  se   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ  Ó(ˆŒ
Ø"&§*¡*×"?Ñ"?ˆÔÜ—Y‘Y˜v×1Ñ1°4·?±?ÈÔOˆŒ
ð 	�‰Õr?   c                 ó.   — | j                   j                  S r1   r`  rd   s    r>   r%  z4Zamba2ForSequenceClassification.get_input_embeddings  ra  r?   c                 ó&   — || j                   _        y r1   r`  r'  s     r>   r(  z4Zamba2ForSequenceClassification.set_input_embeddings
  rc  r?   r)  rð   rÞ   r¢   r*  ru  ré  r1  r+  r,  rŽ   c                 óü  — |
�|
n| j                   j                  }
| j                  ||||||||	|
¬«	      }|d   }| j                  |«      }|�|j                  d   }n|j                  d   }| j                   j
                  €|dk7  rt        d«      ‚| j                   j
                  €d}nÃ|�“|| j                   j
                  k7  j                  |j                  t        j                  «      }t        j                  |j                  d   |j                  t        j                  ¬«      }||z  j                  d«      }n.d}t        j                  | j                  j                   › d�«       |t        j                  ||j                  ¬	«      |f   }d}|��¢|j                  |j                  «      }| j                   j"                  €�| j$                  dk(  rd
| j                   _        nl| j$                  dkD  rL|j&                  t        j(                  k(  s|j&                  t        j*                  k(  rd| j                   _        nd| j                   _        | j                   j"                  d
k(  rIt-        «       }| j$                  dk(  r& ||j/                  «       |j/                  «       «      }nŒ |||«      }n‚| j                   j"                  dk(  r=t1        «       } ||j3                  d| j$                  «      |j3                  d«      «      }n,| j                   j"                  dk(  rt5        «       } |||«      }|
s|f|dd z   }|�|f|z   S |S t7        |||j8                  |j:                  |j<                  ¬«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)rð   rÞ   r¢   r*  ré  r1  r+  r,  r   r%   z=Cannot handle batch sizes > 1 if no padding token is defined.rC   rl   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rn   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrw  )rh   r1  r÷  r‰  rJ   r  r2  rF   rj   r5   Úint32rm  Úargmaxr5  r6  r=   rY   Úproblem_typerˆ  rE   r  rq   r   r†  r   rK   r   r   r¢   rO   r0  )r:   r)  rð   rÞ   r¢   r*  ru  ré  r1  r+  r,  Útransformer_outputsrO   ry  ri   Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsrx  Úloss_fctrÕ  s                         r>   rW   z'Zamba2ForSequenceClassification.forward  s  € ð( &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà"Ÿj™jØØ)Ø%Ø+Ø'ØØ/Ø!5Ø#ð )ó 

Ðð ,¨AÑ.ˆØ—‘˜MÓ*ˆàÐ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×#Ñ#Ð+°
¸a²ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ð+Ø!#ÑØÐ"à%¨¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸL™L¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÑØ—Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù × 2Ñ 2°2°t·±Ó GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨vÓ6�ÙØ#Ð%Ð(;¸A¸BÐ(?Ñ?ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r?   rX  )rY   rZ   r[   r3   r%  r(  r   rY  r   r5   r·   r¶   r
   r   r   r¹   rN  r	   r   rW   r\   r]   s   @r>   r†  r†  í  s0  ø„ ô ò'ò(ñ +Ð+BÓCð 15Ø15Ø37ØKOØ59Ø-1Ø$(Ø,0Ø/3Ø&*ñ[
à˜E×,Ñ,Ñ-ð[
ð ! §¡Ñ.ð[
ð ˜u×/Ñ/Ñ0ð	[
ð
 " %¨¨t°E×4EÑ4EÑ/FÐ(FÑ"GÑHð[
ð   × 1Ñ 1Ñ2ð[
ð ˜×)Ñ)Ñ*ð[
ð ˜D‘>ð[
ð $ D™>ð[
ð ' t™nð[
ð ˜d‘^ð[
ð 
ˆuÐ6Ð6Ñ	7ò[
ó Dô[
r?   r†  )r[  r†  r  rö  )r2  )Nr%   )_r  rM  Ú	itertoolsr   Útypingr   r   r   r   r   r	   r
   r5   r   Útorch.nnr   r   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úmodeling_attn_mask_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   r   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r    r!   Úutils.deprecationr"   Úutils.import_utilsr#   r$   Úconfiguration_zamba2r&   Ú+mamba_ssm.ops.triton.selective_state_updater'   Ú!mamba_ssm.ops.triton.ssd_combinedr(   r)   Úcausal_conv1dr+   r,   Ú
get_loggerrY   r5  r„  ÚModuler/   r_   rg   r¼   r¶   rq   rë   rÖ   rþ   r  r
  r  rD  rG  rT  r‰  rt  rV  rÉ  r×  ræ  rî  rö  ÚZAMBA2_START_DOCSTRINGrY  r  r[  r†  Ú__all__rº   r?   r>   ú<module>r°     sÄ  ðó, Û 	Ý ß D× DÑ Dã Ý ß AÑ Aå !ß .Ý )Ý >Ý Bß qÑ qß Kß FÝ &÷ó õ 1ß TÝ .ñ ÔÝRßmÐmàZjÑWÐÐ5Ð7WáÔßDÐDà-7Ñ*ÐÐ*ð 
ˆ×	Ñ	˜HÓ	%€ð '€ô;˜Ÿ™Ÿ™ô ;ô*J�B—I‘Iô Jô(i ˜|ô i ôX%<˜BŸI™Iô %<ðP	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 ˜UŸ\™\Ñ*ð%ð ð%ð ó%ò4(óô6J)�b—i‘iô J)ð`V U§\¡\ð V¸Só Vò
ò(ñ( Ð4Ð6FÐH\Ð]Ó^Ð ôkO�r—y‘yô kOô\'�—	‘	ô 'ôT= "§)¡)ô =ô@A˜bŸi™iô AôHG˜Ÿ	™	ô GôT$-˜Oô $-ðNÐ ð"BÐ ñJ ØZØóô|Ð'ó |ó	ð|ô@sÐ-¨ô sñl ðð óôm
Ð&;ó m
óðm
ò` k�r?   