Ë
    T^(h¾Þ  ã                   ó>  — d dl Z d dlZd dlmZ d dlmZmZmZmZ d dl	Z	d dl
Z	d dl	mZ ddlmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZmZ ddlmZmZ ddlm Z m!Z!m"Z" ddl#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z- ddl.m/Z/  e«       rd dl0m1Z1 d dl2m3Z3m4Z4 nd\  Z1Z3Z4 e«       r	d dl5m6Z6m7Z7 nd\  Z7Z6 e8e1e6e7f«      Z9dZ: ejv                  e<«      Z= G d„ de	j                  j|                  «      Z? G d„ de,«      Z@ G d„ de(«      ZA G d„ d e«      ZB G d!„ d"e$«      ZC G d#„ d$ej|                  «      ZD G d%„ d&ej|                  «      ZE G d'„ d(e%«      ZF G d)„ d*e*«      ZG G d+„ d,e)«      ZH G d-„ d.e«      ZI G d/„ d0e+eI«      ZJ G d1„ d2e&«      ZK G d3„ d4e'«      ZLg d5¢ZMy)6é    N)Úcycle)ÚCallableÚOptionalÚTupleÚUnion)Únné   )ÚACT2FN)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úlogging)Úis_causal_conv1d_availableÚis_mamba_ssm_availableé   )ÚLlamaRotaryEmbeddingÚapply_rotary_pos_emb)Úpad_tensor_by_sizeÚreshape_into_chunksÚsegment_sum)
ÚZambaAttentionÚZambaAttentionDecoderLayerÚZambaForCausalLMÚZambaForSequenceClassificationÚZambaHybridDynamicCacheÚZambaHybridLayerÚZambaMambaDecoderLayerÚ
ZambaModelÚZambaRMSNormÚeager_attention_forwardé   )ÚZamba2Config)Úselective_state_update)Úmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combined©NNN)Úcausal_conv1d_fnÚcausal_conv1d_update©NNzZyphra/Zamba2-2.7Bc                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚZamba2RMSNormGatedc                 ó˜   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        || _        y ©N)	ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilonÚ
group_size)ÚselfÚhidden_sizer7   ÚepsÚ	__class__s       €úg/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/zamba2/modular_zamba2.pyr1   zZamba2RMSNormGated.__init__J   s6   ø€ Ü‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÔØ$ˆ�ó    c                 ób  — |j                   }|j                  t        j                  «      }|�?|t        j
                  j                  |j                  t        j                  «      «      z  }|j                  �^ }}|| j                  z  } |j                  g |¢|‘| j                  ‘­Ž }|j                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  } |j                  g |¢|| j                  z  ‘­Ž }| j                  |j                  |«      z  S )Nr   éÿÿÿÿT)Úkeepdim)ÚdtypeÚtor3   Úfloat32r   Ú
functionalÚsiluÚshaper7   ÚviewÚpowÚmeanÚrsqrtr6   r5   )	r8   Úhidden_statesÚgateÚinput_dtypeÚprefix_dimsÚlast_dimÚgroup_countÚhidden_states_groupÚvariances	            r<   ÚforwardzZamba2RMSNormGated.forwardP   s  € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØÐØ)¬B¯M©M×,>Ñ,>¸t¿w¹wÄuÇ}Á}Ó?UÓ,VÑVˆMØ!.×!4Ñ!4Ñˆ�hØ $§/¡/Ñ1ˆØ0˜m×0Ñ0Ð\°+Ð\¸{Ð\ÈDÏOÉOÒ\ÐØ&×*Ñ*¨1Ó-×2Ñ2°2¸tÐ2ÓDˆØ1´E·K±KÀÈ4×K`ÑK`Ñ@`Ó4aÑaÐØ0Ð+×0Ñ0Ð]°+Ð]¸{ÈTÏ_É_Ñ?\Ò]ˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r=   )g�íµ ÷Æ°>r/   )Ú__name__Ú
__module__Ú__qualname__r1   rS   Ú__classcell__©r;   s   @r<   r-   r-   I   s   ø„ õ%÷;r=   r-   c                   ó   — e Zd Zy)ÚZamba2RMSNormN©rT   rU   rV   © r=   r<   rZ   rZ   ^   ó   „ Ør=   rZ   c            
       óÎ   — e Zd ZdZej
                  dfdededej                  de	e
   fd„Zded	ej                  d
ej                  dej                  fd„Zd„ Zdde	e   defd„Zy)ÚZamba2HybridDynamicCachea¤  
    A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache
    (which has a constant shape regardless of seq_len).

    This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states`
    and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor
    For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`,
    while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors).
    For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors),
    while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`,
    and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`.
    NÚconfigÚ
batch_sizerA   Údevicec           	      ó.  — || _         |j                  | _        d| _        t        |j                  |j
                  z  «      | _        |j                  | _        |j                  | _
        |j                  | _        g | _        i | _        i | _        i | _        i | _        i | _        t%        |j&                  «      D ]Î  }t)        j*                  || j                  d|j,                  z  |j                  z  z   | j                  ||¬«      | j                   |<   t)        j*                  || j                  |j.                  | j                  ||¬«      | j"                  |<   | j                  |   dk(  sŒ´| j                  j1                  |«       ŒÐ t%        |j&                  «      D �cg c]  }t)        j2                  g g|z  |¬«      ‘Œ c}| _        t%        |j&                  «      D �cg c]  }t)        j2                  g g|z  |¬«      ‘Œ c}| _        y c c}w c c}w )NFr   ©rb   rA   Úhybrid©rb   )rA   Úlayers_block_typeÚhas_previous_stateÚintÚmamba_expandr9   Úintermediate_sizeÚmamba_d_stateÚssm_state_sizeÚmamba_d_convÚconv_kernel_sizeÚn_mamba_headsÚtransformer_layersÚ_modulesÚ_parametersÚ_buffersÚconv_statesÚ
ssm_statesÚrangeÚnum_hidden_layersr3   ÚzerosÚmamba_ngroupsÚmamba_headdimÚappendÚtensorÚ	key_cacheÚvalue_cache)r8   r`   ra   rA   rb   ÚiÚ_s          r<   r1   z!Zamba2HybridDynamicCache.__init__p   sÐ  € ð ˆŒ
Ø!'×!9Ñ!9ˆÔØ"'ˆÔÜ!$ V×%8Ñ%8¸6×;MÑ;MÑ%MÓ!NˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔØ#×1Ñ1ˆÔØ"$ˆÔØˆŒØˆÔØˆŒØˆÔØˆŒÜ�v×/Ñ/Ó0ò 	2ˆAÜ"'§+¡+ØØ×&Ñ&¨¨V×-AÑ-AÑ)AÀF×DXÑDXÑ)XÑXØ×%Ñ%ØØô#ˆD×Ñ˜QÑô "'§¡Ø˜D×.Ñ.°×0DÑ0DÀd×FYÑFYÐbhÐpuô"ˆD�O‰O˜AÑð ×%Ñ% aÑ(¨HÓ4Ø×'Ñ'×.Ñ.¨qÕ1ð	2ô SXÐX^×XpÑXpÓRqÖrÈQœ%Ÿ,™,¨ t¨jÑ'8ÀÖHÒrˆŒÜTYÐZ`×ZrÑZrÓTsÖtÈqœEŸL™L¨"¨°
Ñ):À6ÖJÒtˆÕùò sùÚts   Æ!"HÇ""HÚ	layer_idxÚnew_conv_stateÚcache_positionÚreturnc                 óT  — | j                   |   }|j                  d| j                  dz
  «      }|j                  dd¬«      }|j	                  |j
                  «      |d d …d d …|f<   | j                   |   j                  «        | j                   |xx   |z  cc<   | j                   |   S )Nr   r#   r?   ©ÚshiftsÚdims)ru   Úclampro   ÚrollrB   rb   Úzero_)r8   r‚   rƒ   r„   Ú
conv_states        r<   Úupdate_conv_statez*Zamba2HybridDynamicCache.update_conv_state�   s£   € ð ×%Ñ% iÑ0ˆ
Ø'×-Ñ-¨a°×1FÑ1FÈÑ1JÓKˆà—_‘_¨B°R�_Ó8ˆ
Ø+9×+<Ñ+<¸Z×=NÑ=NÓ+Oˆ
’1’a˜Ð'Ñ(Ø×Ñ˜Ñ#×)Ñ)Ô+Ø×Ñ˜Ó# zÑ1Ó#Ø×Ñ 	Ñ*Ð*r=   c                 ól   — | j                   j                  «        | j                  j                  «        y r/   )ru   rŒ   rv   )r8   s    r<   ÚresetzZamba2HybridDynamicCache.resetœ   s$   € Ø×Ñ×ÑÔ Ø�‰×ÑÕr=   c                 óê   — || j                   vr| j                   d   n|}t        | j                  «      |k  s | j                  |   j                  «       dk(  ry| j                  |   j                  d   S )zYReturns the sequence length of the cached states. A layer index can be optionally passed.r   éþÿÿÿ)rq   Úlenr~   ÚnumelrF   )r8   r‚   s     r<   Úget_seq_lengthz'Zamba2HybridDynamicCache.get_seq_length    sl   € ð 3<À4×CZÑCZÑ2Z�D×+Ñ+¨AÒ.Ð`iˆ	Üˆt�~‰~Ó )Ò+¨t¯~©~¸iÑ/H×/NÑ/NÓ/PÐTUÒ/UØØ�~‰~˜iÑ(×.Ñ.¨rÑ2Ð2r=   )r   )rT   rU   rV   Ú__doc__r3   Úfloat16r$   ri   rA   r   Ústrr1   ÚTensorÚ
LongTensorrŽ   r�   r•   r\   r=   r<   r_   r_   b   s™   „ ñð KPÏ-É-ÐquñuØ"ðuØ03ðuØ<A¿K¹KðuØaiÐjmÑanóuð@
+Øð
+Ø.3¯l©lð
+ØLQ×L\ÑL\ð
+à	�‰ó
+ò ñ3¨°©ð 3¸cô 3r=   r_   c                   ó(   ‡ — e Zd Z	 ddefˆ fd„Zˆ xZS )ÚZamba2RotaryEmbeddingr`   c                 ó†   •— t         ‰| �  ||«       | j                  ||j                  |j                  ¬«      \  }| _        y )N)rb   ÚbaseÚdim)r0   r1   Úrope_init_fnÚ
rope_thetaÚattention_head_dimÚattention_scaling)r8   r`   rb   Úinv_freqr;   s       €r<   r1   zZamba2RotaryEmbedding.__init__ª   sD   ø€ ô
 	‰Ñ˜ Ô(à+/×+<Ñ+<Ø × 1Ñ 1°v×7PÑ7Pð ,=ó ,
Ñ(ˆ�$Õ(r=   r/   )rT   rU   rV   r$   r1   rW   rX   s   @r<   rœ   rœ   ©   s   ø„ ð ñ	
à÷	
ñ 	
r=   rœ   c                   óF  ‡ — e Zd ZdZ	 	 	 ddedee   dee   dee   fˆ fd„Z	 	 	 ddej                  dedeej                     d	ee
   d
eeej                  ej                  f      dee   deej                  eej                     eeej                        f   fd„Zˆ xZS )ÚZamba2AttentionaJ  
    Multi-headed attention from 'Attention Is All You Need' paper.

    Adapted from transformers.models.mistral.modeling_mistral.MistralAttention:
    The input dimension here is attention_hidden_size = 2 * hidden_size, and head_dim = attention_hidden_size // num_heads.
    The extra factor of 2 comes from the input being the concatenation of original_hidden_states with the output of the previous (mamba) layer
    (see fig. 2 in https://arxiv.org/pdf/2405.16712).
    Additionally, replaced
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim) with
    attn_weights = torch.matmul(query_states, key_states.transpose(2, 3)) / math.sqrt(self.head_dim/2)
    Finally, this attention layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this
    layer is tied, un-tied adapters (formally the same as LoRA but used in the base model) modules are added to the q, k, v projectors to increase
    expressivity with a small memory overhead (see Fig. 2 of https://arxiv.org/pdf/2411.15242).
    r`   r‚   Únum_fwd_mem_blocksÚblock_idc           	      óð  •— t         ‰| �  ||«       || _        |j                  | _        || _        |j                  �r…t        j                  g «      | _	        t        j                  g «      | _
        t        j                  g «      | _        t        | j                  «      D �]  }||j                  z  |k(  �r{t        j                  t        j                  | j                   | j"                  j$                  d¬«      t        j                  | j"                  j$                  | j                   d¬«      «      }t        j                  t        j                  | j                   | j"                  j$                  d¬«      t        j                  | j"                  j$                  | j                   d¬«      «      }t        j                  t        j                  | j                   | j"                  j$                  d¬«      t        j                  | j"                  j$                  | j                   d¬«      «      }n<t        j&                  «       }t        j&                  «       }t        j&                  «       }| j                  j)                  |«       | j                  j)                  |«       | j                  j)                  |«       �Œ! t+        | j                  «      D �	�
ci c]  \  }	}
|
|	“Œ
 c}
}	| _        y c c}
}	w )NF©Úbias)r0   r1   r§   Úhybrid_layer_idsÚlayer_block_mapr¨   Úuse_shared_attention_adapterr   Ú
ModuleListÚlinear_q_adapter_listÚlinear_k_adapter_listÚlinear_v_adapter_listrw   Únum_mem_blocksÚ
SequentialÚLinearÚattention_hidden_sizer`   Úadapter_rankÚIdentityr|   Ú	enumerateÚ	layer_dic)r8   r`   r‚   r§   r¨   r€   Úlinear_q_adapterÚlinear_k_adapterÚlinear_v_adapterÚindexÚvaluer;   s              €r<   r1   zZamba2Attention.__init__Æ   s#  ø€ ô 	‰Ñ˜ Ô+Ø"4ˆÔØ%×6Ñ6ˆÔØ ˆŒà×.Ó.Ü)+¯©°rÓ):ˆDÔ&Ü)+¯©°rÓ):ˆDÔ&Ü)+¯©°rÓ):ˆDÔ&ä˜4×2Ñ2Ó3ó D�Ø�v×,Ñ,Ñ,°Ó8Ü')§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ð$ô (*§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ð$ô (*§}¡}ÜŸ	™	 $×"<Ñ"<¸d¿k¹k×>VÑ>VÐ]bÔcÜŸ	™	 $§+¡+×":Ñ":¸D×<VÑ<VÐ]bÔcó(Ñ$ô
 (*§{¡{£}Ð$Ü')§{¡{£}Ð$Ü')§{¡{£}Ð$Ø×*Ñ*×1Ñ1Ð2BÔCØ×*Ñ*×1Ñ1Ð2BÔCØ×*Ñ*×1Ñ1Ð2BÖCð)Dô, <EÀT×EYÑEYÓ;Z×[©<¨5°%˜% ™,Ó[ˆ�ùÓ[s   ËK2rK   Úattention_maskÚpast_key_valueÚposition_embeddingsÚkwargsr…   c                 ó¦  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      }	| j                  |«      }
| j	                  |«      }| j
                  j                  rW| j                  |   }|	 | j                  |   |«      z   }	|
 | j                  |   |«      z   }
| | j                  |   |«      z   }|	j                  |«      j                  dd«      }	|
j                  |«      j                  dd«      }
|j                  |«      j                  dd«      }| j
                  j                  r|\  }}t        |	|
||«      \  }	}
|�|j                  |
||«      \  }
}t         }| j
                  j"                  dk7  r^| j
                  j"                  dk(  r(|j%                  dd«      rt&        j)                  d«       nt*        | j
                  j"                     } || |	|
||f| j,                  sd	n| j.                  | j0                  d
œ|¤Ž\  }} |j2                  g |¢d‘­Ž j5                  «       }| j7                  |«      }||fS )Nr?   r#   r   ÚeagerÚsdpaÚoutput_attentionsFzã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.ç        )ÚdropoutÚscaling)rF   Úhead_dimÚq_projÚk_projÚv_projr`   r®   rº   r°   r±   r²   rG   Ú	transposeÚuse_mem_roper   Úupdater"   Ú_attn_implementationÚgetÚloggerÚwarning_oncer   ÚtrainingÚattention_dropoutrÊ   ÚreshapeÚ
contiguousÚo_proj)r8   rK   r‚   rÀ   rÁ   rÂ   rÃ   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚadapter_layer_idxÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                     r<   rS   zZamba2Attention.forwardï   s\  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1ˆØ—[‘[ Ó/ˆ
Ø—{‘{ =Ó1ˆØ�;‰;×3Ò3Ø $§¡¨yÑ 9ÐØ'Ð*W¨$×*DÑ*DÐEVÑ*WÐXeÓ*fÑfˆLØ#Ð&S d×&@Ñ&@ÐARÑ&SÐTaÓ&bÑbˆJØ'Ð*W¨$×*DÑ*DÐEVÑ*WÐXeÓ*fÑfˆLà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà�;‰;×#Ò#Ø*‰HˆC�Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ%Ø'5×'<Ñ'<¸ZÈÐW`Ó'aÑ$ˆJ˜ä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9¸f¿j¹jÐI\Ð^cÔ>dÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r=   r(   )rT   rU   rV   r–   r$   r   ri   r1   r3   r™   r_   r   r   r   rS   rW   rX   s   @r<   r¦   r¦   ¶   sû   ø„ ñð$ $(Ø,0Ø"&ñ'\àð'\ð ˜C‘=ð'\ð % S™Mð	'\ð
 ˜3‘-õ'\ðZ 26Ø=AØKOñ7)à—|‘|ð7)ð ð7)ð ! §¡Ñ.ð	7)ð
 !Ð!9Ñ:ð7)ð & e¨E¯L©L¸%¿,¹,Ð,FÑ&GÑHð7)ð Ð-Ñ.ð7)ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷7)r=   r¦   c                   óê   ‡ — e Zd ZdZddedee   fˆ fd„Z	 	 ddej                  dee
   deej                     fd„Zddee
   deej                     fd	„Z	 	 ddee
   deej                     fd
„Zˆ xZS )ÚZamba2MambaMixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    r`   r‚   c           	      ó  •— t         ‰| �  «        || _        |j                  | _        |j                  | _        |j                  | _        t        |j                  | j                  z  «      | _
        || _        |j                  | _        d| _        t        j                  «       | _        |j"                  | _        |j$                  | _        |j(                  | _        | j                  j,                  | _        |j0                  | _        |j2                  | _        |j4                  | _        |j6                  | _        | j                  d| j&                  z  | j
                  z  z   | _        t        j:                  | j8                  | j8                  d|j                  | j8                  |j                  dz
  ¬«      | _        | j                  | j8                  z   | j.                  z   }t        j>                  | j                  ||j@                  ¬«      | _!        t        jD                  tG        jH                  | j.                  «      «      | _%        tG        jL                  d| j.                  dz   «      }t        jD                  tG        jN                  |«      «      | _(        d| jP                  _)        tU        | j                  | j                  | j&                  z  d¬«      | _+        t        jD                  tG        jH                  | j.                  «      «      | _,        d| jX                  _)        t        j>                  | j                  | j                  |j@                  ¬«      | _-        t\        st^        ja                  d	«       y y )
NrE   r   Tr#   )Úin_channelsÚout_channelsr«   Úkernel_sizeÚgroupsÚpaddingrª   gñhãˆµøä>)r7   r:   a  The fast path is not available because on of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)1r0   r1   r`   r9   rl   rm   rn   ro   ri   rj   rk   r‚   Úuse_conv_biasÚ
activationr   ÚSiLUÚactÚuse_mem_eff_pathrz   Ún_groupsr{   rË   rp   Ú	num_headsÚ
chunk_sizeÚtime_step_limitÚtime_step_minÚtime_step_maxÚconv_dimÚConv1dÚconv1drµ   Úadd_bias_linearÚin_projr2   r3   r4   Údt_biasÚarangeÚlogÚA_logÚ_no_weight_decayr-   ÚnormÚDÚout_projÚis_fast_path_availablerÔ   rÕ   )r8   r`   r‚   Úprojection_sizeÚAr;   s        €r<   r1   zZamba2MambaMixer.__init__1  s“  ø€ Ü‰ÑÔØˆŒØ!×-Ñ-ˆÔØ$×2Ñ2ˆÔØ &× 3Ñ 3ˆÔÜ!$ V×%8Ñ%8¸4×;KÑ;KÑ%KÓ!LˆÔØ"ˆŒØ#×1Ñ1ˆÔØ ˆŒÜ—7‘7“9ˆŒØ &× 7Ñ 7ˆÔà×,Ñ,ˆŒØ×,Ñ,ˆŒØŸ™×2Ñ2ˆŒØ ×+Ñ+ˆŒà%×5Ñ5ˆÔØ#×1Ñ1ˆÔØ#×1Ñ1ˆÔà×.Ñ.°°T·]±]Ñ1BÀT×EXÑEXÑ1XÑXˆŒÜ—i‘iØŸ™ØŸ™ØØ×+Ñ+Ø—=‘=Ø×'Ñ'¨!Ñ+ô
ˆŒð ×0Ñ0°4·=±=Ñ@À4Ç>Á>ÑQˆÜ—y‘yØ×ÑØØ×'Ñ'ô
ˆŒô —|‘|¤E§J¡J¨t¯~©~Ó$>Ó?ˆŒô �L‰L˜˜DŸN™N¨QÑ.Ó/ˆÜ—\‘\¤%§)¡)¨A£,Ó/ˆŒ
Ø&*ˆ�
‰
Ô#Ü&Ø×"Ñ"¨t×/EÑ/EÈÏÉÑ/VÐ\`ô
ˆŒ	ô —‘œeŸj™j¨¯©Ó8Ó9ˆŒØ"&ˆ�‰ÔäŸ	™	 $×"8Ñ"8¸$×:JÑ:JÐQW×QgÑQgÔhˆŒå%Ü×Ñð>õð &r=   rK   Úcache_paramsrÀ   c                 ó‚  — |j                   \  }}}| j                  | j                  z  }d| j                  z  d| j                  z  | j                  z  z   | j                  z   }|��4|j
                  �r'| j                  |j                  d«      «      }	|	j                   d   |z
  dz  }
|
|
| j                  | j                  | j                  g}t        j                  |	|d¬«      \  }}}}}t        ||j                  | j                     | j                  j                  j                  d«      | j                  j                   | j"                  «      }t        j                  || j                  ||gd¬«      \  }}}t        j$                  | j&                  j)                  «       «       }|d d …d df   d d …d d …d f   j+                  d| j,                  | j                  «      j/                  t        j0                  ¬«      }|d d …d d …d f   j+                  dd| j,                  «      }| j2                  d d …d df   j+                  d| j,                  «      }| j4                  d d …d df   j+                  d| j,                  «      }|j7                  || j                  |j                   d   | j                  z  «      }|j7                  || j                  |j                   d   | j                  z  «      }|j7                  || j                  | j,                  «      }t9        |j:                  | j                     ||||||d |d¬«
      }|j7                  || j                  | j,                  z  «      }| j=                  ||«      }| j?                  |«      d d …d df   }|S |�Bt        j@                  |dk(  «      s*|jB                  }||d d …d d …d f   z  j/                  |«      }| j                  |«      }t        j$                  | j&                  j)                  «       «       }| jD                  €i nd	| jD                  i}|�t        j@                  |dk(  «      }nd}| jF                  rõ| jH                  ré|€ç|råtK        || j                  j                  j                  d«      | j                  j                   | j2                  |f| j4                  | jL                  d | j"                  | j<                  j                  | j<                  jN                  | j>                  j                  | j>                  j                   | j,                  | j                  d
ddœ|¤Ž\  }}|S t        j                  || j                  | j                  | j                  gd¬«      \  }}}|�v|jQ                  dd«      }tR        jT                  jW                  || jX                  |j                   d   z
  df«      }|j                  | j                     j[                  |«       t\        �| j"                  dvrJ| j_                  | j                  |jQ                  dd«      «      jQ                  dd«      d d …d |…f   «      }nyt]        |jQ                  dd«      | j                  j                  j                  d«      | j                  j                   | j"                  ¬«      jQ                  dd«      d d …d |…f   }t        j                  || j                  ||gd¬«      \  }}}|�Bt        j@                  |dk(  «      s*|jB                  }||d d …d d …d f   z  j/                  |«      }ta        |j7                  ||d| j,                  «      |||j7                  ||| j                  d«      |j7                  ||| j                  d«      f| jL                  | j4                  d d d| j2                  ddœ|¤Ž\  }}|�*|�(|j:                  | j                     j[                  |«       |j7                  ||d«      }| j=                  ||«      }| j?                  |«      }|S )Nr   r#   r?   ©rŸ   .©rA   T)Úzrþ   Údt_softplusÚdt_limitF)r  rõ   Úseq_idxrï   Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_statesr   )rE   Úswish)Úxr5   r«   rï   )rõ   r  r  r  r  rþ   r  )1rF   ró   rm   rk   rô   rh   rý   Úsqueezerù   r3   Úsplitr*   ru   r‚   rû   r5   r«   rï   Úexpr  ÚfloatÚexpandrË   rB   rC   rþ   r  rG   r%   rv   r  r  ÚallrA   rö   rò   rÖ   r'   rõ   r6   rÏ   r   rD   Úpadro   Úcopy_r)   rñ   r&   )r8   rK   r	  rÀ   ra   Úseq_lenr�   Úgroups_time_state_sizeÚd_to_removeÚin_projected_statesÚd_mlpÚsplit_projection_dimrL   Úhidden_states_B_CÚdtÚBÚCr  rþ   r  Úhidden_states_reshapedÚoutrA   Úprojected_statesÚdt_limit_kwargsÚinput_not_maskedÚ	ssm_stateÚ	time_stepÚhidden_states_B_C_tr�   Úscan_outputs                                  r<   Úcuda_kernels_forwardz%Zamba2MambaMixer.cuda_kernels_forwardr  sv  € ð "/×!4Ñ!4Ñˆ
�G˜QØ!%§¡°×1DÑ1DÑ!DÐØ˜$×0Ñ0Ñ0°1°t·}±}Ñ3DÀt×GZÑGZÑ3ZÑZÐ]a×]kÑ]kÑkˆð Ñ#¨×(GÓ(GØ"&§,¡,¨}×/DÑ/DÀQÓ/GÓ"HÐØ(×.Ñ.¨rÑ2°[Ñ@ÀQÑFˆEØ$)¨5°$×2HÑ2HÈ$Ï-É-ÐY]×YgÑYgÐ#hÐ Ü05·±Ð<OÐQeÐkmÔ0nÑ-ˆAˆq�$Ð)¨2ä 4Ø!Ø×(Ñ(¨¯©Ñ8Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ Ø—‘ó!Ðô #(§+¡+Ø!Ø×'Ñ'Ð)?ÐAWÐXØô#ÑˆM˜1˜aô
 —‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAà’!�T˜3�,‘¢¢1 d 
Ñ+×2Ñ2°2°t·}±}Àd×FYÑFYÓZ×]Ñ]Ôdi×dqÑdqÐ]ÓrˆAØ’A’q˜$�J‘×&Ñ& r¨2¨t¯}©}Ó=ˆBØ—l‘l¢1 d¨C <Ñ0×7Ñ7¸¸D¿M¹MÓJˆGØ—‘’q˜$ �|Ñ$×+Ñ+¨B°·±Ó>ˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ%2×%7Ñ%7¸
ÀDÇNÁNÐTX×TaÑTaÓ%bÐ"Ü2Ø×'Ñ'¨¯©Ñ7Ø&ØØØØØØØØ ôˆMð *×.Ñ.¨z¸4¿>¹>ÈDÏMÉMÑ;YÓZˆMØ ŸI™I m°TÓ:ˆMØ—-‘- Ó.ªq°$¸¨|Ñ<ˆCðz ˆ
ðu Ð)´%·)±)¸NÈaÑ<OÔ2Pà%×+Ñ+�Ø!.°ÂÂ1ÀdÀ
Ñ1KÑ!K× OÑ OÐPUÓ V�à#Ÿ|™|¨MÓ:ÐÜ—‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAØ$(×$8Ñ$8Ð$@™bÀzÐSW×SgÑSgÐFhˆOØÐ)Ü#(§9¡9¨^¸qÑ-@Ó#AÑ à#'Ð à×$Ò$¨¯ª¸<Ð;OÑTdÜ!AØ$Ø—K‘K×&Ñ&×.Ñ.¨qÓ1Ø—K‘K×$Ñ$Ø—L‘LØð"ð —f‘fØ#Ÿ™Ø Ø#Ÿ™Ø#'§9¡9×#3Ñ#3Ø $§	¡	× :Ñ :Ø#'§=¡=×#7Ñ#7Ø!%§¡×!3Ñ!3Ø ŸM™MØ ŸM™MØ%*Ø(,ñ#"ð$ &ñ%"‘��YðX ˆ
ôm 6;·[±[Ø$Ø×+Ñ+¨T¯]©]¸D¿N¹NÐKØô6Ñ2�Ð'¨ð  Ð+Ø*;×*EÑ*EÀaÈÓ*KÐ'Ü!#§¡×!2Ñ!2Ø+¨d×.CÑ.CÐFY×F_ÑF_Ð`bÑFcÑ.cÐefÐ-gó"�Jð !×,Ñ,¨T¯^©^Ñ<×BÑBÀ:ÔNÜ#Ð+¨t¯©ÐFWÑ/WØ(,¯©ØŸ™Ð$5×$?Ñ$?ÀÀ1Ó$EÓF×PÑPÐQRÐTUÓVÒWXÐZbÐ[bÐZbÐWbÑcó)Ñ%ô )9Ø+×5Ñ5°a¸Ó;Ø#Ÿ{™{×1Ñ1×9Ñ9¸!Ó<Ø!Ÿ[™[×-Ñ-Ø#'§?¡?ô	)÷
  ‘i  1“o¢a¨¨'¨ kñ)3Ð%ô ',§k¡kØ%Ø×+Ñ+Ð-CÐE[Ð\Øô'Ñ#�˜q !ð
 "Ð-´e·i±iÀÐRSÑ@SÔ6Tà)×/Ñ/�EØ%2°^ÂAÂqÈ$ÀJÑ5OÑ%O×$SÑ$SÐTYÓ$Z�MÜ)BØ!×&Ñ& z°7¸BÀÇÁÓNØØØ—F‘F˜: w°·±¸rÓBØ—F‘F˜: w°·±¸rÓBð*ð  $Ÿ™Ø—f‘fØØ Ø(,Ø ŸL™LØ $ñ*ð &ñ*Ñ&�˜Yð Ð(¨\Ð-EØ ×+Ñ+¨D¯N©NÑ;×AÑAÀ)ÔLØ)×.Ñ.¨z¸7ÀBÓG�à"Ÿi™i¨°TÓ:�Ø—m‘m KÓ0�Øˆ
r=   c                 óî  — |j                   \  }}}|j                  }|�-|j                  r!| j                  |j	                  d«      «      }nI|�6t        j                  |dk(  «      s||d d …d d …d f   z  j                  |«      }| j                  |«      }|j                   d   d| j                  z  z
  d| j                  z  | j                  z  z
  | j                  z
  dz  }	|j                  |	|	| j                  | j                  | j                  gd¬«      \  }}}
}}|��w|j                  | j                     j!                  «       }|j                  |j"                  «      }|j                  �r1|
j%                  d«      }
|j&                  | j                     }t        j(                  |dd¬«      }|j*                  dk(  r|d d …dd d …f   n||d d …d d …df<   |j&                  | j                     j-                  |«       t        j.                  |j                  |j"                  «      | j0                  j2                  d d …dd d …f   z  d¬«      }| j4                  r|| j0                  j6                  z  }| j9                  |«      j                  |«      d d …d df   }�n‚|j;                  dd«      }t<        j>                  jA                  || jB                  |j                   d   z
  df«      }|j&                  | j                     j-                  |«       | j9                  | j1                  |«      j;                  dd«      «      d d …d |…d d …f   }|�Ît        j                  |dk(  «      s¶|j                  }||d d …d d …d f   z  j                  |«      }n‹t        jD                  || j                  | jF                  | j                  f|j"                  |¬	«      }| j9                  | j1                  |j;                  dd«      «      dd |…f   j;                  dd«      «      }t        j                  || j                  | j                  | j                  z  | j                  | j                  z  gd¬«      \  }}}t        jH                  | jJ                  jM                  «       «       }|��t|j                  �rg|j*                  dk(  r
|d d …d df   n|d d …dd d …f   d d …d df   }|j;                  dd«      jO                  ||j                   d   | jF                  «      }| jP                  d
   jO                  | jP                  j                   d   | jF                  «      }t
        j<                  j>                  jS                  ||j                  |j                  «      z   «      }t        jT                  || jV                  «      }|d   jO                  | j                  | jF                  | j                  «      j                  t
        jX                  ¬«      }t        jH                  |d
   |z  «      }|j[                  || j                  d«      dd d d …f   }|jO                  || j                  | j                  | j                  z  |j                   d   «      j]                  «       }|j[                  |d|j                   d   «      }|d
   |dd d d …f   z  }|j[                  |d| jF                  «      }||d
   z  }|j                  | j                     j-                  |j                  | j                     |z  |z   «       |j[                  || j                  d«      dd d d …f   }|jO                  || j                  | j                  | j                  z  |j                   d   «      j]                  «       }|j[                  |d|j                   d   «      }|j                  | j                     j                  |j                  «      }|j_                  || j                  z  | jF                  | j                  «      }|j_                  || j                  z  | j                  d«      }t        j`                  ||«      }|j_                  || j                  | jF                  «      }| jb                  d
   jO                  | jb                  j                   d   | jF                  «      }|||z  z   j                  |j                  «      }|j[                  |d«      d d …d df   }�n t<        j>                  jS                  || jP                  z   «      }t        jT                  || jV                  «      }|j[                  ||d| jF                  «      jM                  «       }|j[                  ||d| j                  «      jM                  «       }|j[                  ||d| j                  «      jM                  «       }|je                  dd| j                  | j                  z  d«      }|je                  dd| j                  | j                  z  d«      }| jf                  || jf                  z  z
  | jf                  z  }| jb                  d
   ti        ||«      z  }||d
   z  }|j                  |j                  «      |z  }||||fD �cg c]  }tk        ||| jf                  «      ‘Œ c}\  }}}}|jm                  dddd«      }t        jn                  |d¬«      }t        jH                  tq        |«      «      }|d d …d d …d d …d d d …d d …f   |d d …d d …d d d …d d …d d …f   z  } | j/                  d¬«      }!|!d
   |jm                  ddddd«      d
   z  }"|"j/                  d¬«      }#|#d
   |d d …d d …d f   z  j/                  d«      }$t        jH                  |d d …d d …d d …dd …f   |z
  «      }%||%jm                  dddd«      d
   z  }&|&jm                  ddddd«      d
   |jm                  ddddd«      dd d d …f   z  j/                  d¬«      jm                  ddddd«      }'|�.|j                  r"|j                  | j                     d d …d df   }(nt        jr                  |'d d …d d…f   «      }(t        jt                  |(|'gd¬«      }'t        jH                  tq        t<        j>                  jA                  |d d …d d …d d …df   d«      «      «      })|'jm                  ddddd«      }*|)d   |*d d …d d …d df   z  j/                  d¬«      }+|+jm                  ddddd«      },|,d d …d d…f   |,d d …df   }}'t        jH                  |«      }-|dd d d …f   |'d d …d d …d df   z  }.|-jm                  dddd«      }/|.j/                  d«      |/d
   z  }0|$|0z   }|j[                  |d| j                  | jF                  «      }||z   }|dkD  r|d d …d |…d d …d d …f   }|j[                  ||d«      }|�*|�(|j                  | j                     j-                  |«       | jw                  ||
«      }1| jy                  |1j                  |«      «      }2|2S c c}w )Nr#   r?   r   r  r‡   r	   r   .rd   ).N).NNr  é   )r#   r   )=rF   rA   rh   rý   r  r3   r   rB   rk   ró   rm   rô   r  rù   rv   r‚   Úclonerb   Ú	unsqueezeru   r‹   Úndimr"  Úsumrû   r5   rî   r«   rñ   rÏ   r   rD   r!  ro   ry   rË   r  r  r  r  rþ   ÚsoftplusrŠ   r÷   rC   rØ   rÙ   rG   Úbmmr  Úrepeatrõ   r   r   ÚpermuteÚcumsumr   Ú
zeros_likeÚcatr  r  )3r8   Úinput_statesr	  rÀ   ra   r#  r�   rA   r/  r'  rL   rK   r*  r2  r�   r+  r,  r  rþ   ÚdAÚdBÚdBxrv   Ússm_states_reshapedÚ
C_reshapedÚyr  Úpad_sizeÚ
D_residualÚtÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decay_contractionÚstatesÚprevious_statesÚdecay_chunkÚstates_permutedÚresultÚ
new_statesÚstate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offr5  Úcontextualized_statess3                                                      r<   Útorch_forwardzZamba2MambaMixer.torch_forward	  sj  € Ø!-×!3Ñ!3Ñˆ
�G˜QØ×"Ñ"ˆàÐ#¨×(GÒ(GØ $§¡¨\×-AÑ-AÀ!Ó-DÓ EÑàÐ)´%·)±)¸NÈAÑ<MÔ2Nà$0°>Â!ÂQÈÀ*Ñ3MÑ$M×#QÑ#QÐRWÓ#X�LØ $§¡¨\Ó :ÐØ!×'Ñ'¨Ñ+¨a°$×2HÑ2HÑ.HÑHÈAÐPT×P]ÑP]ÑL]Ð`d×`sÑ`sÑLsÑsÐuy÷  vDñ  vDñ  Dð  IJñ  JˆØ(8×(>Ñ(>Ø˜˜t×5Ñ5¸¿¹ÀtÇ~Á~ÐVÐ\^ð )?ó )
Ñ%ˆˆ1ˆd�M 2ð
 Ñ#Ø$×/Ñ/°·±Ñ?×EÑEÓGˆIØ!Ÿ™ ]×%9Ñ%9Ó:ˆIØ×.Ó.Ø—~‘~ aÓ(�Ø)×5Ñ5°d·n±nÑE�
Ü"ŸZ™Z¨
¸2ÀBÔG�
àAN×ASÑASÐWXÒAX }²Q¸º1°WÒ'=Ð^k�
š1ša ˜8Ñ$Ø×(Ñ(¨¯©Ñ8×>Ñ>¸zÔJÜ %§	¡	¨*¯-©-Ð8H×8OÑ8OÓ*PÐSW×S^ÑS^×SeÑSeÒfgÐijÒlmÐfmÑSnÑ*nÐtvÔ w�Ø×%Ò%Ø! T§[¡[×%5Ñ%5Ñ5�MØ $§¡¨Ó 7× :Ñ :¸5Ó AÂ!ÀTÈ3À,Ñ O’à -× 7Ñ 7¸¸!Ó <�ÜŸ]™]×.Ñ.Ø!Ø×*Ñ*¨]×-@Ñ-@ÀÑ-DÑDÀaÐHó�
ð ×(Ñ(¨¯©Ñ8×>Ñ>¸zÔJØ $§¡¨¯©°]Ó)C×)MÑ)MÈaÐPQÓ)RÓ SÒTUÐW_ÐX_ÐW_ÒabÐTbÑ c�Ø!Ð-´e·i±iÀÐPQÑ@QÔ6RØ)×/Ñ/�Eà%2°^ÂAÂqÈ$ÀJÑ5OÑ%O×$SÑ$SÐTYÓ$Z‘MäŸ™Ø˜TŸ^™^¨T¯]©]¸D×<OÑ<OÐPØ$×+Ñ+°5ôˆIð !ŸH™H T§[¡[°×1HÑ1HÈÈAÓ1NÓ%OÐPSÐU]ÐV]ÐU]ÐP]Ñ%^×%hÑ%hÐijÐlmÓ%nÓoˆMÜ#Ÿk™k¨-¸$×:PÑ:PÐRV×R_ÑR_Ðbf×buÑbuÑRuÐw{÷  xEñ  xEð  HL÷  H[ñ  H[ñ  x[ð  :\ð  bdô  eÑˆ�q˜!Ü�Y‰Y�t—z‘z×'Ñ'Ó)Ó*Ð*ˆØÑ#¨×(GÓ(Gð &(§W¡W°¢\�’A�t˜S�LÒ!°rº!¸QÂ¸'±{Â1ÀdÈCÀ<Ñ7PˆBØ—‘˜a Ó#×*Ñ*¨:°r·x±xÀ±|ÀTÇ]Á]ÓSˆBà—l‘l 9Ñ-×4Ñ4°T·\±\×5GÑ5GÈÑ5JÈDÏMÉMÓZˆGä—‘×$Ñ$×-Ñ-¨b°7·:±:¸b¿h¹hÓ3GÑ.GÓHˆBÜ—‘˜R ×!3Ñ!3Ó4ˆBØ�/Ñ"×)Ñ)¨$¯.©.¸$¿-¹-È×I\ÑI\Ó]×`Ñ`Ôgl×gtÑgtÐ`ÓuˆAä—‘˜2˜i™=¨1Ñ,Ó-ˆBð
 —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAà�I‘  3¨ªa <¡Ñ0ˆBð *×1Ñ1°*¸bÀ$Ç-Á-ÓPˆMØ�} YÑ/Ñ/ˆCð ×#Ñ# D§N¡NÑ3×9Ñ9Ø×'Ñ'¨¯©Ñ7¸"Ñ<¸sÑBôð —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAð &×0Ñ0°·±Ñ@×CÑCÀAÇGÁGÓLˆJà",§/¡/°*¸t¿~¹~Ñ2MÈtÏ}É}Ð^b×^qÑ^qÓ"rÐØŸ™ 
¨T¯^©^Ñ ;¸T×=PÑ=PÐRSÓTˆJÜ—	‘	Ð-¨zÓ:ˆAØ—‘�z 4§>¡>°4·=±=ÓAˆAð —‘�yÑ!×(Ñ(¨¯©¯©°a©¸$¿-¹-ÓHˆAØ�] QÑ&Ñ&×*Ñ*¨1¯7©7Ó3ˆAð —	‘	˜* bÓ)ª!¨T°3¨,Ñ7ŠAô —‘×'Ñ'¨¨T¯\©\Ñ(9Ó:ˆBÜ—‘˜R ×!3Ñ!3Ó4ˆBØ)×1Ñ1°*¸gÀrÈ4Ï=É=ÓY×_Ñ_ÓaˆMØ—	‘	˜* g°°D×4GÑ4GÓH×NÑNÓPˆAØ—	‘	˜* g¨r°4×3FÑ3FÓG×MÑMÓOˆAØ—‘˜˜A˜tŸ~™~°·±Ñ>ÀÓBˆAØ—‘˜˜A˜tŸ~™~°·±Ñ>ÀÓBˆAØŸ™¨'°D·O±OÑ*CÑCÀtÇÁÑVˆHàŸ™ 	Ñ*Ô-?ÀÈxÓ-XÑXˆJð *¨B¨y©MÑ9ˆMØ—‘�]×(Ñ(Ó)¨BÑ.ˆAð cpÐqrÐtuÐwxÐayÖ%zÐ\]Ô&9¸!¸XÀtÇÁÕ&WÒ%zÑ"ˆM˜1˜a ð —	‘	˜!˜Q  1Ó%ˆAÜ—|‘| A¨2Ô.ˆHô —	‘	œ+ a›.Ó)ˆAð šq¢!¢Q¨ªa²Ð2Ñ3°aºº1¸dÂAÂqÊ!Ð8KÑ6LÑLˆNØ×"Ñ" rÐ"Ó*ˆAð ˜y™\¨A¯I©I°a¸¸A¸qÀ!Ó,DÀYÑ,OÑOˆNØ×"Ñ" rÐ"Ó*ˆAð ˜	‘l ]²1²a¸°:Ñ%>Ñ>×CÑCÀAÓFˆFô !Ÿ9™9 hªq²!²Q¸¹¨|Ñ&<¸xÑ&GÓIˆLØ"# l×&:Ñ&:¸1¸aÀÀAÓ&FÀyÑ&QÑ"QÐà)×1Ñ1°!°Q¸¸1¸aÓ@ÀÑKÈ}×OdÑOdÐefÐhiÐklÐnoÐqrÓOsÐtwÐy}ò  @Að  uAñ  PBñ  B÷  Gñ  Gð  LMð  Gó  N÷  Vñ  Vð  WXð  Z[ð  ]^ð  `að  cdó  eˆFØÐ'¨L×,KÒ,KØ".×"9Ñ"9¸$¿.¹.Ñ"IÊ!ÈTÐSVÈ,Ñ"W‘ä"'×"2Ñ"2°6º!¸R¸a¸R¸%±=Ó"A�Ü—Y‘Y °Ð8¸aÔ@ˆFÜŸ)™)¤K´·±×0AÑ0AÀ(Ê1ÊaÒQRÐTVÈ;ÑBWÐY_Ó0`Ó$aÓbˆKà$Ÿn™n¨Q°°1°a¸Ó;ˆOØ! /Ñ2°_ÂQÊÈ4ÐQTÀ_Ñ5UÑU×ZÑZÐ_`ÐZÓaˆFØŸ™¨¨1¨a°°AÓ6ˆJØ *ª1¨c¨r¨c¨6Ñ 2°JºqÀ"¸uÑ4E�IˆFô $Ÿi™i¨Ó1ˆOà  Tª1 ™o°²qº!¸TÀ3°Ñ0GÑGˆNØ'6×'>Ñ'>¸qÀ!ÀQÈÓ'JÐ$Ø#×'Ñ'¨Ó+Ð.FÀyÑ.QÑQˆEð ˜‘ˆAà—	‘	˜* b¨$¯.©.¸$¿-¹-ÓHˆAà�J‘ˆAà˜!Š|Ø’a˜˜'˜¢1¢aÐ'Ñ(�Ø—	‘	˜* g¨rÓ2ˆAØÐ$¨Ð)AØ×'Ñ'¨¯©Ñ7×=Ñ=¸iÔHà—i‘i  4Ó(ˆð
 !%§¡¨k¯n©n¸UÓ.CÓ DÐØ$Ð$ùòI &{s   ê7y2c                 ó²   — t         r?d| j                  j                  j                  j                  v r| j                  |||«      S | j                  |||«      S )NÚcuda)r  rý   r5   rb   Útyper6  rb  )r8   rK   r	  rÀ   s       r<   rS   zZamba2MambaMixer.forwardË  sN   € õ " f°·±×0CÑ0C×0JÑ0J×0OÑ0OÑ&OØ×,Ñ,¨]¸LÈ.ÓYÐYà×!Ñ! -°¸~ÓNÐNr=   r/   r+   )rT   rU   rV   r–   r$   r   ri   r1   r3   r™   r_   r6  rb  rS   rW   rX   s   @r<   rç   rç   )  sÌ   ø„ ññ?˜|ð ?¸À¹õ ?ðH <@Ø15ñ	Tà—|‘|ðTð Ð7Ñ8ðTð ! §¡Ñ.ó	Tñn%¸ÐAYÑ8Zð %ÐqyÐz÷  {Gñ  {Gñ  rHó %ðJ <@Ø15ñ		Oð Ð7Ñ8ð	Oð ! §¡Ñ.÷		Or=   rç   c                   ó8   ‡ — e Zd Zddedee   fˆ fd„Zdd„Zˆ xZS )Ú	Zamba2MLPr`   r¨   c           	      ó   •— t         ‰	| �  «        || _        |j                  | _        |j                  | _        || _        || _        t        j                  | j                  d| j                  z  |j                  ¬«      | _
        t        j                  | j                  | j                  |j                  ¬«      | _        t        |j                     | _        t        j                  g «      | _        t#        | j
                  «      D ]Ï  }||j$                  z  |k(  rŒt        j&                  t        j                  | j                  j                  | j                  j(                  d¬«      t        j                  | j                  j(                  d| j                  z  d¬«      «      }nt        j*                  «       }| j                   j-                  |«       ŒÑ |j.                  }t1        |«      D ��ci c]  \  }}||“Œ
 c}}| _        yc c}}w )aQ  
        This MLP layer contributes to tied transformer blocks aimed to increasing compute without increasing model size. Because this layer
        is tied, un-tied adapter modules (formally same as LoRA, but used in the base model) are added to the up and gate projectors to increase expressivity with a small memory overhead.
        r   rª   FN)r0   r1   r`   r9   rk   r§   r¨   r   rµ   rü   Úgate_up_projÚ	down_projr
   Ú
hidden_actÚact_fnr¯   Úgate_up_proj_adapter_listrw   r³   r´   r·   r¸   r|   r¬   r¹   rº   )
r8   r`   r§   r¨   r€   Úgate_up_proj_adapterr­   r¾   r¿   r;   s
            €r<   r1   zZamba2MLP.__init__Ø  s”  ø€ ô
 	‰ÑÔØˆŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔØ"4ˆÔØ ˆŒäŸI™I d×&6Ñ&6¸¸D×<RÑ<RÑ8RÐY_×YoÑYoÔpˆÔÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRX×RhÑRhÔiˆŒÜ˜V×.Ñ.Ñ/ˆŒä)+¯©°rÓ):ˆÔ&Ü�t×.Ñ.Ó/ò 	HˆAØ�6×(Ñ(Ñ(¨HÒ4Ü')§}¡}Ü—I‘I˜dŸk™k×5Ñ5°t·{±{×7OÑ7OÐV[Ô\Ü—I‘I˜dŸk™k×6Ñ6¸¸D×<RÑ<RÑ8RÐY^Ô_ó(Ñ$ô
 (*§{¡{£}Ð$Ø×*Ñ*×1Ñ1Ð2FÕGð	Hð !×1Ñ1ˆÜ;DÀ_Ó;U×V©<¨5°%˜% ™,ÓVˆ�ùÓVs   Ç3H
c                 óü   — | j                  |«      }| j                  |   }| | j                  |   |«      z   }t        j                  |dd¬«      }| j                  |d   «      |d   z  }| j                  |«      }|S )Nr   r?   r  r   r#   )ri  rº   rm  r3   Úchunkrl  rj  )r8   Úhidden_stater‚   Úgate_up_stateÚoutputs        r<   rS   zZamba2MLP.forwardö  s€   € Ø×)Ñ)¨,Ó7ˆØ—N‘N 9Ñ-ˆ	Ø%Ð(Q¨×(FÑ(FÀyÑ(QÐR^Ó(_Ñ_ˆäŸ™ M°1¸"Ô=ˆØ—{‘{ =°Ñ#3Ó4°}ÀQÑ7GÑGˆØ—‘ Ó-ˆØˆr=   r+   r/   )	rT   rU   rV   r$   r   ri   r1   rS   rW   rX   s   @r<   rg  rg  ×  s%   ø„ ñW˜|ð WÐPXÐY\ÑP]õ W÷<r=   rg  c                   ó4  ‡ — e Zd Zddedee   dee   fˆ fd„Z	 	 	 	 ddej                  dej                  dedeej                     dee	   d	ee
   d
eej                     dee   deej                  eeej                  ej                  f      f   fd„Zˆ xZS )ÚZamba2AttentionDecoderLayerr`   r¨   r‚   c                 ó¬   •— || _         t        |j                  «      }t        ‰| �  ||«       t        |d||¬«      | _        t        |||¬«      | _        y )Nr?   )r‚   r§   r¨   )r§   r¨   )	r¨   r“   r¬   r0   r1   r¦   Ú	self_attnrg  Úfeed_forward)r8   r`   r¨   r‚   Únum_gsr;   s        €r<   r1   z$Zamba2AttentionDecoderLayer.__init__  sO   ø€ Ø ˆŒÜ�V×,Ñ,Ó-ˆÜ‰Ñ˜ Ô+Ü(¨¸2ÐRXÐckÔlˆŒÜ% fÀÐRZÔ[ˆÕr=   rK   Úoriginal_hidden_statesrÀ   rÁ   rÇ   rÂ   rÃ   r…   c           
      óî   — t        j                  ||gd¬«      }| j                  |«      } | j                  d||||||dœ|¤Ž\  }}	| j	                  |«      }| j                  ||«      }|f}
|r|
|	fz  }
|
S )aú  
        Args:
            hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`.
                This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The
                concatenated tensor is then used as input of the pre-attention RMSNorm
                (see fig. 2 in https://arxiv.org/pdf/2405.16712).
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        r?   r  )rK   r‚   rÀ   rÁ   rÇ   rÂ   r\   )r3   ÚconcatenateÚinput_layernormrw  Úpre_ff_layernormrx  )r8   rK   rz  r‚   rÀ   rÁ   rÇ   rÂ   rÃ   Úself_attn_weightsÚoutputss              r<   rS   z#Zamba2AttentionDecoderLayer.forward	  s¦   € ô> ×)Ñ)¨=Ð:PÐ*QÐWYÔZˆØ×,Ñ,¨]Ó;ˆØ+9¨4¯>©>ð ,
Ø'ØØ)Ø)Ø/Ø 3ñ,
ð ñ,
Ñ(ˆÐ(ð ×-Ñ-¨mÓ<ˆØ×)Ñ)¨-¸ÓCˆà Ð"ˆáØÐ)Ð+Ñ+ˆGàˆr=   r+   )NNFN)rT   rU   rV   r$   r   ri   r1   r3   r™   r_   Úboolrš   r   r   r   ÚFloatTensorrS   rW   rX   s   @r<   ru  ru    sî   ø„ ñ\˜|ð \°xÀ±}ð \ÐX`ÐadÑXeõ \ð 26Ø=AØ,1Ø:>ñ3à—|‘|ð3ð !&§¡ð3ð ð	3ð
 ! §¡Ñ.ð3ð !Ð!9Ñ:ð3ð $ D™>ð3ð & e×&6Ñ&6Ñ7ð3ð Ð-Ñ.ð3ð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷3r=   ru  c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚZamba2MambaDecoderLayerr`   r‚   c                 ó–   •— t         ‰| �  ||«       t        ||¬«      | _        t	        |j
                  |j                  ¬«      | _        y )N)r`   r‚   ©r:   )r0   r1   rç   ÚmambarZ   r9   Úrms_norm_epsr}  )r8   r`   r‚   r;   s      €r<   r1   z Zamba2MambaDecoderLayer.__init__@  s;   ø€ Ü‰Ñ˜ Ô+Ü%¨V¸yÔIˆŒ
Ü,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÕr=   )rT   rU   rV   r$   ri   r1   rW   rX   s   @r<   r„  r„  ?  s   ø„ ðZ˜|ð Z¸÷ Zñ Zr=   r„  c                   ól  ‡ — e Zd Zdedej
                  defˆ fd„Z	 	 	 	 	 	 	 	 ddej                  de
ej                     de
e   de
ej                     d	e
ej                     d
e
e   de
e   de
e   de
ej                     deej                   e
eej                   ej                   f      f   fd„Zˆ xZS )ÚZamba2HybridLayerÚshared_transformerÚlinearr‡  c                 ó:   •— t         ‰| �  |||«       | `|| _        y r/   )r0   r1   Úshared_transfr‹  )r8   r‹  rŒ  r‡  r;   s       €r<   r1   zZamba2HybridLayer.__init__G  s%   ø€ ô 	‰ÑÐ+¨V°UÔ;ØÐØ"4ˆÕr=   rK   rz  r‚   rÀ   Úcausal_maskrÁ   rÇ   Ú	use_cacherÂ   r…   c
           	      ó¾   — | j                  |||||||	¬«      }
|
d   }|r|
d   }| j                  |«      }| j                  |||||||	¬«      }
|r|
d   f|
dd z   }
|
S )aX  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with
            hidden activations to form the input of the shared transformer layer.
            layer_idx (`int`): layer number.
            attention_mask (`torch.FloatTensor`, *optional*): attention mask of size
                `(batch, sequence_length)` where padding elements are indicated by 0.
            past_key_value (`Zamba2HybridDynamicCache`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
        )rz  r‚   rÀ   rÁ   rÇ   rÂ   r   r#   )Útransformer_hidden_statesrÀ   rÁ   rÇ   r�  rÂ   r   N)r‹  rŒ  Úmamba_decoder)r8   rK   rz  r‚   rÀ   r�  rÁ   rÇ   r�  rÂ   Úlayer_outputsr’  r  s                r<   rS   zZamba2HybridLayer.forwardN  s­   € ð@ ×/Ñ/ØØ#9ØØ&Ø)Ø/Ø 3ð 0ó 
ˆð %2°!Ñ$4Ð!áØ -¨aÑ 0Ðà$(§K¡KÐ0IÓ$JÐ!à×*Ñ*ØØ&?Ø)Ø)Ø/ØØ 3ð +ó 
ˆñ Ø*¨1Ñ-Ð/@ÐAÀMÐRSÐRTÐDUÑUˆMàÐr=   )NNNNNFFN)rT   rU   rV   ru  r   rµ   r„  r1   r3   r™   r   ri   r_   r�  rš   r   r‚  rS   rW   rX   s   @r<   rŠ  rŠ  F  s  ø„ ð5Ø"=ð5ØGIÇyÁyð5ØYpõ5ð :>Ø#'Ø15Ø.2Ø=AØ,1Ø$)Ø:>ñ>à—|‘|ð>ð !)¨¯©Ñ 6ð>ð ˜C‘=ð	>ð
 ! §¡Ñ.ð>ð ˜eŸl™lÑ+ð>ð !Ð!9Ñ:ð>ð $ D™>ð>ð ˜D‘>ð>ð & e×&6Ñ&6Ñ7ð>ð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷>r=   rŠ  c                   ó>   — e Zd ZeZdZdZddgZdZdZ	dZ
dZdZdZd„ Zy)ÚZamba2PreTrainedModelÚmodelTru  r„  Úpast_key_valuesc                 ó^  — | j                   j                  }t        |t        j                  t        j
                  f«      rY|j                  j                  j                  d|¬«       |j                  �%|j                  j                  j                  «        y y t        |t        j                  «      rf|j                  j                  j                  d|¬«       |j                  �2|j                  j                  |j                     j                  «        y y t        |t        «      �rwd|j                  _        d|j                   _        t#        j$                  t#        j&                  | j                   j(                  «      t+        j,                  | j                   j.                  «      t+        j,                  | j                   j0                  «      z
  z  t+        j,                  | j                   j0                  «      z   «      j3                  | j                   j4                  ¬«      }|t#        j,                  t#        j6                  | «       «      z   }t#        j8                  «       5  |j:                  j=                  |«       d d d «       d|j:                  _        y y # 1 sw Y   ŒxY w)NrÈ   )rI   ÚstdT)Úmin) r`   Úinitializer_rangeÚ
isinstancer   rµ   rú   r5   ÚdataÚnormal_r«   rŒ   Ú	EmbeddingÚpadding_idxrç   r  r  r  r3   r  Úrandrp   Úmathr   rø   r÷   rŠ   Útime_step_floorÚexpm1Úno_gradrþ   r"  Ú
_no_reinit)r8   Úmodulerš  r*  Úinv_dts        r<   Ú_init_weightsz#Zamba2PreTrainedModel._init_weights›  sÜ  € Ø�k‰k×+Ñ+ˆÜ�fœrŸy™y¬"¯)©)Ð4Ô5Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°SÐ&Ô9Ø×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ä˜Ô 0Õ1Ø,0ˆF�L‰LÔ)Ø(,ˆF�H‰HÔ%ä—‘Ü—
‘
˜4Ÿ;™;×4Ñ4Ó5Ü—8‘8˜DŸK™K×5Ñ5Ó6¼¿¹À$Ç+Á+×B[ÑB[Ó9\Ñ\ñ^ä—(‘(˜4Ÿ;™;×4Ñ4Ó5ñ6ó÷ ‰e˜Ÿ™×3Ñ3ˆeÓ4ð	 ð œ%Ÿ)™)¤U§[¡[°"°Ó%5Ð$5Ó6Ñ6ˆFä—‘“ñ -Ø—‘×$Ñ$ VÔ,÷-à(,ˆF�N‰NÕ%ð 2÷-ð -ús   É,J#Ê#J,N)rT   rU   rV   r$   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attn_2Ú_supports_flex_attnÚ_supports_sdpaÚ_supports_cache_classÚ_is_statefulrª  r\   r=   r<   r–  r–  �  sF   „ Ø€LØÐØ&*Ð#Ø6Ð8QÐRÐØ"3ÐØ!ÐØÐØ€NØ ÐØ€Ló-r=   r–  c                   ó  — e Zd ZdZdefd„Zd„ Z	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     d	ee   d
eej                     dee   dee   dee   dee   deej                     deeef   fd„Zy)ÚZamba2Modelzh
    Model consisting of *config.num_hidden_layers* layers.

    Args:
        config: Zamba2Config
    r`   c                 óÄ  — t         j                  | |«       || _        |j                  | _        |j
                  | _        t        j                  |j
                  |j                  | j                  «      | _	        t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ }}g }g }|j                  | _        t        |j                  «      D ]¯  }|j                  |   dk(  r|j                  t!        ||¬«      «       Œ2|j                  |   dk(  sŒE|j                  t        j"                  | j                  j                  | j                  j                  d¬«      «       |j                  t!        ||¬«      «       Œ± t%        |«      }t%        |«      }t'        |«      }| j)                  |||«      }t        j*                  |«      | _        |j.                  | _        t1        |j                  |j2                  ¬«      | _        |j6                  r1|j8                  rt:        j=                  d«       t?        |«      | _         d| _!        | jE                  «        y c c}w )	N)r¨   r‡  ©r‚   re   Frª   r†  ze`use_long_context` set to `True`: using rescaled `rope_theta` and extended `max_position_embeddings`.)#r–  r1   r`   Úpad_token_idr¡  Ú
vocab_sizer   r   r9   Úembed_tokensrw   r³   ru  rg   rx   r|   r„  rµ   Úiterr   Ú
get_layersr¯   ÚlayersrÒ   rZ   rˆ  Úfinal_layernormrÐ   Úuse_long_contextrÔ   rÕ   rœ   Ú
rotary_embÚgradient_checkpointingÚ	post_init)r8   r`   ÚkÚblocksÚmamba_layersÚlinear_layersr€   r¾  s           r<   r1   zZamba2Model.__init__¾  sî  € Ü×&Ñ& t¨VÔ4ØˆŒØ!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸL™L¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜKPÐQW×QfÑQfÓKgÖhÀaÔ-¨f¸qÖAÐhˆÐhØˆØˆØ!'×!9Ñ!9ˆÔÜ�v×/Ñ/Ó0ò 	RˆAØ×'Ñ'¨Ñ*¨gÒ5Ø×#Ñ#Ô$;¸FÈaÔ$PÕQØ×)Ñ)¨!Ñ,°Ó8Ø×$Ñ$¤R§Y¡Y¨t¯{©{×/FÑ/FÈÏÉ×H_ÑH_ÐfkÔ%lÔmØ×#Ñ#Ô$;¸FÈaÔ$PÕQð	Rô ˜LÓ)ˆÜ˜]Ó+ˆÜ�v“ˆØ—‘ ¨¸ÓEˆÜ—m‘m FÓ+ˆŒà$*×$?Ñ$?ˆÔ!Ü,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔØ×ÒØ×&Ò&Ü×#Ñ#Ø{ôô 4°FÓ;ˆDŒOØ&+ˆÔ#ð 	�‰Õùò7 is   ÂIc           
      óx  — g }g | _         d| _        t        | j                  «      D �]  \  }}|dk(  �rê| j                  dk(  r|| _        t	        |«      }| j
                  j                  t        | j
                  j                  «      z  dkD  �r_d|› d�}t        j                  |dz   dz   dz   d	z   d
z   «      }	| j                   j                  |	«       d}
| j                  D ]q  }|dk(  re|
| j
                  j                  z  |j                  k(  r?t        j                  dt        |
«      z   dz   «      }| j                   j                  |«       |
dz  }
Œs | j
                  j                  r‚d}
| j                  D ]q  }|dk(  re|
| j
                  j                  z  |j                  k(  r?t        j                  dt        |
«      z   dz   «      }| j                   j                  |«       |
dz  }
Œs |j                  t        |t	        |«      t	        |«      «      «       �Œ÷|j                  t	        |«      «       �Œ |S )Nr   re   r#   z	^layers\.z\.shared_transformer\.z(?:z3self_attn\.(?:q_proj|k_proj|v_proj|o_proj)\.weight|z1feed_forward\.(?:gate_up_proj|down_proj)\.weight|z,(?:input_layernorm|pre_ff_layernorm)\.weightz)$z>^shared_transformer\.feed_forward\.gate_up_proj_adapter_list\.z\.(?:0|1)\.weight$zg^shared_transformer\.self_attn\.(?:linear_q_adapter_list|linear_k_adapter_list|linear_v_adapter_list)\.)Ú_tied_weights_keysÚfirst_transformer_layer_idr¹   rg   Únextr`   r³   r“   r¬   ÚreÚcompiler|   r¨   r˜   r®   rŠ  )r8   rÅ  rÇ  rÆ  r¾  Úlayer_idÚ
layer_typeÚblockÚprefix_patternÚmain_keys_patternÚ
adapter_idÚ_layer_typeÚadapter_patternÚattn_adapter_patterns                 r<   r½  zZamba2Model.get_layersâ  sH  € ØˆØ"$ˆÔØ*+ˆÔ'Ü$-¨d×.DÑ.DÓ$Eó )	2Ñ ˆH�jØ˜XÓ%Ø×2Ñ2°aÒ7Ø6>�DÔ3Ü˜V›�Ø—;‘;×-Ñ-´°D·K±K×4PÑ4PÓ0QÑQÐTUÓUØ(1°(°Ð;QÐ%R�NÜ(*¯
©
Ø&Ø ñ!àPñQð OñOð Jñ	Jð
  ñ ó)Ð%ð ×+Ñ+×2Ñ2Ð3DÔEà!"�JØ'+×'=Ñ'=ò (˜Ø&¨(Ò2°zÀDÇKÁK×D^ÑD^Ñ7^Ðbg×bpÑbpÒ7pÜ.0¯j©jØ aÜ"% j£/ñ!2à"7ñ!8ó/˜Oð
 !×3Ñ3×:Ñ:¸?ÔKØ" a™™
ð(ð —{‘{×?Ò?Ø%&˜
Ø+/×+AÑ+Aò 	,˜KØ*¨hÒ6¸:ÈÏÉ×HbÑHbÑ;bÐfk×ftÑftÒ;tÜ79·z±zð%qä&)¨*£oñ%6ð '<ñ%<ó8"Ð 4ð !%× 7Ñ 7× >Ñ >Ð?SÔ TØ&¨!™O™Jð	,ð —‘Ô/°´t¸MÓ7JÌDÐQ]ÓL^Ó_Ö`à—‘œd <Ó0Ö1ðS)	2ðT ˆr=   NÚ	input_idsrÀ   Úposition_idsr˜  Úinputs_embedsr�  rÇ   Úoutput_hidden_statesÚreturn_dictr„   r…   c                 ór  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|	�|	n| j                   j                  }	|d u |d uz  rt        d«      ‚| j                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|}t        j                  |«      }|rO|€M|�|j                  d   n|j                  d   }t        | j                   || j                  | j                   ¬«      }|
€R|�|j#                  | j$                  ¬«      nd}t        j&                  |||j                  d   z   |j                   ¬«      }
|€|
j)                  d«      }| j+                  |||
«      }| j                   j,                  r| j/                  ||«      }nd }|rd	nd }|rd	nd }t1        | j2                  «      D ]r  \  }}|r||fz  }| j                  r1| j                  r%| j5                  |j6                  |||||||||«
      }n ||||||||||¬
«	      }|d   }|sŒd|d   €Œj||d   fz  }Œt | j9                  |«      }|r||fz  }|r|j:                  sd|_        t=        ||r|nd ||¬«      }|	r|S |j?                  «       S )NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either onezX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fr   )rA   rb   r¸  r#   rf   r\   )rz  r‚   rÀ   r�  rÁ   rÇ   r�  rÂ   T)Úlast_hidden_stater˜  rK   Ú
attentions) r`   rÇ   rÚ  r�  Úuse_return_dictÚ
ValueErrorrÂ  rÖ   rÔ   rÕ   r»  r3   r9  rF   r_   rA   rb   r•   rÊ  rÿ   r:  Ú_update_causal_maskrÐ   rÁ  r¹   r¾  Ú_gradient_checkpointing_funcÚ__call__r¿  rh   r   Úto_tuple)r8   r×  rÀ   rØ  r˜  rÙ  r�  rÇ   rÚ  rÛ  r„   rK   rz  ra   Úpast_seen_tokensr�  rÂ   Úall_hidden_statesÚall_self_attnsr‚   Úlayerr”  rs  s                          r<   rS   zZamba2Model.forward  s  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	à%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà˜Ð -°tÐ";Ò<ÜØsóð ð ×&Ò&¨4¯=ª=¹YÜ×ÑØjôð ˆIàÐ Ø ×-Ñ-¨iÓ8ˆMà%ˆä!&§¡¨]Ó!;Ðñ ˜Ð0Ø/8Ð/D˜Ÿ™¨Ò+È-×J]ÑJ]Ð^_ÑJ`ˆJÜ6°t·{±{ÀJÐVZ×V`ÑV`Ðim×itÑitÔuˆOàÐ!ð #Ð.ð  ×.Ñ.¸×9XÑ9XÐ.ÔYàð ô
 #Ÿ\™\Ø Ð"2°]×5HÑ5HÈÑ5KÑ"KÐTa×ThÑThôˆNð ÐØ)×3Ñ3°AÓ6ˆLà×.Ñ.¨~¸}ÈnÓ]ˆð �;‰;×#Ò#Ø"&§/¡/°-ÀÓ"NÑà"&Ðá"6™B¸DÐÙ0™°dˆä )¨$¯+©+Ó 6ò "	:ÑˆI�uÙ#Ø! mÐ%5Ñ5Ð!à×*Ò*¨t¯}ª}Ø $× AÑ AØ—N‘NØ!Ø*ØØ"ØØ#Ø%ØØ'ó!‘ñ !&Ø!Ø+AØ'Ø#1Ø +Ø#2Ø&7Ø'Ø(;ô
!�ð *¨!Ñ,ˆMâ Ø  Ñ#Ñ/à" }°QÑ'7Ð&9Ñ9‘NðE"	:ðH ×,Ñ,¨]Ó;ˆñ  Ø -Ð!1Ñ1Ðá ?×#EÒ#EØ15ˆOÔ.ä(Ø+Ù/8™O¸dØ+Ø%ô	
ˆñ %ˆvÐ;¨&¯/©/Ó*;Ð;r=   )
NNNNNNNNNN)rT   rU   rV   r–   r$   r1   r½  r   r3   rš   r™   r_   r‚  r�  r   r   r   rS   r\   r=   r<   r¶  r¶  ¶  s  „ ñð"˜|ó "òH.ðd 15Ø15Ø37Ø>BØ59Ø$(Ø,0Ø/3Ø&*Ø59ñv<à˜E×,Ñ,Ñ-ðv<ð ! §¡Ñ.ðv<ð ˜u×/Ñ/Ñ0ð	v<ð
 "Ð":Ñ;ðv<ð   × 1Ñ 1Ñ2ðv<ð ˜D‘>ðv<ð $ D™>ðv<ð ' t™nðv<ð ˜d‘^ðv<ð ! ×!1Ñ!1Ñ2ðv<ð 
ˆuÐ-Ð-Ñ	.ôv<r=   r¶  c                   ó   — e Zd Zy)ÚZamba2ForCausalLMNr[   r\   r=   r<   rê  rê  ‹  r]   r=   rê  c                   ó   — e Zd Zy)ÚZamba2ForSequenceClassificationNr[   r\   r=   r<   rì  rì  �  r]   r=   rì  )rê  rì  r¶  r–  )Nr£  rÌ  Ú	itertoolsr   Útypingr   r   r   r   r3   Útorch.utils.checkpointr   Úactivationsr
   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   Úutils.import_utilsr   r   Úllama.modeling_llamar   r   Úmamba2.modeling_mamba2r   r   r   Úzamba.modeling_zambar   r   r   r   r   r   r   r    r!   r"   Úconfiguration_zamba2r$   Ú+mamba_ssm.ops.triton.selective_state_updater%   Ú!mamba_ssm.ops.triton.ssd_combinedr&   r'   Úcausal_conv1dr)   r*   r   r  Ú_CONFIG_FOR_DOCÚ
get_loggerrT   rÔ   ÚModuler-   rZ   r_   rœ   r¦   rç   rg  ru  r„  rŠ  r–  r¶  rê  rì  Ú__all__r\   r=   r<   ú<module>r     s�  ðó  Û 	Ý ß 3Ó 3ã Û Ý å !Ý BÝ 7ß FÝ &õ÷÷ Nß YÑ Y÷÷ ÷ õ /ñ ÔÝRßmÐmàZjÑWÐÐ5Ð7WáÔßDÐDà-7Ñ*ÐÐ*áÐ4Ð6FÐH\Ð]Ó^Ð ð '€à	ˆ×	Ñ	˜HÓ	%€ô;˜Ÿ™Ÿ™ô ;ô*	�Lô 	ôD3Ð6ô D3ôN

Ð0ô 

ôp)�nô p)ôfkO�r—y‘yô kOô\'�—	‘	ô 'ôT;Ð"<ô ;ô|ZÐ4ô ZôFÐ(ô FôR$-˜Oô $-ôNR<�*Ð3ô R<ôj	Ð(ô 	ô	Ð&Dô 	ò�r=   