Ë
    T^(hÚÊ  ã                   ó&  — d Z ddlZddlmZ ddlmZmZmZ ddlZddl	Zddlm
Z
 ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZmZ ddlmZmZ ddlmZ  ej<                  e«      Z  e«       rddl!m"Z" ddl#m$Z$m%Z% nd\  Z$Z%Z" e«       r	ddl&m'Z'm(Z( nd\  Z(Z' e)e"e$e%e'e(f«      Z*dZ+dZ,dejZ                  de.fd„Z/d„ Z0d„ Z1d„ Z2 G d„ d«      Z3 G d„ dej                  jh                  «      Z5 G d „ d!e
jh                  «      Z6 G d"„ d#e
jh                  «      Z7 G d$„ d%e
jh                  «      Z8 G d&„ d'e«      Z9e G d(„ d)e«      «       Z:e G d*„ d+e«      «       Z;d,Z<d-Z= ed.e<«       G d/„ d0e9«      «       Z> ed1e<«       G d2„ d3e9e«      «       Z?g d4¢Z@y)5zPyTorch MAMBA2 model.é    N)Ú	dataclass)ÚOptionalÚTupleÚUnion)Únn)ÚCrossEntropyLossé   )ÚACT2FN)ÚGenerationMixin)ÚPreTrainedModel)ÚModelOutputÚadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚlogging)Úis_causal_conv1d_availableÚis_mamba_2_ssm_availableé   )ÚMamba2Config)Úselective_state_update)Úmamba_chunk_scan_combinedÚ mamba_split_conv1d_scan_combined©NNN)Úcausal_conv1d_fnÚcausal_conv1d_update)NNz!mistralai/mamba-codestral-7B-v0.1r   Úinput_tensorÚpad_sizec                 ó°   — t        | j                  «      dk(  r
ddddd|ddfnddd|ddf}t        j                  j                  j                  | |dd¬«      S )z‚
    Padding x tensor with `pad_size` on the seq_len dim (dim=1)

    Assumes that we only have tensors of either size 4 or 3
    é   r   Úconstant)ÚmodeÚvalue)ÚlenÚshapeÚtorchr   Ú
functionalÚpad)r   r   Ú	pad_shapes      úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/mamba2/modeling_mamba2.pyÚpad_tensor_by_sizer*   G   sf   € ô 47°|×7IÑ7IÓ3JÈaÒ3O��A�q˜!˜Q ¨!¨QÑ/ÐVWÐYZÐ\]Ð_gÐijÐlmÐUn€Iä�8‰8×Ñ×"Ñ" <°ÀÐSTÐ"ÓUÐUó    c                 ó  — t        | |«      } t        | j                  «      dk(  r.| j                  | j                  d   d|| j                  d   «      S | j                  | j                  d   d|| j                  d   | j                  d   «      S )zÀ
    Padding input_tensor with `pad_size` on the seq_len dim (dim=1) and
    simultaneously splitting it into chunk sequences.

    Assumes that we only have tensors of either size 4 or 3
    r	   r   éÿÿÿÿé   )r*   r#   r$   Úreshape)r   r   Ú
chunk_sizes      r)   Úreshape_into_chunksr1   R   s“   € ô & l°HÓ=€Lä
ˆ<×ÑÓ !Ò#à×#Ñ# L×$6Ñ$6°qÑ$9¸2¸zÈ<×K]ÑK]Ð^_ÑK`ÓaÐað ×#Ñ#Ø×Ñ˜qÑ! 2 z°<×3EÑ3EÀaÑ3HÈ,×J\ÑJ\Ð]^ÑJ_ó
ð 	
r+   c                 ó"  — | j                  d«      } | d   j                  g | j                  «       ¢|‘­Ž } t        j                  t        j                  ||| j
                  t        j                  ¬«      d¬«      }| j                  | d«      } t        j                  | d¬«      }t        j                  t        j                  ||| j
                  t        j                  ¬«      d¬«      }|j                  | t        j                   «      }|S )zo
    More stable segment sum calculation. Uses cumulative sums and masking instead of direct subtractions.
    r-   ©.N©ÚdeviceÚdtype)Údiagonalr   éþÿÿÿ©Údim)
ÚsizeÚexpandr%   ÚtrilÚonesr5   ÚboolÚmasked_fillÚcumsumÚinf)r   r0   ÚmaskÚtensor_segsums       r)   Úsegment_sumrE   f   sÞ   € ð ×"Ñ" 2Ó&€Jð 2�< 	Ñ*×1Ñ1ÐS°<×3DÑ3DÓ3FÐSÈ
ÒS€Lä�:‰:”e—j‘j ¨ZÀ×@SÑ@SÔ[`×[eÑ[eÔfÐqsÔt€DØ×+Ñ+¨T¨E°1Ó5€Lä—L‘L °2Ô6€Mô �:‰:”e—j‘j ¨ZÀ×@SÑ@SÔ[`×[eÑ[eÔfÐqrÔs€DØ!×-Ñ-¨t¨e´e·i±i°ZÓ@€MØÐr+   c                 ó¦   — |�N|j                   d   dkD  r<|j                   d   dkD  r*| j                  }| |dd…dd…df   z  j                  |«      } | S )zm
    Tunes out the hidden states for padding tokens, see https://github.com/state-spaces/mamba/issues/66
    Nr   r   )r$   r6   Úto)Úhidden_statesÚattention_maskr6   s      r)   Úapply_mask_to_padding_statesrJ   z   sa   € ð Ð! n×&:Ñ&:¸1Ñ&=ÀÒ&AÀn×FZÑFZÐ[\ÑF]Ð`aÒFaØ×#Ñ#ˆØ&¨ºº1¸d¸
Ñ)CÑC×GÑGÈÓNˆàÐr+   c            
       óÊ   — e Zd ZdZej
                  dfdededej                  de	e
   fd„Z	 dded	ej                  d
edej                  fd„Zdedej                  fd„Zd„ Zy)ÚMamba2CacheaÊ  
    Arguments:
        config: Mamba2Config
        batch_size: int
        dtype: torch.dtype
        device: torch.device

    Attributes:
        dtype: (`torch.dtype`):
            The default `dtype` used to initializing the cache.
        conv_kernel_size: (`int`):
            Model's convolution kernel size taken from config.
        n_groups: (`int`):
            Model's number of groups taken from the config - similar to tensor parallel in Transformer.
        state_size: (`int`):
            Model's SSM state size taken from config.
        num_heads: (`int`):
            The number of heads used in the linear attention / SSM.
        head_dim: (`int`):
            The respective dimension of the heads used in the linear attention / SSM.
        intermediate_size: (`int`):
            Model's intermediate_size based on (expand * hidden_dim) from config.
        conv_states: (`torch.Tensor`):
            A tensor of shape `[num_layers, batch_size, conv_kernel_size, intermediate_size + 2 * n_groups * state_size]` that holds convolutional states.
        ssm_states: (`torch.Tensor`):
            A tensor of shape `[num_layers, batch_size, num_heads, head_dim, state_size]` that holds ssm states.
    NÚconfigÚ
batch_sizer6   r5   c           	      óR  — || _         |j                  | _        |j                  | _        |j                  | _        |j
                  | _        |j                  | _        t        |j                  |j                  z  «      | _
        t        j                  |j                  || j                  d| j                  z  | j                  z  z   | j                  ||¬«      | _        t        j                  |j                  || j
                  | j                  | j                  ||¬«      | _        y )Nr.   r4   )r6   Úconv_kernelÚconv_kernel_sizeÚn_groupsÚ
state_sizeÚ	num_headsÚhead_dimÚintr<   Úhidden_sizeÚintermediate_sizer%   ÚzerosÚnum_hidden_layersÚconv_statesÚ
ssm_states)ÚselfrM   rN   r6   r5   s        r)   Ú__init__zMamba2Cache.__init__¢   sê   € ð ˆŒ
Ø &× 2Ñ 2ˆÔØŸ™ˆŒØ ×+Ñ+ˆŒØ×)Ñ)ˆŒØŸ™ˆŒÜ!$ V§]¡]°V×5GÑ5GÑ%GÓ!HˆÔä Ÿ;™;Ø×$Ñ$ØØ×"Ñ" Q¨¯©Ñ%6¸¿¹Ñ%HÑHØ×!Ñ!ØØô
ˆÔô  Ÿ+™+Ø×$Ñ$ØØ�N‰NØ�M‰MØ�O‰OØØô
ˆ�r+   Ú	layer_idxÚnew_conv_stateÚ
cache_initÚreturnc                 óp  — |r3|j                  | j                  j                  «      | j                  |<   ns| j                  |   j                  dd¬«      | j                  |<   |d d …dd d …f   j                  | j                  j                  «      | j                  |   d d …d d …df<   | j                  |   S )Nr-   )ÚshiftsÚdimsr   )rG   r[   r5   Úroll)r]   r_   r`   ra   s       r)   Úupdate_conv_statezMamba2Cache.update_conv_state¿   s©   € ñ Ø*8×*;Ñ*;¸D×<LÑ<L×<SÑ<SÓ*TˆD×Ñ˜YÒ'à*.×*:Ñ*:¸9Ñ*E×*JÑ*JÐRTÐ[]Ð*JÓ*^ˆD×Ñ˜YÑ'Ø4BÂ1ÀaÊÀ7Ñ4K×4NÑ4NÈt×O_ÑO_×OfÑOfÓ4gˆD×Ñ˜YÑ'ªª1¨b¨Ñ1Ø×Ñ 	Ñ*Ð*r+   Únew_ssm_statec                 ó„   — |j                  | j                  j                  «      | j                  |<   | j                  |   S ©N)rG   r\   r5   )r]   r_   rh   s      r)   Úupdate_ssm_statezMamba2Cache.update_ssm_stateÉ   s4   € Ø%2×%5Ñ%5°d·o±o×6LÑ6LÓ%Mˆ�‰˜	Ñ"Ø�‰˜yÑ)Ð)r+   c                 ól   — | j                   j                  «        | j                  j                  «        y rj   )r[   Úzero_r\   ©r]   s    r)   ÚresetzMamba2Cache.resetÍ   s$   € Ø×Ñ×ÑÔ Ø�‰×ÑÕr+   )F)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r%   Úfloat16r   rV   r6   r   Ústrr^   ÚTensorr?   rg   rk   ro   © r+   r)   rL   rL   …   s”   „ ñð: KPÏ-É-Ðquñ
Ø"ð
Ø03ð
Ø<A¿K¹Kð
ØaiÐjmÑanó
ð< PUñ+Øð+Ø.3¯l©lð+ØHLð+à	�‰ó+ð*¨#ð *¸e¿l¹ló *ó r+   rL   c                   ó(   ‡ — e Zd Zdˆ fd„	Zdd„Zˆ xZS )ÚMambaRMSNormGatedc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y rj   ©Úsuperr^   r   Ú	Parameterr%   r>   ÚweightÚvariance_epsilon©r]   rW   ÚepsÚ	__class__s      €r)   r^   zMambaRMSNormGated.__init__Ó   s/   ø€ Ü‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr+   c                 ó¤  — |j                   }|j                  t        j                  «      }|�?|t        j
                  j                  |j                  t        j                  «      «      z  }|j                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S ©Nr.   r-   T)Úkeepdim)r6   rG   r%   Úfloat32r   r&   ÚsiluÚpowÚmeanÚrsqrtr   r~   )r]   rH   ÚgateÚinput_dtypeÚvariances        r)   ÚforwardzMambaRMSNormGated.forwardØ   s¥   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆàÐØ)¬B¯M©M×,>Ñ,>¸t¿w¹wÄuÇ}Á}Ó?UÓ,VÑVˆMØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆà�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r+   ©g�íµ ÷Æ°>rj   ©rp   rq   rr   r^   rŽ   Ú__classcell__©r‚   s   @r)   ry   ry   Ò   s   ø„ õ$÷
	;r+   ry   c            
       ó@  ‡ — e Zd ZdZdedefˆ fd„Z	 	 	 ddej                  de	e
   de	ej                     de	ej                     fd	„Zdde	e
   de	ej                     de	ej                     fd
„Z	 	 	 dde	e
   de	ej                     de	ej                     fd„Zˆ xZS )ÚMamba2Mixeruƒ  
    Compute âˆ†, A, B, C, and D the state space parameters and compute the `contextualized_states`.
    A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective)
    âˆ†, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4,
    and is why Mamba is called **selective** state spaces)
    rM   r_   c           	      ób  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | _        |j                  | _        t        |j                  | j                  z  «      | _
        t        |j                  «      | _        || _        |j                  | _        |j                  | _        t         |j                     | _        |j$                  | _        |j&                  | _        |j(                  | _        |j*                  | _        |j,                  | _        |j.                  | _        |j0                  | _        |j2                  | _        | j                  d| j(                  z  | j
                  z  z   | _        t7        j8                  | j4                  | j4                  |j                  |j                  | j4                  |j                  dz
  ¬«      | _        | j                  | j4                  z   | j                  z   }t7        j<                  | j                  ||j>                  ¬«      | _         t7        jB                  tE        jF                  | j                  «      «      | _$        tE        jJ                  d| j                  dz   «      }t7        jB                  tE        jL                  |«      «      | _'        d| jN                  _(        tS        | j                  | j$                  ¬«      | _*        t7        jB                  tE        jF                  | j                  «      «      | _+        d| jV                  _(        t7        j<                  | j                  | j                  |j>                  ¬«      | _,        |j>                  | _        tZ        st\        j_                  d«       y y )Nr.   r   )Úin_channelsÚout_channelsÚbiasÚkernel_sizeÚgroupsÚpadding©r˜   T©r�   a  The fast path is not available because on of `(selective_state_update, causal_conv1d_fn, causal_conv1d_update)` is None. Falling back to the naive implementation. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d)0r|   r^   rT   rW   rS   Ússm_state_sizerP   rQ   rV   r<   rX   Útime_step_rankr_   Úuse_conv_biasÚ
hidden_actÚ
activationr
   ÚactÚlayer_norm_epsilonÚrms_normrR   rU   r0   Útime_step_limitÚtime_step_minÚtime_step_maxÚconv_dimr   ÚConv1dÚconv1dÚLinearÚuse_biasÚin_projr}   r%   r>   Údt_biasÚarangeÚlogÚA_logÚ_no_weight_decayry   ÚnormÚDÚout_projÚis_fast_path_availableÚloggerÚwarning_once)r]   rM   r_   Úprojection_sizeÚAr‚   s        €r)   r^   zMamba2Mixer.__init__ì   s©  ø€ Ü‰ÑÔØ×)Ñ)ˆŒØ!×-Ñ-ˆÔØ$×/Ñ/ˆÔØ &× 2Ñ 2ˆÔÜ!$ V§]¡]°T×5EÑ5EÑ%EÓ!FˆÔÜ! &×"7Ñ"7Ó8ˆÔØ"ˆŒØ#×1Ñ1ˆÔØ ×+Ñ+ˆŒÜ˜&×+Ñ+Ñ,ˆŒà"(×";Ñ";ˆÔØŸ™ˆŒàŸ™ˆŒØŸ™ˆŒØ ×+Ñ+ˆŒà%×5Ñ5ˆÔØ#×1Ñ1ˆÔØ#×1Ñ1ˆÔà×.Ñ.°°T·]±]Ñ1BÀT×EXÑEXÑ1XÑXˆŒÜ—i‘iØŸ™ØŸ™Ø×%Ñ%Ø×*Ñ*Ø—=‘=Ø×&Ñ&¨Ñ*ô
ˆŒð ×0Ñ0°4·=±=Ñ@À4Ç>Á>ÑQˆÜ—y‘yØ×ÑØØ—‘ô
ˆŒô —|‘|¤E§J¡J¨t¯~©~Ó$>Ó?ˆŒô �L‰L˜˜DŸN™N¨QÑ.Ó/ˆÜ—\‘\¤%§)¡)¨A£,Ó/ˆŒ
Ø&*ˆ�
‰
Ô#Ü% d×&<Ñ&<À$×BYÑBYÔZˆŒ	Ü—‘œeŸj™j¨¯©Ó8Ó9ˆŒØ"&ˆ�‰ÔäŸ	™	 $×"8Ñ"8¸$×:JÑ:JÐQW×Q`ÑQ`ÔaˆŒØŸ™ˆŒå%Ü×Ñð>õð &r+   rH   Úcache_paramsÚcache_positionrI   c                 óÞ  — t        ||«      }| j                  |«      }|j                  \  }}}| j                  | j                  z  }	|j                  d   d| j
                  z  z
  d| j                  z  | j                  z  z
  | j                  z
  dz  }
|��|��|d   dkD  �rú|j                  d«      j                  |
|
| j
                  | j                  | j                  gd¬«      \  }}}}}t        ||j                  | j                     | j                  j                  j                  d«      | j                  j                  | j                   «      }t#        j                  || j
                  |	|	gd¬«      \  }}}t#        j$                  | j&                  j)                  «       «       }|d d …d df   d d …d d …d f   j+                  d| j,                  | j                  «      j/                  t"        j0                  ¬«      }|d d …d d …d f   j+                  dd| j,                  «      }| j2                  d d …d df   j+                  d| j,                  «      }| j4                  d d …d df   j+                  d| j,                  «      }|j7                  || j                  |j                  d   | j                  z  «      }|j7                  || j                  |j                  d   | j                  z  «      }|j7                  || j                  | j,                  «      }t9        |j:                  | j                     ||||||d |d¬	«
      }|j7                  || j                  | j,                  z  «      }| j=                  ||«      }| j?                  |«      d d …d df   }|S t#        j$                  | j&                  j)                  «       «       }| j@                  d
t)        d«      fk(  ri nd| j@                  i}| jB                  rä|€âtE        || j                  j                  j                  d«      | j                  j                  | j2                  |f| j4                  | jF                  d | j                   | j<                  j                  | j<                  jH                  | j>                  j                  | j>                  j                  | j,                  | j                  dddœ|¤Ž}|S |j                  |
|
| j
                  | j                  | j                  gd¬«      \  }}}}}|�l|jK                  dd«      }tL        jN                  jQ                  ||jR                  |j                  d   z
  df«      }|jU                  | j                  |d¬«       | j                   dvrH| jW                  | j                  |jK                  dd«      «      dd |…f   jK                  dd«      «      }nptY        |jK                  dd«      | j                  j                  j                  d«      | j                  j                  | j                   ¬«      jK                  dd«      }t        ||«      }t#        j                  || j
                  |	|	gd¬«      \  }}}t[        |j7                  ||d| j,                  «      |||j7                  ||| j                  d«      |j7                  ||| j                  d«      f| jF                  | j4                  d d d| j2                  ddœ|¤Ž\  }}|�|�|j]                  | j                  |¬«       |j7                  ||d«      }| j=                  ||«      }| j?                  |«      }|S )Nr-   r.   r   r   r9   .©r6   T)Úzr¯   Údt_softplusg        rB   Údt_limitF)rµ   r0   Úseq_idxr¢   Úrmsnorm_weightÚrmsnorm_epsÚoutproj_weightÚoutproj_biasÚheaddimÚngroupsÚnorm_before_gateÚreturn_final_states©r_   r`   ra   )r‡   Úswish)Úxr~   r˜   r¢   )r0   rµ   rÀ   rÃ   rË   r¯   rÁ   ©r_   rh   )/rJ   r®   r$   rR   rž   rX   rT   ÚsqueezeÚsplitr©   r   r[   r_   r«   r~   r˜   r¢   r%   Úexpr²   Úfloatr<   rU   rG   r†   r¯   rµ   Úviewr   r\   r´   r¶   r¦   Útrainingr   r0   r   Ú	transposer   r&   r'   rQ   rg   r£   r   r   rk   )r]   rH   r¼   r½   rI   Úprojected_statesrN   Úseq_lenÚ_Úgroups_time_state_sizeÚd_mlpr‹   Úhidden_states_B_CÚdtÚBÚCr»   r¯   rµ   Úhidden_states_reshapedÚoutÚdt_limit_kwargsÚhidden_states_B_C_transposedr[   Úscan_outputÚ	ssm_states                             r)   Úcuda_kernels_forwardz Mamba2Mixer.cuda_kernels_forward.  sï  € ô 5°]ÀNÓSˆØŸ<™<¨Ó6Ðð "/×!4Ñ!4Ñˆ
�G˜QØ!%§¡°×1DÑ1DÑ!DÐà×"Ñ" 2Ñ&Ø�$×(Ñ(Ñ(ñ)à�$—-‘-Ñ $×"5Ñ"5Ñ5ñ6ð �n‰nñð ñˆð Ñ#¨Ñ(BÀ~ÐVWÑGXÐ[\ÓG\Ø0@×0HÑ0HÈÓ0K×0QÑ0QØ˜˜t×5Ñ5°t·}±}ÀdÇnÁnÐUÐ[]ð 1Ró 1Ñ-ˆAˆq�$Ð)¨2ô
 !5Ø!Ø×(Ñ(¨¯©Ñ8Ø—‘×"Ñ"×*Ñ*¨1Ó-Ø—‘× Ñ Ø—‘ó!Ðô #(§+¡+Ø!Ø×'Ñ'Ð)?ÐAWÐXØô#ÑˆM˜1˜aô —‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAØ’!�T˜3�,‘¢¢1 d 
Ñ+×2Ñ2°2°t·}±}Àd×FYÑFYÓZ×]Ñ]Ôdi×dqÑdqÐ]ÓrˆAØ’A’q˜$�J‘×&Ñ& r¨2¨t¯}©}Ó=ˆBØ—l‘l¢1 d¨C <Ñ0×7Ñ7¸¸D¿M¹MÓJˆGØ—‘’q˜$ �|Ñ$×+Ñ+¨B°·±Ó>ˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ—‘�z 4§=¡=°!·'±'¸!±*ÀÇÁÑ2MÓNˆAØ%2×%7Ñ%7¸
ÀDÇNÁNÐTX×TaÑTaÓ%bÐ"Ü2Ø×'Ñ'¨¯©Ñ7Ø&ØØØØØØØØ ôˆMð *×.Ñ.¨z¸4¿>¹>ÈDÏMÉMÑ;YÓZˆMØ ŸI™I m°TÓ:ˆMð —-‘- Ó.ªq°$¸¨|Ñ<ˆCðz ˆ
ôs —‘˜4Ÿ:™:×+Ñ+Ó-Ó.Ð.ˆAØ$(×$8Ñ$8¸SÄ%ÈÃ,Ð<OÒ$O™bÐV`Ðbf×bvÑbvÐUwˆOð �}Š} Ð!5Ü6Ø$Ø—K‘K×&Ñ&×.Ñ.¨qÓ1Ø—K‘K×$Ñ$Ø—L‘LØðð —f‘fØ#Ÿ™Ø Ø#Ÿ™Ø#'§9¡9×#3Ñ#3Ø $§	¡	× :Ñ :Ø#'§=¡=×#7Ñ#7Ø!%§¡×!3Ñ!3Ø ŸM™MØ ŸM™MØ%*Ø(-ñ#ð$ &ñ%�ðh ˆ
ð} 5E×4JÑ4JØ˜E 4×#9Ñ#9¸4¿=¹=È$Ï.É.ÐYÐ_að 5Kó 5Ñ1��1�dÐ-¨rð  Ð+Ø3D×3NÑ3NÈqÐRSÓ3TÐ0Ü"$§-¡-×"3Ñ"3Ø4Ø%×6Ñ6Ð9U×9[Ñ9[Ð\^Ñ9_Ñ_ÐabÐcó#�Kð !×2Ñ2Ø"&§.¡.ÀÐY]ð 3ô ð —?‘?Ð*;Ñ;Ø(,¯©ØŸ™Ð$5×$?Ñ$?ÀÀ1Ó$EÓFÀsÈHÈWÈHÀ}ÑU×_Ñ_Ð`aÐcdÓeó)Ñ%ô )9Ø+×5Ñ5°a¸Ó;Ø#Ÿ{™{×1Ñ1×9Ñ9¸!Ó<Ø!Ÿ[™[×-Ñ-Ø#'§?¡?ô	)÷
  ‘i  1“oð &ô %AÐARÐTbÓ$cÐ!Ü&+§k¡kØ%Ø×+Ñ+Ð-CÐE[Ð\Øô'Ñ#�˜q !ô *CØ!×&Ñ& z°7¸BÀÇÁÓNØØØ—F‘F˜: w°·±¸rÓBØ—F‘F˜: w°·±¸rÓBð*ð  $Ÿ™Ø—f‘fØØ Ø(,Ø ŸL™LØ $ñ*ð &ñ*Ñ&�˜Yð" Ð(¨\Ð-EØ ×1Ñ1¸D¿N¹NÐZcÐ1Ôdà)×.Ñ.¨z¸7ÀBÓG�à"Ÿi™i¨°TÓ:�ð —m‘m KÓ0�Øˆ
r+   c                 ór  — |j                   \  }}}|j                  }t        ||«      }| j                  |«      }	|	j                   d   d| j                  z  z
  d| j
                  z  | j                  z  z
  | j                  z
  dz  }
|	j                  |
|
| j                  | j                  | j                  gd¬«      \  }}}}}|�ã|�á|d   dkD  rÙ|j                  | j                  |d¬«       |j                  | j                     j                  | j                  j                  j                   ¬«      }t#        j$                  || j                  j                  j'                  d«      z  d¬«      }| j(                  r|| j                  j*                  z   }| j-                  |«      }nµ|�l|j/                  dd«      }t0        j2                  j5                  ||j6                  |j                   d   z
  df«      }|j                  | j                  |d	¬«       | j-                  | j                  |j/                  dd«      «      d
d |…f   j/                  dd«      «      }t        ||«      }t#        j                  || j                  | j
                  | j                  z  | j
                  | j                  z  gd¬«      \  }}}t#        j8                  | j:                  j=                  «       «       }|��¢|��Ÿ|d   dkD  �r–|j>                  j                   }|d d …dd d …f   d d …d d
f   }|j/                  dd«      jA                  ||j                   d   | jB                  «      }| jD                  d   jA                  | jD                  j                   d   | jB                  «      }t"        j0                  j2                  jG                  ||j                  |j                  «      z   «      }t#        jH                  || jJ                  d   | jJ                  d   «      }|d   jA                  | j                  | jB                  | j                  «      j                  t"        jL                  ¬«      }t#        j8                  |d   |z  «      j                  |¬«      }|jO                  || j
                  d«      d
d d d …f   }|jA                  || j
                  | j                  | j
                  z  |j                   d   «      jQ                  «       }|jO                  |d|j                   d   «      }|d   |d
d d d …f   z  }|jO                  |d| jB                  «      }||d   z  j                  |¬«      }|jS                  | j                  |j>                  | j                     |z  |z   ¬«       |jO                  || j
                  d«      d
d d d …f   }|jA                  || j
                  | j                  | j
                  z  |j                   d   «      jQ                  «       }|jO                  |d|j                   d   «      }|j>                  | j                     j                  |j                   |j                  ¬«      }|jU                  || j                  z  | jB                  | j                  «      }|jU                  || j                  z  | j                  d«      }t#        jV                  ||«      }|jU                  || j                  | jB                  «      }| jX                  d   jA                  | jX                  j                   d   | jB                  «      }|||z  z   j                  |j                  «      }|jO                  |d«      d d …d d
f   }�nît0        j2                  jG                  || jD                  z   «      }t#        jH                  || jJ                  d   | jJ                  d   «      }|jO                  ||d| jB                  «      j=                  «       }|jO                  ||d| j                  «      j=                  «       }|jO                  ||d| j                  «      j=                  «       }|j[                  dd| j                  | j
                  z  d«      }|j[                  dd| j                  | j
                  z  d«      }| j\                  || j\                  z  z
  | j\                  z  }| jX                  d   t_        ||«      z  }||d   z  }|j                  |j                  «      |z  }||||fD � cg c]  } ta        | || j\                  «      ‘Œ c} \  }}}}|jc                  dddd«      }t#        jd                  |d¬«      }!t#        j8                  tg        |«      «      }"|d d …d d …d d …d d d …d d …f   |d d …d d …d d d …d d …d d …f   z  }#|#j%                  d¬«      }$|$d   |"jc                  ddddd«      d   z  }%|%j%                  d¬«      }&|&d   |d d …d d …d f   z  j%                  d¬«      }'t#        j8                  |!d d …d d …d d …dd …f   |!z
  «      }(||(jc                  dddd«      d   z  })|)d
d d d …f   |d   z  j%                  d¬«      }*|�F|�D|d   dkD  r<|j>                  | j                     d d …d d
f   j                  |*j                   ¬«      }+nt#        jh                  |*d d …d d…f   «      }+t#        jj                  |+|*gd¬«      }*t#        j8                  tg        t0        j2                  j5                  |!d d …d d …d d …df   d«      «      «      },|,j/                  dd«      },|,d   |*d d …d d …d d
f   z  j%                  d¬«      }-|-d d …d d…f   |-d d …df   }.}*t#        j8                  |!«      }/|d
d d d …f   |*d d …d d …d d
f   z  }0|/jc                  dddd«      }1|0j%                  d«      |1d   z  }2|'|2z   }|jO                  |d| j                  | jB                  «      }||z   }|dkD  r|d d …d |…d d …d d …f   }|jO                  ||d«      }|.�|�|jS                  | j                  |.¬«       | jm                  ||«      }3| jo                  |3j                  |«      «      }4|4S c c} w )Nr-   r.   r9   r   FrÌ   ©r5   r   T.r3   ).NNr¿   rÏ   r4   r	   r   r8   )r   r   )8r$   r6   rJ   r®   rX   rR   rž   rT   rÑ   r©   rg   r_   r[   rG   r«   r~   r5   r%   ÚsumrÐ   r    r˜   r£   rÖ   r   r&   r'   rQ   rÒ   r²   rÓ   r\   r<   rU   r¯   ÚsoftplusÚclampr¦   r†   r/   Ú
contiguousrk   rÔ   Úbmmrµ   Úrepeatr0   r*   r1   ÚpermuterA   rE   Ú
zeros_likeÚcatr´   r¶   )5r]   Úinput_statesr¼   r½   rI   rN   rØ   rÙ   r6   r×   rÛ   r‹   rÜ   rÝ   r[   rã   rH   rÞ   rß   r»   Úcache_devicer¯   ÚdAÚdBÚdBxr\   Ússm_states_reshapedÚ
C_reshapedÚyrµ   r   Ú
D_residualÚtÚA_cumsumÚLÚG_intermediateÚGÚM_intermediateÚMÚY_diagÚdecay_statesÚB_decayÚstatesÚprevious_statesÚdecay_chunkÚ
new_statesrå   Ústate_decay_outÚC_times_statesÚstate_decay_out_permutedÚY_offrä   Úcontextualized_statess5                                                        r)   Útorch_forwardzMamba2Mixer.torch_forwardÑ  sµ  € Ø!-×!3Ñ!3Ñˆ
�G˜QØ×"Ñ"ˆô 4°LÀ.ÓQˆØŸ<™<¨Ó5ÐØ!×'Ñ'¨Ñ+¨a°$×2HÑ2HÑ.HÑHÈ1ÈtÏ}É}ÑK\Ð_c×_rÑ_rÑKrÑrÐsw÷  tBñ  tBñ  Bð  GHñ  HˆØ,<×,BÑ,BØ˜˜t×5Ñ5¸¿¹ÀtÇ~Á~ÐVÐ\^ð -Có -
Ñ)ˆˆ1ˆdÐ% rð
 Ð#¨Ð(BÀ~ÐVWÑGXÐ[\ÒG\Ø×*Ñ*°T·^±^ÐTeÐrwÐ*Ôxð '×2Ñ2°4·>±>ÑB×EÑEÈTÏ[É[×M_ÑM_×MfÑMfÐEÓgˆKä %§	¡	Ø˜dŸk™k×0Ñ0×8Ñ8¸Ó;Ñ;Àô!Ðð ×!Ò!Ø$5¸¿¹×8HÑ8HÑ$HÐ!Ø $§¡Ð):Ó ;Ñð Ð'Ø/@×/JÑ/JÈ1ÈaÓ/PÐ,Ü Ÿm™m×/Ñ/Ø0°<×3PÑ3PÐSo×SuÑSuÐvxÑSyÑ3yÐ{|Ð2}ó�ð ×.Ñ.¸¿¹ÐXcÐptÐ.Ôuà $§¡¨¯©Ð5F×5PÑ5PÐQRÐTUÓ5VÓ)WÐX[Ð]eÐ^eÐ]eÐXeÑ)f×)pÑ)pÐqrÐtuÓ)vÓ wÐä8Ð9JÈNÓ[ÐÜ#Ÿk™kØØ×#Ñ# T§]¡]°T×5HÑ5HÑ%HÈ$Ï-É-ÐZ^×ZmÑZmÑJmÐnØô
Ñˆ�q˜!ô �Y‰Y�t—z‘z×'Ñ'Ó)Ó*Ð*ˆØÑ#¨Ñ(BÀ~ÐVWÑGXÐ[\ÓG\à'×2Ñ2×9Ñ9ˆLð ’A�qš!�G‘šQ  c˜\Ñ*ˆBØ—‘˜a Ó#×*Ñ*¨:°r·x±xÀ±|ÀTÇ]Á]ÓSˆBà—l‘l 9Ñ-×4Ñ4°T·\±\×5GÑ5GÈÑ5JÈDÏMÉMÓZˆGä—‘×$Ñ$×-Ñ-¨b°7·:±:¸b¿h¹hÓ3GÑ.GÓHˆBÜ—‘˜R ×!5Ñ!5°aÑ!8¸$×:NÑ:NÈqÑ:QÓRˆBØ�/Ñ"×)Ñ)¨$¯.©.¸$¿-¹-È×I\ÑI\Ó]×`Ñ`Ôgl×gtÑgtÐ`ÓuˆAä—)‘)˜B˜y™M¨AÑ-Ó.×2Ñ2¸,Ð2ÓGˆBð
 —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAà�I‘  3¨ªa <¡Ñ0ˆBð *×1Ñ1°*¸bÀ$Ç-Á-ÓPˆMØ˜ iÑ0Ñ0×4Ñ4¸LÐ4ÓIˆCð ×)Ñ)ØŸ.™.Ø*×5Ñ5°d·n±nÑEÈÑJÈSÑPð *ô ð —	‘	˜* d§m¡m°RÓ8¸¸dÂA¸ÑFˆAØ—‘˜ T§]¡]°D·N±NÀdÇmÁmÑ4SÐUV×U\ÑU\Ð]_ÑU`Óa×lÑlÓnˆAØ—	‘	˜* b¨!¯'©'°"©+Ó6ˆAð &×0Ñ0°·±Ñ@×CÑCÈ1Ï8É8Ð[\×[bÑ[bÐCÓcˆJà",§/¡/°*¸t¿~¹~Ñ2MÈtÏ}É}Ð^b×^qÑ^qÓ"rÐØŸ™ 
¨T¯^©^Ñ ;¸T×=PÑ=PÐRSÓTˆJÜ—	‘	Ð-¨zÓ:ˆAØ—‘�z 4§>¡>°4·=±=ÓAˆAð —‘�yÑ!×(Ñ(¨¯©¯©°a©¸$¿-¹-ÓHˆAØ�] QÑ&Ñ&×*Ñ*¨1¯7©7Ó3ˆAð —	‘	˜* bÓ)ª!¨T°3¨,Ñ7ŠAô —‘×'Ñ'¨¨T¯\©\Ñ(9Ó:ˆBÜ—‘˜R ×!5Ñ!5°aÑ!8¸$×:NÑ:NÈqÑ:QÓRˆBØ)×1Ñ1°*¸gÀrÈ4Ï=É=ÓY×_Ñ_ÓaˆMØ—	‘	˜* g¨r°4×3FÑ3FÓG×MÑMÓOˆAØ—	‘	˜* g¨r°4×3FÑ3FÓG×MÑMÓOˆAØ—‘˜˜A˜tŸ~™~°·±Ñ>ÀÓBˆAØ—‘˜˜A˜tŸ~™~°·±Ñ>ÀÓBˆAØŸ™¨'°D·O±OÑ*CÑCÀtÇÁÑVˆHàŸ™ 	Ñ*Ô-?ÀÈxÓ-XÑXˆJð *¨B¨y©MÑ9ˆMØ—‘�]×(Ñ(Ó)¨BÑ.ˆAð cpÐqrÐtuÐwxÐayÖ%zÐ\]Ô&9¸!¸XÀtÇÁÕ&WÒ%zÑ"ˆM˜1˜a ð —	‘	˜!˜Q  1Ó%ˆAÜ—|‘| A¨2Ô.ˆHô —	‘	œ+ a›.Ó)ˆAð šq¢!¢Q¨ªa²Ð2Ñ3°aºº1¸dÂAÂqÊ!Ð8KÑ6LÑLˆNØ×"Ñ" rÐ"Ó*ˆAð ˜y™\¨A¯I©I°a¸¸A¸qÀ!Ó,DÀYÑ,OÑOˆNØ×"Ñ" rÐ"Ó*ˆAð ˜	‘l ]²1²a¸°:Ñ%>Ñ>×CÑCÈÐCÓJˆFô !Ÿ9™9 hªq²!²Q¸¹¨|Ñ&<¸xÑ&GÓIˆLØ˜,×.Ñ.¨q°"°b¸!Ó<¸YÑGÑGˆGØ˜c 4ª˜lÑ+¨m¸IÑ.FÑF×KÑKÐPQÐKÓRˆFð Ð'¨NÐ,FÈ>ÐZ[ÑK\Ð_`ÒK`Ø".×"9Ñ"9¸$¿.¹.Ñ"IÊ!ÈTÐSVÈ,Ñ"W×"ZÑ"ZÐbh×boÑboÐ"ZÓ"p‘ä"'×"2Ñ"2°6º!¸R¸a¸R¸%±=Ó"A�Ü—Y‘Y °Ð8¸aÔ@ˆFÜŸ)™)¤K´·±×0AÑ0AÀ(Ê1ÊaÒQRÐTVÈ;ÑBWÐY_Ó0`Ó$aÓbˆKØ%×/Ñ/°°1Ó5ˆKØ% oÑ6¸ÂÂ1ÀdÈCÀÑ9PÑP×UÑUÐZ[ÐUÓ\ˆJØ *ª1¨c¨r¨c¨6Ñ 2°JºqÀ"¸uÑ4E�IˆFô $Ÿi™i¨Ó1ˆOØ  Tª1 ™o°²qº!¸TÀ3°Ñ0GÑGˆNØ'6×'>Ñ'>¸qÀ!ÀQÈÓ'JÐ$Ø#×'Ñ'¨Ó+Ð.FÀyÑ.QÑQˆEð ˜‘ˆAà—	‘	˜* b¨$¯.©.¸$¿-¹-ÓHˆAà�J‘ˆAà˜!Š|Ø’a˜˜'˜¢1¢aÐ'Ñ(�Ø—	‘	˜* g¨rÓ2ˆAð Ð$¨Ð)AØ×-Ñ-¸¿¹ÐV_Ð-Ô`à—i‘i  4Ó(ˆð
 !%§¡¨k¯n©n¸UÓ.CÓ DÐØ$Ð$ùòG &{s   ä<r4c                 óV  — t         r@d| j                  j                  j                  j                  v r| j                  ||||«      S |j                  }|�B|j                  d   dkD  r0|j                  d   dkD  r||d d …d d …d f   z  j                  |«      }| j                  ||||«      S )NÚcudar   r   )
r·   r®   r~   r5   Útyperæ   r6   r$   rG   r  )r]   rH   r¼   r½   rI   r6   s         r)   rŽ   zMamba2Mixer.forward�  sª   € õ " f°·±×0CÑ0C×0JÑ0J×0OÑ0OÑ&OØ×,Ñ,¨]¸LÈ.ÐZhÓiÐiØ×#Ñ#ˆØÐ%¨.×*>Ñ*>¸qÑ*AÀAÒ*EÈ.×J^ÑJ^Ð_`ÑJaÐdeÒJeà*¨^ºAºqÀ$¸JÑ-GÑG×KÑKÈEÓRˆMà×!Ñ! -°¸~È~Ó^Ð^r+   r   )rp   rq   rr   rs   r   rV   r^   r%   rv   r   rL   Ú
LongTensorræ   r  rŽ   r‘   r’   s   @r)   r”   r”   ä   s  ø„ ñð@˜|ð @¸õ @ðJ /3Ø59Ø15ñ`à—|‘|ð`ð ˜{Ñ+ð`ð ! ×!1Ñ!1Ñ2ð	`ð
 ! §¡Ñ.ó`ñF|%¸ÀÑ8Mð |%ÐckÐlq×l|Ñl|Ñc}ð |%ð  U]ð  ^c÷  ^jñ  ^jñ  Ukó |%ðD /3Ø59Ø15ñ_ð ˜{Ñ+ð_ð ! ×!1Ñ!1Ñ2ð	_ð
 ! §¡Ñ.÷_r+   r”   c                   ó&   ‡ — e Zd Zdˆ fd„	Zd„ Zˆ xZS )ÚMamba2RMSNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y)zM
        Mamba2RMSNorm is equivalent to T5LayerNorm and LlamaRMSNorm
        Nr{   r€   s      €r)   r^   zMamba2RMSNorm.__init__¢  s1   ø€ ô 	‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕr+   c                 ó"  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  |j                  |«      z  S r„   )	r6   rG   r%   r†   rˆ   r‰   rŠ   r   r~   )r]   rH   rŒ   r�   s       r)   rŽ   zMamba2RMSNorm.forwardª  sy   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r+   r�   r�   r’   s   @r)   r  r  ¡  s   ø„ õ$ö;r+   r  c                   ót   ‡ — e Zd Zˆ fd„Z	 	 	 ddee   deej                     deej                     fd„Z	ˆ xZ
S )ÚMamba2Blockc                 óÐ   •— t         ‰| �  «        || _        || _        |j                  | _        t        |j                  |j                  ¬«      | _        t        ||¬«      | _
        y )Nr�   ©r_   )r|   r^   rM   r_   Úresidual_in_fp32r  rW   r¤   r´   r”   Úmixer)r]   rM   r_   r‚   s      €r)   r^   zMamba2Block.__init__³  sR   ø€ Ü‰ÑÔØˆŒØ"ˆŒØ &× 7Ñ 7ˆÔÜ! &×"4Ñ"4¸&×:SÑ:SÔTˆŒ	Ü  °9Ô=ˆ�
r+   r¼   r½   rI   c                 ó  — |}| j                  |j                  | j                   j                  j                  ¬«      «      }| j                  r|j                  t
        j                  «      }| j                  ||||¬«      }||z   }|S )Nr¿   ©r¼   r½   rI   )r´   rG   r~   r6   r  r%   r†   r  )r]   rH   r¼   r½   rI   Úresiduals         r)   rŽ   zMamba2Block.forward»  s   € ð !ˆØŸ	™	 -×"2Ñ"2¸¿¹×9IÑ9I×9OÑ9OÐ"2Ó"PÓQˆØ× Ò Ø—{‘{¤5§=¡=Ó1ˆHàŸ
™
Ø¨À^Ðdrð #ó 
ˆð ! =Ñ0ˆØÐr+   r   )rp   rq   rr   r^   r   rL   r%   r  rv   rŽ   r‘   r’   s   @r)   r  r  ²  sO   ø„ ô>ð /3Ø59Ø15ñð ˜{Ñ+ðð ! ×!1Ñ!1Ñ2ð	ð
 ! §¡Ñ.÷r+   r  c                   ó,   — e Zd ZdZeZdZdgZdZdZ	d„ Z
y)ÚMamba2PreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    Úbackboner  Tc                 ó&  — t        |t        «      �rvd|j                  _        d|j                  _        t        j                  t        j                  | j                  j                  «      t        j                  | j                  j                  «      t        j                  | j                  j                  «      z
  z  t        j                  | j                  j                  «      z   «      j                  | j                  j                  ¬«      }|t        j                  t        j                   | «       «      z   }t        j"                  «       5  |j$                  j'                  |«       ddd«       d|j$                  _        t        |t*        j,                  «      rM|j.                  �št1        |j.                  dd«      sƒt*        j2                  j5                  |j.                  «       nYt        |t*        j6                  «      r?t*        j2                  j9                  |j:                  | j                  j<                  ¬«       | j                  j>                  r›|jA                  «       D ]‡  \  }}|dv sŒt*        j2                  jC                  |t        jD                  d«      ¬	«       t        j"                  «       5  |t        jD                  | j                  jF                  «      z  }ddd«       Œ‰ yy# 1 sw Y   �Œ�xY w# 1 sw Y   Œ¢xY w)
zInitialize the weights.T)ÚminNÚ
_no_reinitF)Ústd)zout_proj.weighté   )Úa)$Ú
isinstancer”   r²   r³   rµ   r%   rÒ   ÚrandrM   rT   Úmathr±   r¨   r§   rë   Útime_step_floorÚexpm1Úno_gradr¯   Úcopy_r%  r   r¬   r˜   ÚgetattrÚinitÚzeros_Ú	EmbeddingÚnormal_r~   Úinitializer_rangeÚrescale_prenorm_residualÚnamed_parametersÚkaiming_uniform_ÚsqrtrZ   )r]   ÚmodulerÝ   Úinv_dtÚnameÚps         r)   Ú_init_weightsz#Mamba2PreTrainedModel._init_weightsÚ  s  € ä�fœkÕ*Ø,0ˆF�L‰LÔ)Ø(,ˆF�H‰HÔ%ä—‘Ü—
‘
˜4Ÿ;™;×0Ñ0Ó1Ü—8‘8˜DŸK™K×5Ñ5Ó6¼¿¹À$Ç+Á+×B[ÑB[Ó9\Ñ\ñ^ä—(‘(˜4Ÿ;™;×4Ñ4Ó5ñ6ó÷ ‰e˜Ÿ™×3Ñ3ˆeÓ4ð	 ð œ%Ÿ)™)¤U§[¡[°"°Ó%5Ð$5Ó6Ñ6ˆFÜ—‘“ñ -Ø—‘×$Ñ$ VÔ,÷-à(,ˆF�N‰NÔ%ä�fœbŸi™iÔ(Ø�{‰{Ð&Ü˜vŸ{™{¨L¸%Ô@Ü—G‘G—N‘N 6§;¡;Õ/Ü˜¤§¡Ô-Ü�G‰G�O‰O˜FŸM™M¨t¯{©{×/LÑ/LˆOÔMà�;‰;×/Ò/ð "×2Ñ2Ó4ò F‘��aØÐ.Ò.ô
 —G‘G×,Ñ,¨Q´$·)±)¸A³,Ð,Ô?ÜŸ™›ñ FØœTŸY™Y t§{¡{×'DÑ'DÓEÑE˜÷Fð FñFð 0÷-ñ -ú÷2Fð Fús   ÅK:Ë-LË:LÌL	N)rp   rq   rr   rs   r   Úconfig_classÚbase_model_prefixÚ_no_split_modulesÚsupports_gradient_checkpointingÚ_is_statefulr>  rw   r+   r)   r!  r!  Î  s-   „ ñð
  €LØ"ÐØ&˜ÐØ&*Ð#Ø€Ló(Fr+   r!  c                   ó|   — e Zd ZU dZdZeej                     ed<   dZ	ee
   ed<   dZeeej                        ed<   y)ÚMamba2Outputa%  
    Class for the MAMBA2 model outputs.

    Args:
        last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`):
            Sequence of hidden-states at the output of the last layer of the model.
        cache_params (`Mamba2Cache`):
            The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
            avoid providing the old `input_ids`.

            Includes both the State space model state matrices after the selective scan, and the Convolutional states
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
    NÚlast_hidden_stater¼   rH   )rp   rq   rr   rs   rF  r   r%   ÚFloatTensorÚ__annotations__r¼   rL   rH   r   rw   r+   r)   rE  rE    sH   … ñð$ 6:Ð�x × 1Ñ 1Ñ2Ó9Ø*.€L�(˜;Ñ'Ó.Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ô<r+   rE  c                   ó¤   — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
ee   ed<   dZeeej                        ed<   y)ÚMamba2CausalLMOutputaý  
    Base class for causal language model (or autoregressive) outputs.

    Args:
        loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided):
            Language modeling loss (for next-token prediction).
        logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`):
            Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
        cache_params (`Mamba2Cache`):
            The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to
            avoid providing the old `input_ids`.

            Includes both the State space model state matrices after the selective scan, and the Convolutional states
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, +
            one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the optional initial embedding outputs.
    NÚlossÚlogitsr¼   rH   )rp   rq   rr   rs   rK  r   r%   rG  rH  rL  r¼   rL   rH   r   rw   r+   r)   rJ  rJ    s\   … ñð( )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø*.€FˆH�U×&Ñ&Ñ'Ó.Ø*.€L�(˜;Ñ'Ó.Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ô<r+   rJ  a@  

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads
    etc.)

    This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass.
    Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage
    and behavior.

    Parameters:
        config ([`Mamba2Config`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
a	  
    Args:
        input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`):
            Indices of input sequence tokens in the vocabulary.

            If `cache_params.seqlen_offset>0`, only `input_ids` that do not have their past calculated should be passed as
            `input_ids`.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
            model's internal embedding lookup matrix.
        cache_params (`Mamba2Cache`, *optional*):
            If passed along, the model uses the previous state in all the blocks (which will give the output for the
            `input_ids` provided as if the model add `state_input_ids + input_ids` as context).
        use_cache (`bool`, *optional*):
            If set to `True`, the `cache_params` is returned and can be used to quickly generate the next logits.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        cache_position (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            The position of the current input in the cache. This is used to ensure that the cache is correctly updated.
            If `cache_params` is passed, `cache_position` should also be passed.
        attention_mask (`torch.FloatTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
z`The bare MAMBA2 Model transformer outputting raw hidden-states without any specific head on top.c                   ó*  ‡ — e Zd Zˆ fd„Zd„ Zd„ Zd„ Z ee«       e	e
ee¬«      	 	 	 	 	 	 	 	 ddeej                     deej                     dee   d	ee   d
ee   dee   deej                     deej$                     deeef   fd„«       «       Zˆ xZS )ÚMamba2Modelc           	      óÈ  •— t         ‰| �  |«       t        j                  |j                  |j
                  «      | _        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        d| _        t        |j
                  |j                  ¬«      | _        | j!                  | j"                  «       | j%                  «        y c c}w )Nr  Fr�   )r|   r^   r   r3  Ú
vocab_sizerW   Ú
embeddingsÚ
ModuleListÚrangerZ   r  ÚlayersÚgradient_checkpointingr  r¤   Únorm_fÚ"_register_load_state_dict_pre_hookÚ	load_hookÚ	post_init)r]   rM   Úidxr‚   s      €r)   r^   zMamba2Model.__init__x  s¡   ø€ Ü‰Ñ˜Ô äŸ,™, v×'8Ñ'8¸&×:LÑ:LÓMˆŒÜ—m‘mÔSXÐY_×YqÑYqÓSrÖ$sÈC¤[°À3Ö%GÒ$sÓtˆŒà&+ˆÔ#Ü# F×$6Ñ$6¸F×<UÑ<UÔVˆŒà×/Ñ/°·±Ô?Ø�‰Õùò %ts   Á&Cc                 óf   — |D ],  }d|v sŒ|j                  |«      ||j                  dd«      <    y  y )Nz
embedding.zembeddings.)ÚpopÚreplace)r]   Ú
state_dictÚprefixÚargsÚks        r)   rX  zMamba2Model.load_hook„  s;   € Øò 	ˆAØ˜qÒ ØEOÇ^Á^ÐTUÓEV�
˜1Ÿ9™9 \°=ÓAÑBÙñ	r+   c                 ó   — | j                   S rj   ©rQ  rn   s    r)   Úget_input_embeddingsz Mamba2Model.get_input_embeddingsŠ  s   € Ø�‰Ðr+   c                 ó   — || _         y rj   rc  ©r]   Únew_embeddingss     r)   Úset_input_embeddingsz Mamba2Model.set_input_embeddings�  s	   € Ø(ˆ�r+   ©Ú
checkpointÚoutput_typer?  Ú	input_idsÚinputs_embedsr¼   Ú	use_cacheÚoutput_hidden_statesÚreturn_dictr½   rI   rb   c	                 ó¨  — |�|n| j                   j                  }|�|n#| j                  s| j                   j                  nd}|�|n| j                   j                  }|d u |d uz  rt        d«      ‚|€| j                  |«      }| j                  r| j                  r|rd}|r‚|€st        | j                   |j                  d«      |j                  |j                  ¬«      }t        j                  d| j                   j                  |j                  ¬«      }n|€t        d«      ‚d }|}
|rdnd }| j                  D ]O  }| j                  r,| j                  r | j!                  |j"                  |
|||«      }
n ||
|||¬«      }
|sŒJ||
fz   }ŒQ | j%                  |
«      }
|r||
fz   }|st'        d	„ |
||fD «       «      S t)        |
|r||¬
«      S d |¬
«      S )NFz:You must specify exactly one of input_ids or inputs_embedsr   r4   rè   zîYou have to specify the `cache_position` manually when `use_cache=True` and `cache_params` is passed, you don't have to pass a `cache_params` if you are in prefilling stage because in that case it will be initialized for you automaticallyrw   r  c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrj   rw   )Ú.0Úvs     r)   ú	<genexpr>z&Mamba2Model.forward.<locals>.<genexpr>Û  s   è ø€ Òf˜qÐXYÑXeœÑfùs   ‚Š)rF  r¼   rH   )rM   ro  rÕ   rn  Úuse_return_dictÚ
ValueErrorrQ  rU  rL   r;   r5   r6   r%   r°   rP   rT  Ú_gradient_checkpointing_funcÚ__call__rV  ÚtuplerE  )r]   rl  rm  r¼   rn  ro  rp  r½   rI   ÚkwargsrH   Úall_hidden_statesÚmixer_blocks                r)   rŽ   zMamba2Model.forward�  s  € ð& %9Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘IÐZ^×ZgÒZg¸T¿[¹[×=RÒ=RÐmrˆ	Ø%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà˜Ð -°tÐ";Ò<ÜÐYÓZÐZàÐ Ø ŸO™O¨IÓ6ˆMà×&Ò&¨4¯=ª=¹YØˆIáØÐ#Ü*Ø—K‘K ×!3Ñ!3°AÓ!6¸}×?SÑ?SÐ[h×[nÑ[nô �ô "'§¡¨a°·±×1HÑ1HÐQ^×QeÑQeÔ!f‘ØÐ'ô !ð;óð ð  ˆLà%ˆÙ"6™B¸DÐØŸ;™;ò 	IˆKØ×*Ò*¨t¯}ª}Ø $× AÑ AØ×(Ñ(¨-¸À~ÐWeó!‘ñ !,Ø!Ø!-Ø#1Ø#1ô	!�ò $Ø$5¸Ð8HÑ$HÑ!ð	Ið  Ÿ™ MÓ2ˆáØ 1°]Ð4DÑ DÐáÜÑf ]°LÐBSÐ$TÔfÓfÐfäØ+Ù)2˜Ø+ô
ð 	
à8<Ø+ô
ð 	
r+   )NNNNNNNN)rp   rq   rr   r^   rX  rd  rh  r   ÚMAMBA2_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCrE  Ú_CONFIG_FOR_DOCr   r%   r  rL   r?   rv   r   r   rŽ   r‘   r’   s   @r)   rN  rN  s  s
  ø„ ô

òòò)ñ +Ð+BÓCÙØ&Ø Ø$ôð 15Ø48Ø.2Ø$(Ø/3Ø&*Ø59Ø15ñK
à˜E×,Ñ,Ñ-ðK
ð   × 0Ñ 0Ñ1ðK
ð ˜{Ñ+ð	K
ð
 ˜D‘>ðK
ð ' t™nðK
ð ˜d‘^ðK
ð ! ×!1Ñ!1Ñ2ðK
ð ! §¡Ñ.ðK
ð 
ˆu�lÐ"Ñ	#òK
óó DôK
r+   rN  z�
    The MAMBA2 Model transformer with a language modeling head on top (linear layer with weights not tied to the input
    embeddings).
    c                   ó®  ‡ — e Zd Zg Zˆ fd„Zd„ Zd„ Zd„ Zd„ Z	 	 	 	 	 dde	e
   de	ej                     de	ej                     fd	„Z ee«       eeee¬
«      	 	 	 	 	 	 	 	 	 dde	ej                     de	ej*                     de	e
   de	ej                     de	e   de	e   de	e   de	ej                     de	ej                     deeef   fd„«       «       Zˆ xZS )ÚMamba2ForCausalLMc                 óÆ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  d¬«      | _        | j                  «        y )NFrœ   )
r|   r^   rN  r"  r   r¬   rW   rP  Úlm_headrY  )r]   rM   r‚   s     €r)   r^   zMamba2ForCausalLM.__init__î  sF   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆŒÜ—y‘y ×!3Ñ!3°V×5FÑ5FÈUÔSˆŒà�‰Õr+   c                 ó   — | j                   S rj   ©r„  rn   s    r)   Úget_output_embeddingsz'Mamba2ForCausalLM.get_output_embeddingsõ  s   € Ø�|‰|Ðr+   c                 ó   — || _         y rj   r†  rf  s     r)   Úset_output_embeddingsz'Mamba2ForCausalLM.set_output_embeddingsø  s	   € Ø%ˆ�r+   c                 ó6   — | j                   j                  «       S rj   )r"  rd  rn   s    r)   rd  z&Mamba2ForCausalLM.get_input_embeddingsû  s   € Ø�}‰}×1Ñ1Ó3Ð3r+   c                 ó8   — | j                   j                  |«      S rj   )r"  rh  rf  s     r)   rh  z&Mamba2ForCausalLM.set_input_embeddingsþ  s   € Ø�}‰}×1Ñ1°.ÓAÐAr+   r¼   r½   rI   c                 ó  — |r\|€t        d«      ‚|d   dkD  r|d d …df   d   }|�9d }n6t        j                  d| j                  j                  |j
                  ¬«      }|�|€d|i}nd|i}|j                  ||||dœ«       |S )	Nzé`cache_position` should not be None as it should have been initialized in `model.generate`, you are responsible for passing in a valid `cache_position` if you are calling `prepare_inputs_for_generation` directly with `use_cache=True`r   r-   r3   rè   rm  rl  )rI   r¼   rn  r½   )rw  r%   r°   rM   rP   r5   Úupdate)	r]   rl  rm  rn  r¼   r½   rI   r{  Úmodel_inputss	            r)   Úprepare_inputs_for_generationz/Mamba2ForCausalLM.prepare_inputs_for_generation  s¸   € ñ àÐ%Ü ðeóð ð
 ˜aÑ  1Ò$Ø%¢a¨ eÑ,¨YÑ7�	à!Ð-Ø%)‘Nô "'§¡¨a°·±×1HÑ1HÐQZ×QaÑQaÔ!b�àÐ$¨Ð)=Ø+¨]Ð;‰Là'¨Ð3ˆLà×Ñà"0Ø ,Ø&Ø"0ñ	ô	
ð Ðr+   ri  rl  rm  Úlabelsro  rp  rn  rb   c
           
      óœ  — |�|n| j                   j                  }| j                  ||||||||	¬«      }|d   }| j                  |j	                  | j                  j
                  j                  «      «      j                  «       }d}|��|j	                  |j                  «      }|ddd…dd…f   j                  «       }|ddd…f   j                  «       }t        «       } ||j                  d|j                  d«      «      |j                  d«      «      }|s|f|dd z   }|�|f|z   S |S t        |||j                  |j                  ¬«      S )a³  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)r¼   rm  ro  rp  rn  r½   rI   r   .r-   r   )rK  rL  r¼   rH   )rM   rv  r"  r„  rG   r~   r6   rÓ   r5   rì   r   rÔ   r;   rJ  r¼   rH   )r]   rl  rm  r¼   r�  ro  rp  rn  r½   rI   r{  Úmamba2_outputsrH   rL  rK  Úshift_logitsÚshift_labelsÚloss_fctÚoutputs                      r)   rŽ   zMamba2ForCausalLM.forward0  s_  € ð2 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆàŸ™ØØ%Ø'Ø!5Ø#ØØ)Ø)ð 'ó 	
ˆð ' qÑ)ˆà—‘˜m×.Ñ.¨t¯|©|×/BÑ/B×/HÑ/HÓIÓJ×PÑPÓRˆàˆØÐà—Y‘Y˜vŸ}™}Ó-ˆFà! # s¨ sªA +Ñ.×9Ñ9Ó;ˆLØ! # q¡r '™?×5Ñ5Ó7ˆLä'Ó)ˆHÙ˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÓHÈ,×J[ÑJ[Ð\^ÓJ_Ó`ˆDáØ�Y °°Ð!3Ñ3ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä#ØØØ'×4Ñ4Ø(×6Ñ6ô	
ð 	
r+   )NNNNN)	NNNNNNNNN)rp   rq   rr   Ú_tied_weights_keysr^   r‡  r‰  rd  rh  r   rL   r%   r  rv   r�  r   r~  r   r  rJ  r€  rG  r?   r   r   rŽ   r‘   r’   s   @r)   r‚  r‚  ä  sr  ø„ ð Ðôòò&ò4òBð ØØ.2Ø59Ø15ñ-ð
 ˜{Ñ+ð-ð ! ×!1Ñ!1Ñ2ð-ð ! §¡Ñ.ó-ñ^ +Ð+BÓCÙØ&Ø(Ø$ôð 15Ø59Ø.2Ø-1Ø/3Ø&*Ø$(Ø15Ø15ñ7
à˜E×,Ñ,Ñ-ð7
ð   × 1Ñ 1Ñ2ð7
ð ˜{Ñ+ð	7
ð
 ˜×)Ñ)Ñ*ð7
ð ' t™nð7
ð ˜d‘^ð7
ð ˜D‘>ð7
ð ! §¡Ñ.ð7
ð ! §¡Ñ.ð7
ð 
ˆuÐ*Ð*Ñ	+ò7
óó Dô7
r+   r‚  )r‚  rN  r!  )Ars   r+  Údataclassesr   Útypingr   r   r   r%   Útorch.utils.checkpointr   Útorch.nnr   Úactivationsr
   Ú
generationr   Úmodeling_utilsr   Úutilsr   r   r   r   r   Úutils.import_utilsr   r   Úconfiguration_mamba2r   Ú
get_loggerrp   r¸   Ú+mamba_ssm.ops.triton.selective_state_updater   Ú!mamba_ssm.ops.triton.ssd_combinedr   r   Úcausal_conv1dr   r   Úallr·   r  r€  rv   rV   r*   r1   rE   rJ   rL   ÚModulery   r”   r  r  r!  rE  rJ  ÚMAMBA2_START_DOCSTRINGr~  rN  r‚  Ú__all__rw   r+   r)   ú<module>rª     sï  ðñ ã Ý !ß )Ñ )ã Û Ý Ý %å !Ý )Ý -÷õ ÷ WÝ .ð 
ˆ×	Ñ	˜HÓ	%€ñ ÔÝRßmÐmàZjÑWÐÐ?ÐAWáÔßDÐDà-7Ñ*ÐÐ*áàØ!Ø(ØØðóÐ ð :Ð Ø €ðV U§\¡\ð V¸Só Vò
ò(ò(÷J ñ J ôZ;˜Ÿ™Ÿ™ô ;ô$z_�"—)‘)ô z_ôz;�B—I‘Iô ;ô"�"—)‘)ô ô84F˜Oô 4Fðn ô=�;ó =ó ð=ð0 ô=˜;ó =ó ð=ð6Ð ð $Ð ñN ØfØóôj
Ð'ó j
ó	ðj
ñZ ðð óôB
Ð-¨ó B
óðB
òJ H�r+   