T^(hdZddlZddlmZmZmZmZmZmZm Z ddl Z ddl Z ddl m Z ddl mZmZmZddlmZddlmZmZdd lmZdd lmZdd lmZdd lmZmZmZdd l m!Z!m"Z"ddl#m$Z$ddl%m&Z&ddl'm(Z(m)Z)m*Z*m+Z+ddl,m-Z-ddl.m/Z/m0Z0ddl1m2Z2e0rddl3m4Z4m5Z5ddl6m7Z7nd\Z7Z5Z4e/r ddl8m9Z9m:Z:nd\Z:Z9e;e7e5e9e:e4fZ<e*jze>Z?dZ@Gdde jZBe&jeBde jdeEde jfd ZFGd!d"eZG dDd#e jd$e jd%e jd&e jd'ee jd(eHd)eHfd*ZIGd+d,e jZJGd-d.e jZKGd/d0e jZLGd1d2e jZMGd3d4e jZNGd5d6e jZOd7ZPe(d8ePGd9d:e"ZQd;ZRe(d8ePGd<d=eQZSGd>d?eQeZTe(d@ePGdAdBeQZUgdCZVy)EzPyTorch Zamba model.N)AnyCallableDictListOptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)Cache DynamicCache)GenerationMixin)AttentionMaskConverter)FlashAttentionKwargs)BaseModelOutputWithPastCausalLMOutputWithPast SequenceClassifierOutputWithPast)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)ALL_LAYERNORM_LAYERS)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)deprecate_kwarg)is_causal_conv1d_availableis_mamba_ssm_available) ZambaConfig)mamba_inner_fnselective_scan_fn)selective_state_update)NNN)causal_conv1d_fncausal_conv1d_updateNNr$c,eZdZdfd ZdZdZxZS) ZambaRMSNormct|tjt j ||_||_y)z; ZambaRMSNorm is equivalent to T5LayerNorm N)super__init__r Parametertorchonesweightvariance_epsilon)self hidden_sizeeps __class__s f/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/zamba/modeling_zamba.pyr/zZambaRMSNorm.__init__Rs1 ll5::k#:; #c"|j}|jtj}|j dj dd}|tj ||jzz}|j|j|zS)NT)keepdim) dtypetor1float32powmeanrsqrtr4r3)r5 hidden_states input_dtypevariances r9forwardzZambaRMSNorm.forwardZsy#)) %((7  $$Q',,R,>% Ht?T?T4T(UU {{]--k:::r:c^t|jjd|jS)Nz, eps=)tupler3shaper4r5s r9 extra_reprzZambaRMSNorm.extra_repras* ))*+6$2G2G1HIIr:)gư>)__name__ __module__ __qualname__r/rHrM __classcell__r8s@r9r,r,Qs$;Jr:r,rEn_repreturnc|j\}}}}|dk(r|S|dddddddddfj|||||}|j|||z||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) r#N)rKexpandreshape)rErSbatchnum_key_value_headsslenhead_dims r9 repeat_kvr\iso 2?1D1D.E h z!!Qa"23::5BUW\^bdlmM  (;e(CT8 TTr:c ~eZdZdZej dfdZ ddejdejdede e e e fde ejejff d Zd ejfd Zdde edefd Zde e eje ejffd Zedde e e ej(ddfdZy)ZambaHybridDynamicCachea A dynamic cache that can handle both the attention cache (which has a seq_len dimension) and the mamba cache (which has a constant shape regardless of seq_len). This cache has two sets of lists of tensors: `key_cache` and `value_cache` for attention cache and `conv_states` and `ssm_states` for mamba cache. Each of these lists has `num_layers` tensors. The expected shape for each tensor For attention layers, `key_cache` and `value_cache` have a shape of `(batch_size, num_heads, seq_len, head_dim)`, while `conv_states` and `ssm_states` have a shape of `(batch_size, 0)` (empty tensors). For mamba layers, `key_cache` and `value_cache` have a shape of `(batch_size, 0)` (empty tensors), while `conv_states` represents the convolution state and has a shape of `(batch_size, d_inner, d_conv)`, and `ssm_states` represents the ssm state and has a shape of `(batch_size, d_inner, d_state)`. Nc ||_|j|_d|_|j|jz|_|j |_|j|_ |j|_ g|_ g|_ g|_ i|_i|_i|_t#|j$D]}|xjt'j(||j |j||gz c_ ||j|j |jz|jf}|xjt'j(|||gz c_ |j|dk(s|jj+|t#|j$Dcgc]}t'j,gg|z|c}|_t#|j$Dcgc]}t'j,gg|z|c}|_ycc}wcc}w)NFdevicer?hybridra)r?layers_block_typehas_previous_state mamba_expandr6intermediate_size mamba_d_statessm_state_size mamba_d_convconv_kernel_size n_mamba_heads conv_states ssm_statestransformer_layers_modules _parameters_buffersrangenum_hidden_layersr1zerosappendtensor key_cache value_cache)r5config batch_sizer?rai cache_shape_s r9r/z ZambaHybridDynamicCache.__init__s !'!9!9"'!'!4!4v7I7I!I$22 & 3 3#11"$  v//0 2A    J(>(>@U@U^dlqr!  ""&&$*<*<<## K OO Ke TU UO%%a(H4''..q1 2SXX^XpXpRqrQ%,,tj'8HrTYZ`ZrZrTstqELL" ):6Jtsts "H"H  key_states value_states layer_idx cache_kwargsrTc|j|jddk(r||j|<||j|<nftj|j||gd|j|<tj|j||gd|j|<|j||j|fS)Nr=rr<dim)rxrKryr1cat)r5rrrrs r9updatezZambaHybridDynamicCache.updates >>) $ * *2 .! 3(2DNN9 %*6D  Y '(- 4>>)3Lj2Y_`(aDNN9 %*/))T5E5Ei5PR^4_ef*gD  Y '~~i($*:*:9*EEEr:beam_idxctt|jD]S}|j|j}|j|j d|j ||j|<|j |j}|j |j d|j ||j |<|j|j}|j|j d|j ||j|<|j|j}|j|j d|j ||j|<Vy)zDReorders the cache for beam search, given the selected beam indices.rN) rslenrxra index_selectr@ryrmrn)r5rrras r9 reorder_cachez%ZambaHybridDynamicCache.reorder_cachesDs4>>23 iI^^I.55F(,y(A(N(NqRZR]R]^dRe(fDNN9 %%%i077F*.*:*:9*E*R*RSTV^VaVabhVi*jD  Y '%%i077F*.*:*:9*E*R*RSTV^VaVabhVi*jD  Y '__Y/66F)-)C)P)PQRT\T_T_`fTg)hDOOI & ir:c||jvr|jdn|}t|j|kry|j|jdS)zYReturns the sequence length of the cached states. A layer index can be optionally passed.r)rorrxrK)r5rs r9get_seq_lengthz&ZambaHybridDynamicCache.get_seq_lengthsR3<4CZCZ2ZD++A.`i t~~ ) +~~i(..r22r:ctdNz@ZambaHybridDynamicCache does not have a legacy cache equivalent.NotImplementedErrorrLs r9to_legacy_cachez'ZambaHybridDynamicCache.to_legacy_caches!"deer:past_key_valuesrctdrr)clsrs r9from_legacy_cachez)ZambaHybridDynamicCache.from_legacy_caches!"deer:N)r)rNrOrP__doc__r1float16r/Tensorintrrstrrrr LongTensorrrr classmethod FloatTensorrr:r9r^r^us 27tuJ26 FLLFllF F tCH~. F u||U\\) * F$ ie&6&6 i3 3c3fuU\\':E%,,$>&B\B\$\!'-'E'E$ )d2 !'!9!9ii < X>X[_[h[h>hotu ii < X>X[_[h[h>hotu ii < X>X[_[h[h>hotu ii : :T]] JFL^L^ejk r:rErpast_key_valuerrTc h|jdd}g|d|j}|j|j|j dd}|j |j|j dd} |j |j|j dd} ||j| | |\} } t} |jjdk7r^|jjdk(r(|jddrtjdnt|jj} | ||| | |f|jsd n |j |j"d |\} } | j$g|dj'} |j)| } | | fS) Nr=r#r<eagersdpaoutput_attentionsFz`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rr)rKr[rviewrrrrrrz_attn_implementationgetlogger warning_oncerrrrrWrr)r5rErrrr input_shape hidden_shape query_statesrrattention_interfacerrs r9rHzZambaAttention.forward s$))#2. 88b8$--8 {{=166|DNNqRST [[/44\BLLQPQR {{=166|DNNqRST  %'5'<'d##L '>dkk>^>^&_#$7      % $}}C$2H2HLL %  % ! \*k));;;;FFH kk+. L((r:r)rNrOrPrr$rr/r1rrr^rrrrHrQrRs@r9rrs l{lsl.=A ))||))))!. )) !!89 )) -. )) u||Xell3XeELL>Q5RR S))r:rcleZdZdZdeffd Z d dejdefdZ d defdZ d defdZ xZ S) ZambaMambaMixeruE Compute ∆, A, B, C, and D the state space parameters and compute the `contextualized_states`. A, D are input independent (see Mamba paper [1] Section 3.5.2 "Interpretation of A" for why A isn't selective) ∆, B, C are input-dependent (this is a key difference between Mamba and the linear time invariant S4, and is why Mamba is called **selective** state spaces) This module differs from `transformers.models.mamba.modeling_mamba.MambaMixer` in two ways: - Added multi-head: the output of `self.in_proj` is split into `self.n_mamba_heads` heads, and each head undergoes an independent forward pass, identical to the original `MambaMixer`, up until the pre-activations of `self.out_proj`. The pre-activations, coming from different mamba heads, are then concatenated and fed into `self.out_proj`. rzc t|||_||_|j|_|j |_|j|_|j|jz|_ |j|_ |j|_ |j|jz|_|j|_|j"|_t'j(|j|j|j |j|j|jdz |_|j,|_t0|j,|_|j4|_t'j8|j|jdz|j$|_t'j<t?j@|j|j|j dzz|j|_!t'j<t?j@|j|j|jdz dz|jdzz |_"t'j<t?j@|j|j|_#t?jHd|j dzt>jJdddf}|jM|jdjO}t'j<t?jP|jS|j|jd|_*t'j<t?jV|j|j|_,t'j8|j|j|j$|_-t\st^jadyy) Nr#) in_channels out_channelsr kernel_sizegroupspaddingr<rg?)r?r=apThe fast path is not available because on of `(selective_state_update, selective_scan_fn, causal_conv1d_fn, causal_conv1d_update, mamba_inner_fn)` is None. To install follow https://github.com/state-spaces/mamba/#installation and https://github.com/Dao-AILab/causal-conv1d. If you want to use the naive implementation, set `use_mamba_kernels=False` in the model config)1r.r/rzrr6rhrirjrkrfrg mamba_dt_ranktime_step_rankrlmamba_head_dimmamba_conv_bias use_conv_biasmamba_proj_biasuse_biasr Conv1dconv1dhidden_mamba_act activationractuse_mamba_kernelsuse_fast_kernelsrin_projr0r1ru x_proj_weightdt_proj_weight dt_proj_biasarangerArVrlogrWA_logr2Dout_projis_fast_path_availablerr)r5rzrAr8s r9r/zZambaMambaMixer.__init__Fs   "!--$22 & 3 3!'!4!4v7I7I!I$22#11"448J8JJ#33.. ii..//##--))))A-   !11&112 & 8 8yy!1!143I3IA3MTXTaTab  \\ &&''$*=*=*AA'' !ll [[++T-@-@$BUBU VY\ \ !!3& '  LLT5G5GI\I\)]^ LLD//!35== I$PQ' R HHT++R 0 ; ; =\\%))A,"6"6t7I7I4K^K^`b"cd ejj););T=P=PQR $"8"8$:J:JQUQ^Q^_ %   ^ &r:rE cache_paramsc |j\}}}|duxr|jxr|dk(}|j|jdd}|j |dd|j dd\}} |j dj}| j d} | j||jd|jdd} |jjj |jjjd|jjjd} |ret|j d|j|j| |jj |j"}|j%d}n|,t'j(|dk(s||j%dz}|dt*j,j/||j0|jdz df} |j|jj3| t5|| |jj |j"}|,t'j(|dk(s||j%dz}|jd|j|j6|jdd}|j8dddddddf|zjdd} t'j:| |j<|j>|j>gd\} }}|j@dddf| jddz}t'jB|jDjG }|jH|jHjGnd}t'jJ|d|f|jL|jN}|rtQ|jD]}tS|jT|jdd|f||d df||d df||||dddf||dddf|jV|| |d df||d j%d}t'jX||fd}nAt'jJ|d|j6|j>f|jL|jN}tQ|jD]}t[||||||||jdd||jdd|jV|jG| |||d d \}}t'jX||fdj}t'jX||j%dfd}|*|(|jT|jj3||j]|jdd}|S) Nr#r<r=rr)rrr`.T) dt_softplus)delta_softplusreturn_last_state)/rKrerrrchunksqueezerrWrlrr3sizer)rmrrr unsqueezer1allr rpadrkcopy_r(rrsplitrrirexprfloatremptyrar?rsr'rnrrr&r)r5rErrr{seq_lenr~use_precomputed_statesprojected_statesgate conv_weightsrmssm_parameters time_stepBCdiscrete_time_steprtime_proj_bias scan_outputsn scan_outputs_ ssm_state ssm_state_contextualized_statess r9cuda_kernels_forwardz$ZambaMambaMixer.cuda_kernels_forwards"/!4!4 GQ!-T!9!nl>]>]!nbimnbn << 6@@AF.33JAwOUUVW]^U_ t%--a0;;= ||A||J(:(:BHRRSTVWX{{))..t{{/A/A/F/Fq/I4;;K]K]KbKbcdKef !0%%b)((8    M*33B7M)%))Na>{K,]L$++JZJZgkgvgvwM)%))NajA| |jB| jdz df} | |j&|j<|j7|j/| dd|f} | t)j:|dk(s| |dd| jd dfj9dz} nt)jD||j|jF|jHf| j"|} |,t)j:|dk(s| |j9dz} |j7|j/| dd|f} |,t)j:|dk(s| |j9dz} | jd|j|jF|jdd} |jJdddddddf| zjdd }t)jL||jN|jH|jHgd\}}}|jPdddf|jdd z|jRddddddfz}t<j>jU|}t)jV|jXj[ }t)jV|ddddddddf|dddddddddfz}|dddddddddf|dddddddddfj[z}|| dddddddddfj[z}g}t]|D]}|dddddd|ddfjdd| z|dddddd|ddfjddz} t)j^| jddj!||dddd|ddfj9d}|ja|dddddddft)jb|d}|| |jdddddddfzz}||j7| z}| r| |j|j<|jg|jddj|d|jdd}|S) Nr#r<r=rr)shiftsdims.r`r)4rKr?rrrrrrrWrl isinstancer^rnrrcloner@rarermr1rollsumrr3rrrrrr rrrkrurrirrrrrsoftplusrrrrsrrvstackrr)r5 input_statesrrr{r r~r?r rEr  use_cacher conv_staterrrrrr discrete_A discrete_BdeltaB_urr| scan_outputrs r9 slow_forwardzZambaMambaMixer.slow_forwards!-!3!3 GQ""<< 5??1E.33JAwOUUVW]^U_ t%--a0;;= ||A||J(:(:BHRRSTVWX|-DE 00@FFqIZW}}(33DNNCIIK (33DNNC ! ]%9%9:I//qL ,,T^^<BB1ES)55dnnE "ZZ 2BG '4Q1W'= 1a8$;E ((8 % *t{{7I7I!QPQ'7R*RXZ [ %%!T[[%5%55M $ 7 : :5 A K KB O !-eiiRS@S6T$1N1}GZGZ[]G^F^F`C`4a4k4klm4n$nM]]..}t?T?TWdWjWjkmWn?npq>rs ;E ((8 $])CC'M)R S !-eiiRS@S6T$1N1}GZGZ[]G^F^F`C`4a4k4klm4n$nM T//1D1DdFYFYZ$++I )%))NaN ? E E GG w 9A"1aAq=1;;AqAIMPXYZ\]_`bcefYfPgPqPqrsuvPwwI,,y':':1a'@'C'CE'JAaQRTUWXjMLcLcdfLghK    Aq!QJ 7 8 9kk,B7 !]TVVAtQ=1.2.0' and 'pip install mamba-ssm', or set use_mamba_kernels=False in the model's config.)r)rrrratype ValueErrorrr,)r5rErrs r9rHzZambaMambaMixer.forwardCsm  )V4;M;M;T;T;Y;Y-Y i ,,]LYg,h h   ^ \\r:r*) rNrOrPrr$r/r1rr^rr,rHrQrRs@r9rr9sX ={=@im_%"\\_%9P_%B[%7N[%z ]3J ]r:rc$eZdZfdZdZxZS)ZambaMLPct|||_|j|_|j|_t j |j|jd|_t j |j|jd|_t j |j|jd|_ t|j|_ yNFr) r.r/rzr6rgr r gate_projup_proj down_projr hidden_actact_fnr5rzr8s r9r/zZambaMLP.__init__Qs  !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV 4#9#94;K;KRWXV../ r:c|j|j|j||j|z}|Sr)r7r9r5r6)r5xr7s r9rHzZambaMLP.forward[s6NN4;;t~~a/@#ADLLQRO#ST r:)rNrOrPr/rHrQrRs@r9r2r2Ps 0r:r2ceZdZd dedeeffd Z ddejdejdedeejdee dee d ee d e e d e ejee ejejfffd ZxZS)ZambaAttentionDecoderLayerrzrct|t|||_t ||_t |j|j|_ t |j|j|_ y)Nr7) r.r/r self_attnr2 feed_forwardr,r rms_norm_epsinput_layernormr6pre_ff_layernormrs r9r/z#ZambaAttentionDecoderLayer.__init__as_ ' :$V,+F,H,HfNaNab ,V-?-?VEXEX Yr:rEoriginal_hidden_statesrrrr&rrTc tj||gd}|j|}|jd||||||d|\}} |j |}|j |}|f} |r| | fz } | S)a Args: hidden_states (`torch.FloatTensor`): output of previous Mamba layer of shape `(batch, seq_len, embed_dim)` original_hidden_states (`torch.FloatTensor`): word embedding output of shape `(batch, seq_len, embed_dim)`. This is concatenated with `hidden_states` (which is the output of the previous (mamba) layer). The concatenated tensor is then used as input of the pre-attention RMSNorm (see fig. 2 in https://arxiv.org/pdf/2405.16712). layer_idx (`int`): layer_idx in the forward pass. Used to distinguish Zamba's tied transformer layers. attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. past_key_value (`ZambaHybridDynamicCache`, *optional*): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. r=r)rErrrrr&r)r1 concatenaterDrArErB) r5rErFrrrrr&rself_attn_weightsoutputss r9rHz"ZambaAttentionDecoderLayer.forwardis>))=:P*QWYZ ,,]; +94>>, '))/ , , ( (--m< ))-8  "  )+ +Gr:rNNFF)rNrOrPr$rrr/r1rr^boolrrrrrHrQrRs@r9r>r>`sZ{Zx}Z26<@,1$)3||3!& 3 3 !. 3 !!89 3$D>3D>3-.3 u  (51B1BEDUDU1U+V"WW X3r:r>cteZdZdedeffd Z ddejdeejdeedeejdeejdee d ee d ee d eejd eejd e ejee ejejfffdZxZS)ZambaMambaDecoderLayerrzrct|t|||_t |j |j |_||_y)N)rzrr@) r.r/rmambar,r6rCrDrrs r9r/zZambaMambaDecoderLayer.__init__s> $FiH +F,>,>FDWDWX"r:rErFrrrrr&cache_positiontransformer_hidden_statesrTc  |} | || zn|}|j|}|j|||}d} | |z}|f}|r|| fz }|r||fz }|S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. past_key_value (`ZambaHybridDynamicCache`, *optional*): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. N)rErr)rDrP)r5rErFrrrrrr&rQrRrresidualrIrJs r9rHzZambaMambaDecoderLayer.forwards<! :S9^M5 5dq ,,];  '')# !!=0  "  )+ +G  ( (Gr:) NNNNNFFNN)rNrOrPr$rr/r1rrr^rLrrrrHrQrRs@r9rNrNs#{#s#:>#'15.2<@,1$)59<@:||:!) 6:C= : !. : ell+ :!!89:$D>:D>:!!1!12:$,ELL#9: u  (51B1BEDUDU1U+V"WW X:r:rNcleZdZdedej deffd Z ddejde ejde e de ejd e ejd e e d e e d e e d e ejdeej e eej ej fffdZxZS)ZambaHybridLayer shared_transflinearrPcLt|||_||_||_yr)r.r/rWrX mamba_decoder)r5rWrXrPr8s r9r/zZambaHybridLayer.__init__s% * "r:rErFrrrrrr&rQrTc |j|||||||| } | d} |r| d} |j| } |j|| ||||| } |r | d f| ddz} | S)a Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` original_hidden_states (`torch.FloatTensor`): word embedding output that will be concatenated with hidden activations to form the input of the shared transformer layer. layer_idx (`int`): layer number. attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch, sequence_length)` where padding elements are indicated by 0. past_key_value (`ZambaHybridDynamicCache`, *optional*): cached past key and value projection states output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. )rFrrrrr&rQrr#)rRrrrr&rQr<N)rWrXrZ) r5rErFrrrrrr&rQ layer_outputsrRrIs r9rHzZambaHybridLayer.forwards>** #9&)/)+  %2!$4!  -a 0 $(KK0I$J!** &?))/)+  *1-/@AMRSRTDUUMr:)NNNNNFFN)rNrOrPr>r rrNr/r1rrrr^rLrrrrHrQrRs@r9rVrVs #&@#"))#\r#:>#'15.2<@,1$)59>||>!) 6>C= > !. > ell+ >!!89>$D>>D>>!!1!12> u  (51B1BEDUDU1U+V"WW X>r:rVaJ This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`ZambaConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. zSThe bare Zamba Model outputting raw hidden-states without any specific head on top.c eZdZeZdZdZddgZdZdZ dZ dZ dZ dZ ee d deej"d eeeeeeffd ed effd ZxZS)ZambaPreTrainedModelmodelTr>rNrFc|jj}t|tjtj frY|j jjd||j%|jjjyyt|tjrf|j jjd||j2|j j|jjyyt|tr(d|j_d|j _|j"jjd||jj$dz}tj&j)|j*| ||jj,|jj.z|jj0z}t3j4t3j6|jj0|t9j:|jj<t9j:|jj>z zt9j:|jj>zjA|jjB}|t3j:t3jD|  z}t3jF5|jHjK|dddd|jH_&yy#1swYxYw)Nr)rCstdTr)min)'rzinitializer_rangerr rrr3datanormal_rzero_ Embedding padding_idxrr_no_weight_decayrrrinituniform_rrfr6rlr1rrandmathr time_step_max time_step_minclamptime_step_floorexpm1no_gradrr _no_reinit)r5rra dt_init_stdrdtinv_dts r9 _init_weightsz"ZambaPreTrainedModel._init_weightsKsbkk++ fryy"))4 5 MM   & &CS & 9{{&   &&('  - MM   & &CS & 9!!- ""6#5#56<<>.  0,0FLL )(,FHH %  % % - -3C - @++33T9K GG  V22[L+ N![[55 8O8OOSWS^S^SlSllN 4;;44nE88DKK556$++B[B[9\\^((4;;4456e 33e4  %))U[["%5$566F 2##))&1 2-1F   *'1" 2 2s MM torch_dtype device_maphard_check_onlycheck_device_mapc`t||||||}|s|jdk(rd|_|S)z Overloads `PreTrainedModel._check_and_enable_flash_attn_2` so as to DISABLE Flash Attention 2 by default on Zamba models. Flash attention 2 is currently not supported in the HuggingFace implementation of Zamba v1. )r{r|flash_attention_2r)r._check_and_enable_flash_attn_2r)rrzryrzr{r|r8s r9rz3ZambaPreTrainedModel._check_and_enable_flash_attn_2jsE7 K__o8  6#>#>BU#U*1F ' r:rK)rNrOrPr$ config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_skip_keys_device_placement_supports_flash_attn_2_supports_sdpa_supports_cache_class _is_statefulrxrrr1r?r rrrrLrrQrRs@r9r^r^<s L&*#57OP"3"N L2>.2;? %!& ekk*U3S#X#678     r:r^a Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. If `past_key_values` is used, optionally only the last `input_ids` have to be input (see `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) past_key_values (`ZambaHybridDynamicCache`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): A ZambaHybridDynamicCache object containing pre-computed hidden-states (keys and values in the self-attention blocks and convolution and ssm states in the mamba blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. Key and value cache tensors have shape `(batch_size, num_heads, seq_len, head_dim)`. Convolution and ssm states tensors have shape `(batch_size, d_inner, d_conv)` and `(batch_size, d_inner, d_state)` respectively. See the `ZambaHybridDynamicCache` class for more details. If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, this tensor is not affected by padding. It is used to update the cache in the correct position and to infer the complete sequence length. cDeZdZdZdeffd ZdZdZee  dde e jde e jde e jd e ed e e jd e ed e ed e ede ede e jdeeeffdZdZxZS) ZambaModelz Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`ZambaDecoderLayer`] Args: config: ZambaConfig rzc t |||j|_|j|_t j |j|j|j|_t|}g}g}|j|_ t|jD]}|j|dk(r|jt||2|j|dk(sE|jt j|j j|j jd|jt||t#|}t#|}g}g|_t'|jD]\}}|dk(r_d|d} gd} g|j$| D cgc]} | | z c} |_|jt)|t+|t+|j|jt+|t j,||_|j0|_t3|j|j4 |_d|_|j;ycc} w) NrP)rrbFrzlayers..) z%shared_transf.self_attn.q_proj.weightz%shared_transf.self_attn.k_proj.weightz%shared_transf.self_attn.v_proj.weightz%shared_transf.self_attn.o_proj.weightz+shared_transf.feed_forward.gate_proj.weightz)shared_transf.feed_forward.up_proj.weightz+shared_transf.feed_forward.down_proj.weightz$shared_transf.input_layernorm.weightz%shared_transf.pre_ff_layernorm.weightr@)r.r/ pad_token_idrh vocab_sizer rgr6 embed_tokensr>rdrsrtrvrNrrziter_tied_weights_keys enumeraterVnext ModuleListlayersrr,rCfinal_layernormgradient_checkpointing post_init) r5rzblock mamba_layers linear_layersr|rlayer_id layer_type prefix_name tied_keysrr8s r9r/zZambaModel.__init__s&  !.. ++LL):):F,>FDWDWX&+# Gos. I7c|jSrrrLs r9get_input_embeddingszZambaModel.get_input_embeddingss   r:c||_yrrr5rs r9set_input_embeddingszZambaModel.set_input_embeddingss !r: input_idsr position_idsr inputs_embedsr&routput_hidden_states return_dictrQrTc d||n|jj}||n|jj}||n|jj}| | n|jj} |du|duz r t d|j r%|jr|rtjdd}||j|}|} tj|} |r|tjd| .tj| jd| j} || j!d}|j#||| } |rdnd}|rdnd}t%|j&D]r\}}|r|| fz }|j r1|jr%|j)|j*| | ||| |||| }n|| | ||| |||| }|d} |sd|dj||dfz }t|j-| } |r|| fz }|r|j.sd |_t1| |r|nd|| }| r|S|j3S) NzaYou cannot specify both input_ids and inputs_embeds at the same time, and must specify either onezX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fz{Zamba requires an initialized `ZambaHybridDynamicCache` to return a cache. None was provided, so no cache will be returned.r#rcrr)rFrrrrrr&rQT)last_hidden_staterrE attentions)rzrrr&use_return_dictr0rrrrrr1r rrKrar_update_causal_maskrr_gradient_checkpointing_func__call__rrerto_tuple)r5rrrrrr&rrrrQrErFrall_hidden_statesall_self_attnsrlayerr\outputs r9rHzZambaModel.forward s2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKKk<$D>k<'tnk<d^k<!!1!12k< u-- .k<Ck<\!r:rc eZdZdeffd ZdZdZdZdZdZ dZ e d d d e e eee  ddeej&deej(deej&deedeej,deej&deedeedeedeedeej&d eeej(fdeeeffdZ ddZxZS)ZambaForCausalLMrzc$t||t||_dg|jj|_|j |_t j|j|j d|_ |jy)Nzlm_head.weightFr) r.r/rr_rrr rr6lm_headrr:s r9r/zZambaForCausalLM.__init__so  ' #3"Tdjj6S6S"T ++yy!3!3V5F5FUS  r:c.|jjSrr_rrLs r9rz%ZambaForCausalLM.get_input_embeddingszz&&&r:c&||j_yrrrs r9rz%ZambaForCausalLM.set_input_embeddings"' r:c|jSrrrLs r9get_output_embeddingsz&ZambaForCausalLM.get_output_embeddingss ||r:c||_yrr)r5new_embeddingss r9set_output_embeddingsz&ZambaForCausalLM.set_output_embeddingss % r:c||_yrr_)r5decoders r9 set_decoderzZambaForCausalLM.set_decoders  r:c|jSrrrLs r9 get_decoderzZambaForCausalLM.get_decoders zzr:num_logits_to_keepz4.50logits_to_keep)versionnew_name) output_typerrrrrrlabelsr&rrrrQrTc ||n|jj}| | n|jj} | | n|jj} |j |||||||| | |  }|d}t | t r t| dn| }|j|dd|ddf}d}||j|||jfi| }| s|f|ddz}||f|zS|St|||j|j|jS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. logits_to_keep (`int` or `torch.Tensor`, *optional*): If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that token can save memory, which becomes pretty significant for long sequences or large vocabulary size. If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. This is useful when using packed tensor format (single dimension for batch and sequence length). Returns: Example: ```python >>> from transformers import AutoTokenizer, ZambaForCausalLM >>> model = ZambaForCausalLM.from_pretrained("Zyphra/Zamba-7B-v1") >>> tokenizer = AutoTokenizer.from_pretrained("Zyphra/Zamba-7B-v1") >>> prompt = "Hey, are you conscious? Can you talk to me?" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=30) >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] "Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you." ```N) rrrrrr&rrrQrrr#losslogitsrrEr)rzrrrr_rrslicer loss_functionrrrrEr)r5rrrrrrr&rrrrQr loss_kwargsrJrE slice_indicesrrrs r9rHzZambaForCausalLM.forwardsLf2C1N-TXT_T_TqTq%9$D $++JjJj &1%SV8W~ot4]k mA}a,?@A  %4%%ffdooUUDY,F'+'7D7V# CV C%#33!//))   r:c t|du} | sZ||d|jdk\r|dd|jd df}nc|jd|jdk7rD|dd|f}n:t|j|jd|j|j}|T|R|j j ddz }|j|dk(d| s|dd|jd df}|| rd|i} nd|ji} | j|||||jj|d| S)Nr=r#r)r?rarr)rrr&rrrQ) rKr^rzr?ralongcumsum masked_fill_rrr) r5rrrrrQrr&r empty_past_kv model_inputss r9prepare_inputs_for_generationz.ZambaForCausalLM.prepare_inputs_for_generationsc(4/ )!"%);;%a.*>*>q*A)A)C&CD #~';';A'>>%a&78 5 Y__Q/tzz$++O  %,*>)..077;a?L  % %n&91 = +A 0B/B/D,DE   $+];L')=)=)?@L ,#2&"0"&++"@"@"0   r:) NNNNNNNNNNNr)NNNNNT)rNrOrPr$r/rrrrrrr rrrr_CONFIG_FOR_DOCrr1rrr^rrLr rrrHrrQrRs@r9rrs{'(&)6DTU*+AB+AP_`151537=A59-1$(,0/3&*5934X E,,-X !.X u//0 X ""9: X   1 12 X ))*X D>X $D>X 'tnX d^X !!1!12X c5<</0X  u,, -X aCVX z9r:ra The Zamba Model with a sequence classification head on top (linear layer). [`ZambaForSequenceClassification`] uses the last token in order to do the classification, as other causal models (e.g. GPT-2) do. Since it does classification on the last token, it requires to know the position of the last token. If a `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in each row of the batch). cXeZdZfdZdZdZee ddee jdee jdee jdee e ee jfdee jd ee jd eed eed eed eede eeffdZxZS)ZambaForSequenceClassificationct|||j|_t||_|jj |_t j|j|jd|_ |jyr4) r.r/ num_labelsrr_rr rr6scorerr:s r9r/z'ZambaForSequenceClassification.__init__cse   ++' "&**"?"?YYv114??O  r:c.|jjSrrrLs r9rz3ZambaForSequenceClassification.get_input_embeddingsmrr:c&||j_yrrrs r9rz3ZambaForSequenceClassification.set_input_embeddingsprr:rrrrrrr&rrrrTc | | n|jj} |j|||||||| |  } | d} |j| } ||jd}n|jd}|jj |dk7r t d|jj d}n|||jj k7j| jtj}tj|jd| jtj}||zjd}n.d}tj|jj d| tj|| j |f}d}||j| j}|jj"|j$dk(rd |j_nl|j$dkDrL|j&tj(k(s|j&tj*k(rd |j_nd |j_|jj"d k(rIt-}|j$dk(r&||j/|j/}n|||}n|jj"d k(r=t1}||j3d|j$|j3d}n,|jj"d k(rt5}|||}| s|f| ddz}||f|zS|St7||| j8| j:| j< S)a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). N)rrrrr&rrrrr#z=Cannot handle batch sizes > 1 if no padding token is defined.r=r`z will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rc regressionsingle_label_classificationmulti_label_classificationr)rzrr_rrKrr0r@rar1int32rargmaxrrr8rN problem_typerr?rrr rr rr rrrEr)r5rrrrrrr&rrrtransformer_outputsrErr{last_non_pad_token non_pad_mask token_indices pooled_logitsrloss_fctrs r9rHz&ZambaForSequenceClassification.forwardss(&1%!F!Fr!J !#    >>**+,ZZ  u||Jv}}MOaab   YYv}}-F{{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#M$9$9$;V^^=MND#M6:D))-JJ+- 2 22t GUWY))-II,. v6#%(;AB(??F)-)9TGf$ Ev E/ /??-;;*55   r:r)rNrOrPr/rrrrrr1rrr rrrrLrrrHrQrRs@r9rrSs0 '(++AB151537KO59-1$(,0/3&*[ E,,-[ !.[ u//0 [ "%tE4E4E/F(F"GH [   1 12 [ ))*[ D>[ $D>[ 'tn[ d^[  u66 7[ C[ r:r)rrrr^)r)Wrrmtypingrrrrrrr r1torch.utils.checkpointr torch.nnr r r activationsr cache_utilsrr generationrmodeling_attn_mask_utilsrmodeling_flash_attention_utilsrmodeling_outputsrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsrrrrutils.deprecationr utils.import_utilsr!r"configuration_zambar$&mamba_ssm.ops.selective_scan_interfacer%r&+mamba_ssm.ops.triton.selective_state_updater' causal_conv1dr(r)rr get_loggerrNrrModuler,rvrrr\r^rrrrr2r>rNrVZAMBA_START_DOCSTRINGr^rrrr__all__rr:r9rs( DDD AA!.)C G&1 1-XR@P=-~DD-7**.0@BVXfg   H %J299J(L) UU\\ U# U%,, U[fl[fJ% II% <<% % << % U\\* %  %%4I)RYYI)XS]biiS]n ryy  <<~ARYYAHEryyEP"Y@?@ @FBJYM%M Mbs+_sl m %9m m ` gr: