T^(h  dZddlZddlmZmZmZmZddlZddlm cm Z ddl Zddlm Z ddl mZddlmZddlmZmZmZdd lmZdd lmZdd lmZmZmZdd lmZmZdd l m!Z!m"Z"m#Z#m$Z$m%Z%ddl&m'Z'ddl(m)Z)m*Z*m+Z+e#r ddl,m-Z-ddl.m/Z/e$j`e1Z2dejfde4de5deejfejfffdZ6dejfde4de4dejndejff dZ8Gdde jrZ:Gdde jrZ;Gd d!e jrZ<Gd"d#e jrZ=Gd$d%e=Z>e=e>d&Z?Gd'd(e jrZ@Gd)d*e jrZAGd+d,e jrZBGd-d.e jrZCGd/d0eCZDd1ZEdUd2ZFd3ejfd4e4dejffd5ZGGd6d7e jrZHGd8d9eHZIeCeDd&ZJeHeId&ZKGd:d;e jrZLGd<d=e jrZMGd>d?ejjrZNGd@dAe jrZOGdBdCeZPdDZQdEZRdFZSdGZTe!dHeQGdIdJePZUe!dKeQGdLdMePZVe!dNeQGdOdPePeZWe!dQeQGdRdSePeZXgdTZYy)VzPyTorch Mllama model.N)ListOptionalTupleUnion)nn)PreTrainedModel)ACT2FN)Cache DynamicCache StaticCache)GenerationMixin)AttentionMaskConverter)BaseModelOutputBaseModelOutputWithPastCausalLMOutputWithPast)ROPE_INIT_FUNCTIONSdynamic_rope_update)add_start_docstrings%add_start_docstrings_to_model_forwardis_torch_flex_attn_availableloggingreplace_return_docstrings)deprecate_kwarg) MllamaConfigMllamaTextConfigMllamaVisionConfig) BlockMask)make_flex_block_causal_maskcross_attention_masknum_vision_tokensdtypereturnc|j^}}}|j|d}|j||d}|jd}d|z j |}|j |j t jt j|j}t j|j}||k7jdj|d}||z}||fS)Nrdimrg?).N) shaperepeat_interleaveview unsqueezeto masked_filltorchboolfinfominanytype_as) r!r"r# batch_sizetext_total_length_inverted_cross_attn_masknegative_inf_valuefull_text_row_masked_out_masks h/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/mllama/modeling_mllama.py_prepare_cross_attention_maskr<4s )=(B(B%J!A/AABSYZA[/44ZARTVW/99!<!$&: :>>uE3?? ##EJJ/U1C1G1G U+// !3 388R8@HHI]^_hi"99 !> >>aspect_ratio_mask num_patches target_lengthc||j\}}|j||ddj|}|jdd|d}||z }d|dddd| df<d|z }|j |||zd}||j ddzt j|jz}|jd}|S)Nrrr() r)r+r-repeatreshape transposer/r1r2r,)r>r?r@r#r5 max_num_tilesattention_mask pad_patchess r;$_prepare_aspect_ratio_attention_maskrIPs!2 7 7J &++J q!LOOPUVN#**1aBN +-K*+N1a+&''N$++J 8UWXYN#n&>&>r2&FFUZI[I_I__N#--a0N r=czeZdZddedeffd ZdejdejdejfdZxZ S) %MllamaPrecomputedAspectRatioEmbeddingconfigis_gatedctt||j|_|j|_|j|_||_t j|jdz|j|jz|_|r.t jtjd|_ yyNr) super__init__rF hidden_sizemax_aspect_ratio_idrMr Embedding embedding Parameterr/zerosgateselfrLrM __class__s r;rQz.MllamaPrecomputedAspectRatioEmbedding.__init__ls #11!--#)#=#=   d&>&>&BDDVDVY]YiYiDij  U[[^4DI r= hidden_stateaspect_ratio_idsr$c|j|}|jd|jd|j}|jr||j j z}||z}|S)Nr(r)rUrDrFrRrMrXtanh)rZr\r] embeddingss r;forwardz-MllamaPrecomputedAspectRatioEmbedding.forwardws_^^$45 ''D,>,>4CSCST ==#diinn&66J#j0 r=)T) __name__ __module__ __qualname__rr0rQr/Tensorra __classcell__r[s@r;rKrKks@ 51 5T 5ELLELLUZUaUar=rKcteZdZdeffd Zdej dej dej fdZxZS)"MllamaPrecomputedPositionEmbeddingrLct||j|_|j|_|j|j zdzdz|_|j|_|jdz|_tjtjd|_ tj|j |j}tj|j|z|_tj |jdz|j|j z|jz|_y)Nr)rPrQrFrS image_size patch_sizer?rRscalerrVr/rWrXrandnrUrTtile_embedding)rZrLposition_embeddingr[s r;rQz+MllamaPrecomputedPositionEmbedding.__init__s #11#)#=#= "--1B1BBqH1L!--''- LLQ0 #[[)9)94;K;KLdjj3E&EF!ll  $ $q ($*<*&C&CAq$JZJZ\`\l\l&mm #'"5"56F"G!''* "9"A"A **D,<,N>N# )- (8;R(R%#&CC r=) rbrcrdrrQr/rerarfrgs@r;riris9 1 &ELLELLUZUaUar=ricVeZdZfdZdej dej fdZxZS)MllamaVisionMLPct|||_t|j|_t j|j|j|_ t j|j|j|_ yN) rPrQrLr hidden_act activation_fnrLinearrRintermediate_sizefc1fc2rZrLr[s r;rQzMllamaVisionMLP.__init__sd  #F$5$5699V//1I1IJ99V55v7I7IJr= hidden_statesr$cl|j|}|j|}|j|}|Srz)rr|r)rZrs r;razMllamaVisionMLP.forwards4/ **=9 / r=)rbrcrdrQr/rerarfrgs@r;rxrxs$KU\\ellr=rxc eZdZdeffd Z ddej deej deede ej eej ffdZ xZ S) MllamaVisionAttentionrLct||j|_|j|_|j|jz|_tj|j|j |j zd|_ tj|j|j |j zd|_ tj|j|j |j zd|_ tj|j |j z|jd|_ yNFbias) rPrQrR embed_dimattention_heads num_headshead_dimrr}q_projk_projv_projo_projrs r;rQzMllamaVisionAttention.__init__s ++//**f.D.DD ii0NUZ[ ii0NUZ[ ii0NUZ[ ii >UZ[ r=r\rGoutput_attentionsr$c$|j|}|j|}|j|}|j\}}} |j\} } } |j |||j |j jdd}|j || |j |j jdd}|j || |j |j jdd}tj||jddtj|j z } |#|ddddddd|jdf} | | z} tjj| dtjj!|j"} tj| |} | jddj%} | j'||d} |j)| }|sd} || fS)NrrkrrBr(r'r#)rrrr)r+rrrEr/matmulmathsqrtr functionalsoftmaxfloat32r-r# contiguousrDr)rZr\rGrquerykeyvaluer5 q_seq_lenr7 kv_seq_len attn_weights causal_mask attn_outputoutputs r;razMllamaVisionAttention.forwards  L)kk,' L)#(;; Iq99:q :y$..$--PZZ[\^_`hhz:t~~t}}MWWXY[\] :z4>>4==Q[[\]_`a||E3==A+>?$))DMMBZZ  %(Aq/CIIbM/)ABK'+5L}},,\r,WZZ[`[f[fg ll<7 !++Aq1<<> !))*iD [) L|##r=NN) rbrcrdrrQr/rerr0rrarfrgs@r;rrsm \1 \26,0 #$ll#$!.#$$D> #$ u||Xell33 4 #$r=rc ~eZdZ ddejdeejdeedejffd ZxZS)MllamaVisionSdpaAttentionr\rGrr$c|r'tjdt | |||S|j |}|j |}|j |}|j\}}} |j\} } } |j|||j|j}|j|| |j|j}|j|| |j|j}|jdd}|jdd}|jdd}tj||||} | jddj} | j||d} |j!| } | dfS)NaMllamaModel is using MllamaVisionSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.r\rGrrrk) attn_maskr()logger warning_oncerPrarrrr)r+rrrEFscaled_dot_product_attentionrrDr)rZr\rGrrrrr5rr7rrrr[s r;raz!MllamaVisionSdpaAttention.forwardsk    [ 7?)-"3#   L)kk,' L)#(;; Iq99:q :y$..$--Phhz:t~~t}}M :z4>>4==Q1%mmAq!1%44UCR`a !++Aq1<<> !))*iD [)t|r=r) rbrcrdr/rerr0rarfrgs@r;rrsP 26,0 (ll(!.($D> (  ((r=r)eagersdpacxeZdZddedeffd Z d dejdeejdeefdZ xZ S) MllamaVisionEncoderLayerrLrMct||j|_|j|_||_|j |_t|j||_ t||_ tj|j|j|_tj|j|j|_|rtj"t%j&dt(j*zdz |_tj"t%j&dt(j*zdz |_yy)Nepsr)rPrQrRrnum_attention_headsrMr~MLLAMA_VISION_ATTENTION_CLASSES_attn_implementation self_attnrxmlpr LayerNormnorm_epsinput_layernormpost_attention_layernormrVr/onesrpi gate_attngate_ffnrYs r;rQz!MllamaVisionEncoderLayer.__init__s !--#)#9#9   !'!9!989T9TUV\]"6*!||D,<,<&//R(* T5E5E6??([% \\%**Q-$''*AA*EFDNLLA)@1)DEDM r=r\rGrcn|}|j|}|j||\}}|jr|jj |z}||z}|}|j |}|j |}|jr|jj |z}||z}|f}|r||fz }|S)N)rG)rrrMrr_rrr)rZr\rGrresidualroutputss r;raz MllamaVisionEncoderLayer.forward-s ++L9 %)^^LQ_^%`" l ==>>..0<?L,.  44\B xx - ====--/,>L,. /    &Gr=Fr) rbrcrdrr0rQr/rerrarfrgs@r;rrsTF1FTF*26,0 ll!.$D> r=rceZdZdZd deffd Z d dejdeejdee dee dee d e e e ff d Z xZS) MllamaVisionEncoderz Transformer encoder consisting of `config.num_hidden_layers` self attention layers. Each layer is a [`MllamaEncoderLayer`]. Args: config: MllamaConfig rLc t|||_tjt |Dcgc]}t ||c}|_d|_||_ycc}w)NF) rPrQrLr ModuleListrangerlayersgradient_checkpointing)rZrL num_layersrMr7r[s r;rQzMllamaVisionEncoder.__init__TsU  mmY^_iYj$kTU%=fh%O$kl &+# %lsA!rrGroutput_hidden_states return_dictr$c||n|jj}||n|jj}||n|jj}|rdnd}|rdnd}|jD]\}|r||fz}|j r+|j r|j|j|||} n ||||} |r || dfz}| d}^|r||fz}|std|||fDSt|||S)ad Args: inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. Nrrrc3&K|] }|| ywrzr.0vs r; z.MllamaVisionEncoder.forward..seqWXWdelast_hidden_stater attentions) rLrruse_return_dictrrtraining_gradient_checkpointing_func__call__tupler) rZrrGrrrencoder_statesall_attentions encoder_layer layer_outputss r;razMllamaVisionEncoder.forward[s0<2C1N-TXT_T_TqTq$8$D $++JjJj &1%>Ne]NN$See e+>Vd  r=) F)NNNN)rbrcrd__doc__rrQr/rerr0rrrrarfrgs@r;rrKs126,0/3&* D ||D !.D $D> D 'tn D d^ D  uo% &D r=rc,eZdZdfd ZdZdZxZS)MllamaTextRMSNormct|tjt j ||_||_y)z@ MllamaTextRMSNorm is equivalent to T5LayerNorm N)rPrQrrVr/rweightvariance_epsilon)rZrRrr[s r;rQzMllamaTextRMSNorm.__init__s1 ll5::k#:; #r=c"|j}|jtj}|j dj dd}|tj ||jzz}|j|j|zS)Nrkr(T)keepdim) r#r-r/rpowmeanrsqrtrr)rZr input_dtypevariances r;razMllamaTextRMSNorm.forwardsy#)) %((7  $$Q',,R,>% Ht?T?T4T(UU {{]--k:::r=c^t|jjd|jS)Nz, eps=)rrr)rrZs r; extra_reprzMllamaTextRMSNorm.extra_reprs* ))*+6$2G2G1HIIr=)gư>)rbrcrdrQrarrfrgs@r;rrs$;Jr=rc<eZdZdZ ddeedeeffd Z ddejdeejdee deejd e d ee d eejd e ejeejee ejffd ZxZS)MllamaTextCrossAttentionz=Multi-headed attention from 'Attention Is All You Need' paperrL layer_idxct|||_|jj|_|jj |_|j |_|j|_|j|jz|_||_ |j|j z|_ tj|j|j|jzd|_ tj|j|j |jzd|_tj|j|j |jzd|_tj|j|jz|jd|_t#|j|j$|_t#|j|j$|_y)NFrr)rPrQrLrrnum_key_value_headsdropoutrRrrnum_key_value_groupsrr}rrrrr rms_norm_epsq_normk_normrZrLrr[s r;rQz!MllamaTextCrossAttention.__init__s[  88#';;#B#B ~~ !--**dnn< "$(NNd6N6N$N!ii 0 0$..4==2PW\] ii 0 0$2J2JT]]2Zafg ii 0 0$2J2JT]]2Zafg ii >@P@PW\] ' 6;N;NO ' 6;N;NO r=rcross_attention_statespast_key_valuerGr use_cachecache_positionr$c|j\}} } |j|} | j|| |j|jj dd} |j | } ||j|} |j|} | j|d|j|jj dd} | j|d|j|jj dd} t| |j} t| |j} |j| } |j|j| | |jd|i\} } nF|ddk7r3|j|j|j |j} } n t#dt%j&| | j ddt)j*|jz }|#|ddddddd| j,d f}||z}t.j0j3|dt$j4 j7| j8}t.j0j;||j:|j< }t%j&|| }|j ddj?}|jA|| d}|jC|}|sd}|||fS) #Input shape: Batch x Time x ChannelrrkNr(rr^Cross attention layer can't find neither `cross_attn_states` nor cached values for key/values!rrBrpr)"sizerr+rrrErrrr repeat_kvrrupdater key_cache value_cache ValueErrorr/rrrr)rrrrr-r#rrrrDr)rZrrrrGrrrbszq_lenr7 query_states key_states value_statesrrrs r;raz MllamaTextCrossAttention.forwards&**, UA{{=1 #((eT^^T]]S]]^_abc {{<0 ! -%;L#b$2J2JDMMZddefhijJ',,S"d6N6NPTP]P]^hhijlmnL":t/H/HIJ$\43L3LMLZ0J),:+@+@ dnn?OQ_>`,( LA ! #((8**4>>:%J p ||L*2F2Fq!2LMPTPYPYZ^ZgZgPhh  %(Aq2HJ4D4DR4H2H)HIK'+5L}},,\r,WZZ[g[m[mn }},,\T\\TXTaTa,b ll<> !++Aq1<<> !))#ub9 kk+.  LL.88r=rNNNFNN)rbrcrdrrrintrQr/rer r0 LongTensorrrarfrgs@r;rrsG.2#'P)*PC=P4:>*.15"'$(5999||99!) 699! 99 !. 99  99D>99!!1!1299 u||Xell3XeELL>Q5RR S99r=rceZdZdZ d dej deej deedeej dedeedeejd e ej eej ee ej fffd Z xZ S) MllamaTextCrossSdpaAttentiona Mllama attention module using torch.nn.functional.scaled_dot_product_attention. This module inherits from `MllamaTextCrossAttention` as the weights of the module stays untouched. The only changes are on the forward pass to adapt to SDPA API. rrrrGrrrr$c |r+tjdt| |||||||S|j \}} } |j |} | j || |j|jjdd} |j| } ||j|} |j|} | j |d|j|jjdd} | j |d|j|jjdd} |j|j| | |jd|i\} } nF|ddk7r3|j |j|j"|j} } n t%d t'| |j(} t'| |j(} |j+| } | j,j.d k(r2|0| j1} | j1} | j1} || dkDrd nd }t2j4j6j9| | | ||j:r |j<nd |}|jddj1}|j?|| d}|jA|}|d|fS)raMllamaModel is using MllamaTextCrossSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rrrGrrrrrrkNr(rrrcudaTFr dropout_p is_causal)!rrrPrar rr+rrrErrrrr rrrrr rrdevicetyperr/rrrrrrDr)rZrrrrGrrrrrr7rrrr rr[s r;raz$MllamaTextCrossSdpaAttention.forwards    [ 7?+'=--"3#-# &**, UA{{=1 #((eT^^T]]S]]^_abc {{<0 ! -%;L#b$2J2JDMMZddefhijJ',,S"d6N6NPTP]P]^hhijlmnL),:+@+@ dnn?OQ_>`,( LA ! #((8**4>>:%J p z4+D+DE  t/H/HI [[,     # #v -.2L'224L#..0J'224L+2uqyDe hh))FF   $&*mmdll G "++Aq1<<> !))#ub9 kk+. D.00r=r) rbrcrdrr/rerr r0rrrarfrgs@r;rr s:>*.15"'$(59S1||S1!) 6S1! S1 !. S1  S1D>S1!!1!12S1 u||Xell3XeELL>Q5RR SS1S1r=rc|dd|jddzf}|d|jddzdf}tj| |fdS)z*Rotates half the hidden dims of the input..Nr(rkr&)r)r/cat)xx1x2s r; rotate_halfr(lsZ 3"!''"+"" " #B 3 q " " #B 99rc2YB ''r=c|j|}|j|}||zt||zz}||zt||zz}||fS)aApplies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`, *optional*): Deprecated and unused. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. )r,r()qkcossin position_ids unsqueeze_dimq_embedk_embeds r;apply_rotary_pos_embr2tsY( -- &C -- &C3w;q>C/0G3w;q>C/0G G r=rn_repc|j\}}}}|dk(r|S|dddddddddfj|||||}|j|||z||S)z This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch, num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim) rN)r)expandrD)rr3batchrslenrs r;r r so 2?1D1D.E h z!!Qa"23::5BUW\^bdlmM  (;e(CT8 TTr=c eZdZdedeffd Z d dejdejdejdedef d Z xZ S) MllamaTextSelfAttentionrLrcPt|||_|j|_|j |_|j |_|j|_|j |jz|_|j|jz|_ |j|_ ||_ tj|j |j|jzd|_tj|j |j|jzd|_tj|j |j|jzd|_tj|j|jz|j d|_yr)rPrQrLrrrrRrrr rope_thetarrr}rrrrrs r;rQz MllamaTextSelfAttention.__init__s.  33~~ !--#)#=#= **dnn< $(NNd6N6N$N! ++"ii 0 0$..4==2PW\] ii 0 0$2J2JT]]2Zafg ii 0 0$2J2JT]]2Zafg ii >@P@PW\] r=rrGposition_embeddingsrrc P|j\} } } |j|} |j|} |j|}| j | | |j |j jdd} | j | | |j|j jdd} |j | | |j|j jdd}|\}}t| | ||\} } |'|||d}|j| ||j|\} }t| |j} t||j}tj| | jddt!j"|j z }|#|ddddddd| j$df}||z}t&j(j+|dtj,j/| j0}t&j(j3||j2|j4}tj||}|jddj7}|j | | d}|j9|}|sd}|||fS) Nrrkr-r,rrrBr(rr )r rrrr+rrrErr2r rr rr/rrrr)rrrrr-r#rrrr)rZrrGr<rrrrkwargsrrr7rrrr,r- cache_kwargsrrrs r;razMllamaTextSelfAttention.forwardsh&**, UA{{=1 [[/ {{=1 #((eT^^T]]S]]^_abc __S%1I1I4==Yccdeghi #((eT5M5Mt}}]gghiklm &S#7 jRUWZ#[ j  %#&snUL'5'<' !++Aq1<<> !&&sE26 kk+.  LL.88r=FFNN) rbrcrdrrrQr/rer0rarfrgs@r;r9r9si^/^C^,#(39||39 39#\\ 39  39  39r=r9c zeZdZ ddejdejdejdedef fd ZxZS)MllamaTextSelfSdpaAttentionrrGr<rrc $|r,tjdt| d |||||||d|S|j \} } } |j |} |j |} |j|}| j| | |j|jjdd} | j| | |j|jjdd} |j| | |j|jjdd}|\}}t| | ||\} } |'|||d}|j| ||j|\} }t!| |j"} t!||j"}|}||ddddddd| j$df}| j&j(dk(r2|0| j+} | j+} |j+}|| dkDrdnd }t,j.j0j3| | |||j4r |j6nd | }|jddj+}|j| | d }|j9|}|d|fS)NaMllamaModel is using MllamaTextSelfSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rrGr<rrrrrrkr>rBrTFrrr(r)rrrPrar rrrr+rrrErr2r rr rr)r!r"rr/rrrrrr)rZrrGr<rrrrr?rrr7rrrr,r-r@rr rr[s r;raz#MllamaTextSelfSdpaAttention.forwards    [ 7? +-$7-"3#-   &**, UA{{=1 [[/ {{=1 #((eT^^T]]S]]^_abc __S%1I1I4==Yccdeghi #((eT5M5Mt}}]gghiklm &S#7 jRUWZ#[ j  %#&snUL'5'<' !&&sE26 kk+. D.00r=rA)rbrcrdr/rer0rarfrgs@r;rCrCs^#(M1||M1 M1#\\ M1  M1  M1M1r=rCc$eZdZfdZdZxZS) MllamaTextMLPct|||_|j|_|j|_t j |j|jd|_t j |j|jd|_t j |j|jd|_ t|j|_ yr) rPrQrLrRr~rr} gate_projup_proj down_projr r{act_fnrs r;rQzMllamaTextMLP.__init__<s  !--!'!9!94#3#3T5K5KRWXyy!1!143I3IPUV 4#9#94;K;KRWXV../ r=c|j|j|j||j|z}|Srz)rJrKrHrI)rZr%rJs r;razMllamaTextMLP.forwardGs6NN4;;t~~a/@#ADLLQRO#ST r=)rbrcrdrQrarfrgs@r;rFrF;s  0r=rFceZdZdedeffd Z ddejdeejdeejdeejdee ejejfd eejd ee d ee d ee d eejdee ejejfde ejee ejejfffdZxZS)MllamaSelfAttentionDecoderLayerrLrcHt||j|_t|j|||_t ||_t|j|j|_ t|j|j|_ ||_ y)N)rLrr) rPrQrRMLLAMA_TEXT_ATTENTION_CLASSESrrrFrrrrrrrs r;rQz(MllamaSelfAttentionDecoderLayer.__init__Ns !--6v7R7RS[amvw (01C1CI\I\](9&:L:LRXReRe(f%"r=rrr!rGr:r.rrrrr<r$c |} |j|}|j|||||| | | \}} }| |z}|} |j|}|j|}| |z}|f}|r|| fz }| r||fz }|S)a. Args: hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`torch.FloatTensor`, *optional*): attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1, query_sequence_length, key_sequence_length)` if default attention is used. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*): Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`, with `head_dim` being the embedding dimension of each attention head. kwargs (`dict`, *optional*): Arbitrary kwargs to be ignored, used for FSDP and other methods that injects code into the model )rrGr.rrrrr<)rrrr)rZrrr!rGr:r.rrrrr<rself_attn_weightspresent_key_valuers r;raz'MllamaSelfAttentionDecoderLayer.forwardZsH!,,]; ?Cnn')%)/) 3?M ? ; (*;!=0 !55mD /  =0  "  )+ +G  )+ +Gr=) NNNNNNFFNN)rbrcrdrrrQr/rerrrr r0 FloatTensorrarfrgs@r;rNrNMsY #/ #C #:>7;15UY37*.,1$)59KOC||C!) 6C'u||4 C !. C (0ellELL6P0Q'R Cu//0C!C$D>CD>C!!1!12C&eELL%,,,F&GHC u  (51B1BEDUDU1U+V"WW XCr=rNcreZdZdZdededdffd Z ddejdejd ejd ejd e ejejfd e ejd e e de e de e de ejde ejde ejfdZxZS) MllamaCrossAttentionDecoderLayerzLCross-attention transformer block with tanh-gated attention and feedforward.rLrr$Nct|||_t|j|||_t |j|j|_ tjjtjd|_t||_t |j|j|_tjjtjd|_y)N)rrr)rPrQr#MLLAMA_TEXT_CROSS_ATTENTION_CLASSESr cross_attnrrRrrr/rrVrWcross_attn_attn_gaterFrrcross_attn_mlp_gaters r;rQz)MllamaCrossAttentionDecoderLayer.__init__s "=f>Y>YZ[amvw01C1CI\I\]$)HH$6$6u{{1~$F! ((9&:L:LRXReRe(f%#(88#5#5ekk!n#E r=rrr!rGr:r.rrrrr<c l|} |j|}|j|||||| \}} }| |jj|zz}|} |j |}|j |}| |dddf|z}| |j j|zz}|f}|r|| fz }| r||fz }|S)N)rrGrrrrr)rrYrZr_rrr[)rZrrr!rGr:r.rrrrr<rrrs r;raz(MllamaCrossAttentionDecoderLayer.forwards!,,]; 6:oo'/#9)/) 7F7 3 |^!4#<#<#A#A#Cm#SS  55mD / ( 49!Q$?-OM 4#;#;#@#@#B]#RR  "    &G  ( (Gr=)NNFFNN)rbrcrdrrrrQr/rerrrr r0rarfrgs@r;rVrVs V F/ FC FD F&48*.,1$)596:*||*!& *$ll *  * (-U\\5<<-G'H *u//0*!*$D>*D>*!!1!12*&ell3* u|| *r=rVc^eZdZddeffd Zej edZxZ S)MllamaRotaryEmbeddingrLc^t||jd|_|j|_|j|_||_t|j|_ |j|j|\}|_ |jd|d|j|_ y)N rope_typeinv_freqF) persistent)rPrQ rope_scalingr`max_position_embeddingsmax_seq_len_cachedoriginal_max_seq_lenrLr rope_init_fnattention_scalingregister_bufferraoriginal_inv_freq)rZrLr!rar[s r;rQzMllamaRotaryEmbedding.__init__s ,,[9"("@"@$*$B$B! /?+/+<+>@GG HZHZ[\H]_acde ,QaZ 8 > > @'1!((--'E!((--[`J`ahhmmfk ^^ U C 5&,,.1F1L1L1NNYYZ[]^_E))UEN3C'')d444C'')d444C  5 vvAGGv$cff177f&;;;  5 5s BF  Frz) rbrcrdrrQr/no_gradrrarfrgs@r;r^r^s4 // /U]]_ < .  - KK  Sc  2  1 2 GGOOF2277SO A  B C GGOOF,,11sO ;  8 9foo GGOOF,,11sO ; GGOOFOO00cO :?N 9r=rG input_tensorrpast_key_valuesrc |jjdk(r||dk(jr|Sy|jjdk(r7t|tj r t |}t|tr|S||jnd}t|t}|jjdk(r(|s&|s$tj||||jry|j|j} }|jd} |r|j!} n1t|tj r|jdn|| zdz} |j#|| | || ||jd } |jjdk(rQ|O|jj$d vr7|s5t j&|j(} tj*| | } | S) Nflash_attention_2rflex_attentionrr) inputs_embedspast_key_values_length is_trainingrr()sequence_lengthr@r#r!rr5)rxpu)rLrr3rqr/rer rget_seq_lengthr r_ignore_causal_mask_sdparr#r!r)get_max_cache_shape5_prepare_4d_causal_attention_mask_with_cache_positionr"r1r2_unmask_unattended)rZrGrrrrpast_seen_tokensusing_static_cacher#r!rr@r min_dtypes r;_update_causal_maskz)MllamaPreTrainedModel._update_causal_masks ;; + +/B B)~/D.I.I.K%% ;; + +/? ?.%,,7!<^!L.)4%% @O?Z?99;`a'E ;; + +v 5>PYj%>>*'7 MM  $**L,?,?v&,,Q/ +??AMnell;$$R(%7!; PP +')#))!,Q  KK , , 6*%%**o=%  E*..I0CCKQZ[Kr=rr@r#r!r5c ||jdk(r|}|Stj|j} tj||f| ||}|dk7rtj |d}|tj |||jddkDz}|ddddddfj|ddd}||j}|jd} |ddddddd| f|ddddddfj|jz} | dk(} |ddddddd| fj| | |ddddddd| f<|S) a Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing. Args: attention_mask (`torch.Tensor`): A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape `(batch_size, 1, query_length, key_value_length)`. sequence_length (`int`): The sequence length being processed. target_length (`int`): The target length: when generating with static cache, the mask should be as long as the static cache, to account for the 0 padding, the part of the cache that is not filled yet. dtype (`torch.dtype`): The dtype to use for the 4D attention mask. device (`torch.device`): The device to place the 4D attention mask on. cache_position (`torch.Tensor`): Indices depicting the position of the input sequence tokens in the sequence. batch_size (`torch.Tensor`): Batch size. Nr) fill_valuer#r!r)diagonalr!r(r)r'r/r1r2fulltriuarangerDr5cloner)r-r!r.) rGrr@r#r!rr5r?rr mask_length padding_masks r;rzKMllamaPreTrainedModel._prepare_4d_causal_attention_mask_with_cache_positionbsyD  %.*<*<*>!*C(K*' E*..I** -0Ye\bK!##jjqA 5<< fEH^H^_acdHee eK%dD!Q&67>>z1bRTUK))//1 ,2226 *1aL[L+@ANSTVZ\`bcScDdDgDg&&E   ,q0 5@Aq,;,AV5W5c5c )6 Aq!\k\12r=Nr)rbrcrdr config_classbase_model_prefixsupports_gradient_checkpointing_no_split_modules_supports_cache_class_supports_static_cache_supports_sdpa_supports_quantized_cacherr/rer r0r staticmethodrr#r!rrr=r;rzrzsL&*# !"N $;6#( D DllD D  D  DL7 777{{ 7  7  777r=rzaK This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`MllamaConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, max_num_images, max_num_tiles, channels, image_size, image_size)): The tensors corresponding to the input images. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MllamaImageProcessor.__call__`] for details ([]`MllamaProcessor`] uses [`MllamaImageProcessor`] for processing images). aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*): Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`: - 1 for tiles that are **not masked**, - 0 for tiles that are **masked**. aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*): Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image. These ids correspond to indices in the model's list of supported aspect ratios, offset by 1. For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]: - An image with aspect ratio [1, 1] would have ID 1 - An image with aspect ratio [1, 2] would have ID 2 - An image with aspect ratio [2, 1] would have ID 3 The id 0 is reserved for padding (i.e., no image). If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. a Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. If `past_key_values` is used, optionally only the last `input_ids` have to be input (see `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): Cross-attention mask to control the interaction between text tokens and image tiles. This 4D tensor defines which image tiles each text token should attend to. For each text token (in seq_length): - 1 indicates the token **should attend** to the corresponding image tile - 0 indicates the token **should not attend** to the corresponding image tile cross_attention_states (`torch.FloatTensor`, *optional*): Output of the vision model, used for cross-attention. This tensor contains the processed image features that the language model will attend to. position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. Two formats are allowed: - a [`~cache_utils.Cache`] instance, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy cache format. The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the legacy cache format will be returned. If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, this tensor is not affected by padding. It is used to update the cache in the correct position and to infer the complete sequence length. a Args: input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) pixel_values (`torch.FloatTensor` of shape `(batch_size, max_num_images, max_num_tiles, channels, image_size, image_size)): The tensors corresponding to the input images. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MllamaImageProcessor.__call__`] for details ([]`MllamaProcessor`] uses [`MllamaImageProcessor`] for processing images). aspect_ratio_mask (`torch.Tensor` of shape `(batch_size, max_num_images, max_num_tiles)`, *optional*): Mask to avoid performing attention on padding tiles. Mask values selected in `[0, 1]`: - 1 for tiles that are **not masked**, - 0 for tiles that are **masked**. aspect_ratio_ids (`torch.Tensor` of shape `(batch_size, max_num_images)`, *optional*): Aspect ratio ids used to select the appropriate precomputed tile embeddings based on the aspect ratio of each input image. These ids correspond to indices in the model's list of supported aspect ratios, offset by 1. For example, if the model supports aspect ratios [[1, 1], [1, 2], [2, 1]]: - An image with aspect ratio [1, 1] would have ID 1 - An image with aspect ratio [1, 2] would have ID 2 - An image with aspect ratio [2, 1] would have ID 3 The id 0 is reserved for padding (i.e., no image). If an image has aspect ratio [1, 2], that means it was split into 2 tiles horizontally, and its `aspect_ratio_id` would be 2. attention_mask (`torch.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. If `past_key_values` is used, optionally only the last `input_ids` have to be input (see `past_key_values`). If you want to change padding behavior, you should read [`modeling_opt._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attention_mask (`torch.Tensor` of shape `(batch_size, seq_length, max_num_images, max_num_tiles)`, *optional*): Cross-attention mask to control the interaction between text tokens and image tiles. This 4D tensor defines which image tiles each text token should attend to. For each text token (in seq_length): - 1 indicates the token **should attend** to the corresponding image tile - 0 indicates the token **should not attend** to the corresponding image tile cross_attention_states (`torch.FloatTensor`, *optional*): Output of the vision model, used for cross-attention. This tensor contains the processed image features that the language model will attend to. position_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.n_positions - 1]`. [What are position IDs?](../glossary#position-ids) past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*): Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values` returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`. Two formats are allowed: - a [`~cache_utils.Cache`] instance, see our [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache); - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy cache format. The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the legacy cache format will be returned. If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*): Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`, this tensor is not affected by padding. It is used to update the cache in the correct position and to infer the complete sequence length. z>The Mllama Vision Model which consists of two vision encoders.c<eZdZeZdZdeffd ZdZdejdejfdZ e e e ed  dd ejd ejd ejd eedeedeedeeeejdfffdZxZS)r vision_modelrLct|||j|_|j|_|j|_|j |_|j |_|j|_|j|jzdzdz|_|j dz|_ tj|j |j |j|jdd|_ tj|jtj|j z|_t#||_t'|d|_t'|d|_tj,|j |_tj,|j |_t3||j4d|_t3||j8d|_|j=y) NrkrrlvalidF) in_channels out_channels kernel_sizestridepaddingrT)rM)rPrQrmrnrFrR num_channelsintermediate_layers_indicesr?rorrpatch_embeddingrVr/rprrigated_positional_embeddingrKpre_tile_positional_embeddingpost_tile_positional_embeddingr layernorm_prelayernorm_postrnum_hidden_layers transformernum_global_layersglobal_transformer post_initrs r;rQzMllamaVisionModel.__init__s   ++ ++#11!--"//+1+M+M( OOt>1DqH''- !yy++))??   "||DJJTEUEU9V,VW*LV*T'-RSYdh-i*.STZei.j+ \\$*:*:; ll4+;+;</vv7O7OZ_`"5ff>V>Vae"f r=c|jS)zg This function is used to fetch the first embedding layer to activate grads on inputs. )rrs r;get_input_embeddingsz&MllamaVisionModel.get_input_embeddingss###r=r\r$c|j\}}}|jj|d|}tj||gd}|S)Nrr&)r)rr5r/r$)rZr\r5r7rRrs r;apply_class_embeddingz'MllamaVisionModel.apply_class_embeddingsI%1%7%7" A{..55j![Qyy/rrr.cH||n|jj}||n|jj}||n|jj}|j\}}} } } } |j ||z| z| | | }|j ||zd}|j jj} |j jj}|j |j|| }|jdjdd}|j\}}}|j ||z| d|}|j||}|j ||z| z||}|j|}|dz }|j ||z| ||}|j||}|j!|}d|jddzz dz}ddd|f}t#j$||dd }|dkDr| nd}|j ||zd}t'||j(|jd|j }|j+||zd|}|j-||d | }|d}|j/|}|j ||z| ||z|}|j1||}|j ||z| ||zz|}|j3|||| }|d}|j ||z| ||z|}|ddddd|f}|j ||| ||}|j4Dcgc] }|d| }}t7j8|d }|j ||z| ||zd}|ddddd|f}|j ||| |d}t7j:||gd }|rt=|t=|dz}nd}|r0|rt=|dn t=|d}t=|d|z}nd}|st=d|||fDSt?|||Scc}w)a Returns: Example: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, MllamaVisionModel >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision" >>> model = MllamaVisionModel.from_pretrained(checkpoint) >>> processor = AutoProcessor.from_pretrained(checkpoint) >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(images=image, return_tensors="pt") >>> output = model(**inputs) >>> print(output.last_hidden_state.shape) torch.Size([1, 1, 4, 1025, 7680]) ``` Nr(rkrrBrconstant)moder)r>r?r@r#T)rGrrr&c3&K|] }|| ywrzrrs r;rz,MllamaVisionModel.forward.._s_qQRQ^_rr) rLrrrr)rDrrr#r!r-flattenrErrrrrpadrIr?r+rrrrrr/stackr$rr) rZrr]r>rrrr5num_concurrent_media num_tilesrheightwidth target_dtype target_device patch_embedsr\r7r?r'num_padding_patchesr slice_indexrGr global_outputiall_intermediate_hidden_statesintermediate_hidden_statesr global_attnrs r;razMllamaVisionModel.forwardsH2C1N-TXT_T_TqTq$8$D $++JjJj &1% +00;#++J9M,MyZ\^ab 99,HXY $++J9M,MPY,Y[fhkl 11,? q $++J9M,MyZegjk 66|EUV )),7  !L$6$6r$:Q$>?1Da/0uu\71M .AA.E**4 +22:@T3TVXY=,((&,,Q/**  $((6J)JBPST !! )!%/ "  ay **<8 $++ - -y+H[:[]` :: Z 'tn Z d^Z  ellC&7 88 9Z _KZ r=rzYThe Mllama Text Model which consists of transformer with self and cross attention layers.c!eZdZeZdZdeffd ZdZdZe e e e d dde ejd e ej d e ejd e ej"d e ej d e eej ej fde eeeej"fde ej"de ede ede ede ede ejdeee ffdZxZS)MllamaTextModelzlanguage_model.modelrLct|||j|_|j|_t j |jdz|j|j|_|j|_ g}t|jD]G}||jvr|jt||-|jt||It j||_t#|j|j$|_t)||_d|_|j/y)Nrr)rLF)rPrQ pad_token_idr vocab_sizerrTrR embed_tokenscross_attention_layersrrappendrVrNrrrrnormr^ rotary_embrr)rZrLrrr[s r;rQzMllamaTextModel.__init__ps  !.. ++LL):):Q)>@R@RTXTdTde&,&C&C#v778 RID777 >vyQR =fiPQ  R mmF+ %f&8&8f>Q>QR /v>&+# r=c|jSrzrrs r;rz$MllamaTextModel.get_input_embeddingss   r=c||_yrzrrZrs r;set_input_embeddingsz$MllamaTextModel.set_input_embeddingss !r=rr input_idsrGr.rr!r:rrrrrrrr$c| | n|jj} | | n|jj} | | n|jj} | | n|jj} |du|duz r t d|j r%|jr| rtjdd} ||j|}|}| r | t}| F||jnd}tj|||jdz|j } || j#d}|j%||| || }|j'||}| rdnd}| rdnd}d}t)|j*D]\}}| r||fz }||j,v}|duxs|duxr|j|dk(}|r||rC|j r3|jr'|j/|j0|||||||| | | | }n||||||||| | | | }|d}| r || rd nd}| s||dfz }|j3|}| r||fz }| r|nd}| st5d ||||fDSt7|||| S) aN Returns: Example: ```python >>> from transformers import AutoProcessor, MllamaTextModel >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision" >>> model = MllamaTextModel.from_pretrained(checkpoint) >>> processor = AutoProcessor.from_pretrained(checkpoint) >>> text = "<|image|>If I had to write a haiku for this one" >>> inputs = processor(text=text, return_tensors="pt") >>> output = model(**inputs) >>> print(output.last_hidden_state.shape) torch.Size([1, 13, 4096]) ``` N:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Frrrr) rr!rGr:r.rrrrr<rkc3&K|] }|| ywrzrrs r;rz*MllamaTextModel.forward..stqfgfstr)rrrr)rLrrrrrrrrrrr rr/rr)r!r,rr enumeraterrrrrrr)rZrrGr.rr!r:rrrrrrrrrrr<all_hidden_statesall_self_attnsnext_decoder_cacheidx decoder_layeris_cross_attention_layeris_cross_attention_cache_emptyr next_caches r;razMllamaTextModel.forwardsP2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKK?L] #oom\J#7BD0d!"+DKK"83 6 C#!m%55! (+d.I.I'I $-<-D.t+X0N0Ns0SWX0X +(,B,JOm**t}} $ A A!**!*(1 #%"' ! !.!+A)=#.2O!-#2&7'#1(; ! *!,M%28I1q%Q" =#3"55g3 6j -0  -!1 1 +4'$ t]J@QSa$btt t&+&+%   r=) NNNNNNNNNNNNN)rbrcrdrrrrQrrrMLLAMA_TEXT_INPUTS_DOCSTRINGrrrr/rrerTrrr rr0rarfrgs@r;rrhs $L./(!"++GH+BQcd151537>B7;UYKO59$(,0/3&*59V E,,-V !.V u//0 V !)):): ; V 'u||4 V (0ellELL6P0Q'RV "%tE4E4E/F(F"GHV   1 12V D>V $D>V 'tnV d^V !!1!12V  u-- .V eIV r=rz;The Mllama Text Model with a language modeling head on top.c&zeZdZeZdZdZdgZfdZdZ dZ dZ dZ d Z d Zed d d eeeed d!deej,deej.deej,deej,deej,deeej.ej.fdeeeeej8fdeej8deej,deedeedeedeedeej,d eeej.fdeeeff d ZxZ S)"MllamaForCausalLMTlanguage_modelzlm_head.weightc~t||j|j|_|jj|_t j |j|_tj|jj|jd|_ |jyr) rPrQr~ text_configrr _from_configr{rr}rRlm_headrrs r;rQzMllamaForCausalLM.__init__/s //12!113**55$11$2B2BC yy!1!1!=!=tUZ[  r=c.|jjSrzr{rrs r;rz&MllamaForCausalLM.get_input_embeddings8szz&&&r=c&||j_yrzr rs r;rz&MllamaForCausalLM.set_input_embeddings;s"' r=c|jSrzrrs r;get_output_embeddingsz'MllamaForCausalLM.get_output_embeddings>s ||r=c||_yrzr rZnew_embeddingss r;set_output_embeddingsz'MllamaForCausalLM.set_output_embeddingsAs % r=c||_yrzr{rZdecoders r; set_decoderzMllamaForCausalLM.set_decoderDs  r=c|jSrzrrs r; get_decoderzMllamaForCausalLM.get_decoderGs zzr=num_logits_to_keep4.50logits_to_keepversionnew_namerrrrGr.rr!r:rrlabelsrrrrrr$c @| | n|jj} | | n|jj} | | n|jj} |j ||||||||| | | | | }|d}t |t r t| dn|}|j|dd|ddfj}d}| |j|| |jfi|}| s|f|ddz}||f|zS|St|||j|j|jS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. logits_to_keep (`int` or `torch.Tensor`, *optional*): If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that token can save memory, which becomes pretty significant for long sequences or large vocabulary size. If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. This is useful when using packed tensor format (single dimension for batch and sequence length). Returns: Example: ```python >>> from transformers import AutoTokenizer, MllamaForCausalLM >>> model = MllamaForCausalLM.from_pretrained("Llama-3.2-11B-Vision") >>> tokenizer = AutoTokenizer.from_pretrained("Llama-3.2-11B-Vision") >>> prompt = "If I had to write a haiku, it would be:" >>> inputs = tokenizer(prompt, return_tensors="pt") >>> # Generate >>> generate_ids = model.generate(inputs.input_ids, max_length=40, do_sample=True, temperature=0.6) >>> result = tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0] >>> print(result) If I had to write a haiku, it would be: "Snowflakes gently fall" - simple, yet peaceful. I love the idea of snowflakes gently falling, each one ``` N) rrrGr.r!r:rrrrrrrrrlosslogitsrrr)rLrrrr{rqrslicerrp loss_functionrrrrr)rZrrGr.rr!r:rrrrrrrrr loss_kwargsrr slice_indicesr#r"rs r;razMllamaForCausalLM.forwardJs\p2C1N-TXT_T_TqTq$8$D $++JjJj &1%SV8W~ot4]k mA}a,?@AGGI  %4%%ffdooUUDY,F'+'7D7V# CV C%#33!//))   r=)NNNNNNNNNNNNNNr)!rbrcrdrrrr_tied_weights_keysrQrrr rrrrrMLLAMA_INPUTS_DOCSTRINGrrrr/rrerrr rrTr0rrarfrgs@r;rr%s $L!(*+'(&)6DTU*+BC+APbc151537=A;?UYKO59-1$(,0/3&*5934!^ E,,-^ !.^ u//0 ^ !))9)9 : ^ 'u'7'78 ^ (0ellELL6P0Q'R^ "%tE4E4E/F(F"GH^   1 12^ ))*^ D>^ $D>^ 'tn^ d^^ !!1!12^ c5<</0!^ $ u,, -%^ dDV^ r=rzIThe Mllama model which consists of a vision encoder and a language model.c*eZdZdZdeffd ZdZdZdZdZ dZ d Z e d d d e eeed d$deej&deej(deej*deej*deej*deej*deej*deej&deeeeej(fdeej(deej&deedeedeedeedeej&d eeej*fd eeeff$d!Z d%d"Zfd#ZxZS)&MllamaForConditionalGenerationFrLc*t|||jj|_|jj|_|j j |_|j j|_|jj|jjnd|_ tj|j |_ tj|j|_|jj,|jjDcgc]}d| c}|_t!j"|j j|jjd|_|j'ycc}w)Nr(zlanguage_model.Tr)rPrQrrrR vision_configrFvision_output_dimrLrrrrrrr(rr}multi_modal_projectorr)rZrLr+r[s r;rQz'MllamaForConditionalGeneration.__init__s1   ,,77!--99#11??!'!5!5!G!G8< 8P8P8\DKK44bd-::6;O;OP/<r]rGr!rr.rrrrrrrrr$c | | n|jj} ||n|jj}||n|jj}|du| duz r t d| | t d| | t d|b| t d|j ||||| |}|d}|j |jd|jd |j}|0t||j j|j \}}nd}|||dddd|f}|dddd|f}|jd ||||||| | | || |||d |}d}|d}| 7|j|| |jjj fi|}|s ||f|zS|St#||j$|j&|j(|j* S)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. logits_to_keep (`int` or `torch.Tensor`, *optional*): If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that token can save memory, which becomes pretty significant for long sequences or large vocabulary size. If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension. This is useful when using packed tensor format (single dimension for batch and sequence length). Returns: Example: ```python >>> from PIL import Image >>> import requests >>> from transformers import AutoProcessor, MllamaForConditionalGeneration >>> checkpoint = "meta-llama/Llama-3.2-11B-Vision" >>> model = MllamaForConditionalGeneration.from_pretrained(checkpoint) >>> processor = AutoProcessor.from_pretrained(checkpoint) >>> prompt = "<|image|>If I had to write a haiku for this one" >>> url = "https://www.ilankelman.org/stopsigns/australia.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> inputs = processor(text=prompt, images=image, return_tensors="pt") >>> # Generate >>> output = model.generate(**inputs, max_new_tokens=15) >>> prompt_len = inputs.input_ids.shape[-1] >>> generated_ids = output[:, prompt_len:] >>> generated_text = processor.batch_decode(generated_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False) >>> print(generated_text) [', it would be:.\\nA stop sign in Chinatown.\\n'] ``` NrzdYou cannot specify both pixel_values and inputs_embeds at the same time, and must specify either onezM`pixel_values` and `cross_attention_states` cannot be provided simultaneouslyzA`aspect_ratio_ids` must be provided if `pixel_values` is provided)rr]r>rrrrr(rB)r"r#)rrGr.rr!r:rrrrrrrrr!r)rLrrrrrr/rDr)rRr<r?r#rr%r~rrr#rrr)rZrrr>r]rGr!rr.rrrrrrrrrr&vision_outputsr:rr"r#s r;raz&MllamaForConditionalGeneration.forwardsoF2C1N-TXT_T_TqTq$8$D $++JjJj &1%(Jlm m  #' !dee!..)!1"3%9"3' /N&4A%6 "%)%?%?@V%W%_%_*004d6F6F& " +B_$"&"3"3"?"?jjC ? "? -1 )  +0J#71n8L#M ,I!QP^J^,_ )%$%% )%#9!5*G+'!5/#))  &  %4%%ffdkk6Q6Q6S6^6^nbmnD(,(8D7W$ Eg E%>>#33!//))   r=c z| Y|| d|jdk\r|dd| jd df}n(|jd| jdk7r |dd| f}|t|r|jjddz }|j|dk(d| r9|dd|jd df}|j t j }|| ddk(r|dd}n#|j t j dd}| | |d<|j|| | | ||d| ddk(r||d <||d <||d <|S) Nr(rr) memory_format)rr)rrr)r.rrrrGr!rr]r>)r)longcumsum masked_fill_rr/contiguous_formatr )rZrrrGr.rr]r>r!rrrrr? model_inputss r;prepare_inputs_for_generationz*>q*A)A)C&CD #~';';A'>>%a&78   %,*>)..077;a?L  % %n&91 =+A 0B/B/D,DE  ,11@W@W1X   $):a)?-:NL*3uG^G^)_rvwL  %-;L) * ,"0#2&"0(<    !  !+7L (/?L+ ,0AL, -r=c |jdd}t|d|||d|}|&tj||dddddfgd|d<|S)Nr!)r model_kwargsis_encoder_decoderr(.rr&r)getrP#_update_model_kwargs_for_generationr/r$)rZrrArBr?cross_attention_mask_prevr[s r;rDzBMllamaForConditionalGeneration._update_model_kwargs_for_generations{$0$4$45KT$R!wB %1    % 03899*,Eack,RSYZ4L/ 0r=)NNNNNNNNNNNNNNNNr) NNNNNNNNNFNN) rbrcrdrrrQrrr rrrrrr)rrrr/rrTrerr rr0rrrar?rDrfrgs@r;r+r+s\ !&|(:8;B11)6DTU*+BC+AP^_15484837157;9=37KO59-1$(,0/3&*5934%Q E,,-Q u001Q $ELL1 Q #5<<0 Q !. Q 'u||4Q !) 6Q u//0Q "%tE4E4E/F(F"GHQ   1 12Q ))*Q D>Q $D>Q 'tnQ d^!Q "!!1!12#Q $c5<</0%Q ( u,, -)Q `DVQ j!GRr=r+)r+rrrrzrO)Zrrtypingrrrrr/torch.nn.functionalrrrtorch.utils.checkpointr activationsr cache_utilsr r r generationrmodeling_attn_mask_utilsrmodeling_outputsrrrmodeling_rope_utilsrrutilsrrrrrutils.deprecationrconfiguration_mllamarrr!torch.nn.attention.flex_attentionrintegrations.flex_attentionr get_loggerrbrrerrrr<r#rIModulerKrirxrrrrrrrrr(r2r r9rCrXrPrFrNrVr^rzMLLAMA_START_DOCSTRINGrrr)rrrr+__all__rr=r;rYsK // !;;)>``K1TT !;J   H %?,,?? ? 5<< %& ?8|| ;;   \\ 6BII.""L bii 0$BII0$f* 5*Z-BKd"e.ryy.bT "))T pJ J(S9ryyS9l[1#;[1~(8 UU\\ U# U%,, UE9biiE9PO1"9O1d1IRn&o#*AKf gBII$PbiiPf9uxx9x