Ë
    S^(h/ƒ  ã                   óÂ  — d dl mZ d dlmZmZmZmZ d dlZd dlm	Z	 d dl
ZddlmZ ddlmZmZmZ ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZ ddlmZ ddlmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' dZ( ejR                  e*«      Z+ G d„ de«      Z, G d„ de%«      Z- G d„ de#«      Z.	 	 	 d-de	j^                  dej`                  dej`                  dej`                  deej`                     de1dee1   dee1   deej`                  ej`                  f   fd„Z2 G d „ d!e«      Z3 G d"„ d#e	j^                  «      Z4 G d$„ d%e$«      Z5 G d&„ d'e «      Z6 G d(„ d)e!«      Z7 G d*„ d+e"«      Z8g d,¢Z9y).é    )Úpartial)ÚCallableÚOptionalÚTupleÚUnionNé   )ÚACT2FN)ÚCacheÚHybridCacheÚStaticCache)ÚPretrainedConfig)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úloggingé   )	ÚGemmaAttentionÚGemmaForCausalLMÚGemmaForSequenceClassificationÚGemmaForTokenClassificationÚGemmaMLPÚ
GemmaModelÚGemmaRMSNormÚapply_rotary_pos_embÚ	repeat_kvzgoogle/gemma2-7bc                   ó˜   ‡ — e Zd ZdZdZdgZddddddddœZdgdgfd	d
gd	gfd	gd	gfdœZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ xZ	S )ÚGemma2ConfigaÊ  
    This is the configuration class to store the configuration of a [`Gemma2Model`]. It is used to instantiate an Gemma2
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the Gemma2-7B.
    e.g. [google/gemma2-7b](https://huggingface.co/google/gemma2-7b)
    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.
    Args:
        vocab_size (`int`, *optional*, defaults to 256000):
            Vocabulary size of the Gemma2 model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`Gemma2Model`]
        hidden_size (`int`, *optional*, defaults to 2304):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 9216):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 26):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 8):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*, defaults to 4):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details checkout [this
            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
            `num_attention_heads`.
        head_dim (`int`, *optional*, defaults to 256):
            The attention head dimension.
        hidden_activation (`str` or `function`, *optional*, defaults to `"gelu_pytorch_tanh"`):
            The non-linear activation function (function or string) in the decoder. Will default to `"gelu_pytorch_tanh"`
            if not specified. `"gelu_pytorch_tanh"` uses an approximation of the `"gelu"` activation function.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        rms_norm_eps (`float`, *optional*, defaults to 1e-06):
            The epsilon used by the rms normalization layers.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 0):
            Padding token id.
        eos_token_id (`int`, *optional*, defaults to 1):
            End of stream token id.
        bos_token_id (`int`, *optional*, defaults to 2):
            Beginning of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
            Whether to tie weight embeddings
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        query_pre_attn_scalar (`float`, *optional*, defaults to 256): scaling factor used on the attention scores
        sliding_window (`int`, *optional*, defaults to 4096): in Gemma2, every other layer uses sliding window attention. This is the
            size of the sliding window.
        final_logit_softcapping (`float`, *optional*, defaults to 30.0): scaling factor when applying tanh softcapping on the logits.
        attn_logit_softcapping (`float`, *optional*, defaults to 50.0): scaling factor when applying tanh softcapping on the attention scores.
        cache_implementation (`str`, *optional*, defaults to `"hybrid"`): the cache type to be used with `generate`.

    ```python
    >>> from transformers import Gemma2Model, Gemma2Config
    >>> # Initializing a Gemma2 gemma2-7b style configuration
    >>> configuration = Gemma2Config()
    >>> # Initializing a model from the gemma2-7b style configuration
    >>> model = Gemma2Model(configuration)
    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```Úgemma2Úpast_key_valuesÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormc                 óF  •— t        ‰| �  d||||dœ|¤Ž || _        |	| _        || _        || _        || _        || _        || _        || _	        |
| _
        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        || _        y )N)Úpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddings© )ÚsuperÚ__init__Ú
vocab_sizeÚmax_position_embeddingsÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚhead_dimÚnum_key_value_headsÚinitializer_rangeÚrms_norm_epsÚ	use_cacheÚ
rope_thetaÚattention_biasÚattention_dropoutÚhidden_activationÚquery_pre_attn_scalarÚsliding_windowÚfinal_logit_softcappingÚattn_logit_softcappingÚcache_implementation)Úselfr3   r5   r6   r7   r8   r:   r9   rA   r4   r;   r<   r=   r,   r.   r-   r/   r>   r?   r@   rB   rC   rD   rE   rF   ÚkwargsÚ	__class__s                             €úg/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/gemma2/modular_gemma2.pyr2   zGemma2Config.__init__Ž   sË   ø€ ô8 	‰Ñð 	
Ø%Ø%Ø%Ø 3ñ		
ð
 ò	
ð %ˆŒØ'>ˆÔ$Ø&ˆÔØ!2ˆÔØ!2ˆÔØ#6ˆÔ Ø ˆŒØ#6ˆÔ Ø!2ˆÔØ(ˆÔØ"ˆŒØ$ˆŒØ,ˆÔØ!2ˆÔØ!2ˆÔØ%:ˆÔ"Ø,ˆÔØ'>ˆÔ$Ø&<ˆÔ#Ø$8ˆÕ!ó    )i è i 	  i $  é   é   é   é   Úgelu_pytorch_tanhi    g{®Gáz”?g�íµ ÷Æ°>Tr   é   r   Tg     ˆÃ@Fç        rO   i   g      >@g      I@Úhybrid)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planr2   Ú__classcell__©rI   s   @rJ   r   r   4   sÁ   ø„ ñFðP €JØ#4Ð"5Ðà%.Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð ØØØØØØØ-Ø $ØØØØØØØ ØØØØ!ØØ $Ø#Ø%÷369ñ 69rK   r   c                   ó   — e Zd Zy)ÚGemma2RMSNormN)rT   rU   rV   r0   rK   rJ   r_   r_   Ç   s   „ ØrK   r_   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )Ú	Gemma2MLPc                 óR   •— t         ‰| �  «        t        |j                     | _        y ©N)r1   r2   r	   rA   Úact_fn©rG   ÚconfigrI   s     €rJ   r2   zGemma2MLP.__init__Ì   s    ø€ Ü‰ÑÔÜ˜V×5Ñ5Ñ6ˆ�rK   ©rT   rU   rV   r2   r\   r]   s   @rJ   ra   ra   Ë   s   ø„ ÷7ð 7rK   ra   ÚmoduleÚqueryÚkeyÚvaluer'   ÚdropoutÚscalingÚsoftcapÚreturnc                 ó¸  — |€| j                   dz  }t        || j                  «      }	t        || j                  «      }
t        j                  ||	j                  dd«      «      |z  }|�||z  }t        j                  |«      }||z  }|�#|d d …d d …d d …d |	j                  d   …f   }||z   }t        j                  j                  |dt        j                  ¬«      j                  |j                  «      }t        j                  j                  ||| j                  ¬«      }t        j                  ||
«      }|j                  dd«      j!                  «       }||fS )	Nç      à¿r   r   éþÿÿÿéÿÿÿÿ)ÚdimÚdtype)ÚpÚtrainingrQ   )r9   r   Únum_key_value_groupsÚtorchÚmatmulÚ	transposeÚtanhÚshapeÚnnÚ
functionalÚsoftmaxÚfloat32Útoru   rl   rw   Ú
contiguous)rh   ri   rj   rk   r'   rl   rm   rn   rH   Ú
key_statesÚvalue_statesÚattn_weightsÚcausal_maskÚattn_outputs                 rJ   Úeager_attention_forwardr‰   Ñ   sA  € ð €Ø—/‘/ 4Ñ'ˆä˜3 × ;Ñ ;Ó<€JÜ˜U F×$?Ñ$?Ó@€Lä—<‘<  z×';Ñ';¸A¸qÓ'AÓBÀWÑL€LàÐØ# gÑ-ˆÜ—z‘z ,Ó/ˆØ# gÑ-ˆØÐ!Ø$¢Qªª1Ð.D°
×0@Ñ0@ÀÑ0DÐ.DÐ%DÑEˆØ# kÑ1ˆô —=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÓS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨¸È6Ï?É?Ð(Ó[€LÜ—,‘,˜|¨\Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KØ˜Ð$Ð$rK   c                   ó2  ‡ — e Zd Zdedefˆ fd„Z	 	 ddej                  deej                  ej                  f   de	ej                     de	e
   de	ej                     d	ee   d
eej                  e	ej                     e	eej                        f   fd„Zˆ xZS )ÚGemma2Attentionrf   Ú	layer_idxc                 ó  •— t         ‰| �  ||«       | j                  j                  | _        | j                  j                  | _        d| _        |j                  dz  | _        t        |dz  «      s|j                  | _	        y d | _	        y )NTrq   r   )
r1   r2   rf   rE   r@   Ú	is_causalrB   rm   ÚboolrC   ©rG   rf   rŒ   rI   s      €rJ   r2   zGemma2Attention.__init__õ   sp   ø€ Ü‰Ñ˜ Ô+Ø&*§k¡k×&HÑ&HˆÔ#Ø!%§¡×!>Ñ!>ˆÔØˆŒØ×3Ñ3°TÑ9ˆŒÜ;?À	ÈAÁÔ;N˜f×3Ñ3ˆÕÐTXˆÕrK   r&   Úposition_embeddingsr'   Úpast_key_valueÚcache_positionrH   ro   c                 ó�  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }|\  }}t        |	|
||«      \  }	}
|�~|||| j                  dœ}|j                  |
|| j                  |«      \  }
}|�J| j                  j                  dk(  r1|j                   d   }|
d d …d d …d |…d d …f   |d d …d d …d |…d d …f   }}
t        }| j                  j                  dk7  r^| j                  j                  dk(  r(|j                  dd	«      rt        j!                  d
«       nt"        | j                  j                     } || |	|
||f| j$                  r| j&                  nd| j(                  | j                  | j*                  dœ|¤Ž\  }} |j,                  g |¢d‘­Ž j/                  «       }| j1                  |«      }||fS )Nrs   rQ   r   )ÚsinÚcosr“   rC   Úflash_attention_2ÚeagerÚsdpaÚoutput_attentionsFzã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.rR   )rl   rm   rC   rn   )r}   r9   Úq_projÚviewr{   Úk_projÚv_projr   rC   ÚupdaterŒ   rf   Ú_attn_implementationr‰   ÚgetÚloggerÚwarning_oncer   rw   r@   rm   rE   Úreshaperƒ   Úo_proj)rG   r&   r‘   r'   r’   r“   rH   Úinput_shapeÚhidden_shapeÚquery_statesr„   r…   r–   r•   Úcache_kwargsÚseq_lenÚattention_interfacerˆ   r†   s                      rJ   ÚforwardzGemma2Attention.forwardý   sa  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ%ð ØØ"0Ø"&×"5Ñ"5ñ	ˆLð (6×'<Ñ'<¸ZÈÐW[×WeÑWeÐgsÓ'tÑ$ˆJ˜ð Ð)¨d¯k©k×.NÑ.NÐReÒ.eØ(×.Ñ.¨rÑ2�Ø+5²aº¸H¸W¸HÂaÐ6GÑ+HÈ,ÒWXÒZ[Ð]eÐ^eÐ]eÒghÐWhÑJi˜L�
ä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9¸f¿j¹jÐI\Ð^cÔ>dÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØð%
ð /3¯mªm�D×*Ò*ÀØ—L‘LØ×.Ñ.Ø×/Ñ/ñ%
ð ñ%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rK   )NN)rT   rU   rV   r   Úintr2   ry   ÚTensorr   r   r
   Ú
LongTensorr   r   r¬   r\   r]   s   @rJ   r‹   r‹   ô   sÅ   ø„ ðY˜|ð Y¸õ Yð +/Ø59ñ;)à—|‘|ð;)ð # 5§<¡<°·±Ð#=Ñ>ð;)ð ! §¡Ñ.ð	;)ð
 ! ™ð;)ð ! ×!1Ñ!1Ñ2ð;)ð Ð-Ñ.ð;)ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷;)rK   r‹   c                   ód  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 ddej                  deej                  ej                  f   de	ej                     de	ej                     de	e   d	e	e   d
e	e   de	ej                     dedeej                  e	eej                  ej                  f      f   fd„Zˆ xZS )ÚGemma2DecoderLayerrf   rŒ   c                 ó  •— t         ‰| �  «        |j                  | _        || _        t	        |dz  «       | _        t        ||¬«      | _        t        |«      | _	        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        |j                   | _        y )Nr   )rf   rŒ   )Úeps)r1   r2   r5   rf   r�   Ú
is_slidingr‹   Ú	self_attnra   Úmlpr_   r<   Úinput_layernormÚpost_attention_layernormÚpre_feedforward_layernormÚpost_feedforward_layernormrC   r�   s      €rJ   r2   zGemma2DecoderLayer.__init__<  sË   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØˆŒÜ" 9¨q¡=Ó1Ð1ˆŒÜ(°À)ÔLˆŒÜ˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÔ%ä)6°v×7IÑ7IÈv×ObÑObÔ)cˆÔ&Ü*7¸×8JÑ8JÐPV×PcÑPcÔ*dˆÔ'Ø$×3Ñ3ˆÕrK   r&   r‘   r'   Úposition_idsr’   rš   r=   r“   Úlast_cache_positionro   c
                 ó   — | j                   rõ|�ót        |j                  d   | j                  «      }| j                  j
                  dk(  r|d d …| d …f   }nªt        j                  |j                  «      j                  }t        j                  t        j                  |t        j                  ¬«      | j                   ¬«      }t        j                  |||«      }|	|z
  }t        d|«      }|d d …d d …d d …|||z   …f   }|}| j                  |«      } | j                  d||||||||dœ|
¤Ž\  }}| j!                  |«      }||z   }|}| j#                  |«      }| j%                  |«      }| j'                  |«      }||z   }|f}|r||fz  }|S )Nr   r—   ©ru   )Údiagonal)r&   r‘   r'   r»   r’   rš   r=   r“   r0   )r´   Úmaxr}   rC   rf   r    ry   Úfinforu   ÚminÚtrilÚ	ones_liker�   Úwherer·   rµ   r¸   r¹   r¶   rº   )rG   r&   r‘   r'   r»   r’   rš   r=   r“   r¼   rH   Úeffective_seq_lenÚ	min_dtypeÚsliding_window_maskÚoffsetÚresidualÚself_attn_weightsÚoutputss                     rJ   r¬   zGemma2DecoderLayer.forwardJ  s¸  € ð �?Š?˜~Ð9ä # N×$8Ñ$8¸Ñ$;¸T×=PÑ=PÓ QÐð �{‰{×/Ñ/Ð3FÒFØ!/²Ð4EÐ3EÑ3FÐ0FÑ!G‘ô "ŸK™K¨×(<Ñ(<Ó=×AÑA�	Ü&+§j¡jÜ—O‘O N¼%¿*¹*ÔEÐQU×QdÑQdÐPdô'Ð#ô "'§¡Ð-@À)È^Ó!\�ð -Ð/@Ñ@�ä˜Q ›�Ø!/²²1²a¸À&ÐK\ÑB\Ð9\Ð0\Ñ!]�à ˆà×,Ñ,¨]Ó;ˆð ,:¨4¯>©>ð 
,
Ø'Ø 3Ø)Ø%Ø)Ø/ØØ)ñ
,
ð ñ
,
Ñ(ˆÐ(ð ×5Ñ5°mÓDˆØ  =Ñ0ˆà ˆØ×6Ñ6°}ÓEˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  =Ñ0ˆà Ð"ˆáØÐ)Ð+Ñ+ˆGàˆrK   )NNNFFNr   )rT   rU   rV   r   r­   r2   ry   r®   r   r   r¯   r
   r�   ÚFloatTensorr¬   r\   r]   s   @rJ   r±   r±   ;  s  ø„ ð4˜|ð 4¸õ 4ð$ 26Ø37Ø*.Ø,1Ø$)Ø59Ø#$ñAà—|‘|ðAð # 5§<¡<°·±Ð#=Ñ>ðAð ! §¡Ñ.ð	Að
 ˜u×/Ñ/Ñ0ðAð ! ™ðAð $ D™>ðAð ˜D‘>ðAð ! ×!1Ñ!1Ñ2ðAð !ðAð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷ArK   r±   c                   ó˜  ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee	   deej                     dee   d	ee   d
ee   deej                     dee   dee   defd„Z ej"                  «       dej                  dej                  dej                  de	d	ef
d„«       Zˆ xZS )ÚGemma2Modelrf   c           	      ó¸   •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        y c c}w rc   )r1   r2   r~   Ú
ModuleListÚranger7   r±   r)   r�   s      €rJ   r2   zGemma2Model.__init__�  sD   ø€ Ü‰Ñ˜Ô Ü—m‘mÜDIÈ&×JbÑJbÓDcÖd°yÔ ¨	Õ2Òdó
ˆ�ùÚds   ·Ar$   r'   r»   r!   r%   r=   rš   Úoutput_hidden_statesr“   r¼   Úflash_attn_kwargsro   c                 ó~  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|d u |d uz  rt	        d«      ‚| j
                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|rL|€J| j                  s>|j                  \  }}}t        | j                   |||j                  | j                  ¬«      }|	€F|�|j                  «       nd}t        j                   |||j                  d   z   |j                  ¬«      }	|€|	j#                  d«      }|
€9d}
|�5|j%                  «       dk(  r|j                  d	   n|	d	   j'                  «       }
| j)                  |||	||«      }|}| j+                  ||«      }t        j,                  | j                   j.                  d
z  |j                  ¬«      }||z  }|rdnd }|rdnd }| j0                  d | j                   j2                   D ]t  }|r||fz  }| j
                  r;| j                  r/| j5                  t7        |j8                  fi |¤Ž||||||||	|
«
      }n ||f|||||||	|
dœ|¤Ž}|d   }|sŒl||d   fz  }Œv | j;                  |«      }|r||fz  }t=        ||||¬«      S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)Úmax_batch_sizeÚmax_cache_lenru   Údevicer   rQ   )rØ   r   rs   g      à?r¾   r0   )r‘   r'   r»   r’   rš   r=   r“   r¼   )Úlast_hidden_stater!   r&   Ú
attentions)rf   rš   rÓ   r=   Ú
ValueErrorÚgradient_checkpointingrw   r¢   r£   r(   r}   r   ru   rØ   Úget_seq_lengthry   ÚarangeÚ	unsqueezert   ÚitemÚ_update_causal_maskÚ
rotary_embÚtensorr5   r)   r7   Ú_gradient_checkpointing_funcr   Ú__call__r*   r   )rG   r$   r'   r»   r!   r%   r=   rš   rÓ   r“   r¼   rÔ   Ú
batch_sizerª   Ú_Úpast_seen_tokensr‡   r&   r‘   Ú
normalizerÚall_hidden_statesÚall_self_attnsÚdecoder_layerÚlayer_outputss                           rJ   r¬   zGemma2Model.forward•  s9  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	à˜Ð -°tÐ";Ò<ÜÐYÓZÐZà×&Ò&¨4¯=ª=¹YÜ×ÑØjôð ˆIàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0¸¿ºØ%2×%8Ñ%8Ñ"ˆJ˜ ä)Ø—‘Ø)Ø%Ø#×)Ñ)Ø—{‘{ôˆOð Ð!ØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ"Ÿ\™\Ø Ð"2°]×5HÑ5HÈÑ5KÑ"KÐTa×ThÑThôˆNð ÐØ)×3Ñ3°AÓ6ˆLð Ð&Ø"#ÐØÐ)ð 1?×0BÑ0BÓ0DÈÒ0I�N×(Ñ(¨Ò,È~Ð^`ÑOa×OfÑOfÓOhð $ð ×.Ñ.Ø˜M¨>¸?ÐL]ó
ˆð
 &ˆð #Ÿo™o¨m¸\ÓJÐô
 —\‘\ $§+¡+×"9Ñ"9¸3Ñ">Àm×FYÑFYÔZˆ
Ø%¨
Ñ2ˆñ #7™B¸DÐÙ0™°dˆà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò "	6ˆMÙ#Ø! mÐ%5Ñ5Ð!à×*Ò*¨t¯}ª}Ø $× AÑ AÜ˜M×2Ñ2ÑHÐ6GÑHØ!Ø'ØØ Ø#Ø%ØØ"Ø'ó!‘ñ !.Ø!ð!à(;Ø#.Ø!-Ø#2Ø&7Ø'Ø#1Ø(;ñ!ð (ñ!�ð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5‘ðE"	6ðH Ÿ	™	 -Ó0ˆáØ -Ð!1Ñ1Ðä&Ø+Ø+Ø+Ø%ô	
ð 	
rK   Úinput_tensorc           
      ób  — | j                   j                  dk(  r|S |j                  |j                  }}|j                  d   }t        |t        t        f«      r|j                  «       }	n |�|j                  d   n|j                  d   }	| j                  |||	||||j                  d   ¬«      }
|
S )Nr—   rQ   rs   r   ©Úsequence_lengthÚtarget_lengthru   rØ   r“   ræ   )
rf   r    ru   rØ   r}   Ú
isinstancer   r   Úget_max_cache_shapeÚ5_prepare_4d_causal_attention_mask_with_cache_position)rG   r'   rî   r“   r!   rš   ru   rØ   rñ   rò   r‡   s              rJ   rá   zGemma2Model._update_causal_mask  sÃ   € ð �;‰;×+Ñ+Ð/BÒBØ!Ð!à$×*Ñ*¨L×,?Ñ,?ˆvˆØ&×,Ñ,¨QÑ/ˆÜ�o¬´[Ð'AÔBØ+×?Ñ?ÓA‰Mà8FÐ8R˜N×0Ñ0°Ò4ÐXd×XjÑXjÐklÑXmˆMð ×PÑPØØ+Ø'ØØØ)Ø#×)Ñ)¨!Ñ,ð Qó 
ˆð ÐrK   )
NNNNNNNNNN)rT   rU   rV   r   r2   r   ry   r¯   r®   r   rÍ   r�   r­   r   r   r   r¬   Úno_gradrá   r\   r]   s   @rJ   rÏ   rÏ   Ž  s`  ø„ ð
˜|õ 
ð 15Ø15Ø37Ø15Ø59Ø$(Ø,0Ø/3Ø59Ø-1ñ@
à˜E×,Ñ,Ñ-ð@
ð ! §¡Ñ.ð@
ð ˜u×/Ñ/Ñ0ð	@
ð
 " +Ñ.ð@
ð   × 1Ñ 1Ñ2ð@
ð ˜D‘>ð@
ð $ D™>ð@
ð ' t™nð@
ð ! ×!1Ñ!1Ñ2ð@
ð & c™]ð@
ð $Ð$8Ñ9ð@
ð 
!ó@
ðD €U‡]�]ƒ_ð àŸ™ð ð —l‘lð ð Ÿ™ð	 ð
 %ð ð  ò ó ô rK   rÏ   c                   óZ  ‡ — e Zd Zˆ fd„Z	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee   deej                     deej                     dee
   d	ee
   d
ee
   deej                     deeej                  f   defd„Z	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚGemma2ForCausalLMc                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y rc   ©r1   r2   rÏ   ÚmodelÚ	post_initre   s     €rJ   r2   zGemma2ForCausalLM.__init__<  ó&   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø�‰ÕrK   r$   r'   r»   r!   r%   Úlabelsr=   rš   rÓ   r“   Úlogits_to_keepro   c                 ó.  — | j                   rF| j                  j                  dk7  r-t        j	                  d| j                  j                  › d�«       |�|n| j                  j
                  }|	�|	n| j                  j                  }	 | j                  d||||||||	|
dœ	|¤Ž}|j                  }t        |t        «      rt        | d«      n|}| j                  |dd…|dd…f   «      }| j                  j                  �G|| j                  j                  z  }t        j                  |«      }|| j                  j                  z  }d}|� | j                   ||| j"                  fi |¤Ž}t%        |||j&                  |j(                  |j*                  ¬«      S )aÝ  
            labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
                Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
                config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
                (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

            logits_to_keep (`int` or `torch.Tensor`, *optional*):
                If an `int`, compute logits for the last `logits_to_keep` tokens. If `0`, calculate logits for all
                `input_ids` (special case). Only last token logits are needed for generation, and calculating them only for that
                token can save memory, which becomes pretty significant for long sequences or large vocabulary size.
                If a `torch.Tensor`, must be 1D corresponding to the indices to keep in the sequence length dimension.
                This is useful when using packed tensor format (single dimension for batch and sequence length).

        Returns:

        Example:

        ```python
        >>> from transformers import AutoTokenizer, Gemma2ForCausalLM

        >>> model = Gemma2ForCausalLM.from_pretrained("google/gemma-2-9b")
        >>> tokenizer = AutoTokenizer.from_pretrained("google/gemma-2-9b")

        >>> prompt = "What is your favorite condiment?"
        >>> inputs = tokenizer(prompt, return_tensors="pt")

        >>> # Generate
        >>> generate_ids = model.generate(inputs.input_ids, max_length=30)
        >>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
        "What is your favorite condiment?"
        ```r˜   zhIt is strongly recommended to train Gemma2 models with the `eager` attention implementation instead of `zp`. Use `eager` with `AutoModelForCausalLM.from_pretrained('<path-to-checkpoint>', attn_implementation='eager')`.N)	r$   r'   r»   r!   r%   r=   rš   rÓ   r“   )ÚlossÚlogitsr!   r&   rÚ   r0   )rw   rf   r    r¢   r£   rš   rÓ   rû   rÙ   ró   r­   ÚsliceÚlm_headrD   ry   r|   Úloss_functionr3   r   r!   r&   rÚ   )rG   r$   r'   r»   r!   r%   rþ   r=   rš   rÓ   r“   rÿ   Úloss_kwargsrÌ   r&   Úslice_indicesr  r  s                     rJ   r¬   zGemma2ForCausalLM.forwardA  s¡  € ð^ �=Š=˜TŸ[™[×=Ñ=ÀÒHÜ×ÑðØ#Ÿ{™{×?Ñ?Ð@ð  Aqðrôð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð ,6¨4¯:©:ð ,
ØØ)Ø%Ø+Ø'ØØ/Ø!5Ø)ñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSVÔ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ�;‰;×.Ñ.Ð:Ø˜dŸk™k×AÑAÑAˆFÜ—Z‘Z Ó'ˆFØ˜dŸk™k×AÑAÑAˆFàˆØÐØ%�4×%Ñ% f¨f°d·o±oÑUÈÑUˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ô
ð 	
rK   c	                 ó4  •— t        ‰| �  |f|||||||dœ|	¤Ž}
|�|j                  d   nd|
d<   |€|
j                  dd «      }t	        |t
        «      rÄ|j                  dk(  rµ| j                  j                  dk(  sœ|
d   �#|
d   j                  \  }}}|
d   j                  }n!|
d	   j                  \  }}|
d	   j                  }| j                  j                  |||j                  «       | j                  j                  j                  |||¬
«      }||
d<   |
S )N)r!   r'   r%   r“   r»   r=   rÿ   rs   r   r¼   rÿ   r   r—   r%   r$   rð   r'   )r1   Úprepare_inputs_for_generationr}   Úpopró   r   Úndimrf   r    rØ   rû   rõ   rô   r  Úweightru   )rG   r$   r!   r'   r%   r“   r»   r=   rÿ   rH   Úmodel_inputsrç   ræ   rñ   rØ   rI   s                  €rJ   r	  z/Gemma2ForCausalLM.prepare_inputs_for_generationœ  sU  ø€ ô ‘wÑ<Øð

à+Ø)Ø'Ø)Ø%ØØ)ñ

ð ñ

ˆð KYÐJd¨n×.BÑ.BÀ2Ò.FÐjkˆÐ*Ñ+ØÐ!Ø× Ñ Ð!1°4Ó8ˆAô �¬Ô4Ø×#Ñ# qÒ(Ø—K‘K×4Ñ4Ð8KÒKà˜OÑ,Ð8Ø1=¸oÑ1N×1TÑ1TÑ.�
˜O¨QØ% oÑ6×=Ñ=‘à.:¸;Ñ.G×.MÑ.MÑ+�
˜OØ% kÑ2×9Ñ9�à!ŸZ™Z×]Ñ]ØØ /Ø-×AÑAÓCØ—l‘l×)Ñ)×/Ñ/ØØ-Ø%ð ^ó ˆNð .<ˆLÐ)Ñ*àÐrK   )NNNNNNNNNNr   )NNNNNTN)rT   rU   rV   r2   r   ry   r¯   r®   r   rÍ   r�   r   r­   r   r¬   r	  r\   r]   s   @rJ   rø   rø   ;  s4  ø„ ôð 15Ø15Ø37Ø15Ø59Ø-1Ø$(Ø,0Ø/3Ø59Ø34ñY
à˜E×,Ñ,Ñ-ðY
ð ! §¡Ñ.ðY
ð ˜u×/Ñ/Ñ0ð	Y
ð
 " +Ñ.ðY
ð   × 1Ñ 1Ñ2ðY
ð ˜×)Ñ)Ñ*ðY
ð ˜D‘>ðY
ð $ D™>ðY
ð ' t™nðY
ð ! ×!1Ñ!1Ñ2ðY
ð ˜c 5§<¡<Ð/Ñ0ðY
ð 
 óY
ð| ØØØØØØ÷7ñ 7rK   rø   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚGemma2ForSequenceClassificationc                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y rc   rú   re   s     €rJ   r2   z(Gemma2ForSequenceClassification.__init__×  rý   rK   rg   r]   s   @rJ   r  r  Ö  ó   ø„ ÷ð rK   r  c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚGemma2ForTokenClassificationc                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y rc   rú   re   s     €rJ   r2   z%Gemma2ForTokenClassification.__init__Þ  rý   rK   rg   r]   s   @rJ   r  r  Ý  r  rK   r  )r   rø   rÏ   ÚGemma2PreTrainedModelr  r  )rR   NN):Ú	functoolsr   Útypingr   r   r   r   ry   Útorch.nnr~   Útorch.utils.checkpointÚactivationsr	   Úcache_utilsr
   r   r   Úconfiguration_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   Úgemma.modeling_gemmar   r   r   r   r   r   r   r   r   Ú_CHECKPOINT_FOR_DOCÚ
get_loggerrT   r¢   r   r_   ra   ÚModuler®   Úfloatr‰   r‹   r±   rÏ   rø   r  r  Ú__all__r0   rK   rJ   ú<module>r(     s…  ðõ  ß 3Ó 3ã Ý Û å !ß :Ñ :Ý 3Ý B÷õ 6Ý &Ý ÷
÷ 
õ 
ð )Ð à	ˆ×	Ñ	˜HÓ	%€ôP9Ð#ô P9ôf	�Lô 	ô7�ô 7ð Ø#Ø#ñ %Ø�I‰Ið %à�<‰<ð %ð 
�‰ð %ð �<‰<ð	 %ð
 ˜UŸ\™\Ñ*ð %ð ð %ð �e‰_ð %ð �e‰_ð %ð ˆ5�<‰<˜Ÿ™Ð%Ñ&ó %ôFD)�nô D)ôNP˜Ÿ™ô Pôfj�*ô jôZXÐ(ô XôvÐ&Dô ôÐ#>ô ò�rK   