Ë
    S^(h›|  ã                   óÊ  — d dl mZ d dlmZmZmZ d dlZd dlmZ d dl	Zddl
mZmZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZ ddlmZmZmZmZm Z m!Z!m"Z"m#Z# ddl$m%Z%  ejL                  e'«      Z( G d„ de«      Z) G d„ de!«      Z* G d„ de«      Z+ G d„ deejX                  «      Z- G d„ de«      Z. G d„ de «      Z/ G d„ de%«      Z0 G d„ de«      Z1g d ¢Z2y)!é    )Úpartial)ÚCallableÚOptionalÚTupleNé   )ÚCacheÚHybridCache)ÚPretrainedConfig)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPast)Úrope_config_validation)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úloggingé   )ÚCohereAttentionÚCohereDecoderLayerÚCohereForCausalLMÚCohereLayerNormÚCoherePreTrainedModelÚCohereRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forward)ÚGemma2Modelc                   ó–   ‡ — e Zd ZdZdZdgZddddddddœZdgdgfd	d
gd	gfd	gd	gfdœZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ xZ	S )ÚCohere2ConfigaÂ  
    This is the configuration class to store the configuration of a [`CohereModel`]. It is used to instantiate an Cohere
    model according to the specified arguments, defining the model architecture.

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information. Instantiating a configuration
    with the defaults will yield a similar configuration to that of the [CohereForAI/c4ai-command-r-v01](https://huggingface.co/CohereForAI/c4ai-command-r-v01) model.


    Args:
        vocab_size (`int`, *optional*, defaults to 256000):
            Vocabulary size of the Cohere model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`CohereModel`]
        hidden_size (`int`, *optional*, defaults to 8192):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 22528):
            Dimension of the MLP representations.
        logit_scale (`float`, *optional*, defaults to 0.0625):
            The scaling factor for the output logits.
        num_hidden_layers (`int`, *optional*, defaults to 40):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 64):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details checkout [this
            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 8192):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        layer_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the layer normalization.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 0):
            Padding token id.
        bos_token_id (`int`, *optional*, defaults to 5):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 255001):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `True`):
            Whether to tie weight embeddings
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        rope_scaling (`Dict`, *optional*):
            Dictionary containing the scaling configuration for the RoPE embeddings. NOTE: if you apply new rope type
            and you expect the model to work on longer `max_position_embeddings`, we recommend you to update this value
            accordingly.
            Expected contents:
                `rope_type` (`str`):
                    The sub-variant of RoPE to use. Can be one of ['default', 'linear', 'dynamic', 'yarn', 'longrope',
                    'llama3'], with 'default' being the original RoPE implementation.
                `factor` (`float`, *optional*):
                    Used with all rope types except 'default'. The scaling factor to apply to the RoPE embeddings. In
                    most scaling types, a `factor` of x will enable the model to handle sequences of length x *
                    original maximum pre-trained length.
                `original_max_position_embeddings` (`int`, *optional*):
                    Used with 'dynamic', 'longrope' and 'llama3'. The original max position embeddings used during
                    pretraining.
                `attention_factor` (`float`, *optional*):
                    Used with 'yarn' and 'longrope'. The scaling factor to be applied on the attention
                    computation. If unspecified, it defaults to value recommended by the implementation, using the
                    `factor` field to infer the suggested value.
                `beta_fast` (`float`, *optional*):
                    Only used with 'yarn'. Parameter to set the boundary for extrapolation (only) in the linear
                    ramp function. If unspecified, it defaults to 32.
                `beta_slow` (`float`, *optional*):
                    Only used with 'yarn'. Parameter to set the boundary for interpolation (only) in the linear
                    ramp function. If unspecified, it defaults to 1.
                `short_factor` (`List[float]`, *optional*):
                    Only used with 'longrope'. The scaling factor to be applied to short contexts (<
                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
                    size divided by the number of attention heads divided by 2
                `long_factor` (`List[float]`, *optional*):
                    Only used with 'longrope'. The scaling factor to be applied to long contexts (<
                    `original_max_position_embeddings`). Must be a list of numbers with the same length as the hidden
                    size divided by the number of attention heads divided by 2
                `low_freq_factor` (`float`, *optional*):
                    Only used with 'llama3'. Scaling factor applied to low frequency components of the RoPE
                `high_freq_factor` (`float`, *optional*):
                    Only used with 'llama3'. Scaling factor applied to high frequency components of the RoPE
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        sliding_window (`int`, *optional*, defaults to 4096):
            Size of the sliding window attention context.
        sliding_window_pattern (`int`, *optional*, defaults to 4):
            Pattern for the sliding window attention.
        cache_implementation (`str`, *optional*, defaults to `"hybrid"`): the cache type to be used with `generate`.

    ```python
    >>> from transformers import Cohere2Model, Cohere2Config

    >>> # Initializing a Cohere Nextmodel configuration
    >>> configuration = Cohere2Config()

    >>> # Initializing a model from the Cohere2 configuration
    >>> model = Cohere2Model(configuration) # doctest: +SKIP

    >>> # Accessing the model configuration
    >>> configuration = model.config # doctest: +SKIP
    ```
    Úcohere2Úpast_key_valuesÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormc                 ój  •— || _         |	| _        || _        || _        || _        || _        || _        |€|}|| _        || _        |
| _	        || _
        || _        || _        || _        || _        || _        || _        || _        ||z  | _        || _        t)        | «       t+        ‰| �X  d||||dœ|¤Ž y )N)Úpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddings© )Ú
vocab_sizeÚmax_position_embeddingsÚhidden_sizeÚlogit_scaleÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚ
hidden_actÚinitializer_rangeÚlayer_norm_epsÚ	use_cacheÚ
rope_thetaÚrope_scalingÚattention_biasÚattention_dropoutÚsliding_windowÚsliding_window_patternÚhead_dimÚcache_implementationr   ÚsuperÚ__init__)Úselfr.   r0   r2   r1   r3   r4   r5   r6   r/   r7   r8   r9   r)   r*   r+   r,   r:   r;   r<   r=   r>   r?   rA   ÚkwargsÚ	__class__s                            €úi/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/cohere2/modular_cohere2.pyrC   zCohere2Config.__init__¶   sæ   ø€ ð6 %ˆŒØ'>ˆÔ$Ø&ˆÔØ&ˆÔØ!2ˆÔØ!2ˆÔØ#6ˆÔ ð Ð&Ø"5Ðà#6ˆÔ Ø$ˆŒØ!2ˆÔØ,ˆÔØ"ˆŒØ$ˆŒØ(ˆÔØ,ˆÔØ!2ˆÔØ,ˆÔØ&<ˆÔ#à#Ð':Ñ:ˆŒØ$8ˆÔ!ô 	˜tÔ$ä‰Ñð 	
Ø%Ø%Ø%Ø 3ñ		
ð
 ó	
ó    )i è é    i X  g      °?é(   é@   NÚsilurI   g{®Gáz”?gñhãˆµøä>Tr   é   iä Tg     ˆÃ@NFç        i   é   Úhybrid)
Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planrC   Ú__classcell__©rF   s   @rG   r   r   3   s¾   ø„ ñoðb €JØ#4Ð"5Ðà%.Ø%.Ø%.Ø%.Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð ØØØØØØ ØØ $ØØØØØØØ ØØØØØØ Ø%÷1?
ñ ?
rH   r   c                   ó   — e Zd Zy)ÚCohere2RotaryEmbeddingN©rQ   rR   rS   r-   rH   rG   r\   r\   ø   ó   „ ØrH   r\   c                   ó   — e Zd Zy)ÚCohere2LayerNormNr]   r-   rH   rG   r`   r`   ü   r^   rH   r`   c                   ó2  — e Zd ZdZddedee   fd„Z	 	 ddej                  de
ej                  ej                  f   deej                     d	ee   d
eej                     dee   de
ej                  eej                     ee
ej                        f   fd„Zy)ÚCohere2Attentionz=Multi-headed attention from 'Attention Is All You Need' paperNÚconfigÚ	layer_idxc                 ó  — t         j                  j                  «        || _        || _        t        |d|j                  |j                  z  «      | _        |j                  |j                  z  | _
        | j                  dz  | _        |j                  | _        d| _        t        j                  |j                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  |j                  | j                  z  |j                  ¬«      | _        t        j                  |j                  | j                  z  |j                  |j                  ¬«      | _        | j                  dz   | j                  j(                  z  dk7  r|j*                  | _        y d | _        y )Nr@   g      à¿T)Úbiasé   r   )ÚnnÚModulerC   rc   rd   Úgetattrr0   r4   r@   r5   Únum_key_value_groupsÚscalingr=   Ú	is_causalÚLinearr<   Úq_projÚk_projÚv_projÚo_projr?   r>   )rD   rc   rd   s      rG   rC   zCohere2Attention.__init__  s�  € Ü
�	‰	×ÑÔØˆŒØ"ˆŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfô
ˆŒô —i‘iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfô
ˆŒð '+§n¡n°qÑ&8¸D¿K¹K×<^Ñ<^Ñ%^ÐbcÒ%cˆF×!Ñ!ð 	ÕØimð 	ÕrH   r#   Úposition_embeddingsr$   Úpast_key_valueÚcache_positionrE   Úreturnc                 ó’  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }|\  }}| j                  �t        |	|
||«      \  }	}
|�~||| j                  |dœ}|j                  |
|| j                  |«      \  }
}|�J| j                  j                  dk(  r1|j                   d   }|
d d …d d …d |…d d …f   |d d …d d …d |…d d …f   }}
t        }| j                  j                  dk7  r^| j                  j                  dk(  r(|j                  dd	«      rt        j!                  d
«       nt"        | j                  j                     } || |	|
||f| j$                  sdn| j&                  | j(                  | j                  dœ|¤Ž\  }} |j*                  g |¢d‘­Ž j-                  «       }| j/                  |«      }||fS )Néÿÿÿÿrg   r   )ÚsinÚcosr>   ru   Úflash_attention_2ÚeagerÚsdpaÚoutput_attentionsFzã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.rN   )Údropoutrl   r>   )Úshaper@   ro   ÚviewÚ	transposerp   rq   r>   r   Úupdaterd   rc   Ú_attn_implementationr   ÚgetÚloggerÚwarning_oncer   Útrainingr=   rl   ÚreshapeÚ
contiguousrr   )rD   r#   rs   r$   rt   ru   rE   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrz   ry   Úcache_kwargsÚseq_lenÚattention_interfaceÚattn_outputÚattn_weightss                      rG   ÚforwardzCohere2Attention.forward  sb  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSØ×ÑÐ*Ü';¸LÈ*ÐVYÐ[^Ó'_Ñ$ˆL˜*àÐ%àØØ"&×"5Ñ"5Ø"0ñ	ˆLð (6×'<Ñ'<¸ZÈÐW[×WeÑWeÐgsÓ'tÑ$ˆJ˜ð Ð)¨d¯k©k×.NÑ.NÐReÒ.eØ(×.Ñ.¨rÑ2�Ø+5²aº¸H¸W¸HÂaÐ6GÑ+HÈ,ÒWXÒZ[Ð]eÐ^eÐ]eÒghÐWhÑJi˜L�
ä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9¸f¿j¹jÐI\Ð^cÔ>dÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(rH   ©N)NN)rQ   rR   rS   rT   r   r   ÚintrC   ÚtorchÚTensorr   r   Ú
LongTensorr   r   r•   r-   rH   rG   rb   rb      sÈ   „ ÙGñ
˜}ð 
¸À#¹ó 
ð> +/Ø59ñ:)à—|‘|ð:)ð # 5§<¡<°·±Ð#=Ñ>ð:)ð ! §¡Ñ.ð	:)ð
 ! ™ð:)ð ! ×!1Ñ!1Ñ2ð:)ð Ð-Ñ.ð:)ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	Sô:)rH   rb   c                   óN  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 ddej                  deej                  ej                  f   de	ej                     de	e
   de	e   d	e	e   d
e	ej                     dedee   deej                  e	eej                  ej                  f      f   fd„Zˆ xZS )ÚCohere2DecoderLayerrc   rd   c                 óÀ   •— t         ‰| �  ||«       t        ||«      | _        || _        |dz   | j                  j
                  z  dk7  | _        |j                  | _        y )Nrg   r   )rB   rC   rb   Ú	self_attnrc   r?   Ú
is_slidingr>   )rD   rc   rd   rF   s      €rG   rC   zCohere2DecoderLayer.__init__[  sU   ø€ Ü‰Ñ˜ Ô+Ü)¨&°)Ó<ˆŒØˆŒØ$ q™=¨D¯K©K×,NÑ,NÑNÐRSÑSˆŒØ$×3Ñ3ˆÕrH   r#   rs   r$   rt   r~   r9   ru   Úlast_cache_positionrE   rv   c	                 ó°  — | j                   rõ|�ót        |j                  d   | j                  «      }
| j                  j
                  dk(  r|dd…|
 d…f   }nªt        j                  |j                  «      j                  }t        j                  t        j                  |t        j                  ¬«      | j                   ¬«      }t        j                  |||«      }||
z
  }t        d|«      }|dd…dd…dd…|||
z   …f   }|}| j                  |«      } | j                  d|||||||dœ|	¤Ž\  }}| j!                  |«      }||z   |z   }|f}|r||fz  }|S )a÷  
        Args:
            hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
            position_embeddings (`Tuple[torch.FloatTensor, torch.FloatTensor]`):
                Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
                with `head_dim` being the embedding dimension of each attention head.
            attention_mask (`torch.FloatTensor`, *optional*):
                attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
                query_sequence_length, key_sequence_length)` if default attention is used.
            past_key_value (`Tuple(torch.FloatTensor)`, *optional*): cached past key and value projection states
            output_attentions (`bool`, *optional*):
                Whether or not to return the attentions tensors of all attention layers. See `attentions` under
                returned tensors for more detail.
            use_cache (`bool`, *optional*):
                If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
                (see `past_key_values`).
            cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
                Indices depicting the position of the input sequence tokens in the sequence
            last_cache_position (`int`): equivalent to `cache_position[-1]` but allow indexing without breaking dynamo tracing
        Nr   r{   )Údtype)Údiagonal)r#   rs   r$   rt   r~   r9   ru   r-   )rŸ   Úmaxr€   r>   rc   r„   r˜   Úfinfor¢   ÚminÚtrilÚ	ones_likeÚboolÚwhereÚinput_layernormrž   Úmlp)rD   r#   rs   r$   rt   r~   r9   ru   r    rE   Úeffective_seq_lenÚ	min_dtypeÚsliding_window_maskÚoffsetÚresidualÚhidden_states_attentionÚself_attn_weightsÚhidden_states_mlpÚoutputss                      rG   r•   zCohere2DecoderLayer.forwardb  sƒ  € ðB �?Š?˜~Ð9ä # N×$8Ñ$8¸Ñ$;¸T×=PÑ=PÓ QÐð �{‰{×/Ñ/Ð3FÒFØ!/²Ð4EÐ3EÑ3FÐ0FÑ!G‘ô "ŸK™K¨×(;Ñ(;Ó<×@Ñ@�	Ü&+§j¡jÜ—O‘O N¼%¿*¹*ÔEÐQU×QdÑQdÐPdô'Ð#ô "'§¡Ð-@À)È^Ó!\�ð -Ð/@Ñ@�ä˜Q ›�Ø!/²²1²a¸À&ÐK\ÑB\Ð9\Ð0\Ñ!]�à ˆà×,Ñ,¨]Ó;ˆð 6D°T·^±^ð 	6
Ø'Ø 3Ø)Ø)Ø/ØØ)ñ	6
ð ñ	6
Ñ2ÐÐ!2ð !ŸH™H ]Ó3Ðð !Ð#:Ñ:Ð=NÑNˆà Ð"ˆáØÐ)Ð+Ñ+ˆGàˆrH   )NNFFNr   )rQ   rR   rS   r   r—   rC   r˜   r™   r   r   r   r©   rš   r   r   ÚFloatTensorr•   rY   rZ   s   @rG   rœ   rœ   Z  s  ø„ ð4˜}ð 4¸õ 4ð 26Ø*.Ø,1Ø$)Ø59Ø#$ñRà—|‘|ðRð # 5§<¡<°·±Ð#=Ñ>ðRð ! §¡Ñ.ð	Rð
 ! ™ðRð $ D™>ðRð ˜D‘>ðRð ! ×!1Ñ!1Ñ2ðRð !ðRð Ð-Ñ.ðRð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷RrH   rœ   c                   ó   — e Zd ZeZy)ÚCohere2PreTrainedModelN)rQ   rR   rS   r   Úconfig_classr-   rH   rG   r¸   r¸   ·  s   „ Ø �LrH   r¸   c                   ó  ‡ — e Zd ZdZdefˆ fd„Z	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee
   deej                     d	ee   d
ee   dee   deej                     dee   dee   defd„Zˆ xZS )ÚCohere2Modelzž
    Transformer decoder consisting of *config.num_hidden_layers* layers. Each layer is a [`Cohere2DecoderLayer`]
    Args:
        config: Cohere2Config
    rc   c                 ó’   •— t         ‰| �  |«       t        |j                  |j                  ¬«      | _        t        |¬«      | _        y )N)r0   Úeps)rc   )rB   rC   r`   r0   r8   r'   r\   Ú
rotary_emb©rD   rc   rF   s     €rG   rC   zCohere2Model.__init__Â  s6   ø€ Ü‰Ñ˜Ô Ü$°&×2DÑ2DÈ6×K`ÑK`ÔaˆŒ	Ü0¸Ô?ˆ�rH   r!   r$   Úposition_idsr   r"   r9   r~   Úoutput_hidden_statesru   r    Úflash_attn_kwargsrv   c                 óÒ  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|d u |d uz  rt	        d«      ‚| j
                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|rL|€J| j                  s>|j                  \  }}}t        | j                   |||j                  | j                  ¬«      }|	€F|�|j                  «       nd}t        j                   |||j                  d   z   |j                  ¬«      }	|€|	j#                  d«      }|
€9d}
|�5|j%                  «       dk(  r|j                  d	   n|	d	   j'                  «       }
| j)                  |||	||«      }|}| j+                  ||«      }|rd
nd }|rd
nd }| j,                  D ]r  }|r||fz  }| j
                  r:| j                  r.| j/                  t1        |j2                  fi |¤Ž|||||||	|
«	      }n ||f||||||	|
dœ|¤Ž}|d   }|sŒj||d   fz  }Œt | j5                  |«      }|r||fz  }t7        ||||¬«      S )Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.F)Úmax_batch_sizeÚmax_cache_lenr¢   Údevicer   rg   )rÆ   r   rx   r-   )rs   r$   rt   r~   r9   ru   r    )Úlast_hidden_stater   r#   Ú
attentions)rc   r~   rÁ   r9   Ú
ValueErrorÚgradient_checkpointingrˆ   r†   r‡   r%   r€   r	   r¢   rÆ   Úget_seq_lengthr˜   ÚarangeÚ	unsqueezeÚdimÚitemÚ_update_causal_maskr¾   r&   Ú_gradient_checkpointing_funcr   Ú__call__r'   r   )rD   r!   r$   rÀ   r   r"   r9   r~   rÁ   ru   r    rÂ   Ú
batch_sizer‘   Ú_Úpast_seen_tokensÚcausal_maskr#   rs   Úall_hidden_statesÚall_self_attnsÚdecoder_layerÚlayer_outputss                          rG   r•   zCohere2Model.forwardÇ  sò  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	à˜Ð -°tÐ";Ò<ÜÐYÓZÐZà×&Ò&¨4¯=ª=¹YÜ×ÑØjôð ˆIàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0¸¿ºØ%2×%8Ñ%8Ñ"ˆJ˜ ä)Ø—‘Ø)Ø%Ø#×)Ñ)Ø—{‘{ôˆOð Ð!ØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ"Ÿ\™\Ø Ð"2°]×5HÑ5HÈÑ5KÑ"KÐTa×ThÑThôˆNð ÐØ)×3Ñ3°AÓ6ˆLð Ð&Ø"#ÐØÐ)ð 1?×0BÑ0BÓ0DÈÒ0I�N×(Ñ(¨Ò,È~Ð^`ÑOa×OfÑOfÓOhð $ð ×.Ñ.Ø˜M¨>¸?ÐL]ó
ˆð &ˆð #Ÿo™o¨m¸\ÓJÐñ #7™B¸DÐÙ0™°dˆà!Ÿ[™[ò  	6ˆMÙ#Ø! mÐ%5Ñ5Ð!à×*Ò*¨t¯}ª}Ø $× AÑ AÜ˜M×2Ñ2ÑHÐ6GÑHØ!Ø'ØØ#Ø%ØØ"Ø'ó
!‘ñ !.Ø!ð
!à(;Ø#.Ø#2Ø&7Ø'Ø#1Ø(;ñ
!ð (ñ
!�ð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5‘ðA 	6ðD Ÿ	™	 -Ó0ˆñ  Ø -Ð!1Ñ1Ðä&Ø+Ø+Ø+Ø%ô	
ð 	
rH   )
NNNNNNNNNN)rQ   rR   rS   rT   r   rC   r   r˜   rš   r™   r	   r¶   r©   r—   r   r   r   r•   rY   rZ   s   @rG   r»   r»   »  s  ø„ ñð@˜}õ @ð 15Ø15Ø37Ø15Ø59Ø$(Ø,0Ø/3Ø59Ø-1ñw
à˜E×,Ñ,Ñ-ðw
ð ! §¡Ñ.ðw
ð ˜u×/Ñ/Ñ0ð	w
ð
 " +Ñ.ðw
ð   × 1Ñ 1Ñ2ðw
ð ˜D‘>ðw
ð $ D™>ðw
ð ' t™nðw
ð ! ×!1Ñ!1Ñ2ðw
ð & c™]ðw
ð $Ð$8Ñ9ðw
ð 
!÷w
rH   r»   c                   ó:   ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 	 dd„Zˆ xZS )ÚCohere2ForCausalLMrc   c                 ó$   •— t         ‰| �  |«       y r–   )rB   rC   r¿   s     €rG   rC   zCohere2ForCausalLM.__init__B  s   ø€ Ü‰Ñ˜Õ rH   c	           	      ó  — |�Y|€|d   |j                   d   k\  r|d d …|j                   d    d …f   }n(|j                   d   |j                   d   k7  r	|d d …|f   }|�t|€r|j                  «       j                  d«      dz
  }|j                  |dk(  d«       |r9|d d …|j                   d    d …f   }|j	                  t
        j                  ¬«      }|�|d   dk(  r|d dœ}
n#|j	                  t
        j                  ¬«      d dœ}
|�|j                   d   nd|
d<   t        |t        «      r¿|j                  dk(  r°| j                  j                  d	k(  s—|
d
   �#|
d
   j                   \  }}}|
d
   j                  }n!|
d   j                   \  }}|
d   j                  }| j                  j                  |||j                  «       | j                   j"                  j$                  |||¬«      }|�||
d<   |
j'                  |||||dœ«       |
S )Nrx   rg   r   )Úmemory_format)r"   r!   )r!   r"   r    r   r{   r"   r!   )Úsequence_lengthÚtarget_lengthr¢   rÆ   ru   rÓ   Úlogits_to_keep)rÀ   ru   r   r9   r$   )r€   ÚlongÚcumsumÚmasked_fill_Úcloner˜   Úcontiguous_formatÚ
isinstancer	   Úndimrc   r„   rÆ   ÚmodelÚ5_prepare_4d_causal_attention_mask_with_cache_positionÚget_max_cache_shapeÚlm_headÚweightr¢   rƒ   )rD   r!   r   r$   r"   ru   rÀ   r9   râ   rE   Úmodel_inputsrÓ   rà   rÔ   rÆ   s                  rG   Úprepare_inputs_for_generationz0Cohere2ForCausalLM.prepare_inputs_for_generationE  sR  € ð& Ð&àÐ)Ø! "Ñ%¨¯©¸Ñ);Ò;à%¢a¨.×*>Ñ*>¸qÑ*AÐ)AÑ)CÐ&CÑD‘	Ø—‘ Ñ# ~×';Ñ';¸AÑ'>Ò>Ø%¢a¨Ð&7Ñ8�	ØÐ%¨,Ð*>à)×.Ñ.Ó0×7Ñ7¸Ó;¸aÑ?ˆLØ×%Ñ% n¸Ñ&9¸1Ô=ÙØ+ªA°	·±ÀÑ0BÐ/BÑ/DÐ,DÑE�ð  ,×1Ñ1Ä×@WÑ@WÐ1ÓX�ð Ð$¨¸Ñ):¸aÒ)?Ø-:ÈÑN‰Lð *3¯©Äu×G^ÑG^¨Ó)_ÐrvÑwˆLð KYÐJd¨n×.BÑ.BÀ2Ò.FÐjkˆÐ*Ñ+ô �¬Ô4Ø×#Ñ# qÒ(Ø—K‘K×4Ñ4Ð8KÒKà˜OÑ,Ð8Ø1=¸oÑ1N×1TÑ1TÑ.�
˜O¨QØ% oÑ6×=Ñ=‘à.:¸;Ñ.G×.MÑ.MÑ+�
˜OØ% kÑ2×9Ñ9�à!ŸZ™Z×]Ñ]ØØ /Ø-×AÑAÓCØ—l‘l×)Ñ)×/Ñ/ØØ-Ø%ð ^ó ˆNð Ð%Ø-;ˆLÐ)Ñ*à×Ñà ,Ø"0Ø#2Ø&Ø"0ñô	
ð ÐrH   )NNNNNTN)rQ   rR   rS   r   rC   rð   rY   rZ   s   @rG   rÜ   rÜ   A  s,   ø„ ð!˜}õ !ð ØØØØØØ÷UrH   rÜ   )r   rÜ   r»   r¸   )3Ú	functoolsr   Útypingr   r   r   r˜   Útorch.nnrh   Útorch.utils.checkpointÚcache_utilsr   r	   Úconfiguration_utilsr
   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   Úcohere.modeling_coherer   r   r   r   r   r   r   r   Úgemma2.modeling_gemma2r   Ú
get_loggerrQ   r†   r   r\   r`   ri   rb   rœ   r¸   r»   rÜ   Ú__all__r-   rH   rG   ú<module>r     sá   ðõ  ß ,Ñ ,ã Ý Û ç -Ý 3Ý Bõõ :Ý 5Ý &õ÷	÷ 	ó 	õ 1ð 
ˆ×	Ñ	˜HÓ	%€ôB
Ð$ô B
ôJ	Ð2ô 	ô	�ô 	ôW)�¨¯	©	ô W)ôtZÐ,ô Zôz!Ð2ô !ôC
�;ô C
ôLYÐ*ô Yòx \�rH   