Ë
    T^(h56  ã                   óh  — d dl mZmZmZ d dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZmZ dd
lmZ ddlmZmZmZmZmZ  ej4                  e«      Z G d„ de«      Z G d„ de«      Z ej>                  e«        G d„ de«      Z  G d„ de«      Z! G d„ de«      Z" G d„ de«      Z#g d¢Z$y)é    )ÚCallableÚOptionalÚTupleNé   )ÚCache)ÚALL_ATTENTION_FUNCTIONS)ÚALL_LAYERNORM_LAYERS)Úloggingé   )ÚLlamaRMSNormÚeager_attention_forward)Ú
OlmoConfig)ÚOlmoAttentionÚOlmoDecoderLayerÚOlmoForCausalLMÚ	OlmoModelÚapply_rotary_pos_embc                   óˆ   ‡ — e Zd ZdZdZddddddddœZdgd	gfd
dgd
gfd
gd
gfdœZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„	Zˆ xZS )ÚOlmo2Configaã  
    This is the configuration class to store the configuration of a [`Olmo2Model`]. It is used to instantiate an OLMo2
    model according to the specified arguments, defining the model architecture. Instantiating a configuration with the
    defaults will yield a similar configuration to that of the [allenai/Olmo2-7B-1124-hf](https://huggingface.co/allenai/Olmo2-7B-1124-hf).

    Configuration objects inherit from [`PretrainedConfig`] and can be used to control the model outputs. Read the
    documentation from [`PretrainedConfig`] for more information.


    Args:
        vocab_size (`int`, *optional*, defaults to 50304):
            Vocabulary size of the Olmo2 model. Defines the number of different tokens that can be represented by the
            `inputs_ids` passed when calling [`Olmo2Model`]
        hidden_size (`int`, *optional*, defaults to 4096):
            Dimension of the hidden representations.
        intermediate_size (`int`, *optional*, defaults to 11008):
            Dimension of the MLP representations.
        num_hidden_layers (`int`, *optional*, defaults to 32):
            Number of hidden layers in the Transformer decoder.
        num_attention_heads (`int`, *optional*, defaults to 32):
            Number of attention heads for each attention layer in the Transformer decoder.
        num_key_value_heads (`int`, *optional*):
            This is the number of key_value heads that should be used to implement Grouped Query Attention. If
            `num_key_value_heads=num_attention_heads`, the model will use Multi Head Attention (MHA), if
            `num_key_value_heads=1` the model will use Multi Query Attention (MQA) otherwise GQA is used. When
            converting a multi-head checkpoint to a GQA checkpoint, each group key and value head should be constructed
            by meanpooling all the original heads within that group. For more details checkout [this
            paper](https://arxiv.org/pdf/2305.13245.pdf). If it is not specified, will default to
            `num_attention_heads`.
        hidden_act (`str` or `function`, *optional*, defaults to `"silu"`):
            The non-linear activation function (function or string) in the decoder.
        max_position_embeddings (`int`, *optional*, defaults to 2048):
            The maximum sequence length that this model might ever be used with.
        initializer_range (`float`, *optional*, defaults to 0.02):
            The standard deviation of the truncated_normal_initializer for initializing all weight matrices.
        use_cache (`bool`, *optional*, defaults to `True`):
            Whether or not the model should return the last key/values attentions (not used by all models). Only
            relevant if `config.is_decoder=True`.
        pad_token_id (`int`, *optional*, defaults to 1):
            Padding token id.
        bos_token_id (`int`, *optional*):
            Beginning of stream token id.
        eos_token_id (`int`, *optional*, defaults to 50279):
            End of stream token id.
        tie_word_embeddings (`bool`, *optional*, defaults to `False`):
            Whether to tie weight embeddings
        rope_theta (`float`, *optional*, defaults to 10000.0):
            The base period of the RoPE embeddings.
        rope_scaling (`Dict`, *optional*):
            Dictionary containing the scaling configuration for the RoPE embeddings. Currently supports two scaling
            strategies: linear and dynamic. Their scaling factor must be a float greater than 1. The expected format is
            `{"type": strategy name, "factor": scaling factor}`. When using this flag, don't update
            `max_position_embeddings` to the expected new maximum. See the following thread for more information on how
            these scaling strategies behave:
            https://www.reddit.com/r/LocalLLaMA/comments/14mrgpr/dynamically_scaled_rope_further_increases/. This is an
            experimental feature, subject to breaking API changes in future versions.
        attention_bias (`bool`, defaults to `False`, *optional*, defaults to `False`):
            Whether to use a bias in the query, key, value and output projection layers during self-attention.
        attention_dropout (`float`, *optional*, defaults to 0.0):
            The dropout ratio for the attention probabilities.
        rms_norm_eps (`float`, *optional*, defaults to 1e-05):
            The epsilon used by the rms normalization layers.

    ```python
    >>> from transformers import Olmo2Model, Olmo2Config

    >>> # Initializing a Olmo2 7B style configuration
    >>> configuration = Olmo2Config()

    >>> # Initializing a model from the Olmo2 7B style configuration
    >>> model = Olmo2Model(configuration)

    >>> # Accessing the model configuration
    >>> configuration = model.config
    ```
    Úolmo2Úcolwise_repÚrowwise_repÚcolwiseÚrowwise)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormc                 ó¢   •— t        ‰| �  di d|“d|“d|“d|“d|“d|“d|“d|“d	|	“d
|
“d|“d|“d|“d|“d|“d|“d|“d|“|¤Ž || _        | `y )NÚ
vocab_sizeÚhidden_sizeÚintermediate_sizeÚnum_hidden_layersÚnum_attention_headsÚnum_key_value_headsÚ
hidden_actÚmax_position_embeddingsÚinitializer_rangeÚ	use_cacheÚpad_token_idÚbos_token_idÚeos_token_idÚtie_word_embeddingsÚ
rope_thetaÚrope_scalingÚattention_biasÚattention_dropout© )ÚsuperÚ__init__Úrms_norm_epsÚclip_qkv)Úselfr#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r2   r3   r4   r8   ÚkwargsÚ	__class__s                        €úe/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/olmo2/modular_olmo2.pyr7   zOlmo2Config.__init__v   sÔ   ø€ ô. 	‰Ñò 	
Ù!ð	
á#ð	
ñ 0ð	
ñ 0ð		
ñ
 !4ð	
ñ !4ð	
ñ "ð	
ñ %<ð	
ñ 0ð	
ñ  ð	
ñ &ð	
ñ &ð	
ñ &ð	
ñ !4ð	
ñ "ð	
ñ  &ð!	
ñ" *ð#	
ñ$ 0Øò'	
ð, )ˆÔØ‰Mó    )i€Ä  i   i +  é    r?   NÚsilui   g{®Gáz”?Té   NigÄ  Fg     ˆÃ@NFç        gñhãˆµøä>)	Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú
model_typeÚbase_model_tp_planÚbase_model_pp_planr7   Ú__classcell__©r<   s   @r=   r   r      s©   ø„ ñKðZ €Jà%2Ø%2Ø%2Ø%2Ø"+Ø )Ø"+ñÐð &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð ØØØØØ ØØ $ØØØØØØ!ØØØØØ÷).ñ .r>   r   c                   ó   — e Zd Zy)ÚOlmo2RMSNormN©rC   rD   rE   r5   r>   r=   rM   rM   §   ó   „ Ør>   rM   c                   ó0  ‡ — e Zd Zddedee   fˆ fd„Z	 	 ddej                  de	ej                  ej                  f   deej                     dee
   deej                     d	e	ej                  eej                     ee	ej                        f   fd
„Zˆ xZS )ÚOlmo2AttentionÚconfigÚ	layer_idxc                 óð   •— t         ‰| �  ||¬«       t        |j                  | j                  z  |j
                  «      | _        t        |j                  | j                  z  |j
                  «      | _        y )N©rS   )	r6   r7   rM   r'   Úhead_dimr8   Úq_normr(   Úk_norm©r:   rR   rS   r<   s      €r=   r7   zOlmo2Attention.__init__²   s[   ø€ Ü‰Ñ˜¨9ÐÔ5Ü" 6×#=Ñ#=ÀÇÁÑ#MÈv×ObÑObÓcˆŒÜ" 6×#=Ñ#=ÀÇÁÑ#MÈv×ObÑObÓcˆ�r>   r   Úposition_embeddingsr   Úpast_key_valueÚcache_positionÚreturnc                 óþ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  | j                  |«      «      }	| j	                  | j                  |«      «      }
| j                  |«      }|	j                  |«      j                  dd«      }	|
j                  |«      j                  dd«      }
|j                  |«      j                  dd«      }|\  }}t        |	|
||«      \  }	}
|�'|||dœ}|j                  |
|| j                  |«      \  }
}t        }| j                  j                  dk7  r^| j                  j                  dk(  r(|j                  dd«      rt         j#                  d	«       nt$        | j                  j                     } || |	|
||f| j&                  sd
n| j(                  | j*                  dœ|¤Ž\  }} |j,                  g |¢d‘­Ž j/                  «       }| j1                  |«      }||fS )NéÿÿÿÿrA   r   )ÚsinÚcosr\   ÚeagerÚsdpaÚoutput_attentionsFzã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.rB   )ÚdropoutÚscaling)ÚshaperV   rW   Úq_projrX   Úk_projÚv_projÚviewÚ	transposer   ÚupdaterS   r   rR   Ú_attn_implementationÚgetÚloggerÚwarning_oncer   Útrainingr4   rf   ÚreshapeÚ
contiguousÚo_proj)r:   r   rZ   r   r[   r\   r;   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesra   r`   Úcache_kwargsÚattention_interfaceÚattn_outputÚattn_weightss                     r=   ÚforwardzOlmo2Attention.forward·   sý  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ 4§;¡;¨}Ó#=Ó>ˆØ—[‘[ §¡¨]Ó!;Ó<ˆ
Ø—{‘{ =Ó1ˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ%à#&¨sÀnÑUˆLØ'5×'<Ñ'<¸ZÈÐW[×WeÑWeÐgsÓ'tÑ$ˆJ˜ä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9¸f¿j¹jÐI\Ð^cÔ>dÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r>   )N)NN)rC   rD   rE   r   r   Úintr7   ÚtorchÚTensorr   r   Ú
LongTensorr   rJ   rK   s   @r=   rQ   rQ   ±   s¹   ø„ ñd˜{ð d°xÀ±}õ dð +/Ø59ñ3)à—|‘|ð3)ð # 5§<¡<°·±Ð#=Ñ>ð3)ð ! §¡Ñ.ð	3)ð
 ! ™ð3)ð ! ×!1Ñ!1Ñ2ð3)ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷3)r>   rQ   c                   óf  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 ddej                  deej                     deej                     dee
   dee   d	ee   d
eej                     deeej                  ej                  f      deej                  eeej                  ej                  f      f   fd„Zˆ xZS )ÚOlmo2DecoderLayerrR   rS   c                 óè   •— t         ‰| �  ||¬«       t        |j                  |j                  ¬«      | _        t        |j                  |j                  ¬«      | _        t        ||¬«      | _        | `	y )NrU   ©Úeps)rR   rS   )
r6   r7   rM   r$   r8   Úpost_attention_layernormÚpost_feedforward_layernormrQ   Ú	self_attnÚinput_layernormrY   s      €r=   r7   zOlmo2DecoderLayer.__init__ñ   s_   ø€ Ü‰Ñ˜¨9ÐÔ5Ü(4°V×5GÑ5GÈV×M`ÑM`Ô(aˆÔ%Ü*6°v×7IÑ7IÈv×ObÑObÔ*cˆÔ'Ü'¨vÀÔKˆŒØÑ r>   r   r   Úposition_idsr[   rd   r,   r\   rZ   r]   c	                 óÚ   — |}
 | j                   d||||||||dœ|	¤Ž\  }}| j                  |«      }|
|z   }|}
| j                  |«      }| j                  |«      }|
|z   }|f}|r||fz  }|S )N)r   r   r�   r[   rd   r,   r\   rZ   r5   )r‹   r‰   ÚmlprŠ   )r:   r   r   r�   r[   rd   r,   r\   rZ   r;   ÚresidualÚself_attn_weightsÚoutputss                r=   r   zOlmo2DecoderLayer.forwardø   s±   € ð !ˆð ,:¨4¯>©>ð 
,
Ø'Ø)Ø%Ø)Ø/ØØ)Ø 3ñ
,
ð ñ
,
Ñ(ˆÐ(ð ×5Ñ5°mÓDˆØ  =Ñ0ˆð !ˆØŸ™ Ó/ˆØ×7Ñ7¸ÓFˆØ  =Ñ0ˆà Ð"ˆÙØÐ)Ð+Ñ+ˆGàˆr>   )NNNFFNN)rC   rD   rE   r   r€   r7   r�   r‚   r   rƒ   r   Úboolr   ÚFloatTensorr   rJ   rK   s   @r=   r…   r…   ð   sû   ø„ ð!˜{ð !°sõ !ð 26Ø37Ø*.Ø,1Ø$)Ø59ØKOñ'à—|‘|ð'ð ! §¡Ñ.ð'ð ˜u×/Ñ/Ñ0ð	'ð
 ! ™ð'ð $ D™>ð'ð ˜D‘>ð'ð ! ×!1Ñ!1Ñ2ð'ð & e¨E¯L©L¸%¿,¹,Ð,FÑ&GÑHð'ð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷'r>   r…   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )Ú
Olmo2ModelrR   c           	      ó  •— t         ‰| �  |«       t        |j                  |j                  ¬«      | _        t        j                  t        |j                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        y c c}w )Nr‡   )r6   r7   rM   r$   r8   r!   ÚnnÚ
ModuleListÚranger&   r…   r    rY   s      €r=   r7   zOlmo2Model.__init__%  s^   ø€ Ü‰Ñ˜Ô Ü  ×!3Ñ!3¸×9LÑ9LÔMˆŒ	Ü—m‘mÜCHÈ×IaÑIaÓCbÖc°iÔ˜v yÕ1Òcó
ˆ�ùÚcs   ÁA=)rC   rD   rE   r   r7   rJ   rK   s   @r=   r–   r–   $  s   ø„ ð
˜{÷ 
ñ 
r>   r–   c                   ó   — e Zd Zy)ÚOlmo2ForCausalLMNrN   r5   r>   r=   rœ   rœ   .  rO   r>   rœ   )r   rœ   r–   ÚOlmo2PreTrainedModel)%Útypingr   r   r   r�   Útorch.nnr˜   Úcache_utilsr   Úmodeling_utilsr   Úpytorch_utilsr	   Úutilsr
   Úllama.modeling_llamar   r   Úolmo.configuration_olmor   Úolmo.modeling_olmor   r   r   r   r   Ú
get_loggerrC   rp   r   rM   ÚappendrQ   r…   r–   rœ   Ú__all__r5   r>   r=   ú<module>rª      s©   ðß ,Ñ ,ã Ý å  Ý 5Ý 1Ý ß HÝ 0÷õ ð 
ˆ×	Ñ	˜HÓ	%€ôL�*ô Lô^	�<ô 	ð Ð × Ñ ˜LÔ )ô9)�]ô 9)ô~/Ð(ô /ôh
�ô 
ô	�ô 	ò�r>   