Ë
    S^(h7  ã                   ó´  — d dl Z d dlmZ d dlZd dlmZ d dlZddlmZ ddl	m
Z
mZmZ ddlmZ ddlmZmZmZmZmZ d	d
lmZ  ej.                  e«      Z G d„ dej4                  «      Z G d„ de«      Z G d„ de«      Zd„ Zd"d„Z G d„ de«      Z  G d„ de«      Z! G d„ de«      Z" G d„ de"e«      Z# G d„ de
«      Z$ G d„ de«      Z% G d„ d e«      Z&g d!¢Z'y)#é    N)ÚOptionalé   )Úloggingé   )ÚGemmaForCausalLMÚGemmaForSequenceClassificationÚGemmaForTokenClassification)ÚGraniteAttention)ÚLlamaDecoderLayerÚLlamaMLPÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingé   )ÚHeliumConfigc                   ó,   ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Zˆ xZS )ÚHeliumRMSNormc                 óŠ   •— t         ‰| �  «        t        j                  t	        j
                  |«      «      | _        || _        y ©N)ÚsuperÚ__init__ÚnnÚ	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €úg/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/helium/modular_helium.pyr   zHeliumRMSNorm.__init__.   s/   ø€ Ü‰ÑÔÜ—l‘l¤5§:¡:¨kÓ#:Ó;ˆŒØ #ˆÕó    c                 ó\  — |j                   }|j                  t        j                  «      }|j	                  d«      j                  dd¬«      }|t        j                  || j                  z   «      z  }| j                  j                  t        j                  «      |z  j                  |«      S )Nr   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor   Úfloat32ÚpowÚmeanÚrsqrtr   r   )r   Úhidden_statesÚinput_dtypeÚvariances       r"   ÚforwardzHeliumRMSNorm.forward3   s†   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ó>ˆØ%¬¯©°H¸t×?TÑ?TÑ4TÓ(UÑUˆØ—‘—‘œuŸ}™}Ó-°Ñ=×AÑAÀ+ÓNÐNr#   c                 ó^   — t        | j                  j                  «      › d| j                  › �S )Nz, eps=)Útupler   Úshaper   )r   s    r"   Ú
extra_reprzHeliumRMSNorm.extra_repr:   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr#   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__r   r0   r4   Ú__classcell__©r!   s   @r"   r   r   -   s   ø„ õ$ò
OöJr#   r   c                   ó   — e Zd Zy)ÚHeliumRotaryEmbeddingN©r5   r6   r7   © r#   r"   r;   r;   >   ó   „ Ør#   r;   c                   ó   — e Zd Zy)Ú	HeliumMLPNr<   r=   r#   r"   r@   r@   B   r>   r#   r@   c                 ó|   — | dddd…f   }| dddd…f   }t        j                  | |fd¬«      j                  d«      S )	z*Rotates half the hidden dims of the input..r   Nr   r   r%   ©Údiméþÿÿÿ)r   ÚstackÚflatten)ÚxÚx1Úx2s      r"   Úrotate_halfrJ   F   sJ   € à	
ˆ3���1�ˆ9‰€BØ	
ˆ3���1�ˆ9‰€BÜ�;‰;˜˜˜R�y bÔ)×1Ñ1°"Ó5Ð5r#   c                 óF  — |j                  |«      }|j                  |«      }|dd|j                  d   dz  …f   j                  dd¬«      }|dd|j                  d   dz  …f   j                  dd¬«      }| |z  t        | «      |z  z   }||z  t        |«      |z  z   }||fS )aÛ  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .Nr%   r   rB   )Ú	unsqueezer3   Úrepeat_interleaverJ   )ÚqÚkÚcosÚsinÚposition_idsÚunsqueeze_dimÚq_embedÚk_embeds           r"   Úapply_rotary_pos_embrV   M   s¿   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ó
E€CØ
ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ó
E€Cà�3‰wœ; q›>¨CÑ/Ñ0€GØ�3‰wœ; q›>¨CÑ/Ñ0€Gà�GÐÐr#   c                   ó0   ‡ — e Zd Zddedee   fˆ fd„Zˆ xZS )ÚHeliumAttentionÚconfigÚ	layer_idxc                 óÖ   •— t         ‰| �  ||«       t        j                  |j                  |j                  d¬«      | _        dt        j                  | j                  «      z  | _	        y )NF)Úbiasr   )
r   r   r   ÚLinearr   Úo_projÚmathÚsqrtÚhead_dimÚscaling©r   rY   rZ   r!   s      €r"   r   zHeliumAttention.__init__o   sK   ø€ Ü‰Ñ˜ Ô+Ü—i‘i × 2Ñ 2°F×4FÑ4FÈUÔSˆŒØœ4Ÿ9™9 T§]¡]Ó3Ñ3ˆ�r#   r   ©r5   r6   r7   r   r   Úintr   r8   r9   s   @r"   rX   rX   n   s   ø„ ñ4˜|ð 4¸À¹÷ 4ñ 4r#   rX   c                   ó0   ‡ — e Zd Zddedee   fˆ fd„Zˆ xZS )ÚHeliumDecoderLayerrY   rZ   c                 óÚ   •— t         ‰| �  «        t        |«      | _        t	        |j
                  |j                  ¬«      | _        t	        |j
                  |j                  ¬«      | _        y )N©r    )	r   r   r@   Úmlpr   r   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrc   s      €r"   r   zHeliumDecoderLayer.__init__v   sP   ø€ Ü‰ÑÔä˜VÓ$ˆŒÜ,¨V×-?Ñ-?ÀV×EXÑEXÔYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÔ(bˆÕ%r#   r   rd   r9   s   @r"   rg   rg   u   s#   ø„ ñc˜|ð c¸À¹÷ cñ cr#   rg   c                   ó   — e Zd Zy)ÚHeliumPreTrainedModelNr<   r=   r#   r"   ro   ro   ~   r>   r#   ro   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚHeliumModelrY   c           	      óR  •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        |j                  |j                  ¬«      | _        t        |«      | _        d| _        | j                  «        y c c}w )Nri   F)r   r   r   Ú
ModuleListÚrangeÚnum_hidden_layersrg   Úlayersr   r   rk   Únormr;   Ú
rotary_embÚgradient_checkpointingÚ	post_initrc   s      €r"   r   zHeliumModel.__init__ƒ   sƒ   ø€ Ü‰Ñ˜Ô Ü—m‘mÜDIÈ&×JbÑJbÓDcÖd°yÔ ¨	Õ2Òdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÔNˆŒ	Ü/°Ó7ˆŒØ&+ˆÔ#ð 	�‰Õùò es   ·B$©r5   r6   r7   r   r   r8   r9   s   @r"   rq   rq   ‚   s   ø„ ð
˜|÷ 
ñ 
r#   rq   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚHeliumForCausalLMrY   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r   ©r   r   rq   Úmodelrz   ©r   rY   r!   s     €r"   r   zHeliumForCausalLM.__init__‘   ó&   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø�‰Õr#   r{   r9   s   @r"   r}   r}   �   ó   ø„ ð˜|÷ ñ r#   r}   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚHeliumForSequenceClassificationrY   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r   r   r�   s     €r"   r   z(HeliumForSequenceClassification.__init__˜   r‚   r#   r{   r9   s   @r"   r…   r…   —   rƒ   r#   r…   c                   ó$   ‡ — e Zd Zdefˆ fd„Zˆ xZS )ÚHeliumForTokenClassificationrY   c                 ód   •— t         ‰| �  |«       t        |«      | _        | j	                  «        y r   r   r�   s     €r"   r   z%HeliumForTokenClassification.__init__Ÿ   r‚   r#   r{   r9   s   @r"   rˆ   rˆ   ž   rƒ   r#   rˆ   )ro   rq   r}   r…   rˆ   )Nr   )(r_   Útypingr   r   Útorch.nnr   Útorch.utils.checkpointÚutilsr   Úgemma.modeling_gemmar   r   r	   Úgranite.modeling_graniter
   Úllama.modeling_llamar   r   r   r   r   Úconfiguration_heliumr   Ú
get_loggerr5   ÚloggerÚModuler   r;   r@   rJ   rV   rX   rg   ro   rq   r}   r…   rˆ   Ú__all__r=   r#   r"   ú<module>r–      sç   ðó  Ý ã Ý Û å ÷ñ õ
÷õ õ /ð 
ˆ×	Ñ	˜HÓ	%€ôJ�B—I‘Iô Jô"	Ð0ô 	ô	�ô 	ò6óôB4Ð&ô 4ôcÐ*ô cô	Ð0ô 	ôÐ'¨ô ôÐ(ô ôÐ&Dô ôÐ#>ô ò�r#   