Ë
    S^(hý  ã                   ó  — d dl mZ d dlZd dlmZ d dlZddlmZ ddlm	Z	m
Z
mZmZ ddlmZ dd	lmZ  ej"                  e«      Zd
Z G d„ de«      Zd„ Zdd„Z G d„ de	«      Z G d„ de
«      Z G d„ de«      Z G d„ de«      Zg d¢Zy)é    )ÚOptionalNé   )Úloggingé   )ÚLlamaAttentionÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaForTokenClassification)ÚPhi3MLPé   )Ú	GlmConfigzTHUDM/glm-4-9bc                   ó   — e Zd Zy)ÚGlmMLPN©Ú__name__Ú
__module__Ú__qualname__© ó    úa/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/glm/modular_glm.pyr   r   &   ó   „ Ør   r   c                 ó|   — | dddd…f   }| dddd…f   }t        j                  | |fd¬«      j                  d«      S )	z*Rotates half the hidden dims of the input..r   Nr   r   éÿÿÿÿ©Údiméþÿÿÿ)ÚtorchÚstackÚflatten)ÚxÚx1Úx2s      r   Úrotate_halfr#   *   sJ   € à	
ˆ3���1�ˆ9‰€BØ	
ˆ3���1�ˆ9‰€BÜ�;‰;˜˜˜R�y bÔ)×1Ñ1°"Ó5Ð5r   c                 ó  — |j                  |«      }|j                  |«      }|dd|j                  d   dz  …f   j                  dd¬«      }|dd|j                  d   dz  …f   j                  dd¬«      }|j                  d   }| dd|…f   | d|d…f   }}|dd|…f   |d|d…f   }
}	||z  t        |«      |z  z   }|	|z  t        |	«      |z  z   }t	        j
                  ||gd¬«      }t	        j
                  ||
gd¬«      }||fS )aÛ  Applies Rotary Position Embedding to the query and key tensors.

    Args:
        q (`torch.Tensor`): The query tensor.
        k (`torch.Tensor`): The key tensor.
        cos (`torch.Tensor`): The cosine part of the rotary embedding.
        sin (`torch.Tensor`): The sine part of the rotary embedding.
        position_ids (`torch.Tensor`, *optional*):
            Deprecated and unused.
        unsqueeze_dim (`int`, *optional*, defaults to 1):
            The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
            sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
            that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
            k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
            cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
            the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
    Returns:
        `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
    .Nr   r   r   )Ú	unsqueezeÚshapeÚrepeat_interleaver#   r   Úcat)ÚqÚkÚcosÚsinÚposition_idsÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds                r   Úapply_rotary_pos_embr6   1   sD  € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ó
E€CØ
ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ó
E€Cð —‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€EØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6€Eð �s‰{œ{¨5Ó1°CÑ7Ñ8€GØ�s‰{œ{¨5Ó1°CÑ7Ñ8€Gô �i‰i˜ &Ð)¨rÔ2€GÜ�i‰i˜ &Ð)¨rÔ2€GØ�GÐÐr   c                   ó0   ‡ — e Zd Zddedee   fˆ fd„Zˆ xZS )ÚGlmAttentionÚconfigÚ	layer_idxc                 ó¢   •— t         ‰| �  ||«       t        j                  |j                  | j
                  z  |j                  d¬«      | _        y )NF)Úbias)ÚsuperÚ__init__ÚnnÚLinearÚnum_attention_headsÚhead_dimÚhidden_sizeÚo_proj)Úselfr9   r:   Ú	__class__s      €r   r>   zGlmAttention.__init__\   s<   ø€ Ü‰Ñ˜ Ô+Ü—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆ�r   )N)r   r   r   r   r   Úintr>   Ú__classcell__)rF   s   @r   r8   r8   [   s#   ø„ ñl˜yð l°X¸c±]÷ lñ lr   r8   c                   ó   — e Zd Zy)ÚGlmForCausalLMNr   r   r   r   rJ   rJ   a   r   r   rJ   c                   ó   — e Zd Zy)ÚGlmForSequenceClassificationNr   r   r   r   rL   rL   e   r   r   rL   c                   ó   — e Zd Zy)ÚGlmForTokenClassificationNr   r   r   r   rN   rN   i   r   r   rN   )ÚGlmPreTrainedModelÚGlmModelrJ   rL   rN   )Nr   )Útypingr   r   Útorch.nnr?   Útorch.utils.checkpointÚutilsr   Úllama.modeling_llamar   r   r	   r
   Úphi3.modeling_phi3r   Úconfiguration_glmr   Ú
get_loggerr   ÚloggerÚ_CHECKPOINT_FOR_DOCr   r#   r6   r8   rJ   rL   rN   Ú__all__r   r   r   ú<module>r\      s•   ðõ  ã Ý Û å ÷ó õ )Ý (ð 
ˆ×	Ñ	˜HÓ	%€à&Ð ô	ˆWô 	ò6ó'ôTl�>ô lô	Ð%ô 	ô	Ð#Aô 	ô	Ð ;ô 	ò�r   