Ë
    T^(h»  ã                   óˆ  — d dl mZmZmZ d dlZd dlZd dlmZ ddlmZ ddl	m
Z
 ddlmZ ddlmZ dd	lmZ d
dlmZmZmZmZmZmZmZmZmZ d
dlmZ ddlmZ  ej>                  e «      Z! G d„ de«      Z" G d„ de«      Z# G d„ de«      Z$ G d„ de«      Z% G d„ de«      Z& G d„ de«      Z' G d„ de«      Z( G d„ de«      Z)y)é    )ÚCallableÚOptionalÚTupleN)Únné   )ÚCache)ÚFlashAttentionKwargs)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úloggingé   )	ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaForCausalLMÚLlamaForQuestionAnsweringÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚLlamaMLPÚapply_rotary_pos_embÚeager_attention_forward)ÚMistralModelé   )ÚQwen2Configc                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚQwen2MLPc                 óJ  •— t         ‰| �  |«       t        j                  | j                  | j
                  d¬«      | _        t        j                  | j                  | j
                  d¬«      | _        t        j                  | j
                  | j                  d¬«      | _        y )NF©Úbias)	ÚsuperÚ__init__r   ÚLinearÚhidden_sizeÚintermediate_sizeÚ	gate_projÚup_projÚ	down_proj)ÚselfÚconfigÚ	__class__s     €úe/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/qwen2/modular_qwen2.pyr    zQwen2MLP.__init__   ss   ø€ Ü‰Ñ˜Ô ÜŸ™ 4×#3Ñ#3°T×5KÑ5KÐRWÔXˆŒÜ—y‘y ×!1Ñ!1°4×3IÑ3IÐPUÔVˆŒÜŸ™ 4×#9Ñ#9¸4×;KÑ;KÐRWÔXˆ�ó    )Ú__name__Ú
__module__Ú__qualname__r    Ú__classcell__©r)   s   @r*   r   r      s   ø„ ÷Yð Yr+   r   c                   ó2  ‡ — e Zd Zdedefˆ fd„Z	 	 ddej                  deej                  ej                  f   de	ej                     de	e
   de	ej                     d	ee   d
eej                  e	ej                     e	eej                        f   fd„Zˆ xZS )ÚQwen2Attentionr(   Ú	layer_idxc                 ó  •— t         ‰| �  ||«       t        j                  |j                  |j
                  | j                  z  d¬«      | _        t        j                  |j                  |j                  | j                  z  d¬«      | _	        t        j                  |j                  |j                  | j                  z  d¬«      | _
        t        j                  |j
                  | j                  z  |j                  d¬«      | _        y )NTr   F)r   r    r   r!   r"   Únum_attention_headsÚhead_dimÚq_projÚnum_key_value_headsÚk_projÚv_projÚo_proj©r'   r(   r3   r)   s      €r*   r    zQwen2Attention.__init__'   sÃ   ø€ Ü‰Ñ˜ Ô+Ü—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÔkˆ�r+   Úhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valueÚcache_positionÚkwargsÚreturnc                 óˆ  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }|\  }}t        |	|
||«      \  }	}
|�'|||dœ}|j                  |
|| j                  |«      \  }
}d }| j                  j                  rPt        | j                  dd «      �9| j                  | j                  j                  k\  r| j                  j                  }t        }| j                  j                   dk7  r^| j                  j                   dk(  r(|j#                  dd	«      rt$        j'                  d
«       nt(        | j                  j                      } || |	|
||f| j*                  sdn| j,                  | j.                  |dœ|¤Ž\  }} |j0                  g |¢d‘­Ž j3                  «       }| j5                  |«      }||fS )Néÿÿÿÿr   r   )ÚsinÚcosrA   Úsliding_windowÚeagerÚsdpaÚoutput_attentionsFzã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.g        )ÚdropoutÚscalingrH   )Úshaper6   r7   ÚviewÚ	transposer9   r:   r   Úupdater3   r(   Úuse_sliding_windowÚgetattrÚmax_window_layersrH   r   Ú_attn_implementationÚgetÚloggerÚwarning_oncer
   ÚtrainingÚattention_dropoutrM   ÚreshapeÚ
contiguousr;   )r'   r=   r>   r?   r@   rA   rB   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrG   rF   Úcache_kwargsrH   Úattention_interfaceÚattn_outputÚattn_weightss                      r*   ÚforwardzQwen2Attention.forward.   s/  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆˆSÜ#7¸ÀjÐRUÐWZÓ#[Ñ ˆ�jàÐ%à#&¨sÀnÑUˆLØ'5×'<Ñ'<¸ZÈÐW[×WeÑWeÐgsÓ'tÑ$ˆJ˜àˆà�K‰K×*Ò*Ü˜Ÿ™Ð%5°tÓ<ÐHØ—‘ $§+¡+×"?Ñ"?Ò?à!Ÿ[™[×7Ñ7ˆNä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9¸f¿j¹jÐI\Ð^cÔ>dÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØð
%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘LØ)ñ
%
ð ñ
%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r+   )NN)r,   r-   r.   r   Úintr    ÚtorchÚTensorr   r   r   Ú
LongTensorr   r	   rf   r/   r0   s   @r*   r2   r2   &   sÅ   ø„ ðl˜{ð l°sõ lð +/Ø59ñ8)à—|‘|ð8)ð # 5§<¡<°·±Ð#=Ñ>ð8)ð ! §¡Ñ.ð	8)ð
 ! ™ð8)ð ! ×!1Ñ!1Ñ2ð8)ð Ð-Ñ.ð8)ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷8)r+   r2   c                   ó(   ‡ — e Zd Zdedefˆ fd„Zˆ xZS )ÚQwen2DecoderLayerr(   r3   c                 óæ   •— t         ‰| �  «        t        ||¬«      | _        t	        |«      | _        |j                  r4|j                  dk7  r$t        j                  d|j                  › d�«       y y y )N)r(   r3   Úflash_attention_2z=Sliding Window Attention is enabled but not implemented for `z)`; unexpected results may be encountered.)
r   r    r2   Ú	self_attnr   ÚmlprH   rU   rW   rX   r<   s      €r*   r    zQwen2DecoderLayer.__init__j   sp   ø€ Ü‰ÑÔÜ'¨vÀÔKˆŒÜ˜FÓ#ˆŒØ× Ò  V×%@Ñ%@ÐDWÒ%WÜ×ÑØOÐPV×PkÑPkÐOlð m9ð 9õð &XÐ r+   )r,   r-   r.   r   rg   r    r/   r0   s   @r*   rl   rl   i   s   ø„ ð˜{ð °s÷ ñ r+   rl   c                   ó   — e Zd Zy)Ú
Qwen2ModelN©r,   r-   r.   © r+   r*   rr   rr   u   ó   „ Ør+   rr   c                   ó   — e Zd Zy)ÚQwen2ForCausalLMNrs   rt   r+   r*   rw   rw   y   ru   r+   rw   c                   ó   — e Zd Zy)ÚQwen2ForSequenceClassificationNrs   rt   r+   r*   ry   ry   }   ru   r+   ry   c                   ó   — e Zd Zy)ÚQwen2ForTokenClassificationNrs   rt   r+   r*   r{   r{   �   ru   r+   r{   c                   ó   — e Zd Zy)ÚQwen2ForQuestionAnsweringNrs   rt   r+   r*   r}   r}   …   ru   r+   r}   )*Útypingr   r   r   rh   Útorch.utils.checkpointr   Úcache_utilsr   Úmodeling_flash_attention_utilsr	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   Úmistral.modeling_mistralr   Úconfiguration_qwen2r   Ú
get_loggerr,   rW   r   r2   rl   rr   rw   ry   r{   r}   rt   r+   r*   ú<module>r‰      s¸   ðß ,Ñ ,ã Û Ý å  Ý BÝ 5Ý &Ý ÷
÷ 
õ 
õ 4Ý ,ð 
ˆ×	Ñ	˜HÓ	%€ôYˆxô Yô@)�^ô @)ôF	Ð)ô 	ô	�ô 	ô	Ð'ô 	ô	Ð%Cô 	ô	Ð"=ô 	ô	Ð 9õ 	r+   