Ë
    T^(h/  ã                   óš  — d dl mZ d dlmZmZmZ d dlZd dlmZ ddl	m
Z
mZ ddlmZ ddlmZ ddlmZ dd	lmZ dd
lmZ ddlmZ ddlmZmZmZmZmZmZmZ ddl m!Z!  ejD                  e#«      Z$dZ%dZ& G d„ de«      Z' G d„ de«      Z( G d„ dejR                  «      Z* G d„ de«      Z+ G d„ de«      Z, G d„ de«      Z- G d„ de«      Z.y) é    )Úpartial)ÚCallableÚOptionalÚTupleNé   )ÚCacheÚDynamicCache)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úloggingé   )ÚCLIPMLP)ÚLlamaAttentionÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚ
LlamaModelÚapply_rotary_pos_embÚeager_attention_forwardé   )Ú	PhiConfigzmicrosoft/phi-1r   c                   ó(  ‡ — e Zd Zdedefˆ fd„Z	 	 ddej                  deej                  ej                  f   de	ej                     de	e
   de	ej                     d	eej                  e	ej                     e	eej                        f   fd
„Zˆ xZS )ÚPhiAttentionÚconfigÚ	layer_idxc                 óœ  •— t         ‰| �  ||«       t        j                  |j                  |j
                  | j                  z  d¬«      | _        t        j                  |j                  |j                  | j                  z  d¬«      | _	        t        j                  |j                  |j                  | j                  z  d¬«      | _
        t        j                  |j
                  | j                  z  |j                  d¬«      | _        | `t        | j                  |j                  z  «      | _        |j                   | _        | j                   r}t        j"                  |j                  |j
                  z  |j$                  d¬«      | _        t        j"                  |j                  |j
                  z  |j$                  d¬«      | _        y y )NT©Úbias)ÚepsÚelementwise_affine)ÚsuperÚ__init__ÚnnÚLinearÚhidden_sizeÚnum_attention_headsÚhead_dimÚq_projÚnum_key_value_headsÚk_projÚv_projÚdenseÚo_projÚintÚpartial_rotary_factorÚrotary_ndimsÚqk_layernormÚ	LayerNormÚlayer_norm_epsÚq_layernormÚk_layernorm©Úselfr   r   Ú	__class__s      €úa/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/phi/modular_phi.pyr$   zPhiAttention.__init__#   sb  ø€ Ü‰Ñ˜ Ô+Ü—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—i‘i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÔjˆŒÜ—Y‘Y˜v×9Ñ9¸D¿M¹MÑIÈ6×K]ÑK]ÐdhÔiˆŒ
ØˆKÜ §¡°×0LÑ0LÑ LÓMˆÔØ"×/Ñ/ˆÔØ×ÒÜ!Ÿ|™|Ø×"Ñ" f×&@Ñ&@Ñ@Àf×F[ÑF[Ðptô ˆDÔô  "Ÿ|™|Ø×"Ñ" f×&@Ñ&@Ñ@Àf×F[ÑF[Ðptô ˆDÕð	 ó    Úhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valueÚcache_positionÚreturnc                 ó  — |j                   d d }g |¢d‘| j                  ‘­}| j                  |«      j                  |«      j	                  dd«      }	| j                  |«      j                  |«      j	                  dd«      }
| j                  |«      j                  |«      j	                  dd«      }| j                  r"| j                  |	«      }	| j                  |
«      }
|\  }}|	dd | j                  …f   |	d| j                  d …f   }}|
dd | j                  …f   |
d| j                  d …f   }}t        ||||«      \  }}t        j                  ||fd¬«      }	t        j                  ||fd¬«      }
|�'|||dœ}|j                  |
|| j                  |«      \  }
}t         }| j"                  j$                  dk7  r^| j"                  j$                  dk(  r(|j'                  d	d
«      rt(        j+                  d«       nt,        | j"                  j$                     } || |	|
||f| j.                  sdn| j0                  | j2                  dœ|¤Ž\  }} |j4                  g |¢d‘­Ž j7                  «       }| j9                  |«      }||fS )Néÿÿÿÿr   r   .)Údim)ÚsinÚcosrA   ÚeagerÚsdpaÚoutput_attentionsFzã`torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True`. Falling back to eager attention. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.g        )ÚdropoutÚscaling)Úshaper)   r*   ÚviewÚ	transposer,   r-   r3   r6   r7   r2   r   ÚtorchÚcatÚupdater   r   r   Ú_attn_implementationÚgetÚloggerÚwarning_oncer   ÚtrainingÚattention_dropoutrL   ÚreshapeÚ
contiguousr.   )r9   r=   r>   r?   r@   rA   ÚkwargsÚinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrG   rF   Ú	query_rotÚ
query_passÚkey_rotÚkey_passÚcache_kwargsÚattention_interfaceÚattn_outputÚattn_weightss                         r;   ÚforwardzPhiAttention.forward4   s¦  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà×ÒØ×+Ñ+¨LÓ9ˆLØ×)Ñ)¨*Ó5ˆJà&‰ˆˆSð ˜Ð1 × 1Ñ 1Ð1Ð1Ñ2Ø˜˜d×/Ñ/Ñ1Ð1Ñ2ð ˆ	ð
 �sÐ/˜d×/Ñ/Ð/Ð/Ñ0Ø�s˜D×-Ñ-Ñ/Ð/Ñ0ð ˆô
 2°)¸WÀcÈ3ÓOÑˆ	�7ô —y‘y )¨ZÐ!8¸bÔAˆÜ—Y‘Y ¨Ð2¸Ô;ˆ
àÐ%à#&¨sÀnÑUˆLØ'5×'<Ñ'<¸ZÈÐW[×WeÑWeÐgsÓ'tÑ$ˆJ˜ä(?ÐØ�;‰;×+Ñ+¨wÒ6Ø�{‰{×/Ñ/°6Ò9¸f¿j¹jÐI\Ð^cÔ>dÜ×#Ñ#ðLõô
 '>¸d¿k¹k×>^Ñ>^Ñ&_Ð#á$7ØØØØØð	%
ð  $Ÿ}š}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð *�k×)Ñ)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—j‘j Ó-ˆØ˜LÐ(Ð(r<   )NN)Ú__name__Ú
__module__Ú__qualname__r   r0   r$   rP   ÚTensorr   r   r   Ú
LongTensorri   Ú__classcell__©r:   s   @r;   r   r   "   s¹   ø„ ð˜yð °Sõ ð, +/Ø59ñA)à—|‘|ðA)ð # 5§<¡<°·±Ð#=Ñ>ðA)ð ! §¡Ñ.ð	A)ð
 ! ™ðA)ð ! ×!1Ñ!1Ñ2ðA)ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	S÷A)r<   r   c                   ó   — e Zd Zy)ÚPhiMLPN©rj   rk   rl   © r<   r;   rr   rr   x   ó   „ Ør<   rr   c                   ó€  ‡ — e Zd Zdedefˆ fd„Z	 	 	 	 	 	 	 ddej                  deej                     deej                     dee
ej                        dee   d	ee   d
eej                     dee
ej                  ej                  f      de
ej                  ee
ej                  ej                  f      f   fd„Zˆ xZS )ÚPhiDecoderLayerr   r   c                 ó  •— t         ‰| �  «        t        ||¬«      | _        t	        |«      | _        t        j                  |j                  |j                  ¬«      | _
        t        j                  |j                  «      | _        y )N)r   ©r!   )r#   r$   r   Ú	self_attnrr   Úmlpr%   r4   r'   r5   Úinput_layernormÚDropoutÚresid_pdropÚresid_dropoutr8   s      €r;   r$   zPhiDecoderLayer.__init__}   s]   ø€ Ü‰ÑÔÜ% f¸	ÔBˆŒÜ˜&“>ˆŒÜ!Ÿ|™|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔÜŸZ™Z¨×(:Ñ(:Ó;ˆÕr<   r=   r?   Úposition_idsr@   rJ   Ú	use_cacherA   r>   rB   c	                 óð   — |}
| j                  |«      } | j                  d||||||||dœ|	¤Ž\  }}| j                  |«      }| j                  | j                  |«      «      }||z   |
z   }|f}|r||fz  }|S )N)r=   r?   r€   r@   rJ   r�   rA   r>   rt   )r|   rz   r   r{   )r9   r=   r?   r€   r@   rJ   r�   rA   r>   r[   ÚresidualÚattn_outputsÚself_attn_weightsÚfeed_forward_hidden_statesÚoutputss                  r;   ri   zPhiDecoderLayer.forward„   s²   € ð !ˆà×,Ñ,¨]Ó;ˆð +9¨$¯.©.ð 
+
Ø'Ø)Ø%Ø)Ø/ØØ)Ø 3ñ
+
ð ñ
+
Ñ'ˆÐ'ð ×)Ñ)¨,Ó7ˆà%)×%7Ñ%7¸¿¹ÀÓ8OÓ%PÐ"Ø$Ð'AÑAÀHÑLˆØ Ð"ˆáØÐ)Ð+Ñ+ˆGàˆr<   )NNNFFNN)rj   rk   rl   r   r0   r$   rP   rm   r   rn   r   ÚboolÚFloatTensorri   ro   rp   s   @r;   rw   rw   |   s  ø„ ð<˜yð <°Sõ <ð 26Ø37Ø8<Ø,1Ø$)Ø59ØKOñ%à—|‘|ð%ð ! §¡Ñ.ð%ð ˜u×/Ñ/Ñ0ð	%ð
 !  u§|¡|Ñ!4Ñ5ð%ð $ D™>ð%ð ˜D‘>ð%ð ! ×!1Ñ!1Ñ2ð%ð & e¨E¯L©L¸%¿,¹,Ð,FÑ&GÑHð%ð 
ˆu× Ñ  (¨5°×1BÑ1BÀE×DUÑDUÐ1UÑ+VÑ"WÐWÑ	X÷%r<   rw   c                   ó  ‡ — e Zd Zdefˆ fd„Z	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     dee	   deej                     dee   d	ee   d
ee   deej                     dee   defd„Zˆ xZS )ÚPhiModelr   c           	      ód  •— t         ‰| �  |«       t        j                  t	        |j
                  «      D �cg c]  }t        ||«      ‘Œ c}«      | _        t        j                  |j                  «      | _
        t        j                  |j                  |j                  ¬«      | _        | `y c c}w )Nry   )r#   r$   r%   Ú
ModuleListÚrangeÚnum_hidden_layersrw   Úlayersr}   Ú
embd_pdropÚembed_dropoutr4   r'   r5   Úfinal_layernormÚnormr8   s      €r;   r$   zPhiModel.__init__­   sƒ   ø€ Ü‰Ñ˜Ô Ü—m‘mÜAFÀv×G_ÑG_ÓA`Öa°IŒ_˜V YÕ/Òaó
ˆŒô  ŸZ™Z¨×(9Ñ(9Ó:ˆÔÜ!Ÿ|™|¨F×,>Ñ,>ÀF×DYÑDYÔZˆÔØ‰Iùò	 bs   ·B-Ú	input_idsr?   r€   Úpast_key_valuesÚinputs_embedsr�   rJ   Úoutput_hidden_statesrA   Úflash_attn_kwargsrB   c
                 ó4  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }|d u |d uz  rt	        d«      ‚| j
                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }|r|€
t        «       }|	€F|�|j                  «       nd}t        j                  |||j                  d   z   |j                  ¬«      }	|€|	j!                  d«      }| j#                  |||	||«      }| j%                  |«      }|}| j'                  ||«      }|rdnd }|rdnd }| j(                  d | j                   j*                   D ]r  }|r||fz  }| j
                  r:| j                  r.| j-                  t/        |j0                  fi |
¤Ž|||||||	|«	      }n ||f||||||	|dœ|
¤Ž}|d   }|sŒj||d   fz  }Œt | j3                  |«      }|r||fz  }t5        ||r|nd ||¬	«      S )
Nz:You must specify exactly one of input_ids or inputs_embedszX`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`.Fr   r   )Údevicert   )r?   r€   r@   rJ   r�   rA   r>   )Úlast_hidden_stater–   r=   Ú
attentions)r   rJ   r˜   r�   Ú
ValueErrorÚgradient_checkpointingrW   rU   rV   Úembed_tokensr	   Úget_seq_lengthrP   ÚarangerM   r›   Ú	unsqueezeÚ_update_causal_maskr’   Ú
rotary_embr�   r�   Ú_gradient_checkpointing_funcr   Ú__call__r“   r   )r9   r•   r?   r€   r–   r—   r�   rJ   r˜   rA   r™   Úpast_seen_tokensÚcausal_maskr=   r>   Úall_hidden_statesÚall_self_attnsÚdecoder_layerÚlayer_outputss                      r;   ri   zPhiModel.forward¶   s”  € ð 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	à˜Ð -°tÐ";Ò<ÜÐYÓZÐZà×&Ò&¨4¯=ª=¹YÜ×ÑØjôð ˆIàÐ Ø ×-Ñ-¨iÓ8ˆMá˜Ð0Ü*›nˆOàÐ!ØCRÐC^˜×=Ñ=Ô?ÐdeÐÜ"Ÿ\™\Ø Ð"2°]×5HÑ5HÈÑ5KÑ"KÐTa×ThÑThôˆNð ÐØ)×3Ñ3°AÓ6ˆLà×.Ñ.Ø˜M¨>¸?ÐL]ó
ˆð ×*Ñ*¨=Ó9ˆØ%ˆð #Ÿo™o¨m¸\ÓJÐñ #7™B¸DÐÙ0™°dˆà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÐIò  	6ˆMÙ#Ø! mÐ%5Ñ5Ð!à×*Ò*¨t¯}ª}Ø $× AÑ AÜ˜M×2Ñ2ÑHÐ6GÑHØ!ØØ Ø#Ø%ØØ"Ø'ó
!‘ñ !.Ø!ð
!à#.Ø!-Ø#2Ø&7Ø'Ø#1Ø(;ñ
!ð (ñ
!�ð *¨!Ñ,ˆMâ Ø =°Ñ#3Ð"5Ñ5‘ðA 	6ðD ×,Ñ,¨]Ó;ˆñ  Ø -Ð!1Ñ1Ðä&Ø+Ù/8™O¸dØ+Ø%ô	
ð 	
r<   )	NNNNNNNNN)rj   rk   rl   r   r$   r   rP   rn   rm   r   r‰   rˆ   r   r
   r   ri   ro   rp   s   @r;   r‹   r‹   ¬   sô   ø„ ð˜yõ ð 15Ø15Ø37Ø+/Ø59Ø$(Ø,0Ø/3Ø59ñf
à˜E×,Ñ,Ñ-ðf
ð ! §¡Ñ.ðf
ð ˜u×/Ñ/Ñ0ð	f
ð
 " %™ðf
ð   × 1Ñ 1Ñ2ðf
ð ˜D‘>ðf
ð $ D™>ðf
ð ' t™nðf
ð ! ×!1Ñ!1Ñ2ðf
ð $Ð$8Ñ9ðf
ð 
!÷f
r<   r‹   c                   ó   ‡ — e Zd Zˆ fd„Zˆ xZS )ÚPhiForCausalLMc                 ó†   •— t         ‰| �  |«       t        j                  |j                  |j
                  d¬«      | _        y )NTr   )r#   r$   r%   r&   r'   Ú
vocab_sizeÚlm_head)r9   r   r:   s     €r;   r$   zPhiForCausalLM.__init__   s0   ø€ Ü‰Ñ˜Ô Ü—y‘y ×!3Ñ!3°V×5FÑ5FÈTÔRˆ�r<   )rj   rk   rl   r$   ro   rp   s   @r;   r¯   r¯     s   ø„ ÷Sð Sr<   r¯   c                   ó   — e Zd Zy)ÚPhiForSequenceClassificationNrs   rt   r<   r;   r´   r´   %  ru   r<   r´   c                   ó   — e Zd Zy)ÚPhiForTokenClassificationNrs   rt   r<   r;   r¶   r¶   )  ru   r<   r¶   )/Ú	functoolsr   Útypingr   r   r   rP   Útorch.nnr%   Úcache_utilsr   r	   Úmodeling_flash_attention_utilsr
   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   Úclip.modeling_clipr   Úllama.modeling_llamar   r   r   r   r   r   r   Úconfiguration_phir   Ú
get_loggerrj   rU   Ú_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCr   rr   ÚModulerw   r‹   r¯   r´   r¶   rt   r<   r;   ú<module>rÇ      sÄ   ðÝ ß ,Ñ ,ã Ý ç .Ý Bõõ 6Ý &Ý Ý (÷÷ ñ õ )ð 
ˆ×	Ñ	˜HÓ	%€à'Ð Ø€ôS)�>ô S)ôl	ˆWô 	ô-�b—i‘iô -ô`p
ˆzô p
ôfSÐ%ô Sô	Ð#Aô 	ô	Ð ;õ 	r<   