Ë
    S^(h�
  ã                   ó4  — d dl mZmZ d dlZdej                  dedej                  fd„Z	 	 	 ddej                  j                  dej                  d	ej                  d
ej                  deej                     de	dee	   dee
   deej                  df   fd„Zy)é    )ÚOptionalÚTupleNÚhidden_statesÚn_repÚreturnc                 óª   — | j                   \  }}}}|dk(  r| S | dd…dd…ddd…dd…f   j                  |||||«      } | j                  |||z  ||«      S )zÔ
    This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
    num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
    é   N)ÚshapeÚexpandÚreshape)r   r   ÚbatchÚnum_key_value_headsÚslenÚhead_dims         úf/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/integrations/sdpa_attention.pyÚ	repeat_kvr      so   € ð
 2?×1DÑ1DÑ.€EÐ  hØ�‚zØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐBUÐW\Ð^bÐdlÓm€MØ× Ñ  Ð(;¸eÑ(CÀTÈ8ÓTÐTó    ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingÚ	is_causalc           	      ó   — t        | d«      r,t        || j                  «      }t        || j                  «      }|}	|�-|	j                  dk(  r|	d d …d d …d d …d |j                  d   …f   }	|j                  «       }|j                  «       }|j                  «       }|€|j                  d   dkD  xr |	d u }t        j                  j                  «       r*t        |t        j                  «      r|j                  «       }t        j                  j                  j                  ||||	|||¬«      }
|
j                  dd«      j                  «       }
|
d fS )NÚnum_key_value_groupsé   éþÿÿÿé   r	   )Ú	attn_maskÚ	dropout_pÚscaler   )Úhasattrr   r   Úndimr
   Ú
contiguousÚtorchÚjitÚ
is_tracingÚ
isinstanceÚTensorÚitemÚnnÚ
functionalÚscaled_dot_product_attentionÚ	transpose)r   r   r   r   r   r   r   r   ÚkwargsÚcausal_maskÚattn_outputs              r   Úsdpa_attention_forwardr4      s@  € ô ˆvÐ-Ô.Ü˜˜V×8Ñ8Ó9ˆÜ˜% ×!<Ñ!<Ó=ˆà €KØÐ! k×&6Ñ&6¸!Ò&;Ø!¢!¢Qª¨?¨S¯Y©Y°r©]¨?Ð":Ñ;ˆð ×ÑÓ€EØ
�.‰.Ó
€CØ×ÑÓ€Eð
 ÐØ—K‘K ‘N QÑ&Ò>¨;¸$Ð+>ˆ	ô ‡y�y×ÑÔ¤*¨Y¼¿¹Ô"EØ—N‘NÓ$ˆ	ä—(‘(×%Ñ%×BÑBØØØØØØØð Có €Kð ×'Ñ'¨¨1Ó-×8Ñ8Ó:€Kà˜ÐÐr   )g        NN)Útypingr   r   r'   r+   Úintr   r-   ÚModuleÚfloatÚboolr4   © r   r   ú<module>r;      sÊ   ðß "ã ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ó 	Uð$ Ø#Ø $ñ/Ø�H‰H�O‰Oð/à�<‰<ð/ð 
�‰ð/ð �<‰<ð	/ð
 ˜UŸ\™\Ñ*ð/ð ð/ð �e‰_ð/ð ˜‰~ð/ð ˆ5�<‰<˜ÐÑô/r   