Ë
    S^(hÚ  ã                   ó"  — d dl mZmZ d dlZddlmZmZ  e«       Z	 	 	 	 ddej                  j                  dej                  dej                  dej                  d	eej                     d
edee   dee   dee   deej                  df   fd„Zy)é    )ÚOptionalÚTupleNé   )Ú_flash_attention_forwardÚ!flash_attn_supports_top_left_maskÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚdropoutÚscalingÚsliding_windowÚsoftcapÚreturnc	                 óV  — |j                   d   }
|j                  dd«      }|j                  dd«      }|j                  dd«      }d }|j                  t        j                  k(  rŠt        j
                  «       rt        j                  «       }nat        | j                  d«      r| j                  j                  }n4t        d„ | j                  «       D «       «      j                  j                  }|	j                  dd «       t        ||||f|
| j                  ||||t         |dœ|	¤Ž}|d fS )Nr   é   Ú_pre_quantization_dtypec              3   ój   K  — | ]+  }t        |t        j                  j                  «      sŒ(|–— Œ- y ­w)N)Ú
isinstanceÚtorchÚnnÚLinear)Ú.0Úlayers     úg/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/integrations/flash_attention.pyú	<genexpr>z*flash_attention_forward.<locals>.<genexpr>,   s'   è ø€ Òj¨%ÄzÐRWÔY^×YaÑYa×YhÑYhÕGi¤Ñjùs   ‚)3¬3Ú	is_causal)Úquery_lengthr   r   Úsoftmax_scaler   r   Úuse_top_left_maskÚtarget_dtype)ÚshapeÚ	transposeÚdtyper   Úfloat32Úis_autocast_enabledÚget_autocast_gpu_dtypeÚhasattrÚconfigr   ÚnextÚmodulesÚweightÚpopr   r   Ú_use_top_left_mask)r   r	   r
   r   r   r   r   r   r   ÚkwargsÚseq_lenr"   Úattn_outputs                r   Úflash_attention_forwardr3      s  € ð �k‰k˜!‰n€Gð �O‰O˜A˜qÓ!€EØ
�-‰-˜˜1Ó
€CØ�O‰O˜A˜qÓ!€Eð €LØ‡{�{”e—m‘mÒ#Ü×$Ñ$Ô&Ü ×7Ñ7Ó9‰Lä�V—]‘]Ð$=Ô>Ø!Ÿ=™=×@Ñ@‰LäÑj°6·>±>Ó3CÔjÓj×qÑq×wÑwˆLð ‡J�Jˆ{˜DÔ!ä*ØØØØð	ð
 Ø×"Ñ"ØØØ%ØÜ,Ø!ñð ñ€Kð  ˜ÐÐó    )g        NNN)Útypingr   r   r   Úmodeling_flash_attention_utilsr   r   r/   r   ÚModuleÚTensorÚfloatÚintr3   © r4   r   ú<module>r<      sÃ   ðß "ã ç hñ 7Ó8Ð ð Ø#Ø$(Ø#ñ6Ø�H‰H�O‰Oð6à�<‰<ð6ð 
�‰ð6ð �<‰<ð	6ð
 ˜UŸ\™\Ñ*ð6ð ð6ð �e‰_ð6ð ˜S‘Mð6ð �e‰_ð6ð ˆ5�<‰<˜ÐÑô6r4   