Ë
    S^(h  ã                   ón   — d dl mZ  e«       r
d dlZd dlmZ d dlmZ d„ Z G d„ dej                  «      Z	y)é    )Úis_torch_availableN)ÚLlama4TextMLPc                   ó   — y ©N© )ÚargsÚkwargss     új/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/integrations/compressed_tensors.pyÚskipr      s   € Øó    c                   óZ   ‡ — e Zd ZdZˆ fd„Zdej                  dej                  fd„Zˆ xZS )ÚCompressedExpertsLinearz¤
    A module that implements a compressed version of a list of expert modules.
    This is specifically designed to work with Llama4TextExperts in MoE layers.
    c                 ól  •— t         t        j                  _        t         t        j                  _        t         t        j                  _        t        ‰| �  «        |j                  | _	        t        j                  t        | j                  «      D �cg c]  }t        |«      ‘Œ c}«      | _        y c c}w r   )r   ÚnnÚinitÚkaiming_uniform_Úuniform_Únormal_ÚsuperÚ__init__Únum_local_expertsÚnum_expertsÚ
ModuleListÚranger   Úexpert_modules)ÚselfÚconfigÚ_Ú	__class__s      €r
   r   z CompressedExpertsLinear.__init__"   sq   ø€ ô $(Œ�‰Ô ÜŒ�‰ÔÜŒ�‰ŒÜ‰ÑÔØ!×3Ñ3ˆÔÜ Ÿm™mÌEÐRV×RbÑRbÓLcÖ,dÀq¬]¸6Õ-BÒ,dÓeˆÕùÒ,ds   ÂB1Úhidden_statesÚreturnc                 ó  — |j                  | j                  d|j                  d   «      }g }t        | j                  «      D ])  }|j	                   | j
                  |   ||   «      «       Œ+ t        j                  |d¬«      }|S )Néÿÿÿÿr   )Údim)Úreshaper   Úshaper   Úappendr   ÚtorchÚcat)r   r    Úexpert_routed_out_listÚ
expert_idxÚ
routed_outs        r
   ÚforwardzCompressedExpertsLinear.forward-   sŠ   € ð &×-Ñ-¨d×.>Ñ.>ÀÀM×DWÑDWÐXZÑD[Ó\ˆØ!#ÐÜ × 0Ñ 0Ó1ò 	fˆJØ"×)Ñ)Ð*I¨$×*=Ñ*=¸jÑ*IÈ-ÐXbÑJcÓ*dÕeð	fä—Y‘YÐ5¸1Ô=ˆ
ØÐr   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r(   ÚTensorr-   Ú__classcell__)r   s   @r
   r   r      s-   ø„ ñô
	fð	à—|‘|ð	ð 
�‰÷	r   r   )
Útransformers.utilsr   r(   Útorch.nnr   Ú*transformers.models.llama4.modeling_llama4r   r   ÚModuler   r   r   r
   ú<module>r8      s0   ðõ 2ñ ÔÛÝå Dò	ô˜bŸi™iõ r   