Ë
    S^(hš0  ã                   ó0  — d dl mZ d dlmZmZmZmZ  e«       r
ddlZddlmZ  e«       rddl	m
Z
  e«       rddlZ ej                  e«      Z G d„ dej                  j                   «      Z G d	„ d
ej$                  «      Z	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 dd„Zy)é   )ÚACT2FN)Úis_accelerate_availableÚis_fbgemm_gpu_availableÚis_torch_availableÚloggingé    N)Únn)Úinit_empty_weightsc                   ó<   ‡ — e Zd Zej                  fˆ fd„	Zd„ Zˆ xZS )ÚFbgemmFp8Linearc                 óV  •— t         ‰| �  |||«       || _        || _        t        j
                  j                  t	        j                  ||ft        j                  ¬«      «      | _	        t        j
                  j                  t	        j                  |df|¬«      «      | _
        | j                  dt	        j                  dgt        j                  ¬«      d¬«       |rDt        j
                  j                  t	        j                  | j                  |¬«      «      | _        y d | _        y )N©Údtypeé   Úinput_scale_ubF©Ú
persistent)ÚsuperÚ__init__Úin_featuresÚout_featuresÚtorchr	   Ú	ParameterÚzerosÚfloat8_e4m3fnÚweightÚweight_scaleÚregister_bufferÚfloatÚbias)Úselfr   r   r    Úweight_dtypeÚ	__class__s        €úb/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/integrations/fbgemm_fp8.pyr   zFbgemmFp8Linear.__init__!   sÓ   ø€ Ü‰Ñ˜ l°DÔ9Ø&ˆÔØ(ˆÔä—h‘h×(Ñ(¬¯©°lÀKÐ5PÔX]×XkÑXkÔ)lÓmˆŒÜ!ŸH™H×.Ñ.¬u¯{©{¸LÈ!Ð;LÐT`Ô/aÓbˆÔØ×ÑÐ-¬u¯{©{¸A¸3ÄeÇkÁkÔ/RÐ_dÐÔeáÜŸ™×*Ñ*¬5¯;©;¸×8IÑ8IÐR^Ô+_Ó`ˆD�IàˆD�Ió    c                 óL  — g |j                   d d ¢d‘­}t        j                  j                  j	                  |j                  d|j                   d   «      j                  «       | j                  ¬«      \  }}| j                  j                  t        j                  «      }t        j                  j                  j                  || j                  ||d¬«      }| j                  �|| j                  z   n|}|j                  |j                  «      }|j                  |«      }~~|S )Néÿÿÿÿ)Úscale_ubT©Úuse_fast_accum)Úshaper   ÚopsÚfbgemmÚquantize_fp8_per_rowÚviewÚ
contiguousr   r   ÚtoÚfloat32Úf8f8bf16_rowwiser   r    ÚdeviceÚreshape)r!   ÚxÚoutput_shapeÚx_quantizedÚx_scaleÚweight_scale_float32Úoutputs          r$   ÚforwardzFbgemmFp8Linear.forward/   sû   € à*˜Ÿ™  "˜Ð* rÑ*ˆô  %Ÿy™y×/Ñ/×DÑDØ�F‰F�2�q—w‘w˜r‘{Ó#×.Ñ.Ó0¸4×;NÑ;Nð  Eó  
Ñˆ�Wð  $×0Ñ0×3Ñ3´E·M±MÓBÐÜ—‘×!Ñ!×2Ñ2Ø˜Ÿ™ gÐ/CÐTXð 3ó 
ˆð (,§y¡yÐ'<�˜$Ÿ)™)Ò#À&ˆà—‘˜1Ÿ8™8Ó$ˆØ—‘ Ó-ˆØ˜Øˆr%   ©Ú__name__Ú
__module__Ú__qualname__r   r2   r   r<   Ú__classcell__©r#   s   @r$   r   r       s   ø„ ØEJÇ]Á]õ ör%   r   c                   ó<   ‡ — e Zd Zej                  fˆ fd„	Zd„ Zˆ xZS )ÚFbgemmFp8Llama4TextExpertsc                 ón  •— t         ‰| �  «        |j                  | _        |j                  | _        |j
                  | _        | j                  | _        t        |j                     | _	        t        j                  j                  t        j                  | j                  | j
                  d| j                  z  ft        j                  ¬«      «      | _        t        j                  j                  t        j                  | j                  d| j                  dz  ft        j                   ¬«      «      | _        t        j                  j                  t        j                  | j                  | j                  | j
                  ft        j                  ¬«      «      | _        t        j                  j                  t        j                  | j                  | j
                  dft        j                   ¬«      «      | _        | j)                  dt        j                  dgt        j*                  ¬«      d¬«       y )Nr   r   r   r   Fr   )r   r   Únum_local_expertsÚnum_expertsÚintermediate_sizeÚhidden_sizeÚ
expert_dimr   Ú
hidden_actÚact_fnr   r	   r   r   r   Úgate_up_projr2   Úgate_up_proj_scaleÚ	down_projÚdown_proj_scaler   r   )r!   Úconfigr   r#   s      €r$   r   z#FbgemmFp8Llama4TextExperts.__init__H   sŒ  ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ!'×!9Ñ!9ˆÔØ!×-Ñ-ˆÔØ×0Ñ0ˆŒÜ˜V×.Ñ.Ñ/ˆŒä!ŸH™H×.Ñ.Ü�K‰K˜×)Ñ)¨4×+;Ñ+;¸QÀÇÁÑ=PÐQÔY^×YlÑYlÔmó
ˆÔô #(§(¡(×"4Ñ"4Ü�K‰K˜×)Ñ)¨1¨d¯o©oÀÑ.AÐBÌ%Ï-É-ÔXó#
ˆÔô Ÿ™×+Ñ+Ü�K‰K˜×)Ñ)¨4¯?©?¸D×<LÑ<LÐMÔUZ×UhÑUhÔió
ˆŒô  %Ÿx™x×1Ñ1Ü�K‰K˜×)Ñ)¨4×+;Ñ+;¸QÐ?ÄuÇ}Á}ÔUó 
ˆÔð 	×ÑÐ-¬u¯{©{¸A¸3ÄeÇkÁkÔ/RÐ_dÐÕer%   c           
      óT  — |j                  | j                  d| j                  «      }d}t        j                  |«      }t        | j                  «      D �]›  }||   }|j                  d| j                  «      }t        j                  j                  j                  ||| j                  «      \  }}| j                  j                  d   dz  }	| j                  j                  t        j                  «      }
t        j                  j                  j!                  || j                  |   j#                  dd«      d|	 j%                  «       ||
|   d   d|	 j                  dd«      j%                  «       d¬«      }t        j                  j                  j!                  || j                  |   j#                  dd«      |	d j%                  «       ||
|   d   |	d j                  dd«      j%                  «       d¬«      }|| j'                  |«      z  }t        j                  j                  j                  ||| j                  «      \  }}| j(                  j                  t        j                  «      }t        j                  j                  j!                  || j*                  |   j#                  dd«      j%                  «       |||   j                  dd«      j%                  «       d¬«      }|||<   �Œž |j                  |j,                  «      }|j                  d| j                  «      S )z¸
        Args:
            hidden_states (torch.Tensor): (batch_size * token_num, hidden_size)
        Returns:
            torch.Tensor: (batch_size * token_num, hidden_size)
        r'   Nr   r   r   Tr)   )r/   rG   rI   r   Ú
empty_likeÚranger5   r,   r-   r.   r   rM   r+   rN   r1   r2   r3   Ú	transposer0   rL   rP   rO   r4   )r!   Úhidden_statesÚ
num_tokensÚnext_statesÚiÚexpert_hiddenÚexpert_hidden_reshapedÚexpert_quantizedÚexpert_scaleÚsharded_expert_dimÚgate_up_proj_scale_float32ÚgateÚupÚ	activatedÚactivated_quantizedÚactivated_scaleÚdown_proj_scale_float32Úexpert_outputs                     r$   r<   z"FbgemmFp8Llama4TextExperts.forward`   så  € ð &×*Ñ*¨4×+;Ñ+;¸RÀ×AQÑAQÓRˆØˆ
ô ×&Ñ& }Ó5ˆä�t×'Ñ'Ó(ó *	+ˆAà)¨!Ñ,ˆMØ%2×%:Ñ%:¸2¸t×?OÑ?OÓ%PÐ"ä-2¯Y©Y×-=Ñ-=×-RÑ-RØ&¨
°D×4GÑ4Gó.Ñ*Ð˜lð "&×!2Ñ!2×!8Ñ!8¸Ñ!<ÀÑ!AÐØ)-×)@Ñ)@×)CÑ)CÄEÇMÁMÓ)RÐ&ä—9‘9×#Ñ#×4Ñ4Ø Ø×!Ñ! !Ñ$×.Ñ.¨q°!Ó4Ð5HÐ6HÐI×TÑTÓVØØ*¨1Ñ-¨aÑ0Ð1DÐ2DÐE×JÑJÈ2ÈqÓQ×\Ñ\Ó^Ø#ð 5ó ˆDô —‘×!Ñ!×2Ñ2Ø Ø×!Ñ! !Ñ$×.Ñ.¨q°!Ó4Ð5GÐ5HÐI×TÑTÓVØØ*¨1Ñ-¨aÑ0Ð1CÐ1DÐE×JÑJÈ2ÈqÓQ×\Ñ\Ó^Ø#ð 3ó ˆBð ˜TŸ[™[¨Ó.Ñ.ˆIä38·9±9×3CÑ3C×3XÑ3XØ˜: t×':Ñ':ó4Ñ0Ð ð '+×&:Ñ&:×&=Ñ&=¼e¿m¹mÓ&LÐ#Ü!ŸI™I×,Ñ,×=Ñ=Ø#Ø—‘˜qÑ!×+Ñ+¨A¨qÓ1×<Ñ<Ó>ØØ'¨Ñ*×/Ñ/°°AÓ6×AÑAÓCØ#ð >ó ˆMð +ˆK˜‹NðU*	+ðV "—n‘n ]×%9Ñ%9Ó:ˆØ×Ñ  D×$4Ñ$4Ó5Ð5r%   r=   rB   s   @r$   rD   rD   G   s   ø„ Ø%*§]¡]õ fö0:6r%   rD   c                 óÔ  ‡— ddl }|€g }| j                  «       D �]/  \  }	}
|j                  |	«       t        |
t        j
                  «      rÞ|	|vrÚdj                  |«      Št        ˆfd„|D «       «      sµt        d¬«      5  |
j                  }|
j                  }t        |||
j                  du«      | j                  |	<   d}| j                  |	   j                  d«       ddd«       t        j                   |j"                  gt        j$                  ¬«      | j                  |	   _        |
j(                  j*                  d	k(  r»|	|vr·dj                  |«      Št        ˆfd
„|D «       «      s’t        d¬«      5  d||j-                  dd‰dz   «      <   t/        |j0                  «      | j                  |	<   ddd«       t        j                   |j"                  gt        j$                  ¬«      | j                  |	   _        t3        t5        |
j7                  «       «      «      dkD  rt9        |
|||||||¬«      \  }}|j;                  d«       �Œ2 | |fS # 1 sw Y   �ŒsxY w# 1 sw Y   Œ«xY w)z¶
    Private method that wraps the recursion for module replacement.

    Returns the converted model and a boolean that indicates if the conversion has been successfull or not.
    r   Nú.c              3   ó:   •K  — | ]  }|d z   ‰v xs |‰k(  –— Œ y­w©rh   N© ©Ú.0ÚkeyÚcurrent_key_name_strs     €r$   ú	<genexpr>z2_replace_with_fbgemm_fp8_linear.<locals>.<genexpr>¸   ó.   øè ø€ ò ØY\��s‘Ð2Ð2ÒT¸Ð?SÑ8SÓTñùó   ƒT)Úinclude_buffersFr   ÚLlama4TextExpertsc              3   ó:   •K  — | ]  }|d z   ‰v xs |‰k(  –— Œ y­wrj   rk   rl   s     €r$   rp   z2_replace_with_fbgemm_fp8_linear.<locals>.<genexpr>Î   rq   rr   z\d+Ú*z.down_proj_scale)Úhas_been_replacedÚpre_quantizedrQ   Útp_planr'   )ÚreÚnamed_childrenÚappendÚ
isinstancer	   ÚLinearÚjoinÚanyr
   r   r   r   r    Ú_modulesÚrequires_grad_r   ÚtensorÚactivation_scale_ubr   r   r#   r>   ÚsubrD   Útext_configÚlenÚlistÚchildrenÚ_replace_with_fbgemm_fp8_linearÚpop)ÚmodelÚmodules_to_not_convertÚcurrent_key_nameÚquantization_configrw   rx   rQ   ry   rz   ÚnameÚmoduler   r   Ú_ro   s                 @r$   rŠ   rŠ   �   sR  ø€ ó  àÐØÐà×,Ñ,Ó.ó 4!‰ˆˆfØ×Ñ Ô%ä�vœrŸy™yÔ)¨tÐ;QÑ/Qà#&§8¡8Ð,<Ó#=Ð Üó Ø`vôô ô (¸Ô=ñ ?Ø"(×"4Ñ"4�KØ#)×#6Ñ#6�LÜ+:Ø#Ø$ØŸ™¨4Ð/ó,�E—N‘N 4Ñ(ð
 )-Ð%ð —N‘N 4Ñ(×7Ñ7¸Ô>÷?ô 7<·l±lØ(×<Ñ<Ð=ÜŸ+™+ô7�—‘˜tÑ$Ô3ð ×Ñ×$Ñ$Ð(;Ò;ÀÐLbÑ@bØ#&§8¡8Ð,<Ó#=Ð Üó Ø`vôô ô (¸Ô=ñ Ø^b�G˜BŸF™F 6¨3Ð0DÐGYÑ0YÓZÑ[Ü+EØ×*Ñ*ó,�E—N‘N 4Ñ(÷ô
 7<·l±lØ(×<Ñ<Ð=ÄUÇ[Á[ô7�—‘˜tÑ$Ô3ô Œt�F—O‘OÓ%Ó&Ó'¨!Ò+Ü#BØØ&Ø Ø#Ø"3Ø+ØØô	$Ñ ˆAÐ ð 	×Ñ˜RÖ ði4!ðj Ð#Ð#Ð#÷Y?ñ ?ú÷,ð ús   ÂAIÅ8<IÉI	ÉI'	c           	      óâ   — |€dgn|}|j                   �|j                  |j                   «       t        t        |«      «      }t	        | ||||||¬«      \  } }|st
        j                  d«       | S )aÇ  
    A helper function to replace all `torch.nn.Linear` modules by `FbgemmFp8Linear` modules.
    This will enable running your models using high performance fp8 kernel from FBGEMM library.

    The function will be run recursively and replace all `torch.nn.Linear` modules except for the `lm_head` that should
    be kept as a `torch.nn.Linear` module. The replacement is done under `init_empty_weights` context manager so no
    CPU/GPU memory is required to run this function. Each weight will be quantized along the channel.

    Parameters:
        model (`torch.nn.Module`):
            Input model or `torch.nn.Module` as the function is run recursively.
        modules_to_not_convert (`List[`str`]`, *optional*, defaults to `["lm_head"]`):
            Names of the modules to not convert in `FP8Linear`. In practice we keep the `lm_head` in full precision
            for numerical stability reasons.
        current_key_name (`List[`str`]`, *optional*):
            An array to track the current key of the recursion. This is used to check whether the current key (part of
            it) is not in the list of modules to not convert (for instances modules that are offloaded to `cpu` or
            `disk`).
    Úlm_head)rx   rQ   ry   zÇYou are loading your model using FP8 quantization but no linear modules were found in your model. Please double check your model architecture, or submit an issue on github if you think this is a bug.)r�   Úextendrˆ   ÚsetrŠ   ÚloggerÚwarning)rŒ   r�   rŽ   r�   rx   rQ   ry   rw   s           r$   Úreplace_with_fbgemm_fp8_linearr™   ê   s‰   € ð: -CÐ,J˜i™[ÐPfÐà×1Ñ1Ð=Ø×%Ñ%Ð&9×&PÑ&PÔQÜ!¤#Ð&<Ó"=Ó>ÐÜ>ØØØØØ#ØØô Ñ€EÐñ Ü�‰ðô	
ð €Lr%   )NNNFFNN)NNNFNN)Úactivationsr   Úutilsr   r   r   r   r   r	   Ú
accelerater
   Úfbgemm_gpu.experimental.gen_aiÚ
fbgemm_gpuÚ
get_loggerr>   r—   r~   r   ÚModulerD   rŠ   r™   rk   r%   r$   ú<module>r¡      s¡   ðõ !ß aÓ añ ÔÛÝáÔÝ-áÔÛ)à	ˆ×	Ñ	˜HÓ	%€ô$�e—h‘h—o‘oô $ôNS6 §¡ô S6ðp  ØØØØØØóJ$ð^  ØØØØØô2r%   