Ë
    T^(h2  ã                   óÄ   — d dl Z d dlmZmZmZmZmZ d dlmZ ddl	m
Z
 erddlmZ ddlmZmZmZmZ dd	lmZ  e«       rd dlZ ej*                  e«      Z G d
„ de
«      Zy)é    N)ÚTYPE_CHECKINGÚAnyÚDictÚListÚOptional)Úversioné   )ÚHfQuantizeré   )ÚPreTrainedModel)Úis_accelerate_availableÚis_fbgemm_gpu_availableÚis_torch_availableÚlogging)Úget_module_from_namec                   ó  ‡ — e Zd ZdZdZdZddgZˆ fd„Zd„ Zdd	„Z	d
dddde
dee
ef   fd„Z	 dd
dddde
dddee
ef   deee
      fd„Zd d„Z	 dd
ddeee
      fd„Zdee
   de
dee
   fd„Zd„ Zdd„Zedefd„«       Zˆ xZS )!ÚFbgemmFp8HfQuantizerz/
    FP8 quantization using fbgemm kernels
    TFz
fbgemm-gpuÚ
acceleratec                 ó4   •— t        ‰| �  |fi |¤Ž || _        y ©N)ÚsuperÚ__init__Úquantization_config)Úselfr   ÚkwargsÚ	__class__s      €új/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/quantizers/quantizer_fbgemm_fp8.pyr   zFbgemmFp8HfQuantizer.__init__.   s   ø€ Ü‰ÑÐ,Ñ7°Ò7Ø#6ˆÕ ó    c                 óÀ  — t        «       rHt        j                  t        j                  j                  d«      «      t        j                  d«      k  rt        d«      ‚t        «       st        d«      ‚t        d«      st        d«      ‚t        j                  j                  «       st        d«      ‚t        j                  j                  «       }|\  }}|dk  rt        d	«      ‚|j                  d
d «      }|€t        j!                  d«       y |�N| j"                  sAt%        |t&        «      r0d|j)                  «       v sd|j)                  «       v rt        d«      ‚y y y y )NÚtorchz2.1.0z~Using fbgemm fp8 quantization requires torch > 2.1.0Please install the latest version of torch ( pip install --upgrade torch )zìUsing fbgemm fp8 quantization requires fbgemm-gpu libraryPlease install the latest version of fbgemm-gpu library by following : https://pytorch.org/FBGEMM/fbgemm_gpu-development/InstallationInstructions.html#fbgemm-gpu-install-librariesz0.32.2z`Loading an FP8 quantized model requires accelerate > 0.32.1 (`pip install --upgrade accelerate`)z=Using FP8 quantized models with fbgemm kernels requires a GPUé	   zXFP8 quantized models is only supported on GPUs with compute capability >= 9.0 (e.g H100)Ú
device_mapzÀYou have loaded an FP8 model on CPU and have a CUDA device available, make sure to set your model on a GPU device in order to run your model. To remove this warning, pass device_map = 'cuda'. ÚcpuÚdiskzòYou are attempting to load an FP8 model with a device_map that contains a CPU or disk device.This is not supported when the model is quantized on the fly. Please use a quantized checkpoint or remove the CPU or disk device from the device_map.)r   r   ÚparseÚ	importlibÚmetadataÚImportErrorr   r   r    ÚcudaÚis_availableÚRuntimeErrorÚget_device_capabilityÚ
ValueErrorÚgetÚloggerÚwarning_onceÚpre_quantizedÚ
isinstanceÚdictÚvalues)r   Úargsr   Úcompute_capabilityÚmajorÚminorr"   s          r   Úvalidate_environmentz)FbgemmFp8HfQuantizer.validate_environment2   s`  € Ü!Ô#¤w§}¡}´Y×5GÑ5G×5OÑ5OÐPWÓ5XÓ'YÔ\c×\iÑ\iÐjqÓ\rÒ'rÜð]óð ô 'Ô(ÜðFóð ô
 ' xÔ0ÜØróð ô �z‰z×&Ñ&Ô(ÜÐ^Ó_Ð_ä"ŸZ™Z×=Ñ=Ó?ÐØ)‰ˆˆuØ�1Š9ÜØjóð ð —Z‘Z ¨dÓ3ˆ
ØÐÜ×Ñð|õð Ð#à×&Ò&Ü˜z¬4Ô0Ø˜j×/Ñ/Ó1Ñ1°V¸z×?PÑ?PÓ?RÑ5Rä ðnóð ð 6Sð 1ð 'ð $r   Úreturnc                 ó–   — |€(t         j                  }t        j                  d|«       |S |t         j                  k(  rt        d«      ‚|S )Na  Overriding torch_dtype=%s with `torch_dtype=torch.bloat16` due to requirements of `fbgemm-gpu` to enable model loading in fp8. Pass your own torch_dtype to specify the dtype of the remaining non-linear layers or pass torch_dtype=torch.bfloat16 to remove this warning.zeYou cannot use FP8 with torch_dtype=torch.float16.We recommend you passing torch_dtype=torch.bfloat16)r    Úbfloat16r/   ÚinfoÚfloat16r-   )r   Útorch_dtypes     r   Úupdate_torch_dtypez'FbgemmFp8HfQuantizer.update_torch_dtype_   sV   € ØÐÜŸ.™.ˆKÜ�K‰KðFð ôð Ðð	 œEŸM™MÒ)ÜØwóð ð Ðr   Úmodelr   Úparam_valueztorch.TensorÚ
param_nameÚ
state_dictc                 óR  — ddl m}m} t        ||«      \  }}	t	        ||«      rP| j
                  s|	dk(  r.|	dk(  r(|j                  t        j                  k7  rt        d«      ‚y|	dk(  rt        d«      ‚y	t	        ||«      r(| j
                  s|	dk(  ry|	d
k(  s|	dk(  rt        d«      ‚y	y)Nr   ©ÚFbgemmFp8LinearÚFbgemmFp8Llama4TextExpertsÚbiasÚweightz6Expect quantized weights but got an unquantized weightFÚweight_scalez;Expect unquantized weights but got a quantized weight_scaleTÚgate_up_proj_scaleÚdown_proj_scale)
ÚintegrationsrG   rH   r   r2   r1   Údtyper    Úfloat8_e4m3fnr-   )
r   rA   rB   rC   rD   r   rG   rH   ÚmoduleÚtensor_names
             r   Úcheck_quantized_paramz*FbgemmFp8HfQuantizer.check_quantized_paramo   sµ   € ÷ 	Oä2°5¸*ÓEÑˆ�ä�f˜oÔ.Ø×!Ò! [°FÒ%:Ø (Ò*¨{×/@Ñ/@ÄE×DWÑDWÒ/WÜ$Ð%]Ó^Ð^Øà .Ò0Ü$Ð%bÓcÐcØÜ�fÐ8Ô9Ø×!Ò! [°FÒ%:ØàÐ"6Ò6¸+ÐIZÒ:ZÜ$Ð%bÓcÐcØØr   Útarget_deviceztorch.deviceÚunexpected_keysc                 óü  — ddl m} t        ||«      \  }}	t        ||«      �r€|	dk(  rœ|j	                  dd«      }
|
j
                  }|
j                  d|d   «      }t        j                  j                  j                  |«      \  }}|j                  |«      }|j	                  dd«      }|j                  |d   d|d   «      }n |	dk(  r›|j	                  dd«      }
|
j
                  }|
j                  d|d   «      }t        j                  j                  j                  |«      \  }}|j                  |«      }|j	                  dd«      }|j                  |d   |d   d«      }t        j                  j                  j                  |«      «      |j                  |	› d�<   n‡t        j                  j                  j                  |«      \  }}t        j                  j                  |j                  |j
                  d   d«      j                  |«      «      |j                  |	› d�<   t        j                  j                  j                  |«      «      |j                  |	<   |�||v r|j!                  |«       ~y	)
z@
        Quantizes weights into weight and weight_scale
        r   )rH   Úgate_up_projr	   éÿÿÿÿr   Ú	down_projÚ_scaleN)rN   rH   r   r2   Ú	transposeÚshapeÚreshaper    ÚopsÚfbgemmÚquantize_fp8_per_rowÚnnÚ	ParameterÚtoÚ_parametersÚviewÚremove)r   rA   rB   rC   rT   rD   rU   rH   rQ   rR   Útransposed_paramÚoriginal_shapeÚflattened_paramÚnew_value_flatÚweight_scale_flatÚ	new_valuerK   s                    r   Úcreate_quantized_paramz+FbgemmFp8HfQuantizer.create_quantized_param�   sU  € õ 	>ä2°5¸*ÓEÑˆ�Ü�fÐ8Õ9Ø˜nÒ,ð $/×#8Ñ#8¸¸AÓ#>Ð ð "2×!7Ñ!7�Ø"2×":Ñ":¸2¸~ÈbÑ?QÓ"R�ô 5:·I±I×4DÑ4D×4YÑ4YÐZiÓ4jÑ1�Ð 1ð +×2Ñ2°>ÓB�	Ø%×/Ñ/°°1Ó5�	Ø0×8Ñ8¸ÈÑ9JÈAÈ~Ð^_ÑO`Óa‘Ø Ò+ð $/×#8Ñ#8¸¸AÓ#>Ð ð "2×!7Ñ!7�Ø"2×":Ñ":¸2¸~ÈbÑ?QÓ"R�ô 5:·I±I×4DÑ4D×4YÑ4YÐZiÓ4jÑ1�Ð 1ð +×2Ñ2°>ÓB�	Ø%×/Ñ/°°1Ó5�	Ø0×8Ñ8¸ÈÑ9JÈNÐ[\ÑL]Ð_`Óa�ä9>¿¹×9KÑ9KÈLÏOÉOÐ\iÓLjÓ9kˆF×Ñ + ¨fÐ5Ò6ä&+§i¡i×&6Ñ&6×&KÑ&KÈKÓ&XÑ#ˆI�|Ü9>¿¹×9KÑ9KØ×!Ñ! ,×"4Ñ"4°QÑ"7¸Ó;×>Ñ>¸}ÓMó:ˆF×Ñ + ¨fÐ5Ñ6ô +0¯(©(×*<Ñ*<¸Y¿\¹\È-Ó=XÓ*Yˆ×Ñ˜;Ñ'àÐ&¨:¸Ñ+HØ×"Ñ" :Ô.Ùr   c                 ó   — |S r   © )r   rA   r   s      r   Ú#_process_model_after_weight_loadingz8FbgemmFp8HfQuantizer._process_model_after_weight_loadingÌ   s   € Øˆr   Úkeep_in_fp32_modulesc                 ó&  — ddl m} |j                  }| j                  || j                  j
                  |«      | _        |j                  } ||| j
                  | j                  | j                  ||¬«      }| j                  |j                  _        y )Nr   )Úreplace_with_fbgemm_fp8_linear)Úmodules_to_not_convertr   r1   ÚconfigÚtp_plan)rN   rs   Ú_tp_planÚget_modules_to_not_convertr   rt   ru   r1   )r   rA   rq   r   rs   rv   ru   s          r   Ú$_process_model_before_weight_loadingz9FbgemmFp8HfQuantizer._process_model_before_weight_loadingÏ   s…   € õ 	Bà—.‘.ˆØ&*×&EÑ&EØ�4×+Ñ+×BÑBÐDXó'
ˆÔ#ð —‘ˆÙ.ØØ#'×#>Ñ#>Ø $× 8Ñ 8Ø×,Ñ,ØØô
ˆð ,0×+CÑ+Cˆ�‰Õ(r   Úmissing_keysÚprefixc                 ó@  — ddl m}m} g }|j                  «       D ]h  \  }}t	        ||«      st	        ||«      sŒ|D ]E  }	||	v s
||› d|	› �v sŒ|	j                  d«      rŒ#|	j                  d«      rŒ5|j                  |	«       ŒG Œj |D �
cg c]	  }
|
|vsŒ|
‘Œ c}
S c c}
w )Nr   rF   ú.z.weightz.bias)rN   rG   rH   Únamed_modulesr2   ÚendswithÚappend)r   rA   rz   r{   rG   rH   Únot_missing_keysÚnamerQ   ÚmissingÚks              r   Úupdate_missing_keysz(FbgemmFp8HfQuantizer.update_missing_keysè   s«   € ßNàÐØ!×/Ñ/Ó1ò 	9‰LˆD�&Ü˜& /Ô2´jÀÐIcÕ6dØ+ò 9�Gà ™¨D°v°h¸aÀ¸yÐ4IÒ,IØ '× 0Ñ 0°Õ ;Ø '× 0Ñ 0°Õ 9à(×/Ñ/°Õ8ñ9ð	9ð (ÖE�a¨1Ð4DÒ+D’ÒEÐEùÒEs   Â
	BÂBc                 óä   — i dd“dd“dd“dd“dd“dd“dd	“d
d“dd“dd“dd“dd“dd“dd“dd“dd	“dd“dddddd	ddddœ	¥}|j                  «       �||j                  «       _        |S ||_        |S )Nz layers.*.self_attn.q_proj.weightÚlocal_colwisez&layers.*.self_attn.q_proj.weight_scalez layers.*.self_attn.k_proj.weightz&layers.*.self_attn.k_proj.weight_scalez layers.*.self_attn.v_proj.weightz&layers.*.self_attn.v_proj.weight_scalez layers.*.self_attn.o_proj.weightÚlocal_rowwisezlayers.*.self_attnÚgatherzlayers.*.input_layernorm.weightÚsequence_parallelz(layers.*.post_attention_layernorm.weightznorm.weightz4layers.*.feed_forward.shared_expert.gate_proj.weightz:layers.*.feed_forward.shared_expert.gate_proj.weight_scalez2layers.*.feed_forward.shared_expert.up_proj.weightz8layers.*.feed_forward.shared_expert.up_proj.weight_scalez4layers.*.feed_forward.shared_expert.down_proj.weightzlayers.*.feed_forward.expertsÚlocalÚlocal_packed_rowwise)	zlayers.*.feed_forwardz0layers.*.feed_forward.experts.*.gate_proj.weightz6layers.*.feed_forward.experts.*.gate_proj.weight_scalez.layers.*.feed_forward.experts.*.up_proj.weightz4layers.*.feed_forward.experts.*.up_proj.weight_scalez0layers.*.feed_forward.experts.*.down_proj.weightz*layers.*.feed_forward.experts.gate_up_projz0layers.*.feed_forward.experts.gate_up_proj_scalez'layers.*.feed_forward.experts.down_proj)Úget_text_configÚbase_model_tp_plan)r   ru   Ú	text_plans      r   Úupdate_tp_planz#FbgemmFp8HfQuantizer.update_tp_plan÷   s3  € ð
Ø.°ð
à4°oð
ð /°ð
ð 5°oð	
ð
 /°ð
ð 5°oð
ð /°ð
ð ! (ð
ð .Ð/Bð
ð 7Ð8Kð
ð Ð.ð
ð CÀOð
ð IÈ/ð
ð AÀ/ð
ð GÈð
ð  CÀOð!
ð" ,¨Wð#
ð$ &.Ø@OØFUØ>MØDSØ@Oà:PØ@VØ7Fò7
ˆ	ð: ×!Ñ!Ó#Ð/Ø:CˆF×"Ñ"Ó$Ô7ð ˆð )2ˆFÔ%Øˆr   c                  ó   — y)NTro   )r   Úsafe_serializations     r   Úis_serializablez$FbgemmFp8HfQuantizer.is_serializable  s   € Ør   c                  ó   — y)NFro   )r   s    r   Úis_trainablez!FbgemmFp8HfQuantizer.is_trainable  s   € àr   )r?   útorch.dtyper:   r–   r   )rA   r   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__Ú requires_parameters_quantizationÚrequires_calibrationÚrequired_packagesr   r9   r@   Ústrr   r   rS   r   r   rm   rp   ry   r…   r�   r“   ÚpropertyÚboolr•   Ú__classcell__)r   s   @r   r   r   $   s7  ø„ ñð (,Ð$Ø Ðà% |Ð4Ðô7ò+óZð à ðð $ðð ð	ð
 ˜˜c˜‘NóðJ 04ñ=à ð=ð $ð=ð ð	=ð
 &ð=ð ˜˜c˜‘Nð=ð " $ s¡)Ñ,ó=ó~ð 59ñDà ðDð ' t¨C¡yÑ1óDð2F°t¸C±yð FÈ#ð FÐRVÐWZÑR[ó Fò"óHð ð˜dò ó ôr   r   )r&   Útypingr   r   r   r   r   Ú	packagingr   Úbaser
   Úmodeling_utilsr   Úutilsr   r   r   r   Úquantizers_utilsr   r    Ú
get_loggerr—   r/   r   ro   r   r   ú<module>r©      sT   ðó ß ;Õ ;å å ñ Ý0ç aÓ aÝ 2ñ ÔÛð 
ˆ×	Ñ	˜HÓ	%€ô|˜;õ |r   