Ë
    g^(h  ã                   óò  — d dl Z d dlmZ d dlZd dlmZ ddlmZm	Z	 ddl
mZmZ ddlmZ ddl	mZmZ dd	lmZ dd
lmZmZmZ ddlmZmZmZ ddlmZ  e j:                  e«      Z eej@                  dd¬«      Z! eejD                  jF                  jH                  ddejJ                  ¬«      Z&ejD                  jF                  Z#ejD                  jN                  Z'ejD                  jP                  Z(dd„Z)dd„Z*y)é    N)ÚAny)Úmm_argsé   )ÚconfigÚlowering)ÚCppGemmTemplateÚCppWoqInt4GemmTemplate)Úcreate_epilogue_with_attr)ÚexpandÚregister_lowering)ÚWeightInt4PackMatmul)Úautotune_select_algorithmÚExternKernelChoiceÚrealize_inputs)Úuse_aten_gemm_kernelsÚuse_cpp_gemm_templateÚuse_max_autotune)ÚVzat::_weight_int8pack_mmF)Úhas_out_variantz*at::native::_weight_int4pack_mm_cpu_tensor)r   Úkernel_creatorc                  óZ  — t        j                  t        j                  t        j
                  t        j                  g«       t        j                  t        j                  «       t        j                  t        j
                  «       t        j                  t        j                  «       y ©N)r   Úadd_needs_realized_inputsÚ	quantizedÚ
max_pool2dÚ
_quantizedÚ$wrapped_fbgemm_pack_gemm_matrix_fp16Ú!wrapped_fbgemm_linear_fp16_weightÚmake_fallback© ó    úa/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/_inductor/quantized_lowerings.pyÚregister_quantized_opsr#   '   sn   € Ü×&Ñ&ä× Ñ Ü×;Ñ;Ü×8Ñ8ð	
ôô ×Ñœ9×/Ñ/Ô0Ü×Ñœ:×JÑJÔKÜ×Ñœ:×GÑGÕHr!   c                  ó   — t        t        j                  d ¬«      d dœdt        j                  dt        j                  dt        j                  dt
        dt
        f
d„«       } t        t        j                  d ¬«      d dœdt        j                  dt        j                  d	t        d
t        j                  dt
        dt
        fd„«       }t        j                  t        j                  «       t        j                  t        j                  «       y )N)Útype_promotion_kind)ÚlayoutÚinputÚweightÚscaler&   Úreturnc                óÂ  ‡‡— t        | |‰d¬«      \  }}}Š}}|j                  «       t        j                  t        j                  t        j
                  fv r!|j                  «       t        j                  k(  sJ ‚‰}t        «       rt        j                  ||‰f|«      gng }dt        j                  dt        fˆˆfd„}	t        |||d¬«      rt        j                  ||||‰gd|	¬«       t        |«      dk(  rVt         j"                  rFt        «       s<t$        j'                  d	«       t        j                  ||‰f|«      j)                  «       S t+        d
|||‰g|«      S )NT)r&   Úmat2_transposedÚbufr*   c           
      óZ   •— t        | dt        t        ‰‰j                  «      «      ¬«      S )NÚmul)Úother)r
   r   r   Úsize)r-   r&   r)   s    €€r"   Ú_mul_epiloguez?register_woq_mm_ops.<locals>.int8pack_mm.<locals>._mul_epilogueO   s'   ø€ Ü,Ø�U¤.´¸¸v¿{¹{Ó1KÓ"Lôð r!   )r,   )Útrans_wÚepilogue_creatorr   ú3No choices for GEMM, using ATen backend as fallbackÚ_weight_int8pack_mm)r   Ú	get_dtypeÚtorchÚbfloat16Úfloat16ÚfloatÚint8r   Úaten__weight_int8pack_mmÚbindÚTensorr   r   r   Úadd_choicesÚlenÚinductor_configÚautotune_fallback_to_atenÚlogÚwarningÚoutput_noder   )
r'   r(   r)   r&   Ú_Úmat1Úmat2Úaten_layoutÚchoicesr2   s
     ``      r"   Úint8pack_mmz(register_woq_mm_ops.<locals>.int8pack_mm5   sS  ù€ ô '.Ø�6 &¸$ô'
Ñ#ˆˆ1ˆa�˜˜tð �N‰NÓ¤§¡´·±ÄÇÁÐ LÑLØ—‘Ó ¤E§J¡JÒ.ð	
ð/ð ˆô
 %Ô&ô &×*Ñ*¨D°$¸Ð+>ÀÓLÑMàð 	ð	œuŸ|™|ð 	´ö 	ô
 ! ¨d°DÈ$ÕOÜ×'Ñ'ØØØ�t˜UÐ#ØØ!.õô �‹L˜AÒÜ×9Ò9Ü)Ô+ä�K‰KÐMÔNÜ+×0Ñ0Ø�t˜UÐ# [óç‰k‹mðô )Ø! 7¨T°4¸Ð,?Àó
ð 	
r!   Ú
qGroupSizeÚqScaleAndZerosc                óê  — t        | ||dd¬«      \  }}}}}}|j                  «       t        j                  t        j                  t        j
                  fv r!|j                  «       t        j                  k(  sJ ‚t        j                  j                  t        j                  |t        j                  ¬«      d ¬«      }|}	t        «       rt        j                  ||||f|	«      gng }
t        «       rMt!        |	||dd|¬«      r<|j#                  «       j%                  «       rt&        |   j)                  |
|	||||g«       t+        |
«      dk(  rWt,        j.                  rGt        «       s=t0        j3                  d«       t        j                  ||||f|	«      j5                  «       S dt        j6                  j8                  j:                  d	t        j<                  fd
„}|d„ dœ}t?        d|
||||g|	|¬«      S )NT)r&   Úuse_4x2_dimr,   )Údtype)Úname)r,   Úis_woq_int4Úq_group_sizer   r5   Úxr*   c                 óÒ   — | j                  «       j                  «       sJ ‚| j                  «       }| j                  «       }t	        j
                  dd|t        j                  |¬«      S )Nr   éÿ   )rQ   Údevice)Ú
get_layoutÚis_contiguousÚget_sizeÚ
get_devicer8   ÚrandintÚuint8)rU   ÚshaperX   s      r"   Úget_example_weightzHregister_woq_mm_ops.<locals>.int4pack_mm_cpu.<locals>.get_example_weight¨   sK   € Ø—<‘<“>×/Ñ/Ô1Ð1Ð1Ø—J‘J“LˆEØ—\‘\“^ˆFÜ—=‘=  C¨´e·k±kÈ&ÔQÐQr!   c                 óX   — t         j                  j                  | j                  «          S r   )r   ÚgraphÚ	constantsÚget_name)rU   s    r"   ú<lambda>z>register_woq_mm_ops.<locals>.int4pack_mm_cpu.<locals>.<lambda>°   s   € œŸ™×*Ñ*¨1¯:©:«<Ñ8€ r!   )r   é   Ú_weight_int4pack_mm_for_cpu)Úinput_gen_fns) r   r7   r8   r9   r:   r;   r^   r   rb   Úadd_tensor_constantÚtensorÚint64r   Úaten__weight_int4pack_mm_cpur>   r   r   rY   rZ   r	   r@   rA   rB   rC   rD   rE   rF   Ú	_inductorÚirÚIRNoder?   r   )r'   r(   rM   rN   r&   rG   rH   rI   Ú
group_sizerJ   rK   r`   rh   s                r"   Úint4pack_mm_cpuz,register_woq_mm_ops.<locals>.int4pack_mm_cpuk   sà  € ô '.Ø�6 &°dÈDô'
Ñ#ˆˆ1ˆa�˜˜tð �N‰NÓ¤§¡´·±ÄÇÁÐ LÑLØ—‘Ó ¤E§K¡KÒ/ð	
ð0ô —W‘W×0Ñ0Ü�L‰L˜¬5¯;©;Ô7¸dð 1ó 
ˆ
ð ˆô %Ô&ô	 -×1Ñ1Ø˜4 ¨^Ð<¸kóñð ð 	ô ÔÜ%ØØØØ $Ø Ø'õð —‘Ó!×/Ñ/Ô1ä" :Ñ.×:Ñ:ØØØ�t˜Z¨Ð8ôô �‹L˜AÒÜ×9Ò9Ü)Ô+ä�K‰KÐMÔNÜ/×4Ñ4Ø�t˜Z¨Ð8¸+óç‰k‹mðð	R¤%§/¡/×"4Ñ"4×";Ñ";ð 	RÄÇÁó 	Rð "Ù8ñ
ˆô
 )Ø)ØØ�4˜ ^Ð4ØØ'ô
ð 	
r!   )r   Úatenr6   r8   r?   r   rg   Úintr   r   Ú_dyn_quant_matmul_4bitÚ_dyn_quant_pack_4bit_weight)rL   rq   s     r"   Úregister_woq_mm_opsrv   4   s   € Ü”t×/Ñ/ÀTÔJð ò3
Ü�|‰|ð3
ä—‘ð3
ô �|‰|ð3
ô
 ð3
ô 
ò3
ó Kð3
ôj ”t×7Ñ7ÈTÔRð òM
Ü�|‰|ðM
ä—‘ðM
ô ðM
ô Ÿ™ð	M
ô ðM
ô 
òM
ó SðM
ô^ ×Ñœ4×6Ñ6Ô7Ü×Ñœ4×;Ñ;Õ<r!   )r*   N)+ÚloggingÚtypingr   r8   Ú torch._inductor.kernel.mm_commonr   Ú r   rB   r   Úcodegen.cpp_gemm_templater   r	   Úcodegen.cpp_utilsr
   r   r   Ú	mkldnn_irr   Úselect_algorithmr   r   r   Úutilsr   r   r   Úvirtualizedr   Ú	getLoggerÚ__name__rD   r6   r=   Úopsr   Úint4mm_packed_weight_cpuÚcreaterl   r   rr   r#   rv   r    r!   r"   ú<module>r†      sË   ðÛ Ý ã Ý 4ç 1ß NÝ 8ß /Ý +÷ñ ÷
 RÑ QÝ ð €g×Ñ˜Ó!€á-Ø	×ÑÐ8È%ôÐ ñ  2Ø	‡I�I×Ñ×0Ñ0Ø0ØØ'×.Ñ.ô	 Ð ð �I‰I×Ñ€	Ø�Y‰Y×!Ñ!€
Ø‡y�y‡~�~€ó
IôH=r!   