Ë
    g^(h.$  ã                   óÂ  — d Z ddlZddlmZ ddlmZ ddlZddlmZ ddl	m
Z
 ddlmZ ddlmZmZmZmZmZmZ dd	lmZmZmZmZmZ dd
lmZ ddlmZ d„ Zdej>                  j@                  fd„Z!dej>                  j@                  dee"   fd„Z#dej>                  j@                  dee"   fd„Z$de%fd„Z&de'ee"      fd„Z(d„ Z) G d„ d«      Z* ed e*«       ¬«       dd„Z+y)a�  
This module implements CUDA graphs support for TorchDynamo backends.

CUDA graphs allow for capturing and replaying GPU operations, which can significantly
reduce CPU overhead in GPU-accelerated PyTorch models. This module provides:

- CUDA graph creation and management for both forward and backward passes
- Input mutation detection and handling
- Device compatibility checking
- Stack trace management for debugging
- Integration with TorchInductor's cudagraph trees

The backend supports two main modes:
1. cudagraphs: Full CUDA graph support with both forward and backward pass optimization
2. cudagraphs_inner: Lower-level CUDA graph implementation used for benchmarking

Key components:
- CudagraphsBackend: Main backend class for CUDA graph integration
- Mutation detection utilities to ensure graph safety
- Device mapping and compatibility checks
- Stack trace collection for debugging
é    N)Údefaultdict)ÚOptional)Úconfig)Úaot_autograd)Ú	boxed_nop)ÚBoxedDeviceIndexÚ'check_multiple_devices_or_any_cpu_nodesÚformat_default_skip_messageÚget_mutation_stack_traceÚget_placeholder_infoÚ#log_cudagraph_skip_and_bump_counter)Ú	BoxedBoolÚcount_tangentsÚ%get_first_incompatible_cudagraph_nodeÚnum_fw_fixed_argumentsÚoutput_node)ÚStorageWeakRefé   )Úregister_backendc           	      ót  — d„ }t        t        «      }d}t        «       }| j                  D �]‰  }|j                  dk(  rkt	         ||j
                  «      t        j                  «      r;|t         ||j
                  «      j                  «       «         j                  |«       |dz  }Œ~|j                  dk(  sŒŽt        |j                  d«      sŒ¥|j                  j                  }t        |j                  «      D ]·  \  }}|t!        |j"                  «      k  r|j"                  |   }	n2|j$                  |j&                  vrŒG|j&                  |j$                     }	d}
|j(                  r|j(                  j*                  rd}
|
sŒ‰||t         ||	j
                  «      j                  «       «         z  }Œ¹ �ŒŒ |S )	Nc                 ó   — d| v r| d   S | d   S )NÚvalÚfake_result© )Úmetas    ú_/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/_dynamo/backends/cudagraphs.pyÚmeta_fkz%find_input_mutations.<locals>.meta_fk7   s   € Ø# t™mˆt�E‰{ÐD°°mÑ1DÐDó    r   Úplaceholderr   Úcall_functionÚ_schemaFT)r   ÚsetÚnodesÚopÚ
isinstancer   ÚtorchÚTensorr   Ú_typed_storageÚaddÚhasattrÚtargetr!   Ú	enumerateÚ	argumentsÚlenÚargsÚnameÚkwargsÚ
alias_infoÚis_write)Úgr   ÚinputsÚ	input_idxÚmutated_inputsÚnÚschemaÚiÚargÚargumentÚmut_args              r   Úfind_input_mutationsr>   6   sh  € òEô œÓ€FØ€IÜ“U€NØ�W‰Wó ˆØ�4‰4�=Ò Ü™' !§&¡&›/¬5¯<©<Ô8Ø”~¡g¨a¯f©f£o×&DÑ&DÓ&FÓGÑH×LÑLÈYÔWØ˜‰N‰IØ�T‰T�_Ó$Ü˜1Ÿ8™8 YÔ/Øà—X‘X×%Ñ%ˆFÜ# F×$4Ñ$4Ó5ò ‘��3Ø”s˜1Ÿ6™6“{’?Ø Ÿv™v a™y‘Hà—x‘x q§x¡xÑ/Ø Ø Ÿx™x¨¯©Ñ1�HØ�Ø—>’>Ø—~‘~×.Ò.Ø"&˜Úð # fÜ&¡w¨x¯}©}Ó'=×'LÑ'LÓ'NÓOñ'ñ ‘Nòðð: Ðr   Úgmc                 óê   — i }| j                   j                  D ]W  }|j                  j                  dd «      }t	        |t
        j                  «      sŒ:|j                  |vsŒI|||j                  <   ŒY |S )Nr   )Úgraphr#   r   Úgetr%   r&   r'   Údevice)r?   Údevice_node_mappingr8   Úts       r   Úget_device_node_mappingrF   ]   sd   € Ø=?ÐØ�X‰X�^‰^ò .ˆØ�F‰F�J‰J�u˜dÓ#ˆÜ�aœŸ™Õ&¨1¯8©8Ð;NÒ+NØ,-Ð §¡Ò)ð.ð Ðr   Ú	aot_modelÚreturnc                 óž   — t        | j                  «      t        t        |«      «      z
  }|sy t	        | j                  «      }t        ||«      S ©N)r>   rA   r"   Úranger   r   )rG   Ú	num_fixedÚmutation_indicesÚplaceholderss       r   Ú3check_for_mutation_ignore_cuda_graph_managed_tensorrO   f   sD   € ô ,¨I¯O©OÓ<¼sÄ5ÈÓCSÓ?TÑTÐÙØä'¨	¯©Ó8€LÜ# LÐ2BÓCÐCr   c                 óÀ   — t         j                  st        | |«      x}r|S t        t	        | «      «      x}r|S t        | «      x}rt        d|j                  › d�«      S y )Nzincompatible op (ú))r   Ú(cudagraph_backend_support_input_mutationrO   r	   rF   r   r
   r0   )rG   rL   Úmut_skipÚskipÚnodes        r   Úcheck_for_skiprV   q   sz   € Ü×:Ò:ÜJØ�yó
ð 
ˆ8ð 
ð ˆOä6Ü 	Ó*óð €tð ð ˆä4°YÓ?Ð?€tÐ?Ü*Ð->¸t¿y¹y¸kÈÐ+KÓLÐLàr   c                 óv   — t        t        t        | «      «      «      }|j                  dk(  sJ ‚|j                  S )NÚcuda)ÚnextÚiterrF   ÚtypeÚindex)r?   rC   s     r   Úget_device_indexr]   ƒ   s3   € Ü”$Ô.¨rÓ2Ó3Ó4€FØ�;‰;˜&Ò Ð Ð Ø�<‰<Ðr   c                 ó  — t        | «      }t        |j                  «      dk(  sJ ‚|j                  d   D �cg c]>  }t        |t        j
                  j                  j                  «      r|j                  nd ‘Œ@ c}S c c}w )Nr   r   )	r   r.   r/   r%   r&   ÚfxrU   ÚNodeÚstack_trace)r?   Úoutputr;   s      r   Úget_stack_tracesrc   ‰   sl   € Ü˜‹_€FÜˆv�{‰{Ó˜qÒ Ð Ð ð —;‘;˜q‘>öàô ' s¬E¯H©H¯M©M×,>Ñ,>Ô?ˆ�ŠÀTÑ	Iòð ùò s   ·AA=c                 ó   ‡‡‡‡— ddl mŠ t        d«      Št        d «      Šdˆˆˆˆfd„	}ˆˆˆfd„}t	        ||t        j                  |d¬«      t        j                  j                  j                  ¬«      } || ‰«      S )	Nr   )Úcudagraphify_implTc                 óª  •— t        | |«      }t        t        ‰
«      t        |«      «      }t        | |«      x}r%t	        j
                  ‰	«       t        d|› �«       |S ‰j                  t        | «      «        ‰||t        |«      ‰j                  ddt        | «      t        | j                  «      t        | j                  «      ¬«	      }d|_        |S )Nzskipping cudagraphs due to F©Údevice_indexÚis_backwardÚis_inferenceÚstack_tracesrN   Úmutated_input_idxsT)r   r   r.   rV   r   Údisabler   r"   r]   rK   Úvaluerc   r   rA   r>   Ú_boxed_call)rG   Ú
aot_inputsrj   ÚinterpÚfixedÚskip_msgÚoutÚboxed_device_indexre   Údo_cudagraphsÚdynamo_inputss          €€€€r   Úforward_cudagraphsz&cudagraphs.<locals>.forward_cudagraphs˜   sÅ   ø€ Ü˜9 jÓ1ˆÜ&¤s¨=Ó'9¼3¸z»?ÓKˆÜ% i°Ó7Ð7ˆ8Ð7Ü×Ñ˜mÔ,Ü/Ø-¨h¨ZÐ8ôð ˆMà×ÑÔ/°	Ó:Ô;ÙØØÜ�%‹LØ+×1Ñ1ØØÜ)¨)Ó4Ü-¨i¯o©oÓ>Ü3°I·O±OÓDô

ˆð ˆŒØˆ
r   c                 ó¸  •‡ ‡— t        ‰ |«      }‰
s‰ S t        ‰ «      }t        ‰ |«      x}rTt        d|«       t        j
                  j                  j                  ‰j                  d¬«      Š‰€J ‚ˆ ˆfd„}d|_	        |S  ‰	||t        |«      t        ‰ «      ddt        ‰ «      t        ‰ j                  «      t        ‰ j                  «      ¬«	      }d|_	        |S )Nzskipping cudagraphs due to %sF)Úcreate_if_none_existsc                 ó4   •— ‰j                  «         ‰| «      S rJ   )Úset_to_running_backward)r5   rG   Úmanagers    €€r   Úfnz3cudagraphs.<locals>.backward_cudagraphs.<locals>.fnÂ   s   ø€ Ø×/Ñ/Ô1Ù  Ó(Ð(r   Trg   )r   r   rV   r   r&   Ú	_inductorÚcudagraph_treesÚget_managerrn   ro   rK   r]   rc   r   rA   r>   )rG   rp   rq   rr   rs   r~   rt   r}   ru   re   rv   s   `      @€€€r   Úbackward_cudagraphsz'cudagraphs.<locals>.backward_cudagraphs±   sâ   ú€ Ü˜9 jÓ1ˆÙØÐä˜yÓ)ˆÜ% i°Ó7Ð7ˆ8Ð7Ü/Ø/°ôô
 —o‘o×5Ñ5×AÑAØ"×(Ñ(Àð Bó ˆGð Ð&Ð&Ð&õ)ð "ˆBŒNØˆIáØØÜ�%‹LÜ)¨)Ó4ØØÜ)¨)Ó4Ü-¨i¯o©oÓ>Ü3°I·O±OÓDô

ˆð ˆŒØˆ
r   )rj   )Úfw_compilerÚbw_compilerÚinference_compilerÚkeep_inference_input_mutations)F)Útorch._inductor.cudagraph_treesre   r   r   r   Ú	functoolsÚpartialr&   Ú_dynamor   Ú%cudagraph_backend_keep_input_mutation)Údynamo_modelrw   rx   r‚   Úaot_cudagraphsru   re   rv   s    `   @@@r   Ú
cudagraphsrŽ   ’   so   û€ ÝAä˜d“O€MÜ)¨$Ó/Ð÷ð ö2$ôL "Ø&Ø'Ü$×,Ñ,Ð-?ÈdÔSÜ',§}¡}×';Ñ';×'aÑ'aô	€Nñ ˜,¨Ó6Ð6r   c                   ó0   — e Zd ZdZed„ «       Zed„ «       Zy)ÚCudagraphsBackendrŽ   c                  ó   — ddl m}   | «        y )Nr   ©Úreset_cudagraph_trees)r‡   r“   r’   s    r   ÚresetzCudagraphsBackend.resetã   s   € åIáÕr   c                 ó   — t        | |«      S rJ   )rŽ   )Úmodelr5   s     r   Ú__call__zCudagraphsBackend.__call__é   s   € ä˜% Ó(Ð(r   N)Ú__name__Ú
__module__Ú__qualname__Úcompiler_nameÚstaticmethodr”   r—   r   r   r   r�   r�   à   s-   „ Ø €Màñ ó ð ð
 ñ)ó ñ)r   r�   rŽ   )r0   Úcompiler_fnc                 óœ  ‡‡‡‡‡	— t        |t        t        f«      sJ ‚‰r$|D �cg c]  }t        j                  |«      ‘Œ c}Šnt        |«      Št        j
                  j                  «        t        j
                  j                  «       }|j                  t        j
                  j                  «       «       t        j
                  j                  |«      5   | |Ž  ddd«       |j                  «        t        j
                  j                  «       j                  |«       t        j
                  j                  «        t        j
                  j                  «       Št        j
                  j                  ‰|¬«      5   | ‰Ž Š	ddd«       t        ‰	t        t        f«      s‰	fŠ	ˆˆˆˆˆ	fd„}|S c c}w # 1 sw Y   ŒÚxY w# 1 sw Y   Œ>xY w)zBThis isn't registered as a backend, but is used in some benchmarksN)Ústreamc                  óò   •— t        ‰«      t        | «      k(  sJ ‚‰r%t        ‰| «      D ]  \  }}|j                  |«       Œ ‰j                  «        ‰r‰D �cg c]  }|j	                  «       ‘Œ c}S ‰S c c}w rJ   )r.   ÚzipÚcopy_ÚreplayÚclone)	Ú
new_inputsÚdstÚsrcÚxÚcopy_inputsÚcopy_outputsrA   Ústatic_inputsÚstatic_outputss	       €€€€€r   Úrunzcudagraphs_inner.<locals>.run  sp   ø€ Ü�=Ó!¤S¨£_Ò4Ð4Ð4ÙÜ ¨zÓ:ò ‘��SØ—	‘	˜#•ðà�‰ŒÙØ'5Ö6 !�A—G‘G•IÒ6Ð6à!Ð!ùò 7s   ÁA4)r%   ÚlistÚtupler&   Ú
zeros_likerX   ÚsynchronizeÚStreamÚwait_streamÚcurrent_streamrŸ   Ú	CUDAGraphrA   )
r–   r5   rª   r©   r¨   rŸ   r­   rA   r«   r¬   s
     ``   @@@r   Úcudagraphs_innerr¶   ó   sY  ü€ ä�fœt¤U˜mÔ,Ð,Ð,ÙØ6<Ö=°œ×)Ñ)¨!Õ,Ò=‰ä˜V›ˆô 
‡J�J×ÑÔÜ�Z‰Z×ÑÓ €FØ
×Ñ”u—z‘z×0Ñ0Ó2Ô3Ü	�‰×	Ñ	˜6Ó	"ñ Ùˆv‰÷à
×ÑÔÜ	‡J�J×ÑÓ×+Ñ+¨FÔ3Ü	‡J�J×ÑÔô �J‰J× Ñ Ó"€EÜ	�‰×	Ñ	˜%¨Ð	Ó	/ñ /Ù Ð.ˆ÷/ä�n¤t¬U mÔ4Ø(Ð*ˆ÷	"ð 	"ð €JùòA >÷ð ú÷/ð /ús   ¤F1ÃF6Å?GÆ6F?ÇG)TT),Ú__doc__rˆ   Úcollectionsr   Útypingr   r&   Útorch._dynamor   Útorch._dynamo.backends.commonr   Ú torch._dynamo.backends.debuggingr   Útorch._inductor.cudagraph_utilsr   r	   r
   r   r   r   Útorch._inductor.utilsr   r   r   r   r   Ú torch.multiprocessing.reductionsr   Úregistryr   r>   r_   ÚGraphModulerF   ÚstrrO   rV   Úintr]   r®   rc   rŽ   r�   r¶   r   r   r   ú<module>rÄ      sç   ðñó. Ý #Ý ã Ý  Ý 6Ý 6÷÷ ÷õ õ <å &ò$ðN §¡× 4Ñ 4ó ðDØ�x‰x×#Ñ#ðDàˆc�]óDð˜eŸh™h×2Ñ2ð À(È3Á-ó ð$˜Có ð˜D ¨#¡Ñ/ó òK7÷\)ñ )ñ  �lÑ0AÓ0CÕ Dô$r   