g^(hv dZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlmZddlmZddlmZmZmZmZddlZddlmZddlZddlmZddlmZddlm Z dd l!m"Z"dd l#m$Z$m%Z%d d l&m'Z'd d lm(Z(m)Z)ejTe+Z,edZ-edZ.e.j_Z0e0rddl1Z2gZ3dZ4e0regdZ3e2jjjljnjqjsddZ:djwe3Dcgc]}d|d c}Z4gdZ<GddZ=dZ>dZ?GddZ@ejddZBdd d!ZCdd d"ZDd#ZEd$ZFGd%d&eGZHd'ZIdEd(ZJ dFddd)d*ZKd+ZLd,ZMd-ZN dFddd)d.ZOd/ed0d1d2d3d0fd4ZPd5e-d3eee-ge-ffd6ZQeQejZSeQejd7ZUeQdZVeQdZWeQdZXGd8d9ZYGd:d;ZZGd<d=Z[ dGd>ee\ege\efd?e]d@ee^e]e_fdAee_d3e^e]eff dBZ`dCe]d3ee-ge-ffdDZaycc}w)Ha Debug utilities for TorchDynamo compilation and execution. This module provides various debugging tools and utilities for TorchDynamo, including: - Minification support for reducing test cases while preserving bugs - Input/output handling via InputReader and InputWriter for reproducible testing - Accuracy checking between original and compiled models - Neural network module string conversion via NNModuleToString - Profiling tools and system information collection - Buck build system integration for Meta-internal testing Key classes: - InputReader/InputWriter: Handle serialization of model inputs/outputs - NNModuleToString: Converts nn.Modules to string representations - BuckTargetWriter: Manages Buck build system integration N)Counter) import_module)AnyCallableOptionalTypeVar)Tensor) rand_strided)is_float_dtype)StorageWeakRef)ContentStoreReaderContentStoreWriter)config) clone_inputs get_debug_dirTztorch._inductor.config)z1//caffe2/torch/fb/sparsenn:sparsenn_operators_gpuz-//caffe2/torch/fb/sparsenn:sparsenn_operatorsz///deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpuz+//deeplearning/fbgemm/fbgemm_gpu:sparse_opszfbcode:// ztorch.ops.load_library("z"))buck2runz@mode/dev-nosanc eZdZdZdZddZy)BuckTargetWriterc.tjjtjj|\|_|_|j j dd|_|jj ddd|j|_|j|jjdd|_|jdd|_|j}||jdddd}d|d |j|_ y) Nz.pyr/.zfbcode.zfbcode/r:) ospathsplitabspathsubdirpy_filereplacetargetfind cmd_line_path)selffilenametmps W/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/_dynamo/debug_utils.py__init__zBuckTargetWriter.__init__Rs$&GGMM"''//(2K$L! T\ll**5"5 {{**345Qt{{mD IIdiinnY79: IIabM kk#((9%'(,!#a }5c djtDcgc]}d|d c}}tjd|jd|j dt d|d|jd Scc}w) Nrz "z",za load("@fbcode_macros//build_defs:python_binary.bzl", "python_binary") python_binary( name="z", srcs = ["z"], compile = False, deps = [ "//caffe2:torch", "//caffe2:libtorch", "//caffe2/functorch:functorch", "//triton:triton", "z", ], cpp_deps = [ z ], main_module = "z", par_style = "xar", ) )join extra_depstextwrapdedentr'r% cur_targetr!)r*xextra_cpp_depss r-buildzBuckTargetWriter.build`sz#J!is"$5#JK  ;;-ll^  II;#   $Ks A/cPtjj|jd}t |d5}|j |j dddt|jgz}|r%tjddj||S#1swYFxYw)NTARGETSwzFFound an example that reproduces the error. Run this cmd to repro - %s ) r r!r1r$openwriter8BUCK_CMD_PREFIXr)logwarning)r* print_msg target_filefd cmd_splits r-r>zBuckTargetWriter.writezsggll4;; : +s # #r HHTZZ\ " #$t'9'9&::  KKX#  # #s BB%N)T)__name__ __module__ __qualname__r.r8r>r/r-rrQs 6 4 r/rctjjtd}|+t j dt j}tjj|stj|d|S)Nminifierz /minifier_T)exist_ok) r r!r1rtempfile gettempdirgetpassgetuserexistsmakedirs)r!s r- minifier_dirrSsb 77<<  4D |%%'( 7??3D2EF 77>>$  D4( Kr/ceZdZejj ejj ejjejjejjejjejjejjejjejjejjejj ejj"ejj$ejj&ejj(ejj*ejj,ejj.ejj0ejj2gZedZedZy)NNModuleToStringct}|jD]2\}}t|tjvs"|j |4t |dkDrtjd|y)Nrz-We have not tested reprs of some modules - %sT) setnamed_childrentyperV safe_reprsaddlenr@rA)gm cant_convert_modules r-can_convert_to_stringz&NNModuleToString.can_convert_to_stringsgu **, )IAvF|#3#>#>>  ( ) | q KKG Vr/cddlm}d}tjd}|j D]T\}}|j }t |jd}||jr|d}||dzd|d|d z }V|jjD]\}} |  | jtkr'dd l m} | jtk\sJt!| } nbt#j$| r'd t'| j(d | j*d } n&dt'| j(d | j*d } | jr| d} ||dzd|d| dz }|j,jD]Q\} } |  d}| jrd}dt'| j(d | j*|d} ||dzd| d| d z }S|||j.dd z }|S)Nr) _addindent z from torch.nn import * class Repro(torch.nn.Module): def __init__(self) -> None: super().__init__() z.cuda()zself.z = r) PRINT_OPTSz torch.randn(z, dtype=)ztorch.randint(1, size=zself.register_buffer('z', z) rz, device="cuda"ztorch.nn.Parameter(torch.randn(z))rT)torch.nn.modules.modulerdr3r4rY__repr__next parametersis_cuda_buffersitemsnumelMAX_CONSTANT_NUMEL_INLINEtorch._tensor_strrg thresholdreprtorchis_floating_pointlistshapedtype _parameterscode)r^rdtab model_str module_namera module_str example_param buffer_namebufferrg tensor_str param_nameparam maybe_devices r-convertzNNModuleToString.converts>6OO  $#4#6 I K"OO-.J!!2!2!4d;M(]-B-B *|73 C!G9E+c*RH HI I$&;;#4#4#6  K~||~!::8!++/HHHH!&\ ((0+D,>+?x ~UVW -T&,,-?,@VWX~~ *|73 7)1+c*SQ I# *"$!5!5!7 H J}L}}0 :4 ;L:MXV[VaVaUbcobpprsJ C!G9E*S BG GI H  277A./r22 r/N)rFrGrHrunnLinearConv1dConv2dConv3d BatchNorm1d BatchNorm2d BatchNorm3d LayerNormDropoutSoftmaxReLUGELUIdentity MaxPool2d EmbeddingTanhConvTranspose1dGLULSTMFlattenAdaptiveAvgPool2dr[ staticmethodrbrrIr/r-rVrVs0                            ""+J0  ==r/rVc2tjjsyd} tjddg}|j j d}dj|Dcgc] }|dvsd|d c}}||dz }td ttjjD}|d z }|jD]\}}|d|d |d z }|dz }|Scc}w#ttjf$r|d z }YwxYw)Nz:# torch.cuda.is_available()==False, no GPU info collected z# CUDA Info: nvccz --versionrr)r# z z# nvcc not found c3ZK|]#}tjj|%ywN)rucudaget_device_name).0is r- z,_cuda_system_info_comment..s$*+ ""1%s)+z# GPU Hardware Info: z : )rur is_available subprocess check_outputdecoder"r1FileNotFoundErrorCalledProcessErrorrrange device_countro)r}cuda_version_outcuda_version_linesscomment gpu_namesnamecounts r-_cuda_system_info_commentrs- :: " " $L!I*%22FK3HI-446<.filter..s:V6S=:s)any)r allow_list skip_lists`r-filterz(generate_env_vars_string..filters:z::Ss)?SSr/z os.environ['z'] = ''rz import os z )r environror1)rrrvalue config_lines config_stringrrs @@r-generate_env_vars_stringrs<:JUIT ****, C #; se6%*L IIl+M  sA*c ^ddl}ddl}|ry|jjjj }d|j jj d|jjj d|jjj d|d S)Nrz*# config omitted due to stable_output=Truez~import torch._dynamo.config import torch._inductor.config import torch._functorch.config import torch.fx.experimental._config r) torch._functorch.configtorch._inductor.configfx experimental_configcodegen_config_dynamor _inductor _functorch)rruexperimental_configs r-generate_config_stringr,s"!;((//77FFH $$&'(&&()*'')*+  r/cRtjjtdS)Nzminifier_launcher.py)r r!r1rSrIr/r-get_minifier_repro_pathr@s 77<< (> ??r/cLt}tjd|trt |j  t |d5}|j |dddy#1swYyxYw#t$r&}tjdtd|d}~wwxYw)NzWriting minified repro to: %sr;rz(Could not write to {minified_repro_path}) rr@rAuse_buckrr>r=OSError exceptionNotImplementedError)contentsminified_repro_pathrDes r-helper_for_dump_minifyrDs13KK02EF,-335U %s + r HHX     U b!"LMSTTUs6 A4 A(A4(A1-A41A44 B#=!BB#c eZdZy) AccuracyErrorN)rFrGrHrIr/r-rrSsr/rct|}tt|D]A}t||tj s!||j ||jC|S)z This clone inputs is different from utils clone_input. In case of minifier, all the tensors are leaf tensors while creating a new graph. So, we set the requires_grad field w/o checking the leafness of the tensor. )rrr] isinstancerur requires_grad_ requires_grad)example_inputs cloned_inputsidxs r-clone_inputs_retaining_gradnessrWsb !0MS()Q mC(%,, 7 #  - -nS.A.O.O PQ r/c$ddlm}m}m}t j |}|s t |}t|dr|jdt|dr||n||}|r|S||r||}|j|||d|S)z Runs a forward and possibly backward iteration for a given mod and args. When disable_clone is True, we will use args as-is without cloning. This is higher fidelity but we may destroy the args in the process. r)collect_resultsreduce_to_scalar_lossrequires_bwd_pass zero_gradT _boxed_callN) testingrrrcopydeepcopyrhasattrrbackward) r^argsonly_fwd disable_clonerrroutlosss r-run_fwd_maybe_bwdrdsSR r B .t4r; Tb-0"T(b$iC $S)  2sD$ //r/ require_fp64 ignore_non_fpcddlm}t|||}d}tjr9 t t j|t|\} } t| | |} t|||} ||| |tjd|} | S#t$r%|r tdtjdYYwxYw#t$rtjdYywxYw) aa Check two models have same accuracy. require_fp64: if True, raise an error if we unable to calculate the fp64 reference ignore_non_fp: if True, do not compare outputs which are not floating point. This is mostly useful for the minifier (which wants to avoid quantizing floating point error into integer/boolean error) r)sameNzfCould not generate fp64 outputs, workaround with torch._dynamo.config.same_two_models_use_fp64 = FalsezCould not generate fp64 outputszWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graph.T)tol equal_nanr)utilsrrrsame_two_models_use_fp64 cast_to_fp64rrr Exception RuntimeErrorr@rArrepro_tolerance) r^opt_gmrrrrrreffp64_ref fp64_model fp64_examplesrespassings r-same_two_modelsrs" B 9CH && ;(4 b!#B>#R) %J )]HMH A    " "# G N7 ;"| KK9 :  ;   $ s#8B B< +B98B9<CCc|jjD],}|jdk(r|jtj j jjk(rvt|jdk(sJt|jdrD|jdtjk7r$|jdtjf|_ |jdk(s|jjd}|t|st|j}tj|d<||_ /|jj!|j#|S)N call_functionrfrrry)graphnodesopr'ruopsprimsconvert_element_typedefaultr]rr float64kwargsgetdictlint recompile)modelnodery new_kwargss r-cast_dtype_args_to_fp64r)s !! ) GG & uyyCCKKKtyy>Q& &&diil+ !  0M!YYq\5==9 77o %KKOOG,E ^E%:!$++. &+mm 7#(  ) KK OO Lr/cddlm}|j}tjk(r t |}|fd|}||fS)Nr)tree_mapc~t|tjr!|jr|j S|Sr)rrur rvto)r6rys r-zcast_to..s3 a &1+>+>+@$$u+ r/)torch.utils._pytreer+r-rur r))ryr&inputsr+s` r-cast_tor1sK, HHUOE  (.    F &=r/c8ttj||Sr)r1rur )r&r0s r-r r s 5==% 00r/c |tj|t|}t|||||| S#t$rt j dYywxYw)NrzWhile minifying the program in accuracy minification mode, ran into a runtime exception which is likely an unrelated issue. Skipping this graphF)rrrrr r@r)r^r compiler_fnrrr compiled_gms r-backend_accuracy_failsr6st! MM" >~N #    %'       # s69AAstrideztorch._prims_common.StrideTyperxztorch._prims_common.ShapeTypereturnc4||Stj|Sr)r make_contiguous_strides_for)r7rxs r-_stride_or_defaultr; s '6UU-N-Nu-UUr/dcfdS)Nc||SSrrI)r6r<s r-r.z_mk_defaulter..s!-QQr/rI)r<s`r- _mk_defaulterr?s ..r/cpuc.eZdZddZddddZdZdZy)NopInputReaderNcd|_y)Nrtotal)r*s r-r.zNopInputReader.__init__!s  r/device dtype_hintc.|xjdz c_y)NrrD)r* storage_hashnbytesrGrHs r-storagezNopInputReader.storage$s a r/cyrrIr*rr!s r-tensorzNopInputReader.tensor' r/cyrrIrNs r-symintzNopInputReader.symint*rPr/r8NrFrGrHr.rLrOrRrIr/r-rBrB s7;t  r/rBcDeZdZd dddZddddZ d ddddddZdZy) InputReaderN)pbarcv|tjd| t|nd|_g|_||_y)Nz0no save_dir specified, will generate random data)r@rAr storerrW)r*save_dirrWs r-r.zInputReader.__init__1s9   KKJ K5=5I'1t   r/rFc|j|jjdt|}t|}|jP|N |jj |}||j k7r!tjd||j |Stjd|||jzf}td|}t||||jS#t$rYZwxYw)Nrzdevice mismatch: %s != %sz1could not load %s, generating random data insteadrx)rWupdate_device_or_default_dtype_or_defaultrY read_storagerGr@rAritemsizer;r untyped_storage)r*rJrKrGrHrLrxr7s r-rLzInputReader.storage<s 99 II  Q #F+&z2 :: !l&> **11,?W^^+KK ;VW^^T GV:...0#D6E6:v>NNPP%  s C)) C54C5)storage_offsetryris_leafc t||}t|}t|}t|}t |}t j g||j|} t j5| j||||ddd|snt j5| jt j} dddt j5| j||||dddt jjj| |k(sJt j j#| ||j$j'| | S#1swYxYw#1swYxYw#1swYxYw)Nr\)ryrGr) memory_format)r;_storage_offset_or_defaultr__is_leaf_or_default_requires_grad_or_defaultrurOrGno_gradset_ enable_gradclonepreserve_format _subclasses meta_utils safe_is_leaf_utilsset_tensor_metadatarappend) r*rLrxr7rcryrrdmetadatats r-rOzInputReader.tensorRsC$F%83NC!%(%g.1-@ LL eGNN- ]]_ ; FF7NE6 : ;""$ AGG%*?*?G@ A ?wv> ?  ++88;wFFF ((H5  ; ; A A ? ?s$1E)$!E5!F)E25E>F c<|jj||Sr)rrt)r*vals r-rRzInputReader.symintss  r/rrTrIr/r-rVrV0s> d 7;tQ4  Br/rVcPeZdZdddZdZddddefdZd d Zd Zd d Z d d Z y) InputWriterF stable_hashcg|_tj|_||_| t ||nd|_i|_y)Nr{)_lines itertoolsrstorage_counterrZrrY seen_storages)r*rZr|s r-r.zInputWriter.__init__sG (0  # x[ A  r/crdg}|jd|jD|jd|S)Nzdef load_args(reader):c3&K|] }d| yw)reNrI)rls r-rz$InputWriter.lines..s14s1szload_args._version = 0)extendr~rt)r*rs r-lineszInputWriter.liness8 $  1T[[11 )*r/NrH device_hintr8c Dt|}|jj|}||Sdt|j}d}t dt |k7rd|}d}|j }|jdk(r|J|}td|k7rd|}|j} d} |j4|j jdk7r|jj|} |jj|d| d| ||d||j|<|S) Nbufrz , dtype_hint=metaz , device=z = reader.storage(, rh)r rr"rkrr_rGrZr^rKrY write_storager~rt) r*rbrHrwsvmaybe_dtype_hintrrGrKrJs r-rLzInputWriter.storagesB O ,    " "2 & =H$t++,- . T "&7 &C C!.zn=   '' ;;& * ** F d #v -&vj1L '') :: !o&<&<&A&AV&K::33ODL c#L#32fZ ~N^M__` a "#2r/c tddlm}m}|j|j |j |j }g}||td|j|js1|jtt|jtd|j k7r|jd|j |td|jk(s"|jd|jt j"j%|}|r&|j'd|j)Dt+d|j,k7r|jd|j,t j.j0j3|}t5d|k7r|jd ||j6jd d j9|tt|jg|zd |zy) Nr)statically_known_truesym_eqrr\zdtype=zstorage_offset=c30K|]\}}|d|yw)=NrI)rkrs r-rz%InputWriter.tensor..sIA1#Qqe Iszrequires_grad=zis_leaf=zreader.tensor(r) # )%torch.fx.experimental.symbolic_shapesrrrLrbryrGr;rxr7rtstrtupler_rgrcrurrget_tensor_metadatarrorirrorprqrhr~r1) r*rrvrrrLrtensor_metadatards r-rOzInputWriter.tensorsW,,   AGG $ %d!'':AHHJ G  KKE!((*-. / T "agg - KK& , -$ &t ,0@0@0B B  KK/!*:*:*<)?@ A,,::1=  KKI1F1F1HI I $T *aoo = KK.(;< =##..;;A> t $ / KK(7+. /  ii#eAGGn"5==> ?dVn  r/c |jjd|dt|t|tt fr|jjdt |D]\}}|d|d}t|tjr|j||;t|ttjfr|j||n|j|||jjdyy)Nrz# was unsupported type for dumping: z"""[])r~rtrZrrwr enumeraterur rOintSymIntrR unsupported)r*rargraname_is r-rzInputWriter.unsupporteds Rv%Hc TU cD%= ) KK  u %!# 01 61#Qa.KK*C#67KK*$$VQ/ 0 KK  u % *r/cH|jjd|d|dy)Nz reader.const(rz!, filtered out during compilation)r~rt)r*rs r-constzInputWriter.consts' D85.O P r/ct|tjr|jj}|j j d|d|y)Nzreader.symint(r)rrurr'hintr~rt)r*rrxs r-rRzInputWriter.symints< c5<< (((--C ^C7%v>?r/rS) rFrGrHr.rrrLrOrrrRrIr/r-rzrzs905  6:tPS8 @&$ @r/rzfuncrG sym_shapesdefault_sym_shapec ddlm}|jDcic]\}}|| }}}dj|j }t j |} d|d} d|d} d} Gd d } i}xsifd d tffd }|j}|jD]\}}|d k(r tj| |}|r>|j\}}t|jd}||}|||||<tj| |}|s{|jd||<dt j|j vro| }||d<tj"| | D]J}|j\}}}}t|jd}||}t%|||||L|Scc}}w)a Takes in a function which has been printed with print_readable() and constructs kwargs to run it. Handles Tensor inputs, Symints, and a graph module which might have tensor constants. Consider a function `forward` defined as follows: def forward(self, primals_1: "f32[1001, 6]", primals_2: "f32[s0]", primals_3: "Sym(s0)",): _tensor_constant0: "i64[4190]" = self._tensor_constant0 # Further implementation kwargs = aot_graph_input_parser(forward) forward(**kwargs) r) dtype_abbrs|z(_tensor_constant\d+): \"(z0)\[\s*(.*?)\s*\]\" = self\.(_tensor_constant\d+)(z)\[\s*(.*?)\s*\]z Sym\((s\d+)\)ceZdZdZy)/aot_graph_input_parser..TensorContainerz#Container for tensors as attributesN)rFrGrH__doc__rIr/r-TensorContainerrs-r/rcltjvxsdufdjS)NcdS)Nz; not in symbolic_shapes and default sym shape not passed inrI)rRsr-r.z=aot_graph_input_parser..get_sym_int..'svhYZr/)ru_checkr")rRrrs`r- get_sym_intz+aot_graph_input_parser..get_sym_int$s9 j A$5T$A Z ~~f&788r/r8cg}g}t|D]a\}}|j}d|vr+ |}|j||j|E|sH|jt|c|jrt j nt j}||| }|D]"} t jj|| $|S)Nr)ryrG) rstriprtrrvrurandnzerosr mark_dynamic) rxryresolved_shape dynamic_dimsrdimr constructorrr<rGrs r- gen_tensorz*aot_graph_input_parser..gen_tensor+s & 4FAs))+Ccz$%%a(##A&"))#c(3 4&+%<% Iy)//#./Ei(E&ue4F5M /:6 ' A7F5M8 ""4(333#% "v[[!8&A DE16 .Iy)Q)//#./Ei(E Iy*UE*B C  D MSCs Gr+ctjtjj tjj fd}fd}t j||S)z Decorator to cProfile a given function and save the result to disk on process exit. Args: filename: filename to save profile to cFtjfd}|S)Ncj |i|jS#jwxYwr)enabledisable)rr!fnprofs r-wrapperz3profile_to_file..decorator..wrapperhs1 KKM 4*6*  s+=) functoolswraps)rrrs` r- decoratorz"profile_to_file..decoratorgs%     r/c jtjjt j dddy)Nz! Wrote profile to z+, view with: snakeviz z ) dump_statssysstderrr>r3r4)r+rsr-save_itz profile_to_file..save_itrsK !  OO""*,&Z(  r/)cProfileProfiler r!r# expanduseratexitregister)r+rrrs` @r-profile_to_filer]sR    Dwwrww11(;r s$   #33 ##..;M.g! CL 89  $ $ &    J $$..==?GG SWXJIITA!9!B?TUM644nccLT0/42-2(@ U I  0> 7 7t*$1  RV 5 6V +V& V/Q/8Xa[M1$45/"%--0"<5<<#67*1-)%0#E*   EEhs@s@p+/'+ ` DL>4</ 0` `c3h(` } `  #s(^ `F#c#hsAv&6#eUs I!