g^(h\P"dZddlZddlZddlZddlZddlZddlZddlZddlZddl m Z ddl m Z ddl Z ddlmZddlmZddlmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!ddl"m#Z#ddl$m%Z%d d l&m'Z'd d l(m)Z)m*Z*d d l+m,Z,ejZe.Z/e d Z0e0jcZ2dZ3GddZ4de5fdZ6 d&dddddZ7d&dZ8d&dZ9dZ:e*dejvdefdZdZ?dZ@d ZAdd!ddd"d#d$e eBe5ffd%ZCy)'a Utilities for reproducing and debugging issues in Dynamo after graph capture. This file provides tools and infrastructure for debugging problems that occur after Dynamo has captured the graph but before/during backend compilation. Key components include: - Minification tools to reduce large graphs to minimal failing examples - Accuracy testing to validate compiled graph outputs match eager mode - Repro generation to create standalone reproduction scripts - Debug backends for capturing and analyzing failures - Utilities for saving/loading graph states and inputs The tools here focus specifically on the post-graph-capture stage, making them useful for debugging backend compilation issues, AOTAutograd problems, and accuracy discrepancies between compiled and eager execution. N) import_module)Union) CompiledFn) AccuracyErrorbackend_accuracy_failsBUCK_CMD_PREFIXBuckTargetWriter extra_importsgenerate_config_stringgenerate_env_vars_stringhelper_for_dump_minify InputReader InputWriter minifier_dirNNModuleToStringNopInputReaderrun_fwd_maybe_bwdsame_two_models)fx_placeholder_targets)tqdm)config)lookup_backendregister_debug_backend)clone_inputs_retaining_gradnessztorch._inductor.configcZt|||tjtjS)Nonly_fwd ignore_non_fp)rrrepro_forward_onlyrepro_ignore_non_fp)gmexample_inputs compiler_fns ^/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/_dynamo/repro/after_dynamo.py_accuracy_failsr&Gs* ! **00  c"eZdZdeddfdZdZy)WrapBackendDebug compiler_namereturnNctj||||_||_t |dr|j |_t |dr|j |_t |dr|j|_yy)N__name__r*get_compiler_config) functoolswraps_torchdynamo_orig_callable_compiler_namehasattrr-r*r.)selfunconfigured_compiler_fnr*s r%__init__zWrapBackendDebug.__init__Rsv1 01$7*B'+ +Z 84==DM +_ =4BBDM +-B C'?'S'SD $ Dr'c tj|jfi|}tjdvsJtjdk(rd}tj dk(rt |||jtj dk(r|tj||}t|||rtjdt tj|tj|j||jt!d}||| |tj||}t#|||S|S|||}|S#t$$r}tjdtj d k(r`tjt&|j }|tj|tj|j|n[tj d k(rHt tj|tj|j||j||d}~wwxYw) N)dynamoaotNr8ctjjtd|_t r.add_pathscsU$&GGLLAW$X!'*xx'+C,=,=>LLMN(C$r'zXAccuracy failed for the TorchDynamo produced graph. Creating script to minify the error.zBad accuracy detected.DCompiled Fx GraphModule failed. Creating script to minify the error.r*r)r/partialr1r repro_after repro_leveldump_to_minify_after_dynamor2copydeepcopyr&logwarningfx GraphModulegraphrr Exceptiondump_backend_state) r4r"r#kwargsr$rD compiled_gmrC dump_state_fns r%__call__zWrapBackendDebug.__call__]s''(G(GR6R !!%<<<<    ) !!Q&+B@S@ST!!Q&)$--*;^L "2~{CKKr0r4==+BC&++ ((@ACcNI"-dmmB.?"PK%k>B.{&b.9K-!KK^))Q.(1(9(9.d>Q>Q) &NN2t}}RXX/FG ++q03NN2t}}RXX/FG* // cN%s%(E I%C,II)r- __module__ __qualname__strr6rZr'r%r)r)Qs T T T;r'r)r*ct||S)a A minifier decorator that wraps the TorchDynamo produced Fx graph modules. As opposed to wrap_compiler_debug, this wrapper intercepts at the TorchDynamo produced Fx Graph Module. This makes it backend-agnostic to some level, e.g., it is useful for minifying issues related to Aot Autograd tracing. If an error is found, we minify and save the minified repro in repro.tar.gz. )r))r5r*s r%wrap_backend_debugr`s 4m DDr'Frun) stable_outputsave_dircommandcTtj|}t|d}tt ||D]r\} } t | t tjfr|j| | 9t | tjr|j| | ftd| dj|j} tj dt#|dt%|dt&d|d| d |d |d |d tj(d |dS)zH Generate a repro string for backend-agnostic minified version. T) stable_hashz,arg is neither SymInt/int nor torch.Tensor,  )rbz from math import inf import torch from torch import tensor, device import torch.fx as fx import torch._dynamo from torch._dynamo.testing import rand_strided from torch._dynamo.debug_utils import run_fwd_maybe_bwd z z mod = Repro() z if __name__ == '__main__': from torch._dynamo.repro.after_dynamo import run_repro run_repro(mod, load_args, accuracy=z , command=z, save_dir=z , autocast=z , backend=z) )rconvertrzipr isinstanceinttorchSymIntsymintTensortensor TypeErrorr>linestextwrapdedentr r r is_autocast_enabled) r"argsr*check_accuracyrbrcrd model_strwriter placeholderarg load_argss r%generate_dynamo_fx_repro_stringr}s?!((,It 4F 6r :DAR S cC. / MM+s + U\\ * MM+s +J3%PQ QR &,,.)I ??  67 8m456    ()7'9G;O[)B)B)D(GzR_Qbc-  r'c >tj}tjjtjd}tjj |stj |dtjj|dt |jjd}tjdt |jj|t|d5}|jt|||||d d d tjj|d }tjd ||trt|jt!j"||y #1swYvxYw) z, Saves the repro to a repro.py file checkpointsTexist_ok minified_z _nodes.pyz&Writing checkpoint with %s nodes to %sw)rcNzrepro.pyz Copying %s to %s for convenience)r<getcwdr=r>existsmakedirslenrTnodesrPrQopenwriter}r@r shutilcopyfile) r"rvr*rwcurdirsubdir file_namefd latest_repros r%dump_backend_repro_as_filersYY[F WW\\"))+} 5F 77>>& ! FT* VyRXX^^1D0EY%OPIKK0#bhhnn2Ey i    +D-&   77<< 3LKK2I|L&,,. OOI|,  s > FFcLtj|sJt||||S)a  Dumps the dynamo graph to repro the issue. 1) It tries to convert Fx GraphModule to a string. If we can, it writes to a repro.py file. 2) If we can't convert Fx GraphModule to a string, we use to_folder to save the module and save a tar file. )rcan_convert_to_stringr)r"rvr*rws r%rVrVs)  1 1" 55 5 %b$ ~ NNr'c tjjtd}tjj |stj |dt t|||tjdk(|dy)NrTrrFminify)rwrcrd) r<r=r>rrrr r}rrL)r"rvr*rs r%rMrMsc WW\\,.- 8F 77>>& ! FT*'   !--2   r'r"c Vddlm}t|}|Dcgc]4}t|tj r|j jn|6}} |||}t||tdcc}w#t$r}t|}tjdtjt |} | t#j$|t'j(|j*|tjt,||} |||| | Yd}~|Sd}~wwxYw)NrminifierzNo issue was detectedrGrI)r$ orig_failure module_fails dump_state)functorch.compilerrrjrlrmnodehintr ValueErrorrUr]rPrQr/rJrVrRrSrNrOrT backend_fails) r"r#r*rr$irXrCrrYfails_fns r%dynamo_minifier_backendr(s+ /K DR>?z!U\\2 9N !"n5 +~6011  3x  R ")) m  bnnRrxx)@A>R$$ #%    !$   I) s9A2 A77 D(BD##D(cddlm}t|}|jt |||rt j dtjt|d}tjt|}|tj|tj|j|||||||St jd|S) Nrrz2Accuracy failed for the TorchDynamo produced graphT)r*rw)r$rz*Input graph does not fail accuracy testing)rrrevalr&rPrQr/rJrVrRrSrNrOrTerror)r"r#r*rr$rYrs r% dynamo_accuracy_minifier_backendrRs* /KGGIr>;7 HI!)) mD $$ #  bnnRrxx)@A>R  !$   I >? Ir'cddlm} t|t||||}t|t|y#t$r5}t |}|d||j dkDrYd}~yYd}~yd}~wwxYw)a. Minifier uses this function to identify if the minified graph module fails with the same error. One caveat is that minifier can potentially go into a wrong direction when the resulting graph module fails for a different reason. To avoid this, we save the string for the original exception and check similarity between new and old exception. They can be somewhat different in some cases, when the exception string depends on the failing node information. So, we have a loose similarity metric to guide the minifier path. r)SequenceMatcherNg?TF)difflibrrrrUr]ratio)r"r#r$rrrXe new_failures r%rrqs~("=nMN!"n5 +'F~'VW  !f 4{ ; A A Cc I J  s3< A:&A55A:c|t|dstjdn/|jdkDr tjd|jt }||t d|j 5}t|j|}|||j}ddd|S#1swYSxYw)N_versionzzload_args does not have a _version attribute, please file a bug to PyTorch and describe how you generate this repro scriptrzload_args is version %s, but this version of PyTorch only supports version 0. We will try to run it anyway but there may be an incompatibility; if so, try upgrading your version of PyTorch.zLoading inputs)desctotal)rcpbar) r3rPrQrrrrrrcrv)optionsmodr| nop_readerr input_readerrvs r% run_load_argsrs 9j )  >    ! KK@""   !J j #:+;+; <!"G,<,<4H ,  ! K ! Ks ;,B11B;ct|||}|js td}n td}|j t dt j ||j}tjj||}tjjd|j5||dddy#1swYyxYw)NrrzCompiler name is None - this likely means that a custom compiler was called by torchdynamo. Please remove this error, import your custom compiler function, and replace the backend=None line in run_repro to backend=rIcudaenabled) raccuracyrbackend RuntimeErrorr/rJrl_dynamooptimizeampautocast)rrr|rvr$ropt_mods r% repro_minifyrs #y 1D   $%>? $%GH  I  (//oomm$$%<=cBG   FG,<,<  =s 8CCc tjj|j|}|jdk7r|j |j tj jd|j5t|||}t|||sJdt|||tjtjs td dddytj jd|j5t|||}t|||jd~t|||}t|||jddddy#1swYyxYw#1swYyxYw) NrrzEager itself failedrz Dynamo failedT)r disable_clone)rlrrrrrrrrrrr r!rrr)rrr|rrvs r% repro_runrsUmm$$W__5c:G2    YY  0@0@  A 5 #y9D"3T2 I4I I2"22$88 $O44 5 5YY  0@0@  A  #y9D c4'2B2BRV W #y9D (8(8     5 5  s AE.AE:.E7:Frinductor)rdrrcrrrc b|D]}tjd|durdndurdtjd|d|dd d tj } fd } | j d dd} | j dd} | | | jddd| j dd} | | d}ttjdkr|gtjdd}| j|}ttd}||j|||y)NzPUnrecognized kwarg %s; perhaps this repro was made on a newer version of PyTorchTrFrzAn after_dynamo repro script, typically triggering a bug in Dynamo or AOTAutograd. When run with no arguments, this script defaults to running 'z9'. Extra flags may be available; to find out more, try 'zs --help'. There are also alternate subcommands available, see below. default settings on this script: accuracy=z save_dir=rg) descriptionformatter_classc|j}|jddddd|jdddd |jd td d |jddddd|jddddd|jddd|jdddd|jd td!d" y)#Nz --no-accuracyr store_constrz>do not test accuracy, just run the module and see if it errors)destactionconstdefaulthelpz --accuracyz test accuracy)rrrrz --save-dirDIRz!directory where saved inputs live)typermetavarrz --no-save-dirrcz(don't use any directory for saved inputs)rrrrz --no-isolateisolate store_falseFz1no isolate (doesn't do anything for after_dynamo))rrrrz --autocast store_truezuse torch.cuda.amp.autocast)rrrz --no-autocastrz!don't use torch.cuda.amp.autocast)rrrz --backendBACKENDztorch.compile backend to use)add_mutually_exclusive_group add_argumentr])parseraccuracy_grouprrrrcs r% common_flagszrun_repro..common_flagss,<<>##  Q $  ##   $   4     ;     D    .    4   /  r'rdz {run,minify})rrrequiredrazjust run the repro)rz --only-fwdrz+don't run backwards compilation for testing)rrrzrun the minifier on the reprorH)rra)rPrQargparseArgumentParserRawTextHelpFormatteradd_subparsers add_parserrrsysargv parse_argsrrrd)rr|rdrrcrrrWkrr subparsers parser_run parser_minifyrvr COMMAND_FNSs ```` r% run_reprors  ^   4 U   $ $  CG9M  +  + !55 F8 t&& 'J&& !'J : ))6*M D 388}'#((12,'%GK!K #y9r')F)D__doc__rrNr/loggingr<rrrs importlibrtypingrrltorch.fxrRtorch._dynamo.backends.registryrtorch._dynamo.debug_utilsrrrr r r r r rrrrrrr%torch.fx.experimental.symbolic_shapesr torch.hubrrrbackends.registryrr debug_utilsr getLoggerr-rPinductor_config is_fbcoder@r&r)r]r`r}rrVrMrSrrrrrrboolrr^r'r%rsU$  # 6"IF9g! 89  $ $ &GGT E E* 5   5p-8 O$,& &7A&&R<>46F !#   {: D#I {:r'