g^(h_>UddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZddlmZmZddlmZmZmZmZddlmZddlZddlmZddlmZddl m!Z!ejDe#Z$ejDd Z%ejLZ'd Z(d Z)d Z*d Z+dZ,dZ-da.ede/d<da0eGddZ1eGddZ2e1Z3e2a4ejjejjejjejjddddejjd Z6dddddddddddddddddddddddddddddddddddddddddddddddd/dee7dee7dee7dee7dee7dee7d ee7d!ee7d"ee7d#ee7d$ee7d%ee7d&e8d'e8d(e8d)e8d*e8d+e8d,e8d-e8d.e8d/e8d0e8d1e8d2e8d3e8d4e8d5e8d6e8d7e8d8e8d9e8d:e8d;e8de8d?ee7d@ee9e:ee7e8ffdAe8dBe8dCe8dDe8dEe8dFe8dGe8dHe8f^dIZ;dJZdMZ?dNZ@dOZAdPdQje je@e jeAgzdRzZDdSZEdTZFdUZGddVZHddWZIejdXZKdYZLdZZMd[ZNd\ZOGd]d^ejZQd_ZReRZSd`ZTeZUdaZVdbZWdcZXddZYdeZZdfZ[ddgZ\GdhdejZ^ejddiZ_GdjdkZ`eeaabe9e:eafe/dl<dmeadndfdoZcdneeafdpZddqfdre:dse:dtegeee:eeffdndfduZfe9fdvdddddwdre:dxegee9e:efege:e7fffdtegeee:eeffdye8dze8d{e8d|eedndfd}Zhe9fd~dddddre:dxegee9e:efege:e7fffdtegeee:eeffdye8dze8d{e8f dZiddlZddlZddljmkZly)N) defaultdict) dataclassfield)AnyCallableOptionalUnion)WeakSet) CompileId)log_trace_structured_event)CapturedTracebackz torch.__trace TORCH_LOGSTORCH_LOGS_OUTTORCH_LOGS_FORMATTORCH_LOGS_TRACE_ID_FILTER TORCH_TRACE TORCH_DTRACELazyTraceHandlerLOG_TRACE_HANDLERFceZdZUeeZeeeefed<ee Z e eed<ee Z e eed<ee Z e eed<ee Z e eed<eeZeeefed<ee Ze eed<eeZeeej$fed <d Zd Zd eeeeffd ZdZdZdZde efdZdZdZdZy) LogRegistrydefault_factorylog_alias_to_log_qnamesartifact_log_qnameschild_log_qnamesartifact_namesvisible_artifactsartifact_descriptionsoff_by_default_artifact_namesartifact_log_formattersc||jvSNrselfnames V/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/_logging/_internal.py is_artifactzLogRegistry.is_artifact\t****c||jvSr#)r)r&aliass r(is_logzLogRegistry.is_log_s4444r+ log_qnamescHt|tr|g}||j|<yr#) isinstancestrr)r&r-r/s r( register_logzLogRegistry.register_logcs# j# &$J.8$$U+r+c|jj||r|jj|||j|<|r|jj||#t j ||j|<yyr#)raddrrr logging Formatterr!)r&r' descriptionvisibleoff_by_default log_formats r(register_artifact_namez"LogRegistry.register_artifact_nameis~ %   " " & &t ,+6""4(   . . 2 24 8  !181B1B:1ND ( ( . "r+c:|jj|yr#)rr5)r&artifact_log_qnames r(register_artifact_logz!LogRegistry.register_artifact_log|s   $$%78r+c:|jj|yr#)rr5)r& log_qnames r(register_child_logzLogRegistry.register_child_logs !!),r+returncttjj|jj Sr#)set itertoolschain from_iterablervaluesr&s r(get_log_qnameszLogRegistry.get_log_qnamess+9??001M1M1T1T1VWXXr+c,t|jSr#)rErrJs r(get_artifact_log_qnamesz#LogRegistry.get_artifact_log_qnamess4++,,r+c,t|jSr#)rErrJs r(get_child_log_qnamesz LogRegistry.get_child_log_qnamess4(())r+c||jvSr#)r )r&artifact_qnames r(is_off_by_defaultzLogRegistry.is_off_by_defaults!C!CCCr+N)__name__ __module__ __qualname__rdictrr2list__annotations__rErrrrrr r!r6r7r)r.r r3r<r?rBrKrMrOrRr+r(rr6s! 5:$4OT#tCy.1O %*#$>S> "'s!;c#h; %S9NCH9#("<s3x<-2$,G4S>G /4C.H!3s8H=BRV`_ An artifact is a particular type of log message. Each artifact is assigned to a parent component. A component can emit many different kinds of artifacts. In general, an artifact is emitted if either its corresponding setting in the argument list below is turned on or if its parent component is set to a log level less than or equal to the log level of the artifact. Keyword args: all (:class:`Optional[int]`): The default log level for all components. Default: ``logging.WARN`` dynamo (:class:`Optional[int]`): The log level for the TorchDynamo component. Default: ``logging.WARN`` aot (:class:`Optional[int]`): The log level for the AOTAutograd component. Default: ``logging.WARN`` autograd (:class:`Optional[int]`): The log level for autograd. Default: ``logging.WARN`` inductor (:class:`Optional[int]`): The log level for the TorchInductor component. Default: ``logging.WARN`` dynamic (:class:`Optional[int]`): The log level for dynamic shapes. Default: ``logging.WARN`` distributed (:class:`Optional[int]`): Whether to log c10d communication operations and other debug info from PyTorch Distributed components. Default: ``logging.WARN`` c10d (:class:`Optional[int]`): Whether to log c10d communication operations related debug info in PyTorch Distributed components. Default: ``logging.WARN`` ddp (:class:`Optional[int]`): Whether to log debug info related to ``DistributedDataParallel``(DDP) from PyTorch Distributed components. Default: ``logging.WARN`` fsdp (:class:`Optional[int]`): Whether to log debug info related to ``FullyShardedDataParallel``(FSDP) in PyTorch Distributed components. Default: ``logging.WARN`` dtensor (:class:`Optional[int]`): Whether to log debug info related to ``DTensor``(DTensor) in PyTorch Distributed components. Default: ``logging.WARN`` onnx (:class:`Optional[int]`): The log level for the ONNX exporter component. Default: ``logging.WARN`` bytecode (:class:`bool`): Whether to emit the original and generated bytecode from TorchDynamo. Default: ``False`` aot_graphs (:class:`bool`): Whether to emit the graphs generated by AOTAutograd. Default: ``False`` aot_joint_graph (:class:`bool`): Whether to emit the joint forward-backward graph generated by AOTAutograd. Default: ``False`` ddp_graphs (:class:`bool`): Whether to emit graphs generated by DDPOptimizer. Default: ``False`` graph (:class:`bool`): Whether to emit the graph captured by TorchDynamo in tabular format. Default: ``False`` graph_code (:class:`bool`): Whether to emit the python source of the graph captured by TorchDynamo. Default: ``False`` graph_breaks (:class:`bool`): Whether to emit the graph breaks encountered by TorchDynamo. Default: ``False`` graph_sizes (:class:`bool`): Whether to emit tensor sizes of the graph captured by TorchDynamo. Default: ``False`` guards (:class:`bool`): Whether to emit the guards generated by TorchDynamo for each compiled function. Default: ``False`` recompiles (:class:`bool`): Whether to emit a guard failure reason and message every time TorchDynamo recompiles a function. Default: ``False`` recompiles_verbose (:class:`bool`): Whether to emit all guard failure reasons when TorchDynamo recompiles a function, even those that are not actually run. Default: ``False`` trace_source (:class:`bool`): Whether to emit when TorchDynamo begins tracing a new line. Default: ``False`` trace_call (:class:`bool`): Whether to emit detailed line location when TorchDynamo creates an FX node corresponding to function call. Python 3.11+ only. Default: ``False`` trace_bytecode (:class:`bool`): Whether to emit bytecode instructions and traced stack state as TorchDynamo traces bytecode. Default: ``False`` output_code (:class:`bool`): Whether to emit the TorchInductor output code on a per-graph basis. Default: ``False`` kernel_code (:class:`bool`): Whether to emit the TorchInductor output code on a per-kernel bases. Default: ``False`` schedule (:class:`bool`): Whether to emit the TorchInductor schedule. Default: ``False`` perf_hints (:class:`bool`): Whether to emit the TorchInductor perf hints. Default: ``False`` pre_grad_graphs (:class:`bool`): Whether to emit the graphs before inductor grad passes. Default: ``False`` post_grad_graphs (:class:`bool`): Whether to emit the graphs generated by after post grad passes. Default: ``False`` ir_pre_fusion (:class:`bool`): Whether to emit the graphs before inductor fusion passes. Default: ``False`` ir_post_fusion (:class:`bool`): Whether to emit the graphs after inductor fusion passes. Default: ``False`` onnx_diagnostics (:class:`bool`): Whether to emit the ONNX exporter diagnostics in logging. Default: ``False`` fusion (:class:`bool`): Whether to emit detailed Inductor fusion decisions. Default: ``False`` overlap (:class:`bool`): Whether to emit detailed Inductor compute/comm overlap decisions. Default: ``False`` sym_node (:class:`bool`): Whether to emit debug info for various SymNode opterations. Default: ``False`` export (:class:`Optional[int]`): The log level for export. Default: ``logging.WARN`` benchmarking (:class:`bool`): Whether to emit detailed Inductor benchmarking information. Default: ``False`` modules (dict): This argument provides an alternate way to specify the above log component and artifact settings, in the format of a keyword args dictionary given as a single argument. There are two cases where this is useful (1) if a new log component or artifact has been registered but a keyword argument for it has not been added to this function and (2) if the log level for an unregistered module needs to be set. This can be done by providing the fully-qualified module name as the key, with the log level as the value. Default: ``None`` cudagraph_static_inputs (:class:`bool`): Whether to emit debug info for cudagraph static input detection. Default: ``False`` autotuning (:class:`bool`): Autotuning choice logs, such as kernel source, perf, and tuning parameters. Default: ``False`` graph_region_expansion (:class:`bool`): Whether to emit the detailed steps of the duplicate graph region tracker expansion algorithm. Default: ``False`` Example:: >>> # xdoctest: +SKIP >>> import logging # The following changes the "dynamo" component to emit DEBUG-level # logs, and to emit "graph_code" artifacts. >>> torch._logging.set_logs(dynamo=logging.DEBUG, graph_code=True) # The following enables the logs for a different module >>> torch._logging.set_logs(modules={"unregistered.module.name": logging.DEBUG}) zQUsing TORCH_LOGS environment variable for log settings, ignoring call to set_logsNctj|jjD])\}}| tj |r:t |t std|d||sCtj|Ytj|s|tjvr|tjvrVtd|d|ddjtjjDcgc] }t!|c}tj#tj$j'|||td|t)ycc}w)Nz!Expected bool to enable artifact z , received zUnrecognized log level for log z: z, valid level values are: ,z6Unrecognized log or artifact name passed to set_logs: )rFrGrf log_registryr)r1bool ValueError log_stater_r.rr6 _levelToNamejoinkeysr2rdrget _init_logs)kwargsr-valkrs r( _set_logszset_logs.._set_logssG#//&,,.'--/J JE3{''.!#t,$;E7+cUS--e4$$U+u 8U8U/Ug222$9%3%H #':N:N:S:S:U)VQ#a&)V WXZ $$ 88<*?/@$A/B&C/DE/F*G/HI/JK/LM/NO/PQ/R,S/T#>r+cBtjt|duSr#)r fullmatchr)settingss r(_validate_settingsrms <<+-x 8 DDr+c rd d}|rtj}ntj}|rd}nd}dgttjj zt|Dcgc] }||dtj |"c}z}ddj|z}d}d |d |d |d }|Scc}w) NcJt||ksJ|d|t|z zzS)N )len)slengths r(pad_tozhelp_message..pad_tors,1v3&3q6/***r+zAll registered namesz?Visible registered names (use TORCH_LOGS='+help' for full list)rs z z a Examples: TORCH_LOGS="+dynamo,aot" will set the log level of TorchDynamo to logging.DEBUG and AOT to logging.INFO TORCH_LOGS="-dynamo,+inductor" will set the log level of TorchDynamo to logging.ERROR and TorchInductor to logging.DEBUG TORCH_LOGS="aot_graphs" will enable the aot_graphs artifact TORCH_LOGS="+dynamo,schedule" will enable set the log level of TorchDynamo to logging.DEBUG and enable the schedule artifact TORCH_LOGS="+some.random.module,schedule" will set the log level of some.random.module to logging.DEBUG and enable the schedule artifact TORCH_LOGS_FORMAT="%(levelname)s: %(message)s" or any provided format string will set the output format Valid keys are "levelname", "message", "pathname", "levelno", "lineno", "filename" and "name". TORCH_LOGS_OUT=/tmp/output.txt will output the logs to /tmp/output.txt as well. This is useful when the output is long. z TORCH_LOGS Info z  ))rrrsortedrrrr) verboserprinted_artifactsheadingr'lines setting_infoexamplesmsgs r( help_messagerqs+(77(::(S  55::< = > . $<.<#E#Ed#K"LM    &++e,,LH0       C JM s"%B4 cdgttjjzttjz}dj t |}d|d|d}|S)Nrsz, z Invalid log settings: z, must be a comma separated list of fully qualified module names, registered log names or registered artifact names. For more info on various settings, try TORCH_LOGS="help" Valid settings: r)rWrrrrrr)rrvalid_settingsrs r(_invalid_settings_err_msgrsz  |3388: ; < |** + , YYvn56Nj!  C Jr+c|dk(riS|dk(rttd|dk(rttdt|stt|t j dd|}|j d}d }t}|D]}||\}}|d k(rd }tj|r*|Jtj|}|j||Ttj|r|j|{t|rJt|stj!||ntj#||j||tt||S) NhelpF)rz+helpTz\s+rc|jtd}|jtd}|dtk(rtj}||fS|dtk(rtj }||fStj }||fS)Nrr)replaceINCR_VERBOSITY_CHARDECR_VERBOSITY_CHARr6rERRORINFO)r' clean_namelevels r(get_name_level_pairz0_parse_log_settings..get_name_level_pairs\\"5r: ''(;R@ 7) )MME 5  !W+ +MME5  LLE5  r+rsr)rrrrrsubsplitr[rr.rrdr)r__is_valid_module_has_registered_parentr3rB)r log_namesrrr'rr/s r(_parse_log_settingsrs]2~ 6e455 W d344 h '28<==vvfb(+Hs#I ! IB)$/ e 5=D   t $$ $$%==dCJ  U 3  % %d +  % %d + d #)$/))$5//5  u -6x@A A'B* r+cHtjj|}|duSr#) importlibutil find_spec)rspecs r(rrs! >> # #E *D t r+chtjjtd}| t |ayyr#)rrrrrr) log_settings r(_update_log_state_from_envrs+**..d3K' 4 r+ctj|}tj}|jr(|j |vry|j}|jr(y)NTF)r6rrrKparentr')rAcur_logregistered_log_qnamess r(rrsO *G(779 .. <<0 0.. .. r+ctj|j}tjD] } |j |}t |cSt |S#t$rY9wxYw)z Given an absolute filepath corresponding to a Python module which was loaded via normal import mechanisms using sys.path, convert it into a relative path relative to one of the Python search paths. )pathlibPathresolvesyspath relative_tor2r)abs_pathrrel_paths r(make_module_path_relativer  so||H%--/H! !++D1Hx= ! x=    sA"" A.-A.cBeZdZddddedeeeffdZdZxZ S)TorchLogsFormatterFN)tracetrace_id_filterrrc>t|||_||_yr#)super__init__ _is_trace_trace_id_filter)r&rr __class__s r(rzTorchLogsFormatter.__init__"s  /r+c4 ttj|jdd}|3tj j |d}||j|S|j|_ |j|d|_ |j}|jr,|js |j|j|_ |jr|dddk7r|dz}||jz}|jr+|dddk7r|dz}||j!|jz}d|_|j$sEt'j(r1t'j*rdt'j,d|_d|_|j$s6t0j2j4j7x} d|d |_d d d d dd}|j |j8|j8}d|_| d|d |_t=|j>}|j@r'|j.jC|j@vry|j"||jdtE|jFdzdd|jHd|d|jJd |j.|j: |j$rn|dk(sJ dtMjN|jP} |jX4| dj[d|jXj]dDz } | S|j]d} dj[ fd| DS#tR$r"tTjWd|jPwxYw)Nr^z %m%d %H:%M:%Srrz[rankz]:z []VIWEC)rrWARNINGrCRITICALz [__.i06dr:zfailing metadata: %rc3&K|] }d| yw)z NrY).0ls r( z,TorchLogsFormatter.format..tsLAtA3ZLsc3,K|] }d| yw)rNrY)r%r&prefixs r(r'z,TorchLogsFormatter.format..xs<xq_QUV  $!-!E!E!I!It" "-)0088**,A NN ????"&"6"6v"G ??v~HFOO#A   v~HD$$V%6%677A~~$"3"3"5$:M:M:O"' '8 ;F "]]99JJLL "(1-FN  (++F,<,N>NO "  $&*=/$;F !,V__=  ! !$$&d.C.CC  !*fnn-=Qs6<\\]^d^l^l]mmnj}}oQv~~.v/D/D.E G  >>7N7 ha 6?? ;<=~~)RWWL1E1Ed1KLLLHGGDME99s r( _reset_logsrrs!002  * W__%  (oo,,. 0Q0Q0S 23 W^^$  IIr+ctSr#rrYr+r(_get_log_staterus r+c|ayr#rt)states r(_set_log_staterxsIr+cPtttjj t d}||t jD]<}|dk(r tj|}|jtj>tjD]+\}}tj|}|j|-t jD]C}tj|}ttj|5tfd|Et j!D]"}tj|}t#|$tjj t$d}tjj t&dx}rda|}t* t-|at.jtj0t3t*}|j5t7dt.j9|y)Nrc.tjSr#)r6 FileHandler) log_file_namesr(z_init_logs..s++M:r+T)r)rrrrrrLOG_OUT_ENV_VARrrKr6rrrprrgrd StreamHandlerrMr TRACE_ENV_VARDTRACE_ENV_VARGET_DTRACE_STRUCTUREDrrrqrr^r_rra) r|outrArrr>trace_dir_namedtrace_dir_nametrace_log_handlers ` r(rrsM **..$ /C  "002%     * W^^$ %&99; 5 * U "002   *  ! !   $ :  +BBD$ 23s#$ ZZ^^M48N**..>>> $( ,^<w}}%&'89""#5D#AB *+r+c:eZdZdZdeefdZdZfdZxZ S)rzRLike FileHandler, but the file is allocated lazily only upon the first log messageroot_dirct||_tjj|d|_t |_yr#)rr6Handlerrstreamopen _builtin_open)r&rs r(rzLazyTraceHandler.__init__s,!   & !r+c|j |jrA |j|j}d|_t|dr|j  t j j | |jy#|j}d|_t|dr|j wwxYw#t j j |wxYw#|jwxYw)Nclose)acquirerflushhasattrrr6rrelease)r&rs r(rzLazyTraceHandler.closes   2;;+ !%&* "673"LLN %%++D1 LLN"&&* "673"LLN4%%++D1 LLNs3 CB0C C*2CC!C''C**C<c|j|jd}ddlm}t |dr+t j dtjdntjjdstjdnxt jj|stjd|nBt j|t jstjd |n||_|jt j |jd d }t#j$r,t#j&rd t#j(d}t+j,ddd||jd|_tjd|jj.nt0j3|y|jrt4|m|yy)Nz/logsr git_versionMAST_HPC_JOB_NAMEz>LazyTraceHandler: disabled because not fbcode or conda on mastzpytorch/trace:enablezYLazyTraceHandler: disabled because justknobs_check('pytorch/trace:enable') returned Falsez4LazyTraceHandler: disabled because %s does not existz6LazyTraceHandler: disabled because %s is not writeableT)exist_okrrank__zw+z.logdedicated_log_torch_trace_F)modesuffixr)dirdeletezLazyTraceHandler: logging to %s)rr torch.versionversionrrgetenvrinfor_utils_internaljustknobs_checkrexistsaccessW_OKmakedirsr6r7r8r9tempfileNamedTemporaryFiler'rqrlremit)r&rK TRACE_LOG_DIR torch_version ranksuffixrs r(rzLazyTraceHandler.emit.s ;; }}$ ' 5M=9 "56>HHX..>>?UVHHs 6HHN%="'':HHP% %2DM}}( DMMD9 $$&4+>+>+@#((9!;J&99!7 |D   :DKK.s  r+) metadata_fnr)trace_structured)r'rrs`` r(trace_structured_artifactrs   r+cyr#rYrYr+r(r}r}rr+)rsuppress_contextexpect_trace_idrecord_logging_overheadrrrrrrcHddvsJt|sJdt|t|sJdt|tjrOt j }i}|||<|s0t jr+t jrt j|d<tjjj} |rS| Q|Otjjj!t#j$dj'|d <nt| r | j(n|} | Q| j*| j*|d <| j,| j,|d <| j.| j.|d <| r| j0|d <|} | t3| t4sJt3| t6rddj9d| Dzdz} nd} t;j<| | d} t?j@d} | jC| jEd| jG|d<tjId|| ddtK|||r&t j |z dz }tM|yyy)z metadata is an arbitrary JSON compatible struct, but it's expected to not be too long (e.g., less than 1MB) payload is an arbitrary string, which can be arbitrarily long (but expected to have newlines so no lines are too long) r')rankcompiled_autograd_idrrattemptz(metadata_fn should be callable, but got z'payload_fn should be callable, but got rN)skipstackrrrrz[ z, c3FK|]}tj|ywr#)rFrG)r%is r(r'z#trace_structured..s0P1A0Ps!z ]cht|tr t|Stdt |d)NzObject of type z is not JSON serializable)r1rErWrItype)objs r( json_defaultz&trace_structured..json_defaults4%c3/#'9,'-d3i[8QRr+r)defaultindentF)usedforsecurityzutf-8 has_payloadr)rHrJ)extra stacklevelgeA)'callablerrqrftimetime_nsr6r7r8r9rr;r<r=_logging structuredfrom_tracebackr extractsummaryrrrrrr1r2rWrrFrGhashlibmd5updateencode hexdigestdebugr r)r'rrrrrr start_timerKrMcidrJrhstructured_logging_overhead_ss r(rrs&     F 1${2C1DEF   D 0j1A0BCD  \\^ $&"}t   "t':':'1   #40 "-1\\^j-HC,O ) +,I J #qr+cyr#rYrYr+r(r}r}rr+rrrrc4trt||||||yy)zu For logging more detailed information used for debugging. This may result in the program becoming slow. rN)rr)r'rrrrrs r(dtrace_structuredr s&  !-+$;  r+)FFN)Fr#)m functoolsrimportlib.utilrrFrFr6ros.pathrrrrr collectionsr dataclassesrrtypingrrrr weakrefr torch._logging.structuredr torch._guardsr torch._utils_internalr torch.utils._tracebackr rrSrrqrDEFAULT_LOG_LEVELrr~rWrXrrrrXrrr[rrrDEFAULT_LOGGINGrBrrVr2rrr3rrrrrescapeVERBOSITY_REGEXrrrrr lru_cacherrrrr r7rr\r`rdrfr^rjrnrrrurxrrrrrfloatrrrobjectrtuplerrtorch.distributedrur6rYr+r(rsr    #(11 #<4g! G  o . OO "(2 268./6 VDVD VDr #$#$ #$L} J ll << LL||  "!"!%!!$ !" " 59#&+$)#(aW #W SMW # W sm W c] WsmW#W 3-W #W 3-Wc]W 3-WWW !W"#W$ %W&'W()W*+W, -W./W01W23W45W67W89W:;W<=W>?W@AWBCWDEWFGWHIWJ KWLMWN SMOWPd3c4i 001 2QWRSWTUWVWWX $YWZ"[W\]W^_W`!aWt Q6PT"  hh  -.  :M0NOPQ   ?E8v" 22j 5 *W=**W=t &'(" 9   # ( D,NR!w,,R!jT((441J    XeCK&899:  &IM ]K >J" $(&*]K ]K"eDcNE#s(O$CDDE ]K XeCK&899: ]K]K]K"]K#]K ]KHIM  >J" $(  "eDcNE#s(O$CDDE  XeCK&899:    " 4 r+