Ë
    g^(hWA  ã                   ó2  — d Z ddlZddlZddlmZ ddlZddlmZ ddlm	Z	 ddl
mZmZ ddlmZmZmZmZmZmZ dd	lmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dlmZ d
dl m!Z! erddl"m#Z#  G d„ de$«      Z% G d„ de$«      Z& ee'd«      Z(d„ Z) G d„ de!«      Z*y)a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
é    N)ÚTYPE_CHECKING)ÚgetArtifactLogger)Útree_map_onlyé   )ÚGuardBuilderÚinstall_guard)Ú
AttrSourceÚConstDictKeySourceÚDictGetItemSourceÚGetItemSourceÚGlobalWeakRefSourceÚ
GradSource)ÚGLOBAL_KEY_PREFIXé   )ÚVariableTracker)ÚConstantVariable)ÚConstDictVariable)ÚListVariable)ÚGetAttrVariable)ÚUserDefinedObjectVariable)ÚInstructionTranslatorc                   ó   — e Zd Zy)ÚArgMappingExceptionN©Ú__name__Ú
__module__Ú__qualname__© ó    ú_/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/_dynamo/variables/optimizer.pyr   r   7   ó   „ Ør   r   c                   ó   — e Zd Zy)ÚGuardInstallExceptionNr   r   r   r    r#   r#   ;   r!   r   r#   Ú
perf_hintsc                 óþ   — ddl m} | j                  rk || j                  j                  d«      }t
        j                  j                  j                  | «      d u}|r|xs |j                  j                  | «      S |S y)Nr   )Úget_managerFT)Útorch._inductor.cudagraph_treesr&   Úis_cudaÚdeviceÚindexÚtorchÚ_dynamoÚutilsÚget_static_address_typeÚcurrent_nodeÚ_is_cuda_graph_recorded_tensor)Úxr&   ÚmanagerÚis_static_addresss       r    Ú_is_static_for_cudagraphsr4   B   ss   € Ý;à‡y‚yÙ˜aŸh™hŸn™n¨eÓ4ˆÜ!ŸM™M×/Ñ/×GÑGÈÓJÐRVÐVÐÙà!ò JØ×'Ñ'×FÑFÀqÓIðð
 %Ð$ð r   c                   ó¨   ‡ — e Zd Zdddhej                  £Z	 	 	 d	 dˆ fd„Z	 	 	 	 	 	 dˆ fd„Zdˆ fd„Zd„ Zd„ Z	d	„ Z
d
„ Zd„ Zdd„Z	 	 dd„Zd„ Zˆ xZS )ÚOptimizerVariableÚgrad_to_sourceÚtensor_to_sourceÚstatic_tensor_namesc                 óx   •— t        ‰| �  |fi |¤Ž |xs i | _        |xs i | _        |xs
 t	        «       | _        y ©N)ÚsuperÚ__init__r7   r8   Úsetr9   )ÚselfÚvaluer7   r9   r8   ÚkwargsÚ	__class__s         €r    r=   zOptimizerVariable.__init__\   s@   ø€ ô 	‰Ñ˜Ñ) &Ò)Ø,Ò2°ˆÔØ 0Ò 6°BˆÔØ#6Ò#?¼#»%ˆÕ r   c                 ó  •— |dk(  rÓ	 | j                  |«       | j                  «         | j                  |i |¤Ž\  }} | j                  j                  |i |¤Ž}| j                  |«       | j                  |||||«       dt        | j                  «      › �}|j                  || j                  «       | j                  |«       t        j                  |«      S t        ‰
| �=  ||||«      S # t        t        f$ r
}	Y d}	~	Œ&d}	~	ww xY w)zVThis is an optimization to avoid tracing the very slow initialization of the optimizerÚ_init_groupÚ__optimizer_N)Úgraph_break_if_pending_mutationÚmove_step_if_cpuÚget_python_argsr@   rD   Úmap_sources_and_install_guardsÚupdate_list_argsÚidÚstore_global_weakref_by_idÚcreate_finalizerr   Úcreater   r#   r<   Úcall_method)r?   ÚtxÚnameÚargsrA   Úpy_argsÚ	py_kwargsÚret_valÚmangled_nameÚ_rB   s             €r    rO   zOptimizerVariable.call_methodi   s  ø€ ð �=Ò ðØ×4Ñ4°RÔ8Ø×%Ñ%Ô'Ø%9 T×%9Ñ%9¸4Ð%JÀ6Ñ%JÑ"�˜Ø0˜$Ÿ*™*×0Ñ0°'ÐG¸YÑG�Ø×3Ñ3°BÔ7Ø×%Ñ% b¨$°¸ÀÔKð ".¬b°·±«nÐ-=Ð>�Ø×-Ñ-¨l¸D¿J¹JÔGØ×%Ñ% bÔ)ô (×.Ñ.¨wÓ7Ð7ô
 ‰wÑ" 2 t¨T°6Ó:Ð:øô	 (Ô)>Ð?ò äûðús   ˆCC, Ã,DÄ Dc                 ó  •— |dv r"t        | |t        | j                  |«      ¬«      S |dk(  rDddlm} | j
                  j                  D ]  }|d   D ]
  } ||«       Œ Œ | j                  |«       t        ‰| �%  ||«      S )N)rD   Ústep)ÚsourceÚparam_groupsr   ©Úmark_static_addressÚparams)
r   r	   rZ   Ú
decoratorsr]   r@   r[   Ú_set_capturabler<   Úvar_getattr)r?   rP   rQ   r]   ÚgroupÚprB   s         €r    ra   zOptimizerVariable.var_getattrŠ   s‰   ø€ ð Ð*Ñ*Ü" 4¨´jÀÇÁÈdÓ6SÔTÐTà�>Ò!Ý8àŸ™×0Ñ0ò +�Ø˜x™ò +�AÙ'¨Õ*ñ+ð+ð × Ñ  Ô$ä‰wÑ" 2 tÓ,Ð,r   c                 ó  — | j                   j                  D ]i  }|d   D ]_  }|j                  j                  }|j                  j                  t        |«      d «      }|sŒA|j                  |«      sŒSddlm	}  |d«      ‚ Œk y )Nr^   r   )ÚUnsupportedzPending mutation on parameter)
r@   r[   ÚoutputÚside_effectsÚid_to_variableÚgetrK   Úhas_pending_mutationÚexcre   )r?   rP   Úgrc   rg   Úvariablere   s          r    rF   z1OptimizerVariable.graph_break_if_pending_mutationœ   s~   € ð
 —‘×(Ñ(ò 	GˆAØ�x‘[ò G�Ø!Ÿy™y×5Ñ5�Ø'×6Ñ6×:Ñ:¼2¸a»5À$ÓG�Ú × AÑ AÀ(Õ KÝ1á%Ð&EÓFÐFñGñ	Gr   c                 óà  ‡ — ddl m} ˆ fd„}‰ j                  j                  D ]  } ||«      sŒd|d<   Œ ‰ j                  xr t        ‰ j                  d«      }|j                  t        j                  |‰ j                  j                  |«      «      }|j                  D ]L  }t        j                  t        j                  d«      «      }t        j                  d«      |j                  |<   ŒN y )Nr   ©ÚLazyVariableTrackerc                 óÂ   •— d}d}| j                  dg «      D ]:  }||j                  xs |j                  z  }||‰j                  j                  vz  }Œ< d| v xr |xr |S )NTr^   Ú
capturable)ri   r(   Úis_xpur@   Ústate)rb   Úall_uninitializedÚall_gpurc   r?   s       €r    Úsafe_to_set_capturablezAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturable¯   sq   ø€ Ø $ÐØˆGà—Y‘Y˜x¨Ó,ò ?�Ø˜1Ÿ9™9Ò0¨¯©Ñ0�Ø! Q¨d¯j©j×.>Ñ.>Ð%>Ñ>Ñ!ð?ð   5Ð(ÒJÐ->ÒJÀ7ÐJr   Trr   r[   )Ú rp   r@   r[   rZ   r	   Úrealize_allr   ÚbuildÚitemsr   Ú_HashableTrackerr   rN   )	r?   rP   rp   rw   rb   rZ   Úparam_groups_vtÚparam_group_vtÚkeys	   `        r    r`   z!OptimizerVariable._set_capturableª   sÔ   ø€ Ý)ô	Kð —Z‘Z×,Ñ,ò 	+ˆEÙ% eÕ,Ø&*��lÒ#ð	+ð —‘ÒH¤¨D¯K©K¸Ó!HˆØ-×9Ñ9Ü×!Ñ! " d§j¡j×&=Ñ&=¸vÓFó
ˆð .×3Ñ3ò 	FˆNÜ#×4Ñ4Ü ×'Ñ'¨Ó5óˆCô )9×(?Ñ(?ÀÓ(EˆN× Ñ  Ò%ñ		Fr   c                 ó¨   ‡ — ˆ fd„}|D �cg c]
  } ||«      ‘Œ }}|j                  «       D ��ci c]  \  }}| ||«      “Œ }}}||fS c c}w c c}}w )z9Get python values equivalent to the variable tracker argsc                 óÈ  •— t        | t        «      r| j                  «       S t        | t        «      r| j                  sg S t        | t
        «      rŽt        | j                  t        «      rtt        | j                  j                  t        «      rP| j                  j                  j                  dk(  r-‰j                  j                  | j                  j                     S t        ‚)Nr[   )Ú
isinstancer   Úas_python_constantr   r{   r   rZ   r   Úbaser	   Úmemberr@   r[   r*   r   )Úargr?   s    €r    Úmap_argz2OptimizerVariable.get_python_args.<locals>.map_argÍ   s˜   ø€ Ü˜#Ô/Ô0Ø×-Ñ-Ó/Ð/Ü˜C¤Ô.°s·y²yØ�	ä˜3Ô 1Ô2Ü˜sŸz™z¬=Ô9Ü˜sŸz™zŸ™´
Ô;Ø—J‘J—O‘O×*Ñ*¨nÒ<à—z‘z×.Ñ.¨s¯z©z×/?Ñ/?Ñ@Ð@ä%Ð%r   )r{   )	r?   rR   rA   r‡   r†   Únew_argsÚkÚvÚ
new_kwargss	   `        r    rH   z!OptimizerVariable.get_python_argsÊ   sY   ø€ ô	&ð -1Ö1 S‘G˜C•LÐ1ˆÐ1Ø06·±³×?©¨¨1�a™ ›‘mÐ?ˆ
Ñ?à˜Ð#Ð#ùò 2ùÛ?s
   ‹A	¯Ac                 óÈ   — | j                   j                  j                  «       D ];  \  }}d|v sŒ|d   j                  sŒ|d   j	                  |j
                  «      |d<   Œ= y )NrY   )r@   rt   r{   Úis_cpuÚtor)   )r?   rc   rt   s      r    rG   z"OptimizerVariable.move_step_if_cpuæ   sW   € ØŸ
™
×(Ñ(×.Ñ.Ó0ò 	;‰HˆAˆuØ˜Š 5¨¡=×#7Ó#7Ø % f¡× 0Ñ 0°·±Ó :��f’ñ	;r   c                 ó	  ‡— ddl mŠ ddlm} i | _        i | _        ˆfd„}t        t        j                  || j                  j                  «       | j                  xr t        | j                  d«      }|j                  t        j                  || j                  j                   |«      «      }| j                  xr t        | j                  d«      }t        j                  || j                  j                  |«      }|j#                  «        |j$                  j&                  j)                  |j+                  «       «       t-        | j                  j                   |j.                  «      D �]  \  }}	t1        |d   «      d	kD  r«|d   D ]£  }
|
j2                  €Œd }t5        | j                  j                  j7                  «       «      D ]  \  }}||
u sŒ|} n |sŒR|j                  t        j                  || j                  j                  |
   t9        |t;        ||«      «      «      «        n |	j=                  |t?        j@                  d«      «      }d
}g }t5        t-        |d   |jC                  |«      «      «      D ]¦  \  }\  }}|j                  }|| j
                  |<   tE        |d«      }|j2                  �C|| j                  |j2                  <   tG        |j2                  «      rŒkd}|jI                  |«       ŒtK        |jM                  tN        jP                  «      «       Œ¨ |r�ŒÀtR        jU                  tV        jX                  «      s�Œå|D �cg c]  }|j+                  «       ‘Œ }}tR        j[                  d|«       �Œ t5        | j                  j                  j/                  «       «      D ]Ñ  \  }\  }}t9        |t;        ||«      «      }|j$                  j&                  j)                  |j+                  «       «       t5        |j/                  «       «      D ]d  \  }\  }}t]        |t        j                  «      sŒ$|| j                  vsŒ3|| j
                  vsŒBt9        |t;        ||«      «      | j
                  |<   Œf ŒÓ y c c}w )Nr   r\   r   ro   c                 ó   •—  ‰| «       y r;   r   )r1   r]   s    €r    Úmark_staticzEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_staticú   s   ø€ Ù Õ"r   r[   rt   r^   r   TÚgradF)zÏGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.)/r_   r]   Úlazyrp   r7   r8   r   r+   ÚTensorr@   rt   rZ   r	   ry   r   rz   r[   Úrealizerf   Úguard_on_key_orderÚaddrQ   Úzipr{   Úlenr’   Ú	enumerateÚkeysr   r
   Úgetitem_constr   rN   Úunpack_var_sequencer   r4   Úappendr   Ú
make_guardr   ÚCONSTANT_MATCHÚperf_hint_logÚisEnabledForÚloggingÚDEBUGÚwarningr‚   )r?   rP   rp   r‘   Úparams_groups_sourcer}   Ústate_sourceÚstate_vtrb   Úgroup_vtÚparamÚ	key_indexÚir‰   Ú	params_vtÚ
all_staticÚnon_static_gradsÚp_indrc   Úp_vtÚparam_sourceÚgrad_sourceÚsrcÚidxr@   Úp_state_sourceÚ	inner_idxrŠ   r]   s                               @r    rI   z0OptimizerVariable.map_sources_and_install_guardsë   sæ  ø€ Ý4Ý-à ˆÔØ "ˆÔô	#ô 	”e—l‘l K°·±×1AÑ1AÔBð  $Ÿ{™{ÒV¬z¸$¿+¹+À~Ó/VÐØ-×9Ñ9Ü×!Ñ! " d§j¡j×&=Ñ&=Ð?SÓTó
ˆð —{‘{ÒG¤z°$·+±+¸wÓ'Gˆä"×(Ñ(¨¨T¯Z©Z×-=Ñ-=¸|ÓLˆð 	×ÑÔØ
�	‰	×$Ñ$×(Ñ(¨×):Ñ):Ó)<Ô=ô  # 4§:¡:×#:Ñ#:¸O×<QÑ<QÓRó 8	‰OˆE�8ô �5˜‘?Ó# aÒ'Ø" 8™_ò "�EØ—z‘zÑ-Ø$(˜	Ü$-¨d¯j©j×.>Ñ.>×.CÑ.CÓ.EÓ$Fò &™D˜A˜qØ  EšzØ,- 	Ù %ð&ò %Ø/×;Ñ;Ü /× 5Ñ 5Ø$&Ø$(§J¡J×$4Ñ$4°UÑ$;Ü$5Ø(4Ü(:¸<ÈÓ(Só%&ó!"ô	ñ "ð%"ð( !×.Ñ.¨rÔ3C×3JÑ3JÈ8Ó3TÓUˆIØˆJØ!ÐÜ$-Ü�E˜(‘O Y×%BÑ%BÀ2Ó%FÓGó%ò WÑ �‘y˜˜4ð  $Ÿ{™{�Ø+7�×%Ñ% aÑ(Ü(Ø Øó�ð
 —6‘6Ð%Ø2=�D×'Ñ'¨¯©Ñ/Ü4°Q·V±VÕ<Ø%*˜
Ø(×/Ñ/°Õ<ä! +×"8Ñ"8¼×9TÑ9TÓ"UÕVð!Wó( ¤-×"<Ñ"<¼W¿]¹]Ö"KØ:JÖ#K°3 C§H¡H¥JÐ#KÐ Ð#KÜ×%Ñ%ðð
 %öðc8	ôx  )¨¯©×)9Ñ)9×)?Ñ)?Ó)AÓBò 	‰OˆC‘�!�UÜ.ØÔ0°¸sÓCóˆNð �I‰I×(Ñ(×,Ñ,¨^×-@Ñ-@Ó-BÔCÜ%.¨u¯{©{«}Ó%=ò Ñ!�	™6˜A˜qä˜q¤%§,¡,Õ/Ø ×!4Ñ!4Ò4Ø ×!6Ñ!6Ò6ä/@Ø&Ô(:¸>È9Ó(Uó0�D×)Ñ)¨!Ò,ññ	ùò $Ls   ÍRc                 ó  — ddl m} || j                  v rZ ||«       | j                  |   }| j                  j	                  |j
                  j                  |j                  «       «      «       n‰|| j                  v r| j                  |   }nk ||«       |j                  t        |«      }t        |«      }| j                  j	                  |j
                  j                  |j                  «       «      «       t        j                  |||«      S )z%Wrap state tensor in a TensorVariabler   r\   )r_   r]   r8   r9   r—   rf   Úmodule_key_namerQ   r7   rL   r   r   r   rz   )r?   rP   Útensor_valuer]   rZ   Úglobal_names         r    Úwrap_tensorzOptimizerVariable.wrap_tensor\  sÙ   € å4ð ˜4×0Ñ0Ñ0á Ô-Ø×*Ñ*¨<Ñ8ˆFØ×$Ñ$×(Ñ(¨¯©×)BÑ)BÀ6Ç;Á;Ã=Ó)QÕRØ˜T×0Ñ0Ñ0Ø×(Ñ(¨Ñ6‰Fñ   Ô-à×7Ñ7Ô8IÈ<ÓXˆKÜ(¨Ó5ˆFØ×$Ñ$×(Ñ(¨¯©×)BÑ)BÀ6Ç;Á;Ã=Ó)QÔRä×$Ñ$ R¨°vÓ>Ð>r   c           	      ó$  — t        ||«      D �]   \  }}t        |t        «      sŒt        |t        «      sJ d«       ‚t	        |«      D ]Ä  \  }}	|j
                  j                  j                  |«       t        |	t        j                  «      r,|j                  j                  | j                  ||	«      «       Œq|j                  xr t        |j                  |«      }
|j                  j                  t        j                   ||	|
«      «       ŒÆ �Œ y)z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r˜   r‚   r   Úlistrš   rf   rg   Úmutationr+   r”   r{   rž   r¼   rZ   r   r   rz   )r?   rP   rR   rA   rS   rT   r†   Úpy_argr¬   ÚvalrZ   s              r    rJ   z"OptimizerVariable.update_list_argsv  sß   € ô ˜t WÓ-ó 	Q‰KˆC�Ü˜#œ|Õ,Ü! &¬$Ô/ð ØCóÐ/ô (¨Ó/ò Q‘F�A�sØ—I‘I×*Ñ*×3Ñ3°CÔ8Ü! #¤u§|¡|Ô4ØŸ	™	×(Ñ(¨×)9Ñ)9¸"¸cÓ)BÕCà!$§¡Ò!L´¸c¿j¹jÈ!Ó0L˜ØŸ	™	×(Ñ(¬×)>Ñ)>¸rÀ3ÈÓ)OÕPòQñ	Qr   c                 óª   ‡‡‡— | j                   Š| j                  Š|j                  j                  Šˆˆˆfd„}|j                  j	                  |«       y )Nc                 óB   •‡ — ˆ ˆˆfd„}t        j                  ‰|«       y )Nc                  ó  •— ‰D ]‡  } ‰j                   j                  | d «       ‰j                  j                  | d «       ‰j                  r‰j                  j	                  «        ‰j
                  sŒn‰j
                  j	                  «        Œ‰ y r;   )Ú_buffersÚpopÚ_parametersÚparams_flatÚclearÚparams_flat_unwrap_subclasses)rQ   ÚgmÚnames_to_deleteÚtcs    €€€r    Úclear_static_tensor_refsz\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs�  sl   ø€ Ø+ò A�DØ—K‘K—O‘O D¨$Ô/Ø—N‘N×&Ñ& t¨TÔ2Ø—~’~ØŸ™×,Ñ,Ô.Ø×7Ó7Ø×8Ñ8×>Ñ>Õ@ñAr   )ÚweakrefÚfinalize)rË   rÎ   rÌ   rÍ   r@   s   ` €€€r    Úinit_finalizerz:OptimizerVariable.create_finalizer.<locals>.init_finalizerŒ  s   ù€ öAô ×Ñ˜UÐ$<Õ=r   )r9   r@   rf   Útracing_contextÚadd_graph_finalizer)r?   rP   rÑ   rÌ   rÍ   r@   s      @@@r    rM   z"OptimizerVariable.create_finalizer‡  s?   ú€ Ø×2Ñ2ˆØ—
‘
ˆØ�Y‰Y×&Ñ&ˆö
	>ð 	�	‰	×%Ñ% nÕ5r   )NNN)ÚreturnN)rR   zlist[VariableTracker]rA   zdict[str, VariableTracker]rÔ   r   )rP   r   )r   r   r   r   Ú_nonvar_fieldsr=   rO   ra   rF   r`   rH   rG   rI   r¼   rJ   rM   Ú__classcell__)rB   s   @r    r6   r6   T   sŸ   ø„ àØØðð 
#×	1Ñ	1ð	€Nð Ø Øð@ð 
õ@ð;ð &ð	;ð
 -ð;ð 
õ;õB-ò$GòFò@$ò8;ò
oób?ð4QØ)óQö"6r   r6   )+Ú__doc__r£   rÏ   Útypingr   r+   Útorch._loggingr   Útorch.utils._pytreer   Úguardsr   r   rZ   r	   r
   r   r   r   r   r-   r   r„   r   Úconstantr   Údictsr   Úlistsr   Úmiscr   Úuser_definedr   Útorch._dynamo.symbolic_convertr   Ú	Exceptionr   r#   r   r¡   r4   r6   r   r   r    ú<module>rã      s„   ðñó, Û Ý  ã Ý ,Ý -ç 0÷÷ õ &Ý !Ý &Ý $Ý Ý !Ý 3ñ ÝDô	˜)ô 	ô	˜Iô 	ñ " (¨LÓ9€òô$D6Ð1õ D6r   