Ë
    g^(hv6  ã                   óþ   — U d dl Z d dlZd dlmZmZ d dlmZ d dlmZm	Z	m
Z
mZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZ g Zee   ed<    e j4                  e«      Z G d	„ d
ej:                  «      Zdee   defd„Zy)é    N)Ú
CollectionÚMapping)Údeepcopy)ÚAnyÚCallableÚOptionalÚoverloadÚUnion)Úoptim)ÚShardedTensor)ÚFullyShardedDataParallelÚ__all__c                   ó¶  — e Zd ZdZ	 	 ddeeeej                  e	f   f   de
j                  deeeeef         deej                      ddf
d„Zd	„ Zdeeef   fd
„Zeddd„«       Zedeg ef   defd„«       Zddeeg ef      dee   fd„Zedeej                  ef   fd„«       Zdeeef   ddfd„Zdeeef   ddfd„Zdd„Zdeeef   fd„Zdeeef   fd„Zy)Ú_NamedOptimizeraì  
    ``_NamedOptimizer`` takes a dict of parameters and exposes ``state_dict`` by parameter key.

    We replace the original key (number) in an optim to the
    fully qualified name (FQN) string. User can initialize the optim as they
    initialize a PyTorch optim, the only difference is that they also need to
    pass in the FQN of each parameters.

    Args:
        named_parameters (Mapping[str, Union[torch.Tensor, ShardedTensor]]):
            Mapping from FQN to parameter.
        optimizer_class (optim.Optimizer):
            The class of optimizer to instantiate.
        param_groups (Collection[Mapping[str, Any]]):
            `param_groups` to pass to optimizer if specified.
            The key of the inner map needs to be FQNs.
            Default: None
        module (nn.Module): the module whose parameters to updated
            by the optimizer.
        args: arguments to pass to the optimizer constructor.
        kwargs: arguments to pass to the optimizer constructor.

    Example::
        >>> # xdoctest: +SKIP("distributed")
        >>> from torch import optim
        >>> from torch.distributed.optim import _NamedOptimizer
        >>>
        >>> # Define the named optimizer.
        >>> m = Model(...)
        >>> named_optim = _NamedOptimizer(m.named_parameters(), optim.SGD)
        >>> # Forward pass + backward pass.
        >>> named_optim.step()
        >>> ...
        >>> # Call state_dict for the named optimizer returns a FQN state_dict.
        >>> named_optim.state_dict()

    Warning: This API is still in development and subject to change.

    TODO: Add tutorial for _NamedOptimizer.
    TODO: Add documentation in the docstring for the public attributes
          like self.param_groups and self.named_parameters.
    NÚnamed_parametersÚoptimizer_classÚparam_groupsÚmoduleÚreturnc                 óš  — t         j                  j                  d«       || _        | j	                  «        t        |«      | _        |€| j                  j                  «       n|} ||g|¢­i |¤Ž| _        || _	        |€)t        | j                  j                  «       «      | _        nƒt        j                  d«       | j                  j                  «       D ��	ci c]  \  }}	|	|“Œ
 }
}}	g }|D ]3  }|d   D ])  }	|	|
vrt!        d|	› d�«      ‚|j#                  |
|	   «       Œ+ Œ5 || _        | j                  j                  | _        y c c}	}w )Nz'torch.distributed.optim._NamedOptimizerzvSince we pass in param_groups, we will use param_groups to initialize the optimizer, not all parameters of the module.ÚparamszExpect param name z% found in param group but is missing.)ÚtorchÚ_CÚ_log_api_usage_oncer   Ú_param_groups_checkÚdictr   ÚvaluesÚ
_optimizerr   ÚlistÚkeysÚordered_param_keysÚwarningsÚwarnÚitemsÚ
ValueErrorÚappend)Úselfr   r   r   r   ÚargsÚkwargsÚparams_for_optimizerÚkeyÚparamÚparam_to_keyr!   Úgroups                úe/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/distributed/optim/named_optimizer.pyÚ__init__z_NamedOptimizer.__init__@   sh  € ô 	�‰×$Ñ$Ð%NÔOØ;GˆÔØ× Ñ Ô"Ü $Ð%5Ó 6ˆÔà.:Ð.BˆD×!Ñ!×(Ñ(Ô*Èð 	ñ *Ø ð
àò
ð ñ
ˆŒð
 ˆŒØÐÜ&*¨4×+@Ñ+@×+EÑ+EÓ+GÓ&HˆDÕ#ä�M‰MðNôð :>×9NÑ9N×9TÑ9TÓ9V×W©:¨3°˜E 3™JÐWˆLÑWØ!#ÐØ%ò C�Ø" 8™_ò C�EØ LÑ0Ü(Ø0°°Ð7\Ð]óð ð '×-Ñ-¨l¸5Ñ.AÕBñCðCð '9ˆDÔ#à ŸO™O×8Ñ8ˆÕùó Xs   ÃEc                 ól  — | j                   �¨| j                   D ]˜  }t        |t        «      sJ d«       ‚d|v sJ d«       ‚|d   }t        |t        j                  «      r|g}t        |«      }|D ]=  }t        |t        j                  «      rŒt        dt        j                  |«      z   «      ‚ ||d<   Œš y y )Núparam group must be a dictr   z#param group must contain key paramsz>optimizer can only optimize Tensors, but one of the params is )r   Ú
isinstancer   r   ÚTensorr   Ú	TypeErrorÚtypename)r'   Úparam_groupr   r,   s       r/   r   z#_NamedOptimizer._param_groups_checkj   sÇ   € Ø×ÑÐ(Ø#×0Ñ0ò /�Ü! +¬tÔ4ÐRÐ6RÓRÐ4Ø ;Ñ.ÐUÐ0UÓUÐ.Ø$ XÑ.�Ü˜f¤e§l¡lÔ3Ø$˜X�FÜ˜f›�Ø#ò �EÜ% e¬U¯\©\Õ:Ü'ð8Ü:?¿.¹.ÈÓ:OñPóð ðð )/�˜HÒ%ñ/ð )ó    c                 óÈ  — | j                   j                  «       }|d   }|d   j                  «       D ��ci c]  \  }}| j                  |   |“Œ }}}g }|D ]k  }|d   D �cg c]  }| j                  |   ‘Œ }	}dt	        |	«      i}
|j                  «       D ]  \  }}|dk7  sŒt        |«      |
|<   Œ |j                  |
«       Œm | j                  ||dœ«      S c c}}w c c}w )zµ
        Return the ``state_dict`` of the optimizer.

        Instead of using number to index
        parameters, we will use module fully qualified name (FQN) as the key.
        r   Ústater   )r:   r   )r   Ú
state_dictr$   r!   Úsortedr   r&   Ú_post_state_dict)r'   r;   r   Úst_keyÚ	state_valÚ	ret_stateÚ
ret_groupsr.   r,   Ú
param_keysÚ	ret_groupÚkÚvs                r/   r;   z_NamedOptimizer.state_dict{   s  € ð —_‘_×/Ñ/Ó1ˆ
Ø! .Ñ1ˆð &0°Ñ%8×%>Ñ%>Ó%@÷
á!�˜	ð ×#Ñ# FÑ+¨YÑ6ð
ˆ	ñ 
ð
 ˆ
Ø!ò 	)ˆEØFKÈHÁoÖV¸U˜$×1Ñ1°%Ó8ÐVˆJÐVØ!¤6¨*Ó#5Ð6ˆIØŸ™›ò /‘��1Ø˜“=Ü#+¨A£;�I˜a’Lð/ð ×Ñ˜iÕ(ð	)ð ×$Ñ$¨yÈ*Ñ%UÓVÐVùó
ùò Ws   ¶CÁ!CÚclosurec                  ó   — y ©N© ©r'   rF   s     r/   Ústepz_NamedOptimizer.step•   s   € Ø14r8   c                  ó   — y rH   rI   rJ   s     r/   rK   z_NamedOptimizer.step˜   s   € Ø;>r8   c                 ó:   — | j                   j                  |¬«      S )z’
        Perform a single optimization step.

        This will call :meth:`torch.optim.Optimizer.step` on the wrapped
        optimizer.
        ©rF   )r   rK   rJ   s     r/   rK   z_NamedOptimizer.step›   s   € ð �‰×#Ñ#¨GÐ#Ó4Ð4r8   c                 ó.   — | j                   j                  S rH   )r   r:   )r'   s    r/   r:   z_NamedOptimizer.state¤   s   € à�‰×$Ñ$Ð$r8   r;   c                 óÈ  — | j                   j                  «       }| j                  |«      }|d   }|d   }t        |«      dk(  rt	        d«      ‚t        | j                  «      D �]Ü  \  }}||j                  «       vrŒt        ||   «      t        ||   «      k7  r,t	        dt        ||   «      › d|› dt        ||   «      › �«      ‚||   j                  «       D �]c  \  }}|||   vrt	        d|› d|› d�«      ‚||   |   }	t        |t        «      rÄt        |	t        «      sJ ‚t        |j                  «       «      }
t        |	j                  «       «      }|
|k7  rt	        d	|› d
|
› d|› d|› �«      ‚t        |j                  «       |	j                  «       «      D ]8  \  }}|j                  j                  «       j                  |j                  «       Œ: Œüt        |t         j"                  «      r=t        |	t         j"                  «      sJ ‚|j                  «       j                  |	«       �ŒSt%        |	«      ||   |<   �Œf �Œß |d   }|d   }i }|D ]  }t'        |d   «      }||t)        |«      <   Œ  i }|D ]:  }g }|d   D ]   }|j+                  | j                  |   «       Œ" ||t)        |«      <   Œ< |j                  «       D ]�  \  }}||vrŒ||   }t        |«      t        |«      k7  r't	        dt        |«      › d|› d
t        |«      › d�«      ‚|D ]/  }||vrt	        d|› d|› d�«      ‚|dk7  sŒt%        ||   «      ||<   Œ1 Œƒ | j                   j-                  |«       y)aè  
        Define the default behavior to load a state_dict for ``_NamedOptimizer``.

        Sample Code
        ```
            my_model = MyModule()
            optimizer = _NamedOptimizer(my_model.named_parameters(), Adagrad)
            ...

            optim_state_dict = optimizer.state_dict()
            ...
            ...

            optimizer.load_state_dict(optim_state_dict)
            ...
        ```
        Args:
            state_dict (Dict[str, Any]) : A ``state_dict`` to load into the optimizer.
                Note that this state dict update is performed in place.

        .. note:: PyTorch is using lazy init to initialize the optim states.
            So it is possible that there is no optim state when user call
            ``load_state_dict`` and for ``_NamedOptimizer`` we make it stricter
            that users can only call ``load_state_dict`` after the state is initialized.
            By doing this, we can validate the optim ``state_dict`` to be loaded.
        r:   r   zJExpects the optim to be initialized before load but found not initialized.zExpects equal length as z for parameter z but found: zExpects state z but not found.z"Expects equal number of shards as z but found z for ú/r   r   z"Expects equal param_group size as z for group ú.zExpects group key z to be in group z  in `state_dict` but is missing.N)r   r;   Ú_pre_load_state_dictÚlenr%   Ú	enumerater!   r    r$   r3   r   Úlocal_shardsÚzipÚtensorÚdetachÚcopy_r   r4   r   r   Ú_gen_param_group_keyr&   Úload_state_dict)r'   r;   Únew_state_dictr:   Ú	new_stateÚidxÚ	param_keyÚ	state_keyr?   Úsrc_state_valÚ
num_shardsÚnum_new_shardsÚshardÚ	src_shardÚsrc_param_groupsÚnew_param_groupsÚsrc_group_mapr.   rB   Únew_group_mapÚ	new_groupÚ	group_keyÚ	src_grouprD   s                           r/   r\   z_NamedOptimizer.load_state_dict¨   s  € ð6 Ÿ™×3Ñ3Ó5ˆØ×.Ñ.¨zÓ:ˆ
Ø˜7Ñ#ˆØ" 7Ñ+ˆ	Üˆy‹>˜QÒÜØ\óð ô (¨×(?Ñ(?Ó@ó  	H‰NˆC�à §
¡
£Ñ,ØÜ�5˜Ñ#Ó$¬¨I°c©NÓ(;Ò;Ü Ø.¬s°9¸S±>Ó/BÐ.CÀ?ÐS\ÐR]Ð]iÔjmÐnsÐt}Ñn~Ójð  jAð  Bóð ð )2°#©×(<Ñ(<Ó(>ó HÑ$�	˜9Ø E¨)Ñ$4Ñ4Ü$Ø(¨¨°?À9À+È_Ð]óð ð !& iÑ 0°Ñ ;�Ü˜i¬Ô7Ü% m´]ÔCÐCÐCÜ!$ Y×%;Ñ%;Ó%=Ó!>�JÜ%(¨×)CÑ)CÓ)EÓ%F�NØ! ^Ò3Ü(Ø@ÀÐ@PÐP[Ð\fÐ[gÐglÐmvÐlwÐwxð  zCð  yDð  Eóð ô -0Ø!×.Ñ.Ó0°-×2LÑ2LÓ2Nó-ò FÑ(˜˜yð Ÿ™×+Ñ+Ó-×3Ñ3°I×4DÑ4DÕEñFô   	¬5¯<©<Ô8Ü% m´U·\±\ÔBÐBÐBØ×$Ñ$Ó&×,Ñ,¨]Ö;ä08¸Ó0G�I˜c‘N 9Ó-ò/Hð 	HðF & nÑ5ÐØ)¨.Ñ9ÐàˆØ%ò 	DˆEÜ˜e H™oÓ.ˆJØ>CˆMÔ.¨zÓ:Ò;ð	Dð ˆØ)ò 	HˆIØˆJØ& xÑ0ò F�	Ø×!Ñ! $×"9Ñ"9¸)Ñ"DÕEðFà>GˆMÔ.¨zÓ:Ò;ð		Hð
 %2×$7Ñ$7Ó$9ò 	:Ñ ˆI�yð  Ñ-ØØ% iÑ0ˆIÜ�9‹~¤ Y£Ò/Ü Ø8¼¸Y»Ð8HÈÐT]ÐS^Ð^iÔjmÐnwÓjxÐiyÐyzÐ{óð ð ò :�Ø˜IÑ%Ü$Ø,¨Q¨CÐ/?À	¸{ÐJjÐkóð ð ˜“=Ü#+¨I°a©LÓ#9�I˜a’Lñ:ð	:ð$ 	�‰×'Ñ'¨Õ7r8   r7   c                 óÚ  — t        |t        «      sJ d«       ‚|d   }t        |t        j                  «      r|g|d<   nt	        |«      |d<   | j
                  j                  «       D ��ci c]  \  }}||“Œ
 }}}|d   D ]/  }||vrt        d«      ‚| j                  j                  ||   «       Œ1 | j                  j                  |«       | j                  j                  | _        yc c}}w )zŸ
        Add a param group to the :class:`_NamedOptimizer` s `param_groups`.

        Warning: This API is still in development and subject to change.
        r2   r   z%some parameters are not in the moduleN)r3   r   r   r4   r   r   r$   r%   r!   r&   r   Úadd_param_groupr   )r'   r7   r   r+   r,   r-   s         r/   ro   z_NamedOptimizer.add_param_group  så   € ô ˜+¤tÔ,ÐJÐ.JÓJÐ,à˜XÑ&ˆÜ�fœeŸl™lÔ+Ø%+ HˆK˜Ò!ä$(¨£LˆK˜Ñ!à59×5JÑ5J×5PÑ5PÓ5R×S¡z s¨E˜˜s™
ÐSˆÑSØ  Ñ*ò 	@ˆEØ˜LÑ(Ü Ð!HÓIÐIØ×#Ñ#×*Ñ*¨<¸Ñ+>Õ?ð	@ð
 	�‰×'Ñ'¨Ô4à ŸO™O×8Ñ8ˆÕùó Ts   Á)C'c                 óò   — | j                   j                  «       D ]H  }|j                  sŒt        j                  |«      }t        j
                  j                  |«      |_        ŒJ | j                  d¬«       y)z×
        Run a dummy optimizer step, which allows to initialize optimizer state because we do lazy init for most optimizers.

        This allows doing in-place loading of optimizer state from a checkpoint.
        NrN   )	r   r   Úrequires_gradr   Ú
zeros_likeÚautogradÚVariableÚgradrK   )r'   r,   Úts      r/   Ú
init_statez_NamedOptimizer.init_state(  sa   € ð ×*Ñ*×1Ñ1Ó3ò 	8ˆEØ×"Ó"Ü×$Ñ$ UÓ+�Ü"Ÿ^™^×4Ñ4°QÓ7�•
ð	8ð
 	�	‰	˜$ˆ	Õr8   c                 ó”   — t        | j                  t        «      r-t        j                  | j                  | j                  |d¬«      S |S )NT)Úis_named_optimizer)r3   r   ÚFSDPÚoptim_state_dict_to_loadr   ©r'   r;   s     r/   rS   z$_NamedOptimizer._pre_load_state_dict5  s>   € ô �d—k‘k¤4Ô(Ü×0Ñ0Ø—‘˜TŸ_™_¨jÈTôð ð Ðr8   c                 ó�   — t        | j                  t        «      r+t        j                  | j                  | j                  |«       |S rH   )r3   r   rz   Úoptim_state_dictr   r|   s     r/   r=   z _NamedOptimizer._post_state_dict>  s2   € ô �d—k‘k¤4Ô(Ü×!Ñ! $§+¡+¨t¯©À
ÔKØÐr8   )NN).)rF   Nr   NrH   )r   N) Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ústrr
   r   r4   r   r   Ú	Optimizerr   r   r   ÚnnÚModuler0   r   r   r;   r	   rK   r   ÚfloatÚpropertyr:   r\   ro   rw   rS   r=   rI   r8   r/   r   r      sƒ  „ ñ)ð^ AEØ&*ñ(9à! # u¨U¯\©\¸=Ð-HÑ'IÐ"IÑJð(9ð Ÿ™ð(9ð ˜z¨'°#°s°(Ñ*;Ñ<Ñ=ð	(9ð
 ˜Ÿ™Ñ#ð(9ð 
ó(9òT/ð"W˜D  c ™Nó Wð4 Û4ó Ø4àØ>˜H R¨ YÑ/Ð>°EÒ>ó Ø>ñ5˜H X¨b°%¨iÑ%8Ñ9ð 5ÀXÈeÁ_ó 5ð ð%�w˜uŸ|™|¨SÐ0Ñ1ò %ó ð%ðf8¨'°#°s°(Ñ*;ð f8Àó f8ðP9¨7°3¸°8Ñ+<ð 9Àó 9ó0 ð°$°s¸C°x±.ó ð¨d°3¸°8©nô r8   r   rB   r   c                 ó6   — dj                  t        | «      «      S )zGConcatenate all param keys as a unique indentifier for one param group.rQ   )Újoinr<   )rB   s    r/   r[   r[   F  s   € à�8‰8”F˜:Ó&Ó'Ð'r8   ) Úloggingr"   Úcollections.abcr   r   Úcopyr   Útypingr   r   r   r	   r
   r   Útorch.nnr…   r   Ú'torch.distributed._shard.sharded_tensorr   Útorch.distributed.fsdpr   rz   r   r   rƒ   Ú__annotations__Ú	getLoggerr   Úloggerr„   r   r[   rI   r8   r/   ú<module>r•      sr   ðä Û ß /Ý ß ;Õ ;ã Ý Ý Ý AÝ Cð €ˆˆc‰Ó à	ˆ×	Ñ	˜8Ó	$€ôo�e—o‘oô oðd	( T¨#¡Yð (°3ô (r8   