Ë
    g^(h;  ã            	       ó  — d dl Z d dlZd dlmZ d dlmZ d dlmZ d dlm	Z	 d dl
Z
d dlmZ d dlmc mc mZ d dlmZmZmZ  e j,                  e«      Z G d„ d«      Zd	e
j4                  j6                  d
eeeeee   f   f   fd„Z y)é    N)Údefaultdict)ÚIterator)Úcontextmanager)ÚEnum)Ú_apply_to_modulesÚ_get_module_fsdp_stateÚclean_tensor_namec                   ó¾   — e Zd ZU  G d„ dee«      Z ee«      Ze	eef   e
d<    e«       Zee   e
d<   edd„«       Zeededed   fd	„«       «       Zed
eddfd„«       Zy)ÚSimpleProfilerc                   ó$   — e Zd ZdZdZdZdZdZdZy)úSimpleProfiler.TypeÚallÚ
all_gatherÚall_gather_objectÚ
reshardingÚH2DÚD2HN)	Ú__name__Ú
__module__Ú__qualname__ÚALLÚ	ALLGATHERÚALLGATHER_OBJÚ
RESHARDINGr   r   © ó    úa/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/distributed/fsdp/_debug_utils.pyÚTyper      s    „ ØˆØ ˆ	Ø+ˆØ!ˆ
ØˆØ‰r   r   ÚresultsÚ	profilingÚreturnNc                 ól   — | j                   j                  «        | j                  j                  «        y )N)r   Úclearr    )Úclss    r   ÚresetzSimpleProfiler.reset"   s"   € à�‰×ÑÔØ�‰×ÑÕr   Úprofile_typec              #   óÒ  K  — || j                   vs
J |› d�«       ‚| j                   j                  |«       t        j                  «       }	 d –— t        j                  «       }| j                  |xx   ||z
  z  cc<   | j                   j                  |«       y # t        j                  «       }| j                  |xx   ||z
  z  cc<   | j                   j                  |«       w xY w­w)Nzk is already being profiled. SimpleProfiler does not support profiling multiple instances at the same time. )r    ÚaddÚtimeÚ	monotonicr   Úremove)r$   r&   ÚbeginÚends       r   ÚprofilezSimpleProfiler.profile'   sÇ   è ø€ ð  3§=¡=Ñ0ð 	
Øˆnð ð ó	
Ð0ð 	�‰×Ñ˜,Ô'Ü—‘Ó ˆð	/Ûä—.‘.Ó"ˆCØ�K‰K˜Ó%¨¨u©Ñ4Ó%Ø�M‰M× Ñ  Õ.øô —.‘.Ó"ˆCØ�K‰K˜Ó%¨¨u©Ñ4Ó%Ø�M‰M× Ñ  Õ.üs    ‚AC'ÁB ÁA
C'ÂAC$Ã$C'Úmsgc                 óò   — t        j                  «       dk(  rPt        j                  «       t         j                  j                  k(  r!t
        j                  d|| j                  «       | j                  «        y )Nr   z%s %s)	ÚdistÚget_rankÚget_debug_levelÚ
DebugLevelÚINFOÚloggerÚinfor   r%   )r$   r/   s     r   Údump_and_resetzSimpleProfiler.dump_and_reset9   sI   € ô �=‰=‹?˜aÒ¤D×$8Ñ$8Ó$:¼d¿o¹o×>RÑ>RÒ$RÜ�K‰K˜  c§k¡kÔ2Ø�	‰	�r   )r!   N)r   r   r   Ústrr   r   r   Úfloatr   ÚdictÚ__annotations__Úsetr    Úclassmethodr%   r   r   r.   r8   r   r   r   r   r      sœ   … ôˆs�Dô ñ !,¨EÓ 2€GˆT�#�u�*ÑÓ2Ù›%€Iˆs�3‰xÓàòó ðð Øð/ 3ð /¨8°D©>ò /ó ó ð/ð  ð ð ¨ò ó ñr   r   Úmodelr!   c                 ó†   — d„ }d„ }dg}i }t        | ||| j                  «       D ��cg c]  \  }}|‘Œ	 c}}||«      S c c}}w )aô  
    It is used for composable fully_shard() code path, it returns
      1. sharded module tree info: each line reprents a submodule name that contats the
    submodule's FQN and its submodule class name, if the submodule is sharded by `fully_shard`,
    the submodule name will add a postfix with ' FULLY SHARDED'. Each increased tree
    level adds 4 spaces before the printed name. A printed sharded module tree info for a toy model
    is like this:
        [CompositeModel] FULLY SHARDED
            l1[Linear]
            u1[UnitModule] FULLY SHARDED
                u1.l1[Linear]
                u1.seq[Sequential]
                    u1.seq.0[ReLU]
                    u1.seq.1[Linear]
                    u1.seq.2[ReLU]
                u1.l2[Linear]
            u2[UnitModule] FULLY SHARDED
                u2.l1[Linear]
                u2.seq[Sequential]
                    u2.seq.0[ReLU]
                    u2.seq.1[Linear]
                    u2.seq.2[ReLU]
                u2.l2[Linear]
            l2[Linear]
      2. a dict mapping from the concated module FQN and class name to a list of its managed
    original parameters' FQNs. An example of the dict for the above toy sharded model is like this:
            {'[CompositeModel]': ['l1.weight', 'l1.bias', 'l2.weight', 'l2.bias'],
             'u1[UnitModule]': ['u1.l1.weight', 'u1.l1.bias', 'u1.seq.1.weight', 'u1.seq.1.bias', 'u1.l2.weight', 'u1.l2.bias'],
             'u2[UnitModule]': ['u2.l1.weight', 'u2.l1.bias', 'u2.seq.1.weight', 'u2.seq.1.bias', 'u2.l2.weight', 'u2.l2.bias']
            }
    All FQNs are prefixed starting from ``model``.

    Args:
        model (torch.nn.Module): Root module (which may or may not be passed to
                                 composable `fully_shard()`).
    c                 ó<  — |dz  }t        |«      dkD  r|d   dk(  r|d d n|}|dz   | j                  j                  z   dz   }d|z  |z   }t        | «      }	|	€|dxx   |dz   z  cc<   y |	j                  j                  | d «      }
|
r|dxx   |d	z   dz   z  cc<   n|dxx   |dz   z  cc<   |
rl|
j                  }t        |t        j                  «      sJ ‚|j                  D �cg c]  }t        ||z   «      ‘Œ }}||v r||   j                  |«       y |||<   y y c c}w )
Né   r   éÿÿÿÿú.ú[ú]ú ú
z FULLY SHARDED)ÚlenÚ	__class__r   r   Ú_fully_sharded_module_to_handleÚgetÚ
flat_paramÚ
isinstanceÚflat_param_fileÚFlatParameterÚ_fqnsr	   Úextend)ÚmoduleÚprefixÚ
tree_levelÚsharded_tree_infoÚsharded_module_name_to_fqnsÚ
num_spacesÚtrimed_prefixÚprefixed_module_nameÚprinted_prefixed_module_nameÚstateÚhandleÚparamÚnameÚglobal_fqnss                 r   Ú	module_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.module_fnj   sa  € ð   !‘^ˆ
ä ›K¨!šO°°r±
¸cÒ0AˆF�3�B‰KÈð 	ð  -¨sÑ2°V×5EÑ5E×5NÑ5NÑNÐQTÑTÐØ'*¨ZÑ'7Ð:NÑ'NÐ$ä& vÓ.ˆØˆ=Ø˜aÓ Ð$@À4Ñ$GÑGÓ Øà×6Ñ6×:Ñ:¸6À4ÓHˆáØ˜aÓ Ø,Ð/?Ñ?À$ÑFñÔ ð ˜aÓ Ð$@À4Ñ$GÑGÓ áØ×%Ñ%ˆEÜ˜e¤_×%BÑ%BÔCÐCÐCà=B¿[¹[öØ59Ô! &¨4¡-Õ0ðˆKð ð $Ð'BÑBØ+Ð,@ÑA×HÑHÈÕUàDOÐ+Ð,@ÒAð ùòs   Ã"Dc                 ó   — | d   |fS )Nr   r   )rV   rW   s     r   Ú	return_fnzD_get_sharded_module_tree_with_module_name_to_fqns.<locals>.return_fnŽ   s   € Ø  Ñ#Ð%@Ð@Ð@r   Ú )r   Únamed_parameters)r?   ra   rc   rV   rW   ÚkeyÚ_s          r   Ú1_get_sharded_module_tree_with_module_name_to_fqnsrh   B   sa   € òP"PòHAð
 	ð$Ðð 9;ÐÜØØØØ ×1Ñ1Ó3×4‘��aŠÓ4ØØ#óð ùó 	5s   §=)!Úloggingr)   Úcollectionsr   Úcollections.abcr   Ú
contextlibr   Úenumr   ÚtorchÚtorch.distributedÚdistributedr1   Ú"torch.distributed.fsdp._flat_paramÚfsdpÚ_flat_paramrO   Ú$torch.distributed.fsdp._common_utilsr   r   r	   Ú	getLoggerr   r6   r   ÚnnÚModuleÚtupler9   r;   Úlistrh   r   r   r   ú<module>rz      sƒ   ðã Û Ý #Ý $Ý %Ý ã Ý  ß <Ó <÷ñ ð 
ˆ×	Ñ	˜8Ó	$€÷)ñ )ðX[Ø�8‰8�?‰?ð[à
ˆ3��S˜$˜s™)�^Ñ$Ð$Ñ%ô[r   