Ë
    g^(h  ã                   óZ  — d dl Z d dlZd dlZd dlmZ d dlZd dlmZ d dl	m
Z
 d dlmZ d dlmZmZmZmZ d dlmZ d dlmZmZmZmZ d„ Z	 dd	ej2                  d
edededej6                  deej8                     defd„Zd	ej2                  d
ededefd„Zd	edee   dej2                  fd„Zy)é    N)ÚOptional)Ú_get_device_module)Údistributed_c10d)ÚShardÚShardedTensorÚShardedTensorMetadataÚTensorProperties)ÚShardMetadata)Ú
DeviceMeshÚDTensorÚ	Replicater   c                 ó¾   — |j                  «       dk(  rd| › d|› �S |j                  «       dk(  r"d| › d|› dt        |«      j                  «       › �S d| › d|› d| |z  › �S )NÚcpuzrank:ú/Úhpuú:)Úlowerr   Úcurrent_device)ÚrankÚdevice_typeÚnum_devices_per_nodes      úa/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/distributed/fsdp/_shard_utils.pyÚ_get_remote_device_strr      s€   € Ø×ÑÓ˜eÒ#Ø�t�f˜A˜k˜]Ð+Ð+Ø	×	Ñ	Ó	 Ò	%Ø�t�f˜A˜k˜]¨!Ô,>¸{Ó,K×,ZÑ,ZÓ,\Ð+]Ð^Ð^à�t�f˜A˜k˜]¨!¨DÐ3GÑ,GÐ+HÐIÐIó    Útensorr   Ú
world_sizer   ÚpgÚdeviceÚreturnc                 óî  — | j                  |d¬«      }t        |«      |kD  rx||   j                  «       }| j                  «       D �cg c]  }d‘Œ }	}t	        j
                  | j                  «       d   |z  «      |z  |	d<   t        j                  ||	|«      g}
ng }
|D �cg c]  }t        |j                  «       «      ‘Œ }}dgt        t        j                  |D �cg c]  }|d   ‘Œ	 c}«      «      dd z   }dgt        |d   «      dz
  z  }	|D �cg c]  }|g|	z   ‘Œ
 }}|€t        j                  |«      j                  n|j                  }t        t        |«      «      D �cg c]#  }t        t!        j"                  ||«      ||«      ‘Œ% }}t        |«      t        |«      cxk(  rt        |«      k(  sJ ‚ J ‚t%        |||«      D ���cg c]  \  }}}t'        |||«      ‘Œ }}}}t)        || j                  «       t+        | j,                  | j.                  dt0        j2                  | j5                  «       ¬«      ¬«      }t7        j8                  |
||¬	«      S c c}w c c}w c c}w c c}w c c}w c c}}}w )
z 
    Shard a tensor to chunks along the first dimension. The local rank will gets its
    corresponding chunk as the local shard to create a ShardedTensor.
    r   )ÚdimNéÿÿÿÿé   F)ÚdtypeÚlayoutÚrequires_gradÚmemory_formatÚ
pin_memory)Úshards_metadataÚsizeÚtensor_properties)Úsharded_tensor_metadataÚprocess_group)ÚchunkÚlenÚcloner*   ÚmathÚceilr   Úfrom_tensor_and_offsetsÚlistÚ	itertoolsÚ
accumulater   Ú_get_pg_default_deviceÚtypeÚranger   ÚdistÚget_global_rankÚzipr
   r   r	   r$   r%   ÚtorchÚcontiguous_formatÚ	is_pinnedr   Ú+_init_from_local_shards_and_global_metadata)r   r   r   r   r   r   ÚchunksÚlocal_shardÚ_ÚoffsetsÚlocal_shardsr.   Úchunk_sizesÚ
chunk_sizeÚdim0_offsetsÚd0Úchunk_offsetsr   ÚrÚ
placementsÚoffsetr*   Ú	placementÚshard_metadatar,   s                            r   Ú_create_chunk_sharded_tensorrP      su  € ð �\‰\˜*¨!ˆ\Ó,€FÜ
ˆ6ƒ{�TÒØ˜T‘l×(Ñ(Ó*ˆØ$Ÿk™k›mÖ,˜’1Ð,ˆÐ,Ü—Y‘Y˜vŸ{™{›}¨QÑ/°*Ñ<Ó=ÀÑDˆ�‰
Ü×5Ñ5°kÀ7ÈDÓQÐR‰àˆð 4:Ö:¨%”4˜Ÿ
™
›Õ%Ð:€KÐ:Ø�3œÜ×Ñ¸kÖJ°
˜j¨›mÒJÓKóà	€rðñ €Lð ˆc”S˜ Q™Ó(¨1Ñ,Ñ-€GØ.:Ö;¨�b�T˜G“^Ð;€MÐ;ð ˆ>ô 	×/Ñ/°Ó3×8Ò8à�[‰[ð ô ”s˜;Ó'Ó(öð ô 	Ü× Ñ   QÓ'ØØ õ	
ð€Jð ô ˆ{Óœs =Ó1ÔD´S¸³_ÒDÐDÑDÐDÐDô (+¨=¸+ÀzÓ'R÷ð á#ˆF�D˜)ô 	�f˜d IÕ.ð€Nò ô 4Ø&Ø�[‰[‹]Ü*Ø—,‘,Ø—=‘=ØÜ×1Ñ1Ø×'Ñ'Ó)ô
ô
Ðô ×DÑDØÐ.EÐUWôð ùòU -ùò ;ùâJùò <ùòùôs$   Á	IÂ  IÃI!ÄI&Å#(I+Ç	I0Údevice_meshc                 óh  — | j                  «       j                  «       } t        |j                  «      D �cg c]  }t	        «       ‘Œ }}t        |j                  «      D �cg c]  }t	        «       ‘Œ }}t        d«      |d<   t        j                  | ||d¬«      j                  |¬«      S c c}w c c}w )z›
    Shard a tensor to chunks along the first dimension. The local rank will gets its
    corresponding chunk as the local tensor to create a DTensor.
    r   r"   F)Ú	run_check)rL   )	Údetachr0   r9   Úndimr   ÚDShardr   Ú
from_localÚredistribute)r   r   rQ   rC   Úreplicate_placementsÚshard_placementss         r   Ú_create_chunk_dtensorr[   \   s£   € ð �]‰]‹_×"Ñ"Ó$€Fô 27°{×7GÑ7GÓ1HÖI¨AœI�KÐIÐÐIÜ-2°;×3CÑ3CÓ-DÖE¨œ	�ÐEÐÐEÜ! !›9Ð�RÑä×ÑØ�Ð1¸Uôç�lØ#ð ó ðùò	 JùÚEs   ¶B*Á B/Ú	root_meshc                 óò   — || j                   k(  sJ d«       ‚t        t        j                  | j                  «      «      }t        «       |d<   | j                  | j                   |¬«      } | j                  «       S )zT
    All gather a DTensor in its sharded dimension and return the local tensor.
    z2The device mesh of a tensor should be a root mesh.r"   )rQ   rL   )rQ   r4   ÚcopyÚdeepcopyrL   r   rX   Úto_local)r   r\   rL   s      r   Ú_all_gather_dtensorra   u   sz   € ð ˜×*Ñ*Ò*ð Ø<óÐ*ô ”d—m‘m F×$5Ñ$5Ó6Ó7€Jô “[€Jˆr�NØ× Ñ Ø×&Ñ&Øð !ó €Fð
 �?‰?ÓÐr   )N) r^   r5   r1   Útypingr   r=   Útorch.distributedÚdistributedr:   Útorch._utilsr   r   Ú'torch.distributed._shard.sharded_tensorr   r   r   r	   Ú&torch.distributed._shard.sharding_specr
   Útorch.distributed.tensorr   r   r   rV   r   ÚTensorÚintÚProcessGroupr   rP   r[   ra   © r   r   ú<module>rm      sî   ðã Û Û Ý ã Ý  Ý +Ý .÷ó õ Aß TÓ TòJð &*ñ;Ø�L‰Lð;à
ð;ð ð;ð ð	;ð
 	×Ñð;ð �U—\‘\Ñ"ð;ð ó;ð|Ø�L‰Lðà
ðð ðð ó	ð2Øðà˜
Ñ#ðð ‡\�\ôr   