Ë
    g^(h\  ã                   óš  — U d dl mZmZ d dlmZmZ d dlZd dlmZ	 d dl
mZ d dlmZ d dlmZmZmZ d dlmZmZ  G d„ d	e«      Zdaee   ed
<   deddfd„Z	 ddej2                  dee   deej2                  ee   f   fd„Z	 ddej2                  dedee   dej2                  fd„Z	 ddej2                  dededede	j<                  dee   dej2                  fd„Z	 ddej2                  dededee   dej2                  f
d„Z 	 ddej2                  dee   deej2                  e!e   f   fd„Z"	 ddedee   dee   dej2                  fd„Z#y)é    )ÚABCÚabstractmethod)ÚAnyÚOptionalN)ÚShardedTensor)ÚShard)Ú_all_gather_dtensorÚ_create_chunk_dtensorÚ_create_chunk_sharded_tensor)Ú
DeviceMeshÚDTensorc                   ó  — e Zd ZdZedej                  deej                  ee	   f   fd„«       Z
edej                  de	dej                  fd„«       Ze	 ddej                  ded	ed
edej                  deej                     dej                  fd„«       Zedej                  dededej                  fd„«       Zedej                  deej                  ee   f   fd„«       Zededee   dej                  fd„«       Zy)ÚFSDPExtensionszä
    This enables some customizable hooks to enable composability with tensor
    parallelism. To activate these hooks, use :func:`_set_fsdp_extensions` to
    set a custom :class:`FSDPExtensions` that implements the hooks.
    ÚtensorÚreturnc                  ó   — y)z6E.g. converting ``DistributedTensor`` to local tensor.N© ©Úselfr   s     úe/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/distributed/fsdp/_fsdp_extensions.pyÚpre_flatten_transformz$FSDPExtensions.pre_flatten_transform   s   € ð 	ó    Úparam_extensionc                  ó   — y)z6E.g. converting local tensor to ``DistributedTensor``.Nr   )r   r   r   s      r   Úpost_unflatten_transformz'FSDPExtensions.post_unflatten_transform   s   € ð 	r   NÚrankÚ
world_sizeÚnum_devices_per_nodeÚpgÚdevicec                  ó   — y)z6Shards a tensor to chunks and returns the local chunk.Nr   )r   r   r   r   r   r   r    s          r   Úchunk_tensorzFSDPExtensions.chunk_tensor(   ó   € ð 	r   Údevice_meshc                  ó   — y)zAShards a tensor/DTensor to DTensor and returns the local DTensor.Nr   )r   r   r   r$   s       r   Úchunk_dtensorzFSDPExtensions.chunk_dtensor5   s   € ð 	r   c                  ó   — y)z£
        This is to be called before loading a *sharded* model state dict and
        should return the tensor and list of shards from which to load data.
        Nr   r   s     r   Úpre_load_state_dict_transformz,FSDPExtensions.pre_load_state_dict_transform?   s   € ð 	r   Úparent_meshc                  ó   — y)z³
        This is to be called before loading a *sharded* DTensor state dict.
        This gathers tensor in FSDP dimension and returns local tensor of
        TP DTensor.
        Nr   )r   r   r)   s      r   Úall_gather_dtensorz!FSDPExtensions.all_gather_dtensorJ   r#   r   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚtorchÚTensorÚtupler   r   r   r   ÚintÚdistÚProcessGroupr    r"   r   r&   Úlistr   r(   r   r+   r   r   r   r   r      s¦  „ ñð ðà—‘ðð 
ˆu�|‰|˜X c™]Ð*Ñ	+òó ðð ðà—‘ðð ðð 
�‰ò	ó ðð ð *.ñ
à—‘ð
ð ð
ð ð	
ð
 "ð
ð ×Ñð
ð ˜Ÿ™Ñ&ð
ð 
�‰ò
ó ð
ð ðà—‘ðð ðð  ð	ð
 
�‰òó ðð ðà—‘ðð 
ˆu�|‰|˜T %™[Ð(Ñ	)òó ðð ð
àð
ð ˜jÑ)ð
ð 
�‰ò	
ó ñ
r   r   Ú_extensionsÚ	flattenerr   c                 ó   — | a y r,   )r8   )r9   s    r   Ú_set_fsdp_extensionsr;   [   s   € à�Kr   r   Úfsdp_extensionc                 óB   — |�|j                  | «      \  }}|�||fS | d fS r,   )r   )r   r<   Ú
new_tensorr   s       r   Ú_ext_pre_flatten_transformr?   `   s:   € ð Ð!Ø&4×&JÑ&JÈ6Ó&RÑ#ˆ
�OØÐ&Ø˜Ð.Ð.Ø�4ˆ<Ðr   r   c                 ó2   — |�|�|j                  | |«      S | S r,   )r   )r   r   r<   s      r   Ú_ext_post_unflatten_transformrA   k   s'   € ð
 Ð! oÐ&AØ×6Ñ6°v¸ÓOÐOØ€Mr   r   r   r   r   c                 óB   — |�|j                   nt        } || ||||«      S r,   )r"   r   )r   r   r   r   r   r<   Úchunk_tensor_fns          r   Ú_ext_chunk_tensorrD   u   s;   € ð Ð%ð 	×#Ò#ä)ð ñ
 ØØØØØ
óð r   r$   c                 ó>   — |�|j                   nt        } || ||«      S r,   )r&   r
   )r   r   r$   r<   Úchunk_dtensor_fns        r   Ú_ext_chunk_dtensorrG   ‹   s5   € ð Ð%ð 	×$Ò$ä"ð ñ
 ØØØóð r   c                 óv   — |�|j                  | «      S t        | «      t        u sJ ‚| j                  «       }| |fS r,   )r(   Útyper   Úlocal_shards)r   r<   Úshardss      r   Ú"_ext_pre_load_state_dict_transformrL   �   sF   € ð Ð!Ø×;Ñ;¸FÓCÐCä�‹<œ=Ñ(Ð(Ð(Ø× Ñ Ó"€FØ�FÐÐr   r)   c                 ó<   — |�|j                   nt        } || |«      S r,   )r+   r	   )r   r)   r<   Úall_gather_dtensor_fns       r   Ú_ext_all_gather_dtensorrO   ©   s,   € ð Ð%ð 	×)Ò)ä ð ñ
 ! ¨Ó5Ð5r   r,   )$Úabcr   r   Útypingr   r   r1   Útorch.distributedÚdistributedr5   Ú+torch.distributed._shard.sharded_tensor.apir   Ú-torch.distributed._shard.sharded_tensor.shardr   Ú#torch.distributed.fsdp._shard_utilsr	   r
   r   Útorch.distributed.tensorr   r   r   r8   Ú__annotations__r;   r2   r3   r?   rA   r4   r6   rD   rG   r7   rL   rO   r   r   r   ú<module>rY      sõ  ðß #Ð #ß  ã Ý  Ý EÝ ?÷ñ ÷
 9ôE�Sô EðP )-€ˆX�nÑ%Ó ,ð Nð °tó ð 04ñØ�L‰Lðà˜^Ñ,ðð ˆ5�<‰<˜ #™Ð&Ñ'óð 04ñØ�L‰Lðàðð ˜^Ñ,ðð ‡\�\ó	ð  04ñØ�L‰Lðà
ðð ðð ð	ð
 	×Ñðð ˜^Ñ,ðð ‡\�\óð4 04ñ	Ø�L‰Lðà
ðð ðð ˜^Ñ,ð	ð
 ‡\�\óð( 04ñ	Ø�L‰Lð	à˜^Ñ,ð	ð ˆ5�<‰<˜˜e™Ð$Ñ%ó	ð 04ñ
6Øð
6à˜*Ñ%ð
6ð ˜^Ñ,ð
6ð ‡\�\ô	
6r   