Ë
    [^(h‰â  ã            	       ó’  — d dl Z d dlZd dlZd dlZd dlZd dlZd dlmZmZ d dl m	Z	 d dl
mZ d dlmZmZ d dlmZ d dlmZmZmZmZmZmZ d dlmZ d dlZd dlmZ d dlmZ d dl mc m!Z" d d	l#m$Z$ d d
l%m&Z& d dl'm(Z(m)Z)m*Z+ d dl,m-Z- d dl.m/Z/m0Z0 d dl1m2Z2 d dl3m4Z4m5Z5m6Z6 d dl7m8Z8 d dl9m:Z:m;Z;m<Z< d dl=m>Z>m?Z?m@Z@ d dlAmBZBmCZCmDZDmEZE d dlmFZFmGZG d dlHmIZJ d dlKmLZLmMZMmNZNmOZO d dlPmQZQmRZRmSZSmTZTmUZU d dlVmWZW dZXeSrdZYdZZej¶                  j¹                  «       ZXn.eTrdZYdZZn'eUrdZYdZZejº                  j¹                  «       ZXnd ZYd!ZZd"ZX G d#„ d$e«      Z^ G d%„ d&e«      Z_ G d'„ d(ejÀ                  e«      Zad)ejÀ                  d*ejÄ                  d+efd,„Zcd-„ Zd	 	 dtd)ejÀ                  d.eefd/„Zfdud0„Zgd1„ Zhd2„ Zidvd)ejÀ                  d3eefd4„Zjd)ejÀ                  d5eefd6„Zkd)ejÀ                  d7eefd8„Zl G d9„ d:«      Zm G d;„ d<ea«      Zn G d=„ d>ea«      Zo G d?„ d@eo«      Zp G dA„ dBeo«      Zq G dC„ dDea«      Zr G dE„ dFer«      Zs G dG„ dHejÀ                  «      Zt G dI„ dJeo«      Zu G dK„ dLejÀ                  «      Zv G dM„ dNejî                  «      Zx G dO„ dPejÀ                  «      Zye jô                  dQefdR„«       Z{e jô                  dSefdT„«       Z|e jô                  dUefdV„«       Z}ee jô                  dWefdX„«       «       Z~ee jô                  dYefdZ„«       «       Zee jô                  d[efd\„«       «       Z€ee jô                  d]efd^„«       «       Z�d_ed+ed`edaefdb„Z‚	 dwdcejÀ                  ddejÀ                  deeƒe„dff   fdg„Z… G dh„ dieM«      Z† G dj„ dkeL«      Z‡dxdleeˆ   fdm„Z‰ G dn„ doejÀ                  «      ZŠ G dp„ dqejÀ                  «      Z‹ G dr„ dsejÀ                  «      ZŒy)yé    N)ÚABCÚabstractmethod)Únullcontext)Údeepcopy)ÚautoÚEnum©Úwraps)ÚAnyÚCallableÚcastÚno_type_checkÚOptionalÚUnion)Úmock)Ú
checkpoint)Ú
DeviceMesh)Ú
CPUOffloadÚfully_shardÚFullyShardedDataParallel)ÚTrainingState)ÚFSDPParamGroupÚRegisterPostBackwardFunction)Ú#NO_RESHARD_AFTER_FORWARD_STRATEGIES)ÚBackwardPrefetchÚMixedPrecisionÚShardingStrategy)ÚShardedGradScaler)Úalways_wrap_policyÚModuleWrapPolicyÚwrap)Údistribute_tensorÚDTensorÚShard)ÚColwiseParallelÚparallelize_moduleÚRowwiseParallelÚSequenceParallel)ÚTransformerDecoderLayerÚTransformerEncoderLayer)ÚDistributedDataParallel)ÚMultiProcessTestCaseÚMultiThreadedTestCaseÚrun_subtestsÚ
TEST_SKIPS)ÚFILE_SCHEMAÚget_cycles_per_msÚ	TEST_CUDAÚTEST_HPUÚTEST_XPU)Ú
has_tritoné   ÚcudaÚncclzhpu:0ÚhcclÚxpuÚxcclÚcpuÚglooé   c                   ó(   — e Zd Z e«       Z e«       Zy)ÚFSDPInitModeN)Ú__name__Ú
__module__Ú__qualname__r   ÚNO_FSDPÚ	RECURSIVE© ó    úa/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/testing/_internal/common_fsdp.pyr@   r@   V   s   „ á‹f€Gá“�IrG   r@   c                   ó6   — e Zd Z e«       Z e«       Z e«       Zy)ÚDEVICEInitModeN)rA   rB   rC   r   ÚDEVICE_BEFOREÚDEVICE_AFTERÚDEVICE_NEVERrF   rG   rH   rJ   rJ   _   s   „ á“F€Má“6€Lá“6�LrG   rJ   c                   ó¼   — e Zd ZdZedeej                  df   fd„«       Zedej                  fd„«       Z	edd„«       Z
eeded	edej                  fd
„«       «       Zy)ÚFSDPTestModelzZThis defines the interface expected from all models used commonly for
    FSDP unit tests.Úreturn.c                  ó   — y)z+Returns an input for the model as as tuple.NrF   ©ÚselfÚdevices     rH   Ú	get_inputzFSDPTestModel.get_inputl   ó   € ð 	rG   c                  ó   — y)z,Returns the loss given the input and output.NrF   )rS   ÚinputÚoutputs      rH   Úget_losszFSDPTestModel.get_lossq   rV   rG   Nc                  ó   — y)z<Runs the backward pass (e.g. including ``loss.backward()``).NrF   ©rS   Úlosss     rH   Úrun_backwardzFSDPTestModel.run_backwardv   rV   rG   ÚargsÚkwargsc                   ó   — y)z&Initializes an instance of this model.NrF   )r_   r`   s     rH   ÚinitzFSDPTestModel.init{   s   € ð 	rG   ©rP   N)rA   rB   rC   Ú__doc__r   ÚtupleÚtorchÚTensorrU   rZ   r^   Ústaticmethodr   ÚnnÚModulerb   rF   rG   rH   rO   rO   h   s˜   „ ñð ð 5¨¯©°sÐ):Ñ#;ò ó ðð ð¨¯©ò ó ðð òó ðð Øð�Cð  3ð ¨2¯9©9ò ó ó ñrG   rO   ÚmodelÚprocess_groupÚ	assert_fnc                 ó,  — | j                  «       D ��cg c]%  \  }}||j                  «       j                  «       f‘Œ' }}}|| j                  «       D ��cg c]%  \  }}||j                  «       j                  «       f‘Œ' c}}z  }t	        j
                  |«      }t        |«      D �	cg c]  }	d‘Œ }
}	t	        j                  |
||¬«       |
d   }|€J ‚|
dd D ])  }|€J ‚t        ||«      D ]  \  \  }	}\  }	} |||«       Œ Œ+ yc c}}w c c}}w c c}	w )a  
    All-gathers module states across ranks and calls ``assert_fn`` on each pair
    of corresponding states from rank 0 and a nonzero rank. For example, if
    ``assert_fn`` is ``self.assertEqual()``, then this checks that all module
    states are equal across ranks.
    N©Úgroupr   r>   )	Únamed_parametersÚdetachr<   Únamed_buffersÚdistÚget_world_sizeÚrangeÚall_gather_objectÚzip)rk   rl   rm   Ú
param_nameÚparamÚnamed_module_statesÚbuffer_nameÚbufferÚ
world_sizeÚ_ÚolistÚrank0_statesÚstateÚp1Úp2s                  rH   Ú_assert_module_statesr…   ‚   s6  € ð "'×!7Ñ!7Ó!9÷áˆJ˜ð 
�U—\‘\“^×'Ñ'Ó)Ò*ðÐñ ð à#(×#6Ñ#6Ó#8÷áˆK˜ð 
�f—m‘m“o×)Ñ)Ó+Ò,óñ Ðô ×$Ñ$ ]Ó3€JÜ  Ó,Ö-�aŠTÐ-€EÐ-Ü×Ñ˜5Ð"5¸]ÕKØ˜‘8€LØÐ#Ð#Ð#Ø�q�r�ò ˆØÐ Ð Ð Ü # L°%Ó 8ò 	Ñ‰GˆQ�‘W�a˜Ù�b˜"Õñ	ñùóùóùò
 .s   ”*DÁ*DÂ'	Dc                  ó4   — t        j                  t        «      S ©N)rf   rT   ÚDEVICE_TYPErF   rG   rH   Úget_devtyper‰   ¡   s   € Ü�<‰<œÓ$Ð$rG   Úzero_buffersc                 ó°  — |rt        j                  | «      n	t        «       }|5  | j                  «       D ]/  }t	        j
                  «       5  |j                  «        ddd«       Œ1 |rB| j                  «       D ]/  }t	        j
                  «       5  |j                  «        ddd«       Œ1 ddd«       y# 1 sw Y   ŒˆxY w# 1 sw Y   ŒPxY w# 1 sw Y   yxY w)zBZeros the parameters and optionally buffers of ``model`` in place.N)ÚFSDPÚsummon_full_paramsr   Ú
parametersrf   Úno_gradÚzero_Úbuffers)rk   rŠ   Úsummon_fullÚctxrz   r}   s         rH   Ú_zero_modelr”   ¥   sÁ   € ñ -8Œ$×
!Ñ
! %Ô
(¼[»]€CØ	ñ #Ø×%Ñ%Ó'ò 	ˆEÜ—‘“ñ Ø—‘”÷ð ð	ñ ØŸ-™-›/ò #�Ü—]‘]“_ñ #Ø—L‘L”N÷#ð #ð#÷#ð #÷ð ú÷#ð #ú÷#ð #ús;   ¤(CÁB4Á3CÂC Â!
CÂ4B=Â9CÃ C	ÃCÃCc                 ót   — |s| j                  t        «      } |r| j                  «        | j                  «       S r‡   )Útorˆ   ÚhalfÚ
state_dict)rk   Úcpu_offloadr—   s      rH   Ú_get_state_dictrš   ¶   s.   € ÙØ—‘œÓ%ˆÙØ�
‰
Œà×ÑÓÐrG   c           	      ój   — dj                  |D �cg c]  }|�| t        |«         nd‘Œ c}«      S c c}w )Nr   Únone)ÚjoinÚstr)Útest_name_mappingr_   Úss      rH   Úsubtest_namer¡   ¿   s7   € Ø�8‰8ØIMÖNÀA a mÐ	œ3˜q›6Ò	"¸Ñ	?ÒNóð ùÚNs   �0c                 ó”  — |j                  «       D ];  \  }}|j                  t        j                  d«      k7  sŒ)|j                  «       ||<   Œ= | dk(  r|nd g}t	        j
                  |«       t        t        t        t        j                  f   |d   «      }|j                  «       D ]  }||   j                  t        «      ||<   Œ |S )Nr<   r   )ÚitemsrT   rf   r<   rt   Úbroadcast_object_listr   Údictrž   rg   Úkeysr–   rˆ   )Úrankr˜   ry   rz   r€   s        rH   Ú_broadcast_state_dictr¨   Å   s¼   € ð (×-Ñ-Ó/ò 1Ñˆ
�EØ�<‰<œ5Ÿ<™<¨Ó.Ó.Ø%*§Y¡Y£[ˆJ�zÒ"ð1ð   1š9‰Z¨$Ð/€EÜ×Ñ˜uÔ%Ü”dœ3¤§¡Ð,Ñ-¨u°Q©xÓ8€Jà —o‘oÓ'ò Hˆ
Ø!+¨JÑ!7×!:Ñ!:¼;Ó!Gˆ
�:ÒðHàÐrG   Úrecursec                 ó    — t        j                  | |¬«      5  t        t        | j	                  «       «      «      cddd«       S # 1 sw Y   yxY w)a[  
    Returns the full unsharded parameters of ``model``. Any FSDP-managed
    parameters offloaded to CPU are moved to GPU in the returned list.

    Args:
        recurse (bool): If ``False``, only unshards the parameters immediate to
            ``model``; if ``True``, recurses through the module hierarchy
            rooted at ``model``.
    )r©   N)rŒ   r�   r   ÚlistrŽ   )rk   r©   s     rH   Úget_full_paramsr¬   Ö   s?   € ô 
×	 Ñ	  °Ô	8ñ 2Üœ˜U×-Ñ-Ó/Ó0Ó1÷2÷ 2ò 2ús   ˜"AÁAÚmove_to_devicec                 ó4   — |r| j                  t        «      S | S r‡   )r–   rˆ   )rk   r­   s     rH   Ú_move_to_devicer¯   ä   s   € Ù$2ˆ5�8‰8”KÓ Ð=¸Ð=rG   Ú	wrap_fsdpc                 ó(   — |s| S t        | g|¢­i |¤ŽS r‡   ©rŒ   )rk   r°   r_   r`   s       rH   Ú_maybe_wrap_fsdpr³   è   s   € Ù!ˆ5ÐC¤t¨EÐ'C°DÒ'C¸FÑ'CÐCrG   c                   ó:   — e Zd Zdedefd„Zdefd„Zdefd„Zd„ Zy)	ÚDummyProcessGroupr§   Úsizec                 ó    — || _         || _        y r‡   )Ú_rankÚ_size)rS   r§   r¶   s      rH   Ú__init__zDummyProcessGroup.__init__í   s   € ØˆŒ
Øˆ�
rG   rP   c                 ó   — | j                   S r‡   )r¸   ©rS   s    rH   r§   zDummyProcessGroup.rankñ   ó   € Ø�z‰zÐrG   c                 ó   — | j                   S r‡   )r¹   r¼   s    rH   r¶   zDummyProcessGroup.sizeô   r½   rG   c                 óB   — t        j                  «       }d„ }||_        |S )Nc                  ód   — t         j                  j                  «       } | j                  d«       | S )Nr>   )rf   ÚfuturesÚFutureÚ
set_result)Úfutures    rH   Ú
get_futurez/DummyProcessGroup.allreduce.<locals>.get_futureú   s'   € Ü+0¯=©=×+?Ñ+?Ó+AˆFØ×Ñ˜aÔ ØˆMrG   )r   ÚMockrÅ   )rS   r_   r`   Ú	dist_waitrÅ   s        rH   Ú	allreducezDummyProcessGroup.allreduce÷   s"   € Ü—I‘I“Kˆ	ò	ð
  *ˆ	ÔØÐrG   N)rA   rB   rC   Úintrº   r§   r¶   rÈ   rF   rG   rH   rµ   rµ   ì   s2   „ ð˜Sð ¨ó ð�có ð�có ó	rG   rµ   c                   óÚ   ‡ — e Zd Zdej                  dededefˆ fd„Zd„ Zd„ Z	d„ Z
d	„ Ze	 	 	 ddej                  d
ededeeeef      dededeej(                  ef   fd„«       Zd„ Zˆ xZS )ÚTransformerWithSharedParamsrp   Údevice_init_modeÚadd_bnÚdeterministicc                 óª  •— t         ‰| �  «        |j                  «       | _        |j                  «       | _        |rt        j                  d«       d}d}t        j                  ||«      | _	        t        j                  |dddd¬«      | _        t        j                  ||«      | _        | j                  j                  | j                  _        | j                  d| j                  j                  j!                  |f«      «       | j                  d	t        j"                  | j$                  t
        j&                  ¬
«      «       d| _        |r)t
        j                  j+                  | j(                  «      nt
        j                  j-                  «       | _        |t0        j2                  k(  r| j5                  t6        «      } |r| j9                  «        y y )Nr   é   é   é   é   gš™™™™™¹?)Úd_modelÚnum_encoder_layersÚnum_decoder_layersÚdim_feedforwardÚdropoutÚ
vocab_biasÚlong_buffer)Údtype)Úsuperrº   r§   r¶   r~   rf   Úmanual_seedri   Ú	EmbeddingÚembed_tokensÚTransformerÚtransformerÚLinearÚoutput_projÚweightÚregister_bufferÚnew_onesÚ
zeros_likerÙ   ÚlongÚbsÚBatchNorm1dÚIdentityÚbnrJ   rK   r–   rˆ   Úeval)rS   rp   rÌ   rÍ   rÎ   Úd_vocabrÔ   Ú	__class__s          €rH   rº   z$TransformerWithSharedParams.__init__  sa  ø€ ô 	‰ÑÔØ—J‘J“LˆŒ	ØŸ*™*›,ˆŒÙÜ×Ñ˜aÔ ØˆØˆäŸL™L¨°'Ó:ˆÔÜŸ>™>ØØ Ø ØØô
ˆÔô Ÿ9™9 W¨gÓ6ˆÔð #'×"3Ñ"3×":Ñ":ˆ×ÑÔØ×ÑØ˜$×+Ñ+×2Ñ2×;Ñ;¸W¸JÓGô	
ð 	×ÑØÜ×Ñ˜TŸ_™_´E·J±JÔ?ô	
ð
 ˆŒÙ39”%—(‘(×&Ñ& t§w¡wÔ/¼u¿x¹x×?PÑ?PÓ?RˆŒØœ~×;Ñ;Ò;Ø—7‘7œ;Ó'ˆDÙØ�I‰I�Kð rG   c                 ó,  — t        j                  d| j                  z   «       t        j                  d|¬«      j	                  d| j
                  «      }t        j                  | j
                  dz  |¬«      j	                  d| j
                  «      }||fS )Nr>   é   ©rT   é   r6   )rf   rÝ   r§   ÚarangeÚviewré   )rS   rT   ÚsrcÚtgts       rH   rU   z%TransformerWithSharedParams.get_input.  sl   € Ü×Ñ˜!˜dŸi™i™-Ô(Ü�l‰l˜2 fÔ-×2Ñ2°1°d·g±gÓ>ˆÜ�l‰l˜4Ÿ7™7 Q™;¨vÔ6×;Ñ;¸A¸t¿w¹wÓGˆØ�SˆzÐrG   c                 ó  — | j                  |«      }|| j                  z   | j                  j                  |«      z   }| j                  |«      }| j	                  |«      }| j                  ||«      }| j                  |«      S r‡   )rß   rÙ   rÚ   Útype_asrì   rá   rã   )rS   Úsrc_idsÚtgt_idsrö   r÷   Úxs         rH   Úforwardz#TransformerWithSharedParams.forward4  sv   € Ø×Ñ Ó(ˆØ�D—O‘OÑ# d×&6Ñ&6×&>Ñ&>¸sÓ&CÑCˆØ×Ñ Ó(ˆØ�g‰g�c‹lˆØ×Ñ˜S #Ó&ˆØ×Ñ Ó"Ð"rG   c                 ó¬   — |\  }}t         j                  j                  |j                  d|j	                  d«      «      |j                  d«      d¬«      S )NéÿÿÿÿÚsum)Ú	reduction)ri   Ú
functionalÚcross_entropyrõ   r¶   )rS   rX   rY   r   r÷   s        rH   rZ   z$TransformerWithSharedParams.get_loss<  sI   € Ø‰ˆˆ3Ü�}‰}×*Ñ*Ø�K‰K˜˜FŸK™K¨›OÓ,¨c¯h©h°r«lÀeð +ó 
ð 	
rG   c                 ó$   — |j                  «        y r‡   ©Úbackwardr\   s     rH   r^   z(TransformerWithSharedParams.run_backwardB  ó   € Ø�‰�rG   Úfsdp_init_modeÚfsdp_kwargsrP   c                 óN  — |€i }|t         j                  k(  r&t        | t        «      r| d   }n| }t	        ||||«      S |t         j
                  k(  rÈd|vrt        t        t        h«      }n|j                  d«      }d|v r8|d   t        j                  t        j                  hv rt        | t        «      sd}n| }t        | t        «      r| d   }	n| }	t	        |	|||«      }
t        |
|fd|i|¤Ž}|t        j                  k(  r|j!                  t"        «      }|S t%        d|› �«      ‚)au  
        Initializes a :class:`TransformerWithSharedParams` instance.

        Args:
            fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap
                any modules with FSDP. If ``RECURSIVE``, then wraps with
                top-level FSDP. By default, the top-level FSDP uses the
                ``ModuleWrapPolicy`` for encoder and decoder layers, but a
                different auto wrap policy may be specified via
                ``fsdp_kwargs``.
            device_init_mode (DEVICEInitMode): Determines model movement to DEVICE.
            fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments
                forwarded to the FSDP constructor.
            deterministic (bool): Whether to make the model deterministic
                across constructions.
            add_bn (bool): Whether to include batch norm in the model.
        Nr   Úauto_wrap_policyÚsharding_strategyúUnsupported FSDP init mode: )r@   rD   Ú
isinstancere   rË   rE   r    r*   r)   Úpopr   ÚHYBRID_SHARDÚ_HYBRID_SHARD_ZERO2rŒ   rJ   rL   r–   rˆ   Ú
ValueError)rp   r  rÌ   r	  rÎ   rÍ   Úpgr  Úfsdp_pgÚ
tformer_pgÚmÚ
fsdp_models               rH   rb   z TransformerWithSharedParams.initE  sX  € ð6 ÐØˆKØœ\×1Ñ1Ò1Ü˜%¤Ô'Ø˜1‘X‘à�Ü.ØÐ$ f¨móð ð œ|×5Ñ5Ò5à!¨Ñ4Ü#3ä/Ü/ðó$Ñ ð $/§?¡?Ð3EÓ#FÐ ð $ {Ñ2ØÐ 3Ñ4Ü$×1Ñ1Ô3C×3WÑ3WÐXñYä" 5¬%Ô0à‘à�ä˜%¤Ô'Ø" 1™X‘
à"�
ä+ØÐ,¨f°móˆAô ØØñð "2ðð ñ	ˆJð  ¤>×#>Ñ#>Ò>Ø'Ÿ]™]¬;Ó7�
ØÐÜÐ7¸Ð7GÐHÓIÐIrG   c                 ó   — | j                   gS r‡   )rá   r¼   s    rH   Úget_ignored_modulesz/TransformerWithSharedParams.get_ignored_modules“  s   € Ø× Ñ Ð!Ð!rG   )NFT)rA   rB   rC   rt   ÚProcessGrouprJ   Úboolrº   rU   rý   rZ   r^   rh   r@   r   r¥   rž   r   r   ri   rj   rŒ   rb   r  Ú__classcell__©rï   s   @rH   rË   rË     sâ   ø„ ð(à× Ñ ð(ð )ð(ð ð	(ð
 õ(òTò#ò
òð ð
 15Ø#ØñKJØ× Ñ ðKJà$ðKJð )ðKJð ˜d 3¨ 8™nÑ-ð	KJð
 ðKJð ðKJð 
ˆr�y‰y˜$ˆÑ	òKJó ðKJöZ"rG   rË   c                   óÄ   ‡ — e Zd Zdej                  dededefˆ fd„Zd„ Zd„ Z	d„ Z
d	„ Ze	 	 ddej                  d
ededeeeef      dedej&                  fd„«       Zˆ xZS )ÚNestedWrappedModulerp   r°   rÌ   rÎ   c                 ó�  •‡‡‡— t         ‰| �  «        ‰j                  «       | _        ‰j                  «       | _        |t
        j                  k(  }ˆˆˆfd„}|rt        j                  d«       t        j                  t        t        j                  dd«      |«       |t        j                   |t        t        j                  dd«      |«      «      t        t        j                  dd«      |«      «      «       |t        t        j                  dd«      |«      «      t        t        j                  dd«      |«      «      | _        y )Nc                 ó&   •— ‰rt        | ‰fi ‰¤ŽS | S r‡   r²   ©Úlayerr	  rp   r°   s    €€€rH   Ú_maybe_wrapz1NestedWrappedModule.__init__.<locals>._maybe_wrap¥  ó   ø€ ÙÜ˜E 5Ñ8¨KÑ8Ð8ØˆLrG   r   rÓ   r6   rÑ   )rÜ   rº   r§   r¶   r~   rJ   rK   rf   rÝ   ri   Ú
Sequentialr¯   râ   Úmodule©	rS   rp   r°   rÌ   rÎ   r	  r­   r$  rï   s	    ``  `  €rH   rº   zNestedWrappedModule.__init__˜  sé   û€ ô 	‰ÑÔØ—J‘J“LˆŒ	ØŸ*™*›,ˆŒØ)¬^×-IÑ-IÑIˆö	ñ
 Ü×Ñ˜aÔ Ü—m‘mÜœBŸI™I a¨›O¨^Ó<ÙÜ—‘Ù¤´·	±	¸!¸RÓ0@À.Ó QÓRÜ#¤B§I¡I¨b°"Ó$5°~ÓFóóñ œ¬¯	©	°"°aÓ(8¸.ÓIÓJÜœBŸI™I a¨›O¨^Ó<ó

ˆ�rG   c                 óx   — t        j                  d| j                  z   «       t        j                  dd|¬«      fS )Nr>   r6   rÓ   rò   )rf   rÝ   r§   ÚrandrR   s     rH   rU   zNestedWrappedModule.get_input¸  s.   € Ü×Ñ˜!˜dŸi™i™-Ô(Ü—
‘
˜1˜a¨Ô/Ð1Ð1rG   c                 ó$   — | j                  |«      S r‡   ©r'  ©rS   rü   s     rH   rý   zNestedWrappedModule.forward¼  ó   € Ø�{‰{˜1‹~ÐrG   c                 ó&   — |j                  «       }|S r‡   )r   ©rS   rX   rY   r]   s       rH   rZ   zNestedWrappedModule.get_loss¿  s   € Ø�z‰z‹|ˆØˆrG   c                 ó$   — |j                  «        y r‡   r  r\   s     rH   r^   z NestedWrappedModule.run_backwardÃ  r  rG   r  r	  rP   c                 ó  — |€i }|t         j                  k(  rt        | d||¬«      S |t         j                  k(  r:t        | fd||dœ|¤Ž}|t        j
                  k(  r|j                  t        «      }|S t        d|› �«      ‚)aò  
        Initializes a :class:`NestedWrappedModule` instance.

        Args:
            fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap
                any modules with FSDP. If ``RECURSIVE``, then wraps some nested
                modules with FSDP but not the top-level module. The model may
                later be wrapped with a top-level FSDP external to this method
                if desired.
            device_init_mode (DEVICEInitMode): Determines model movement to DEVICE.
            fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments
                forwarded to the FSDP constructor.
            deterministic (bool): Whether to make the model deterministic
                across constructions.
        F©r°   rÌ   rÎ   Tr  )	r@   rD   r  rE   rJ   rL   r–   rˆ   r  )rp   r  rÌ   r	  rÎ   r  s         rH   rb   zNestedWrappedModule.initÆ  s£   € ð. ÐØˆKØœ\×1Ñ1Ò1Ü&ØØØ!1Ø+ô	ð ð œ|×5Ñ5Ò5ä,ØðàØ!1Ø+ñ	ð
 ñˆJð  ¤>×#>Ñ#>Ò>Ø'Ÿ]™]¬;Ó7�
ØÐÜÐ7¸Ð7GÐHÓIÐIrG   ©NF)rA   rB   rC   rt   r  r  rJ   rº   rU   rý   rZ   r^   rh   r@   r   r¥   rž   r   ri   rj   rb   r  r  s   @rH   r  r  —  s¼   ø„ ð
à× Ñ ð
ð ð
ð )ð	
ð
 õ
ò@2òòòð ð
 15Ø#ñ+JØ× Ñ ð+Jà$ð+Jð )ð+Jð ˜d 3¨ 8™nÑ-ð	+Jð
 ð+Jð 
�‰ò+Jó ô+JrG   r  c                   óh   ‡ — e Zd Ze	 	 ddej
                  dededee	e
ef      def
ˆ fd„«       Zˆ xZS )ÚAlwaysWrapNestedWrappedModulerp   r  rÌ   r	  rÎ   c                 ó0  •— t         t        t        �  	 | t        j                  |||¬«      }|t        j                  k(  r|S |t        j
                  k(  rB|xs i }t        |fdt        i|¤Ž}|t        j                  k(  r|j                  t        «      }|S y)zþ
        Initializes a :class:`NestedWrappedModule` instance, but unlike
        :meth:`NestedWrappedModule.init`, for the ``RECURSIVE`` init mode, this
        wraps with top-level FSDP and the ``always_wrap_policy()`` auto wrap
        policy.
        )rp   r  rÌ   r	  rÎ   r  N)rÜ   r6  rb   r@   rD   rE   rŒ   r   rJ   rL   r–   rˆ   )rp   r  rÌ   r	  rÎ   rk   r  rï   s          €rH   rb   z"AlwaysWrapNestedWrappedModule.initö  s£   ø€ ô Ü)Ô+Hñà
ØÜ'×/Ñ/Ø-Ø#Ø'ð ó 
ð 	ð œ\×1Ñ1Ò1ØˆLØœ|×5Ñ5Ò5Ø%Ò+¨ˆKÜ˜eÑXÔ6HÐXÈKÑXˆJØ¤>×#>Ñ#>Ò>Ø'Ÿ]™]¬;Ó7�
ØÐð 6rG   r4  )rA   rB   rC   rh   rt   r  r@   rJ   r   r¥   rž   r   r  rb   r  r  s   @rH   r6  r6  õ  s^   ø„ Øð
 15Ø#ñØ× Ñ ðà$ðð )ðð ˜d 3¨ 8™nÑ-ð	ð
 ôó ôrG   r6  c                   ó¦   ‡ — e Zd Zdej                  dededefˆ fd„Zed
d„«       Z	e	 	 ddej                  de
dedeeeef      def
d	„«       Zˆ xZS )ÚNonUniformReqGradNWMrp   r°   rÌ   rÎ   c                 ó¾  •‡‡‡— t         t        | �  «        ‰j                  «       | _        ‰j	                  «       | _        |t        j                  k(  }ˆˆˆfd„}|rt        j                  d«       t        j                  t        t        j                  dd«      |«       |t        j                   |t        t        j                  dd«      |«      «      t        t        j                  dd«      |«      «      «       |t        j                  t        t        j                  dd«      |«      t        t        j                  dd«      |«      «      «      «      | _        y )Nc                 ó&   •— ‰rt        | ‰fi ‰¤ŽS | S r‡   r²   r"  s    €€€rH   r$  z2NonUniformReqGradNWM.__init__.<locals>._maybe_wrap+  r%  rG   r   rÓ   r6   rÑ   )rÜ   r  rº   r§   r¶   r~   rJ   rK   rf   rÝ   ri   r&  r¯   râ   r'  r(  s	    ``  `  €rH   rº   zNonUniformReqGradNWM.__init__  sý   û€ ô 	Ô! 4Ñ1Ô3ð —J‘J“LˆŒ	ØŸ*™*›,ˆŒØ)¬^×-IÑ-IÑIˆö	ñ
 Ü×Ñ˜aÔ Ü—m‘mÜœBŸI™I a¨›O¨^Ó<ÙÜ—‘Ù¤´·	±	¸!¸RÓ0@À.Ó QÓRÜ#¤B§I¡I¨b°"Ó$5°~ÓFóóñ Ü—‘Ü#¤B§I¡I¨b°!Ó$4°nÓEÜ#¤B§I¡I¨a°£O°^ÓDóóó
ˆ�rG   c                 ó„   — | j                  «       D ]-  \  }}t        j                  ||«      rŒ|j                  d«       Œ/ y r4  )rq   ÚreÚmatchÚrequires_grad_)rk   Úreq_grad_maskÚnÚps       rH   Ú_set_nonuniform_req_gradz-NonUniformReqGradNWM._set_nonuniform_req_gradB  s:   € à×*Ñ*Ó,ò 	(‰DˆAˆqÜ—8‘8˜M¨1Õ-Ø× Ñ  Õ'ñ	(rG   r  r	  c                 óŠ  — t        j                  d«      }|t        j                  k(  r't	        | d||¬«      }t        j                  ||«       |S |t        j                  k(  rT|€i }t	        | fd||dœ|¤Ž}|t        j                  k(  r|j                  t        «      }t        j                  ||«       |S t        d|› �«      ‚)a  
        Initializes a :class:`NestedWrappedModule` instance, but unlike
        :meth:`NestedWrappedModule.init`, it wraps a second :class:`torch.nn.Sequential`
        container to enable the desired non-uniform ``requires_grad``
        ``use_orig_params=True`` tests. For both ``RECURSIVE`` and ``NO_FSDP``
        init modes, freezes all parameters except the last two to validate
        ``ShardedGradScaler`` support for ranks with no (non-zero sized) local shards in
        FSDP ``use_orig_params=True`` mode.
        zmodule\.2.*\.1.*Fr3  Tr  )r=  Úcompiler@   rD   r9  rC  rE   rJ   rL   r–   rˆ   r  )rp   r  rÌ   r	  rÎ   Úreq_grad_patternÚ	ddp_modelr  s           rH   rb   zNonUniformReqGradNWM.initH  sØ   € ô( Ÿ:™:Ð&9Ó:ÐØœ\×1Ñ1Ò1Ü,ØØØ!1Ø+ô	ˆIô !×9Ñ9¸)ÐEUÔVØÐØœ|×5Ñ5Ò5ØÐ"Ø �Ü-ØðàØ!1Ø+ñ	ð
 ñˆJð  ¤>×#>Ñ#>Ò>Ø'Ÿ]™]¬;Ó7�
Ü ×9Ñ9¸*ÐFVÔWØÐÜÐ7¸Ð7GÐHÓIÐIrG   rc   r4  )rA   rB   rC   rt   r  r  rJ   rº   rh   rC  r@   r   r¥   rž   r   rb   r  r  s   @rH   r9  r9    s­   ø„ ð(
à× Ñ ð(
ð ð(
ð )ð	(
ð
 õ(
ðT ò(ó ð(ð
 ð
 15Ø#ñ+JØ× Ñ ð+Jà$ð+Jð )ð+Jð ˜d 3¨ 8™nÑ-ð	+Jð
 ò+Jó ô+JrG   r9  c                   óˆ   ‡ — e Zd ZdZdej
                  dedefˆ fd„Zd„ Zd„ Z	d„ Z
d	„ Zed
ee   dedededef
d„«       Zˆ xZS )ÚModuleWithDelayz†This class wraps a :class:`FSDPTestModel` to optionally add a delay
    after computing the loss and/or before the gradient reduction.r'  Údelay_after_loss_msÚdelay_before_reduction_msc                 óL   •— t         ‰| �  «        || _        || _        || _        y r‡   )rÜ   rº   rJ  rK  r'  )rS   r'  rJ  rK  rï   s       €rH   rº   zModuleWithDelay.__init__{  s'   ø€ ô 	‰ÑÔØ#6ˆÔ Ø)BˆÔ&Øˆ�rG   c                 ó8   — | j                   j                  |«      S r‡   )r'  rU   rR   s     rH   rU   zModuleWithDelay.get_input†  s   € Ø�{‰{×$Ñ$ VÓ,Ð,rG   c                 ó$   — | j                  |«      S r‡   r,  r-  s     rH   rý   zModuleWithDelay.forward‰  r.  rG   c                 óB  — | j                   j                  ||«      }| j                  dkD  rst        st        r$t        j                  | j                  dz  «       |S t        r=t        j                  j                  t        | j                  t        «       z  «      «       |S ©Nr   éè  )r'  rZ   rJ  r3   r4   ÚtimeÚsleepr2   rf   r7   Ú_sleeprÉ   r1   r0  s       rH   rZ   zModuleWithDelay.get_lossŒ  sy   € Ø�{‰{×#Ñ# E¨6Ó2ˆØ×#Ñ# aÒ'Ý�8Ü—
‘
˜4×3Ñ3°dÑ:Ô;ð ˆõ Ü—
‘
×!Ñ!¤# d×&>Ñ&>ÔARÓATÑ&TÓ"UÔVàˆrG   c                 óÔ   ‡ ‡— t         j                  j                  Šˆˆ fd„}t        j                  d|«      5  ‰ j
                  j                  |«       d d d «       y # 1 sw Y   y xY w)Nc                  ó  •— ‰j                   dkD  rrt        r>t        j                  j	                  t        ‰j                   t        «       z  «      «       n.t        st        r"t        j                  ‰j                   dz  «        ‰| i |¤ŽS rP  )rK  r2   rf   r7   rT  rÉ   r1   r3   r4   rR  rS  )r_   r`   Úorig_reduce_scatterrS   s     €€rH   Ú_delayed_reduce_scatterz=ModuleWithDelay.run_backward.<locals>._delayed_reduce_scatter™  sk   ø€ Ø×-Ñ-°Ò1ÝÜ—J‘J×%Ñ%Ü˜D×:Ñ:Ô=NÓ=PÑPÓQõõ ¥Ü—J‘J˜t×=Ñ=ÀÑDÔEÙ&¨Ð7°Ñ7Ð7rG   z'torch.distributed.reduce_scatter_tensor)rf   ÚdistributedÚreduce_scatter_tensorr   Úpatchr'  r^   )rS   r]   rX  rW  s   `  @rH   r^   zModuleWithDelay.run_backward–  sW   ù€ Ü#×/Ñ/×EÑEÐõ	8ô �Z‰ZØ5Ð7Nó
ñ 	+ð �K‰K×$Ñ$ TÔ*÷	+÷ 	+ñ 	+ús   ¹AÁA'Úmodule_classÚ
model_argsÚmodel_kwargsc                ó<   — t         | j                  |i |¤Ž||«      S )aA  
        Args:
            module_class (Type[FSDPTestModel]): Wrapped module class to which
                to add delays.
            model_args: Positional arguments forwarded to the ``module_class``
                ``init()``.
            delay_after_loss_ms (int): Delay after computing the loss/before
                the optimizer step (in ms).
            delay_before_reduction_ms (int): Delay before reduce-scattering
                gradients (in ms).
            model_kwargs: Keyword arguments forwarded to the ``module_class``
                ``init()``.
        )rI  rb   )r\  rJ  rK  r]  r^  s        rH   rb   zModuleWithDelay.init¨  s,   € ô* ØˆL×Ñ˜zÐ:¨\Ñ:ØØ%ó
ð 	
rG   )rA   rB   rC   rd   ri   rj   rÉ   rº   rU   rý   rZ   r^   rh   ÚtyperO   r   rb   r  r  s   @rH   rI  rI  w  s‰   ø„ ñFð	à—	‘	ð	ð !ð	ð $'õ		ò-òòò+ð$ ð
Ø˜=Ñ)ð
àð
ð !ð
ð $'ð	
ð
 ò
ó ô
rG   rI  c                   ó~   — e Zd Zeej
                  ddddfdej                  dedede	e
eef      ded	ed
efd„«       Zy)ÚNestedWrappedModuleWithDelayNFr   rp   r  rÌ   r	  rÎ   rJ  rK  c           
      óD   — t         j                  t        | ||||||¬«      S )N©rp   r  rÌ   r	  rÎ   rJ  rK  )rI  rb   r  rd  s          rH   rb   z!NestedWrappedModuleWithDelay.initÅ  s4   € ô ×#Ñ#ÜØØ)Ø-Ø#Ø'Ø 3Ø&?ð $ó 	
ð 		
rG   )rA   rB   rC   rh   rJ   rL   rt   r  r@   r   r¥   rž   r   r  rÉ   rb   rF   rG   rH   rb  rb  Ä  s€   „ Øð ,:×+FÑ+FØ04Ø#Ø#$Ø)*ñ
Ø× Ñ ð
à$ð
ð )ð
ð ˜d 3¨ 8™nÑ-ð	
ð
 ð
ð !ð
ð $'ò
ó ñ
rG   rb  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚDummyDDPc                 ó0   •— t         ‰| �  «        || _        y r‡   )rÜ   rº   r'  )rS   r'  rï   s     €rH   rº   zDummyDDP.__init__Ü  s   ø€ Ü‰ÑÔØˆ�rG   c                 ó&   —  | j                   |i |¤ŽS r‡   r,  ©rS   r_   r`   s      rH   rý   zDummyDDP.forwardà  s   € Øˆt�{‰{˜DÐ+ FÑ+Ð+rG   ©rA   rB   rC   rº   rý   r  r  s   @rH   rf  rf  Û  s   ø„ ôö,rG   rf  c                   óª   ‡ — e Zd Zdej                  dedededef
ˆ fd„Zd„ Z	d„ Z
e	 	 	 ddej                  d	eded
eeeef      dedefd„«       Zˆ xZS )ÚMixtureOfExpertsrp   r°   rÌ   Údelay_before_free_msrÎ   c                 ó   •— t         ‰| �  ||||¬«       || _        || _        || _        |t
        j                  k(  | _        |r"t        j                  d| j                  z   «       d}d}d}	t        t        j                  ||«      | j                  «      }
t        d„ |
j                  «       D «       «      | _        |
j                  «       D ]	  }d|_        Œ |rt        j                  d«       t        t        j                  ||«      | j                  «      }|rHt        j$                  j'                  |j                  «       g«      }t)        |
|fi |¤Ž}
t)        ||fi |¤Ž}t        j*                  t        t        j                  |	|«      | j                  «      ||
t        t        j                  ||	«      | j                  «      «      | _        y )	N)rp   r°   rÌ   rÎ   é*   rÐ   rñ   rÓ   c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­wr‡   )Únumel)Ú.0rB  s     rH   ú	<genexpr>z,MixtureOfExperts.__init__.<locals>.<genexpr>   s   è ø€ Ò$L°1 Q§W¡W§YÑ$Lùó   ‚Tr   )rÜ   rº   rp   rm  r°   rJ   rK   r­   rf   rÝ   r§   r¯   ri   râ   r   rŽ   Únum_expert_paramsÚexpertrY  Ú	new_grouprŒ   r&  r'  )rS   rp   r°   rÌ   rm  rÎ   r	  Úd_expertÚd_sharedÚd_inputrv  rB  ÚsharedÚexpert_grouprï   s                 €rH   rº   zMixtureOfExperts.__init__å  s’  ø€ ô 	‰ÑØØØ-Ø'ð	 	ô 	
ð ˆŒ
Ø$8ˆÔ!Ø"ˆŒØ.´.×2NÑ2NÑNˆÔÙä×Ñ˜b 4§9¡9™nÔ-ØˆØˆØˆÜ ¤§¡¨8°XÓ!>À×@SÑ@SÓTˆä!$Ñ$L¸×8IÑ8IÓ8KÔ$LÓ!LˆÔØ×"Ñ"Ó$ò 	ˆAØˆA�Hð	ñ ä×Ñ˜aÔ ä ¤§¡¨8°XÓ!>À×@SÑ@SÓTˆáä ×,Ñ,×6Ñ6Ø—‘“�óˆLô ˜& ,Ñ>°+Ñ>ˆFÜ˜& %Ñ7¨;Ñ7ˆFä—m‘mÜœBŸI™I g¨xÓ8¸$×:MÑ:MÓNØØÜœBŸI™I h°Ó8¸$×:MÑ:MÓNó	
ˆ�rG   c                 óf  ‡ ‡— ‰ j                   dkD  r„‰ j                  d   }t        |t        «      ret        j
                  j                  j                  j                  Šˆˆ fd„}t        j                  d|«      5  ‰ j                  |«      cd d d «       S ‰ j                  |«      S # 1 sw Y   ŒxY w)Nr   rÒ   c                  óø   •— t         r>t        j                  j                  t	        ‰j
                  t        «       z  «      «       n.t        st        r"t        j                  ‰j
                  dz  «        ‰| i |¤ŽS )NrQ  )r2   rf   r7   rT  rÉ   rm  r1   r3   r4   rR  rS  )r_   r`   Úorig_reshardrS   s     €€rH   Ú_delayed_reshardz2MixtureOfExperts.forward.<locals>._delayed_reshard  s]   ø€ Ý ÜŸ
™
×)Ñ)Ü × 9Ñ 9Ô<MÓ<OÑ OÓPõõ "¥XÜŸ
™
 4×#<Ñ#<¸tÑ#CÔDá'¨Ð8°Ñ8Ð8rG   z.torch.distributed.fsdp._runtime_utils._reshard)rm  r'  r  rŒ   rf   rY  ÚfsdpÚ_runtime_utilsÚ_reshardr   r[  )rS   rü   rv  r€  r  s   `   @rH   rý   zMixtureOfExperts.forward  s•   ù€ Ø×$Ñ$ qÒ(Ø—[‘[ ‘^ˆFÜ˜&¤$Ô'Ü$×0Ñ0×5Ñ5×DÑD×MÑM�õ9ô —Z‘ZØDÐFVóñ *ð  Ÿ;™; q›>÷*ñ *ð
 �{‰{˜1‹~Ð÷*ð *ús   Á;B'Â'B0c                 ó¤  — |j                  «        | j                  s¨t        j                  «       5  | j	                  «       D ]v  }t        |d«      rŒ|j                  €Œ|j                  j                  | j                  «       t        j                  j                  |j                  | j                  ¬«       Œx 	 d d d «       y y # 1 sw Y   y xY w)Nrv  ro   )r  r°   rf   r�   rŽ   ÚhasattrÚgradÚdiv_r~   rY  Ú
all_reducerp   )rS   r]   rB  s      rH   r^   zMixtureOfExperts.run_backward1  s£   € Ø�‰Œà�~Š~Ü—‘“ñ OØŸ™Ó*ò O�AÜ˜q (Ô+Ø Ø—v‘vÑ)ØŸ™Ÿ™ D§O¡OÔ4Ü×)Ñ)×4Ñ4°Q·V±VÀ4Ç:Á:Ð4ÕNñO÷Oð Oð ÷Oð Oús   ±-CÁACÃCr  r	  c                 ó  — |€i }|t         j                  k(  rt        | d|||¬«      S |t         j                  k(  r;t        | fd|||dœ|¤Ž}|t        j
                  k(  r|j                  t        «      }|S t        d|› �«      ‚)a—  
        Initializes a :class:`MixtureOfExperts` instance.

        Args:
            fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap
                any modules with FSDP. If ``RECURSIVE``, then wraps some nested
                modules with FSDP, including the expert and shared layers, but
                not the top-level module. The model may later be wrapped with a
                top-level FSDP external to this method if desired.
            device_init_mode (DEVICEInitMode): Determines model movement to DEVICE.
            fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments
                forwarded to the FSDP constructor.
            deterministic (bool): Whether to make the model deterministic
                across constructions.
            delay_before_free_ms (int): Delay before resharding expert
                parameters in the forward pass (in ms).
        F)r°   rÌ   rm  rÎ   Tr  )	r@   rD   rl  rE   rJ   rL   r–   rˆ   r  )rp   r  rÌ   r	  rÎ   rm  r  s          rH   rb   zMixtureOfExperts.init=  s©   € ð4 ÐØˆKØœ\×1Ñ1Ò1Ü#ØØØ!1Ø%9Ø+ôð ð œ|×5Ñ5Ò5ä)ØðàØ!1Ø%9Ø+ñð ñˆJð  ¤>×#>Ñ#>Ò>Ø'Ÿ]™]¬;Ó7�
ØÐÜÐ7¸Ð7GÐHÓIÐIrG   )NFr   )rA   rB   rC   rt   r  r  rJ   rÉ   rº   rý   r^   rh   r@   r   r¥   rž   r   rb   r  r  s   @rH   rl  rl  ä  s¼   ø„ ð2
à× Ñ ð2
ð ð2
ð )ð	2
ð
 "ð2
ð õ2
òhò0
Oð ð
 15Ø#Ø$%ñ0JØ× Ñ ð0Jà$ð0Jð )ð0Jð ˜d 3¨ 8™nÑ-ð	0Jð
 ð0Jð "ò0Jó ô0JrG   rl  c                   óš   ‡ — e Zd Z	 dddddœdedeej                     deded	ef
ˆ fd
„Zdej                  dej                  fd„Z
d„ Zˆ xZS )ÚMLPTFr6   )ÚbiasÚwith_bufferÚdim_multiplierÚdimrT   rŒ  r�  rŽ  c                ó
  •— t         ‰| �  «        t        j                  |||z  ||¬«      | _        t        j                  ||z  |||¬«      | _        |r)| j                  dt        j                  |f|¬«      «       y d | _	        y )N)rT   rŒ  r}   rò   )
rÜ   rº   ri   râ   Úin_projÚout_projrå   rf   Úrandnr}   )rS   r�  rT   rŒ  r�  rŽ  rï   s         €rH   rº   zMLP.__init__r  so   ø€ ô 	‰ÑÔÜ—y‘y  n°sÑ&:À6ÐPTÔUˆŒÜŸ	™	 .°3Ñ"6¸ÀFÐQUÔVˆŒÙØ× Ñ  ¬5¯;©;¸°vÀfÔ+MÕNàˆD�KrG   rü   rP   c                 óÔ   — | j                  |«      }t        j                  |«      }| j                  |«      }t        j                  |«      }| j                  �|| j                  z   }|S r‡   )r‘  ÚFÚrelur’  r}   )rS   rü   Úzs      rH   rý   zMLP.forwardƒ  sS   € Ø�L‰L˜‹OˆÜ�F‰F�1‹IˆØ�M‰M˜!ÓˆÜ�F‰F�1‹IˆØ�;‰;Ð"Ø�D—K‘K‘ˆAØˆrG   c                 ó„   — | j                   �4t        j                  j                  j	                  | j                   «       y y r‡   )r}   rf   ri   rb   Únormal_r¼   s    rH   Úreset_parameterszMLP.reset_parametersŒ  s+   € Ø�;‰;Ð"Ü�H‰H�M‰M×!Ñ! $§+¡+Õ.ð #rG   r‡   )rA   rB   rC   rÉ   r   rf   rT   r  rº   rg   rý   rš  r  r  s   @rH   r‹  r‹  q  sv   ø„ ð *.ðð
 Ø!Øòàðð ˜Ÿ™Ñ&ðð
 ðð ðð õð"˜Ÿ™ð ¨%¯,©,ó ö/rG   r‹  c                   óF   ‡ — e Zd Zddœdedefˆ fd„Zdededed	d fd
„Zˆ xZS )ÚMLPStackF)Úwith_seq_parallelÚmlp_dimr�  c                óÄ   •— t        |d¬«      t        |«      t        |d¬«      g}|r&|j                  t        j                  |d¬«      «       t	        ‰| �  |Ž  || _        y )Né   )rŽ  F©rŒ  )r‹  Úappendri   Ú	LayerNormrÜ   rº   r�  )rS   rž  r�  Úmodulesrï   s       €rH   rº   zMLPStack.__init__’  sX   ø€ ô �¨Ô*Ü�‹LÜ�¨Ô*ð	$
ˆñ Ø�N‰Nœ2Ÿ<™<¨°eÔ<Ô=Ü‰Ñ˜'Ñ"Ø!2ˆÕrG   Útp_meshÚdp_meshÚuse_activation_checkpointingrP   c           
      ó¶  — t        d¬«      t        d¬«      t        d¬«      t        d¬«      t        d¬«      | j                  rt        t        d«      ¬«      n	t        «       dœ}| j                  rt	        d¬«      |d<   t        | ||¬«       | D ]8  }t        |t        j                  «      rŒ|rt        |«       t        |fd	|i|¤Ž Œ: t        | fd	|i|¤Ž | S )
NF)Úuse_local_outputr>   )Úoutput_layouts)z	0.in_projz
0.out_projz	1.in_projz
1.out_projz	2.in_projz
2.out_proj)Úsequence_dimÚ3)Údevice_meshÚparallelize_planÚmesh)r%   r'   r�  r$   r(   r&   r  ri   r£  r   r   )rS   r¥  r¦  r§  r	  r®  r'  s          rH   ÚparallelizezMLPStack.parallelizež  sÏ   € ô )¸%Ô@Ü)¸5ÔAÜ(¸%Ô@Ü)¸5ÔAÜ(¸%Ô@à×%Ò%ô *¼¸q»ÕBä Ó"ñ
Ðð ×!Ò!Ü$4À!Ô$DÐ˜SÑ!Ü˜4¨WÐGWÕXØò 	=ˆFÜ˜&¤"§,¡,Ô/ØÙ+Ü˜6Ô"Ü˜Ñ< WÐ<°Ó<ð	=ô 	�DÑ6˜wÐ6¨+Ò6ØˆrG   )	rA   rB   rC   rÉ   r  rº   r   r°  r  r  s   @rH   rœ  rœ  ‘  sD   ø„ ØBGò 
3 ð 
3¸4õ 
3ðàðð ðð '+ð	ð 
÷rG   rœ  c                   ó¢   ‡ — e Zd ZdZddedefˆ fd„Zdej                  de	e
ej                  ej                  f   ej                  f   fd„Zˆ xZS )	ÚDoubleLinearz”
    This can be used for returning multiple outputs from a module
    (``use_second_linear=True``) or for having an unused module (``False``).
    r�  Úuse_second_linearc                 óÎ   •— t         ‰| �  «        t        j                  ||«      | _        t        j                  ||«      | _        t        j                  «       | _        || _        y r‡   )	rÜ   rº   ri   râ   Úlin1Úlin2ÚReLUr–  r³  )rS   r�  r³  rï   s      €rH   rº   zDoubleLinear.__init__Ä  sG   ø€ Ü‰ÑÔÜ—I‘I˜c 3Ó'ˆŒ	Ü—I‘I˜c 3Ó'ˆŒ	Ü—G‘G“IˆŒ	Ø!2ˆÕrG   rü   rP   c                 óÚ   — | j                   r@| j                  | j                  |«      «      | j                  | j                  |«      «      fS | j                  | j                  |«      «      S r‡   )r³  r–  rµ  r¶  r-  s     rH   rý   zDoubleLinear.forwardË  sQ   € ð ×!Ò!Ø—9‘9˜TŸY™Y q›\Ó*¨D¯I©I°d·i±iÀ³lÓ,CÐCÐCØ�y‰y˜Ÿ™ 1›Ó&Ð&rG   ©T)rA   rB   rC   rd   rÉ   r  rº   rf   rg   r   re   rý   r  r  s   @rH   r²  r²  ¾  sT   ø„ ññ
3˜Cð 3°Dõ 3ð'Ø—‘ð'à	ˆu�U—\‘\ 5§<¡<Ð/Ñ0°%·,±,Ð>Ñ	?÷'rG   r²  Únew_all_gather_into_tensorc              #   óú   K  — t         j                  }t        j                  «        | t         _        	 d –— t        j                  «        |t         _        y # t        j                  «        |t         _        w xY w­wr‡   )rt   Úall_gather_into_tensorÚbarrier)rº  Úorig_all_gathers     rH   Úpatch_all_gatherr¿  Ù  sO   è ø€ ä×1Ñ1€OÜ‡L�L„NØ"<„DÔð6Ûä�‰ŒØ&5ŒÕ#øô 	�‰ŒØ&5ŒÕ#üó   ‚0A;³A · A;Á!A8Á8A;Únew_reduce_scatter_tensorc              #   óú   K  — t         j                  }t        j                  «        | t         _        	 d –— t        j                  «        |t         _        y # t        j                  «        |t         _        w xY w­wr‡   )rt   rZ  r½  )rÁ  rW  s     rH   Úpatch_reduce_scatterrÃ  å  sP   è ø€ ä×4Ñ4ÐÜ‡L�L„NØ!:„DÔð9Ûä�‰ŒØ%8ŒÕ"øô 	�‰ŒØ%8ŒÕ"ürÀ  Únew_all_reducec              #   óú   K  — t         j                  }t        j                  «        | t         _        	 d –— t        j                  «        |t         _        y # t        j                  «        |t         _        w xY w­wr‡   )rt   rˆ  r½  )rÄ  Úorig_all_reduces     rH   Úpatch_all_reducerÇ  ñ  sJ   è ø€ ä—o‘o€OÜ‡L�L„NØ$„D„Oð*Ûä�‰ŒØ)Œ�øô 	�‰ŒØ)Œ�ürÀ  Únew_unshardc              #   óú   K  — t         j                  }t        j                  «        | t         _        	 d –— t        j                  «        |t         _        y # t        j                  «        |t         _        w xY w­wr‡   )r   Úunshardrt   r½  )rÈ  Úorig_unshards     rH   Úpatch_unshardrÌ  ý  óQ   è ø€ ô "×)Ñ)€LÜ‡L�L„NØ(„NÔð.Ûä�‰ŒØ!-ŒÕøô 	�‰ŒØ!-ŒÕürÀ  Únew_reshardc              #   óú   K  — t         j                  }t        j                  «        | t         _        	 d –— t        j                  «        |t         _        y # t        j                  «        |t         _        w xY w­wr‡   )r   Úreshardrt   r½  )rÎ  r  s     rH   Úpatch_reshardrÑ  
  rÍ  rÀ  Únew_post_backwardc              #   óú   K  — t         j                  }t        j                  «        | t         _        	 d –— t        j                  «        |t         _        y # t        j                  «        |t         _        w xY w­wr‡   )r   Úpost_backwardrt   r½  )rÒ  Úorig_post_backwards     rH   Úpatch_post_backwardrÖ    sR   è ø€ ô (×5Ñ5ÐÜ‡L�L„NØ#4„NÔ ð:Ûä�‰ŒØ'9ŒÕ$øô 	�‰ŒØ'9ŒÕ$ürÀ  Únew_backwardc              #   óú   K  — t         j                  }t        j                  «        | t         _        	 d –— t        j                  «        |t         _        y # t        j                  «        |t         _        w xY w­wr‡   )r   r  rt   r½  )r×  Úorig_backwards     rH   Ú*patch_register_post_backward_hook_backwardrÚ  $  sT   è ø€ ô 1×9Ñ9€MÜ‡L�L„NØ,8Ô Ô)ð>Ûä�‰ŒØ0=Ô$Õ-øô 	�‰ŒØ0=Ô$Õ-ürÀ  rW  r_   r`   c                 ó€   — t        |«      dkD  r|d   }nd|v r|d   }nt        d|› d|› �«      ‚ ||«        ||i |¤ŽS )Nr   rY   z,Cannot get reduce-scatter output from
args: z	
kwargs: )ÚlenÚAssertionError)ÚclsrW  rm   r_   r`   rY   s         rH   Úreduce_scatter_with_assertrß  1  sa   € ô ˆ4ƒy�1‚}Ø�a‘‰Ø	�VÑ	Ø˜Ñ!‰äØ;¸D¸6ÀÈFÈ8ÐTó
ð 	
ñ ˆfÔÙ Ð/¨Ñ/Ð/rG   Úreplicated_moduleÚsharded_moduleÚprefixes_to_ignore.c                 ó¼  — t        |j                  «       |j                  «       «      D �]°  \  \  }}\  }}|}|D ]  }	|j                  |	d«      }Œ | j                  ||«       | j	                  |t
        «       t        |t
        «      sJ ‚|j                  |j                  }}
t        |«      t        d«      t        d«      fk(  rt        d«      ‚t        ||
|«      }| j                  |j                  «       |j                  «       «       |j                  €| j                  |j                  «       �Œ| j!                  |j                  «       t        |j                  |
|«      }| j	                  |j                  t
        «       t        |j                  t
        «      sJ ‚| j                  |j                  j                  «       |j                  «       «       �Œ³ y )NÚ r   zmFSDP's (Shard(0), Shard(0)) layout differs from distribute_tensor(), so we cannot check for equality using it)rx   rq   ÚreplaceÚassertEqualÚassertIsInstancer#   r  r­  Ú
placementsre   r$   rÝ  r"   Úto_localr†  ÚassertIsNoneÚassertIsNotNone)rÞ  rà  rá  râ  Úreplicated_nameÚreplicated_paramÚsharded_nameÚsharded_paramÚclean_sharded_nameÚprefixr¯  rè  Úsharded_ref_paramÚsharded_ref_grads                 rH   Úcheck_sharded_parityrô  D  s³  € ô ORØ×*Ñ*Ó,¨n×.MÑ.MÓ.OóOó TÑJÑ+ˆÐ*Ñ-J¨l¸Mð *ÐØ(ò 	HˆFØ!3×!;Ñ!;¸FÀBÓ!GÑð	Hà�‰˜Ð);Ô<Ø×Ñ˜]¬GÔ4Ü˜-¬Ô1Ð1Ð1Ø(×4Ñ4°m×6NÑ6NˆjˆÜ�Ó¤ q£¬5°«8Ð 4Ò4Ü ð;óð ô .Ð.>ÀÀjÓQÐØ�‰˜×.Ñ.Ó0Ð2C×2LÑ2LÓ2NÔOØ× Ñ Ð(Ø×Ñ˜]×/Ñ/Ô0ÙØ×Ñ˜M×.Ñ.Ô/Ü,Ð-=×-BÑ-BÀDÈ*ÓUÐØ×Ñ˜]×/Ñ/´Ô9Ü˜-×,Ñ,¬gÔ6Ð6Ð6Ø�‰˜×*Ñ*×3Ñ3Ó5Ð7G×7PÑ7PÓ7RÖSñ1TrG   c                   ó@   ‡ — e Zd Zed„ «       Zˆ fd„Zd„ Zd„ Zd„ Zˆ xZ	S )ÚFSDPTestMultiThreadc                 ó   — t         S r‡   ©ÚDEVICE_COUNTr¼   s    rH   r~   zFSDPTestMultiThread.world_sizef  ó   € äÐrG   c                 óB   •— t         ‰| �  «        | j                  «        y r‡   )rÜ   ÚsetUpÚ_spawn_threads©rS   rï   s    €rH   rü  zFSDPTestMultiThread.setUpj  s   ø€ Ü‰‰ŒØ×ÑÕrG   c                 ó    — t        | g|¢­i |¤ŽS r‡   ©r.   ri  s      rH   r.   z FSDPTestMultiThread.run_subtestsn  ó   € Ü˜DÐ2 4Ò2¨6Ñ2Ð2rG   c                 ó@   — t         j                  j                  «        y r‡   ©rf   Ú_dynamoÚresetr¼   s    rH   ÚperThreadSetUpz"FSDPTestMultiThread.perThreadSetUpq  ó   € Ü�‰×ÑÕrG   c                 ó@   — t         j                  j                  «        y r‡   r  r¼   s    rH   ÚperThreadTearDownz%FSDPTestMultiThread.perThreadTearDownt  r  rG   )
rA   rB   rC   Úpropertyr~   rü  r.   r  r	  r  r  s   @rH   rö  rö  e  s)   ø„ Øñó ðôò3òörG   rö  c            $       ó¨  ‡ — e Zd Zˆ fd„Zed„ «       Zed„ «       Zedefd„«       Zed„ «       Z	d„ Z
d„ Zd	„ Zd
„ Zed„ «       Z	 	 	 	 	 	 	 d'dej"                  dedededee   dedee   dededeeeef      fd„Zddd e«       dddddddddfdee   dededee   deded ed!ee    d"ee!   dee   d#ed$ededed%eeeef      deeeef      f d&„Z"ˆ xZ#S )(ÚFSDPTestc                 óh   •— t         ‰| �  «        dt        j                  d<   | j	                  «        y )NÚ0ÚTORCH_NCCL_DESYNC_DEBUG)rÜ   rü  ÚosÚenvironÚ_spawn_processesrþ  s    €rH   rü  zFSDPTest.setUpy  s)   ø€ Ü‰‰Œð 14Œ�
‰
Ð,Ñ-Ø×ÑÕrG   c                 ó   — t         S r‡   rø  r¼   s    rH   r~   zFSDPTest.world_size�  rú  rG   c                 ó>   — t         j                  j                  «       S r‡   )rt   Údistributed_c10dÚ_get_default_groupr¼   s    rH   rl   zFSDPTest.process_group…  s   € ä×$Ñ$×7Ñ7Ó9Ð9rG   rP   c                  ó   — yr4  rF   r¼   s    rH   Údestroy_pg_upon_exitzFSDPTest.destroy_pg_upon_exit‰  s   € ð rG   c                 ó*   — t         › | j                  › �S r‡   )r0   Ú	file_namer¼   s    rH   Úinit_methodzFSDPTest.init_methodŽ  s   € ä�˜tŸ~™~Ð.Ð/Ð/rG   c                 ó<   — | j                  ||j                  «       y r‡   )ræ  r™   )rS   r  r™   s      rH   Ú_check_cpu_offloadzFSDPTest._check_cpu_offload’  s   € Ø×Ñ˜ j×&<Ñ&<Õ=rG   c                 ó<   — | j                  ||j                  «       y r‡   )ræ  Úbackward_prefetch)rS   r  r  s      rH   Ú_check_backward_prefetchz!FSDPTest._check_backward_prefetch•  s   € Ø×ÑÐ*¨J×,HÑ,HÕIrG   c                 ó<   — | j                  ||j                  «       y r‡   )ræ  Úforward_prefetch)rS   r  r"  s      rH   Ú_check_forward_prefetchz FSDPTest._check_forward_prefetch˜  s   € Ø×ÑÐ)¨:×+FÑ+FÕGrG   c                 ó    — t        | g|¢­i |¤ŽS r‡   r   ri  s      rH   r.   zFSDPTest.run_subtests›  r  rG   c                 ó   —  | |«      }||_         ||_        |j                  dd«      }t        d|j                   › d|j                  › �«       	 |r`t
        j                  j                  j                  j                  j                  «       }t        j                  d|j                  ||¬«       nDt        j                  |j                  t        t        |j                  «      |j                   ¬«       d }
|j                   t,        z  }t.        st0        rt
        j2                  j5                  |«       |g}
t        j6                  |
¬«       t
        j8                  j;                  «        |j=                  ||«       t
        j8                  j;                  «        t        j6                  |
¬«       t        j>                  «        y # t         $ r=}	d|	j"                  d	   v r&t%        j&                  t(        d
   j*                  «       ‚ d }	~	ww xY w)NÚfake_pgFzdist init r=z, world=Úfake)Úbackendr~   r§   Ústore)r  r(  r~   r§   Ú	recompiler   Úbackend_unavailable)Ú
device_ids) r§   r  ÚgetÚprintr~   rf   ÚtestingÚ	_internalrY  r&  Ú	FakeStorert   Úinit_process_groupr  ÚDISTRIBUTED_BACKENDrÉ   ÚRuntimeErrorr_   ÚsysÚexitr/   Ú	exit_coderù  r2   r4   ÚacceleratorÚset_device_indexr½  r  r  Úrun_testÚdestroy_process_group)rÞ  r§   Ú	test_namer  Úpiper`   rS   r&  r)  Úer,  Ú	device_ids               rH   Ú_runzFSDPTest._runž  s†  € á�9‹~ˆØˆŒ	Ø"ˆŒØ—*‘*˜Y¨Ó.ˆä�˜TŸY™Y˜K x°·±Ð/@ÐAÔBð	ÙÜŸ™×/Ñ/×;Ñ;×CÑC×MÑMÓO�Ü×'Ñ'Ø"Ø#Ÿ™ØØö	ô ×'Ñ'Ø $× 0Ñ 0Ü/Ü" 4§?¡?Ó3ØŸ™õ	ð ˆ
Ø—I‘I¤Ñ,ˆ	Ý�Ü×Ñ×.Ñ.¨yÔ9Ø�[ˆ
ô
 	�‰ 
Õ+ä�‰×ÑÔØ�‰�i Ô&Ü�‰×ÑÔä�‰ 
Õ+ä×"Ñ"Õ$øô/ ò 	Ø˜aŸf™f Q™iÑ'Ü—‘œÐ$9Ñ:×DÑDÔEàûð		ús   ÁB&G Ç	HÇ8HÈHNFrk   Ú	num_stepsÚautocastÚlrÚfsdp_cpu_offloadÚ
save_modelÚmixed_precisionÚenable_sharded_grad_scalerÚuse_pure_fp16Úsharded_grad_scaler_kwargsc           	      óÜ  — |xr |j                   }t        |j                  «       «      j                  }|
€i }
t	        dd|i|
¤Ž}t
        j                  j                  |j                  «       |d¬«      }t        |«      D �]  }|j                  «        t
        j                  j                  t        |¬«      5  |j                  j                  t        j                  t        «      «      }|	s|rMt        |t         «      s=t        |t
        j"                  «      r|j%                  «       }nt'        d„ |D «       «      } ||Ž }|rft        |t         «      rV|j(                  t*        vrD|j                  «       D ]1  }| j-                  |j                  t        j                  d«      «       Œ3 |j                  j/                  ||«      j1                  |«      }d d d «       |j3                  «      }|s&|	s$|j4                  t
        j6                  k(  s™J d«       ‚|	r+| j-                  |j4                  t
        j8                  «       net        |t         «      r+|€J ‚| j-                  |j4                  |j:                  «       n*| j-                  |j4                  t
        j6                  «       |j                  j=                  |«       |rTt        |t         «      rD|j                  «       D ]1  }| j-                  |j                  t        j                  d«      «       Œ3 |j?                  |«       |jA                  «        |s�Œ¾|jC                  «       jE                  «       D ��ci c]  \  }}||jG                  «       “Œ }}}tI        |«       |jK                  |«       �Œ t        |t         «      r|jM                  tN        jP                  «       jS                  «       S # 1 sw Y   �ŒxY wc c}}w )	NÚenabledgÍÌÌÌÌÌì?)rC  Úmomentum)rK  c              3   ó<   K  — | ]  }|j                  «       –— Œ y ­wr‡   )r—   )rr  rü   s     rH   rs  z4FSDPTest._train_for_several_steps.<locals>.<genexpr>ó  s   è ø€ Ò%>°1 a§f¡f§hÑ%>ùrt  r<   zeloss data type should be float32, as the original                     parameter data type is float32.rF   )*Úoffload_paramsÚnextrŽ   rT   r   rf   ÚoptimÚSGDrv   Ú	zero_gradÚamprB  rˆ   r'  rU   r  rŒ   rg   r—   re   r  r   ræ  rZ   r–   ÚscalerÛ   Úfloat32Úfloat16Úparam_dtyper^   ÚstepÚupdater˜   r£   Úcloner”   Úload_state_dictÚ_assert_stater   ÚIDLErr   )rS   rk   rA  rB  rC  rD  rE  rF  rG  rH  rI  Úcpu_offload_paramsÚmodel_deviceÚsharded_grad_scalerrP  r   rX   rY   rB  r]   ÚkÚvr˜   s                          rH   Ú_train_for_several_stepsz!FSDPTest._train_for_several_stepsÒ  s4  € ð .ÒQÐ2B×2QÑ2QÐä˜E×,Ñ,Ó.Ó/×6Ñ6ˆØ%Ð-Ø)+Ð&Ü/ñ 
Ø.ð
Ø2Lñ
Ðô
 —‘—‘ × 0Ñ 0Ó 2°rÀC�ÓHˆÜ�yÓ!ó 9	2ˆAØ�O‰OÔÜ—‘×#Ñ#¤K¸Ð#ÓBñ MàŸ™×.Ñ.¬u¯|©|¼KÓ/HÓI�Ù ¡_¼ZÈÌtÔ=TÜ! %¬¯©Ô6Ø %§
¡
£™ä %Ñ%>¸Ô%>Ó >˜Ù ˜�ñ 'Ü" 5¬$Ô/ð ×/Ñ/Ü>ñ?ð #×-Ñ-Ó/ò H˜à×(Ñ(¨¯©´5·<±<ÀÓ3FÕGðHð —|‘|×,Ñ,¨U°FÓ;×>Ñ>¸|ÓL�÷-Mð. '×,Ñ,¨TÓ2ˆDá"©=à—J‘J¤%§-¡-Ò/ð5ð5ó5Ø/ñ !Ø×$Ñ$ T§Z¡Z´·±Õ?ä ¤tÔ,Ø*Ð6Ð6Ð6Ø×$Ñ$ T§Z¡Z°×1LÑ1LÕMà×$Ñ$ T§Z¡Z´·±Ô?Ø�L‰L×%Ñ% dÔ+á!¤j°¼Ô&=Ø×)Ñ)Ó+ò D�Aà×$Ñ$ Q§X¡X¬u¯|©|¸EÓ/BÕCðDð  ×$Ñ$ UÔ+à×&Ñ&Ô(ãØ7<×7GÑ7GÓ7I×7OÑ7OÓ7Q×R©t¨q°!˜a §¡£™lÐR�
ÑRô ˜EÔ"Ø×%Ñ% jÖ1ðs9	2ôv �eœTÔ"Ø×Ñ¤× 2Ñ 2Ô3Ø�{‰{‹}Ð÷wMñ Müóf Ss   Â9DOÍO(ÏO%	rÒ   TÚmodel_classr  rÌ   Úref_init_fnÚ	num_itersr™   r  r  r"  Úuse_orig_paramsÚinit_kwargsc                 ó–  — |t         j                  k7  sJ d«       ‚|€i }d}| j                  j                  «       } |j                  | j                  t         j                  t
        j                  fddi|¤Ž}|€.t        rt        |t        gt        ¬«      }nt        ||g|¬«      }n ||«      }|r|j                  «       }| j                  |||
du|||
|||¬«	      }t        |j                  «       «      }|j                  |||	|
||dœ«       	  |j                  | j                  |||fddi|¤Ž}t%        |t&        «      st'        || j                  fi |¤Ž}|r|j                  «       }|t
        j(                  k(  r|j+                  t        «      }|duxr |j,                  }|xr |t
        j(                  k(  }|xr |t
        j(                  k7  }|rFt/        j0                  d«      }|j                  «       D ]  }| j3                  |j0                  |«       Œ  |r| j5                  t6        dt        › �«      n	t9        «       }|5  | j                  ||d||||
|||¬«
      } ddd«       |ry|r[t/        j0                  d«      }|j                  «       D ]  }| j3                  |j0                  |«       Œ   j+                  t        «      } t;        |«      }!t.        j<                  j?                  | d¬«       |
€|s| j3                  ||!dd¬«       yyy# t        $ r }t!        d	|› d
t#        |«      › �«      |‚d}~ww xY w# 1 sw Y   ŒÞxY w)aå  
        Tests FSDP training against a reference, which defaults to DDP but
        may be customized with ``ref_init_fn``.

        Args:
            model_class (Type[FSDPTestModel]): A model class that inherits from
                ``FSDPTestModel``, which defines the expected interface.
            fsdp_init_mode (FSDPInitMode): The mode to initialize the
                FSDP-wrapped model. This should not be ``NO_FSDP``.
            ref_init_fn (Optional[Callable]): A callable to invoke that wraps a
                non-wrapped model to construct the reference model, where this
                wrapper should provide data parallel semantics. If ``None``,
                then the callable defaults to the DDP constructor.
        z.Expects an FSDP init mode that wraps with FSDPNç{®Gáz„?rÎ   T)r,  Úoutput_device)rB  rC  rD  rF  rG  rH  rI  )r™   r  r  rF  r"  rg  zInitializing z raised error r<   zOAn FSDP-managed module with parameter CPU offloading enabled has parameters on F)rB  rC  rD  rE  rF  rG  rH  rI  )Úcheck_dtypezFSDP did not match DDP)Úexact_deviceÚmsg) r@   rD   rl   r§   rb   rJ   rK   r3   ÚDDPrˆ   r—   rc  r«   rŽ   rY  Ú	Exceptionr  rž   r  rŒ   rL   r–   rN  rf   rT   ræ  ÚassertRaisesRegexr4  r   r¬   r/  Úassert_close)"rS   rd  r  rÌ   re  rf  rE  r™   r  r  rF  r"  rg  rG  rH  rh  rI  r	  rC  r§   rk   Ú	ref_modelÚref_lossÚ
ddp_paramsr  r>  rN  Úexpects_device_errorÚexpects_cpu_deviceÚ
cpu_devicerz   ÚcontextÚ	fsdp_lossÚfsdp_unsharded_paramss"                                     rH   Ú_test_fsdp_parityzFSDPTest._test_fsdp_parity)  s¹  € ðF œl×2Ñ2Ò2ð	<à;ó	<Ø2àÐØˆKØˆØ×!Ñ!×&Ñ&Ó(ˆà �× Ñ Ø×ÑÜ× Ñ Ü×(Ñ(ñ
ð ð	
ð
 ñ
ˆð ÐÝÜØ¤{ mÄ;ô‘	ô   °4°&ÈÔM‘	á# EÓ*ˆIÙØ!Ÿ™Ó(ˆIØ×0Ñ0ØØØ$¨DÐ0ØØ(Ø+Ø'AØ'Ø'Að 1ó 

ˆô ˜)×.Ñ.Ó0Ó1ˆ
à×Ñà*Ø%6Ø%6Ø#2Ø$4Ø#2ñô		
ð
	YØ)˜×)Ñ)Ø×"Ñ"ØØ Øñ	ð
 #ðð ñˆJô ˜*¤dÔ+ô ˜j¨$×*<Ñ*<ÑLÀÑLˆJÙà#Ÿ™Ó*ˆJØœ~×:Ñ:Ò:Ø#Ÿ™¤{Ó3ˆJØ$¨DÐ0ÒO°[×5OÑ5Oˆð
 ÒNÐ/´>×3NÑ3NÑNð 	ð ÒNÐ/´>×3NÑ3NÑNð 	ñ ÜŸ™ eÓ,ˆJØ#×.Ñ.Ó0ò ;�Ø× Ñ  §¡¨zÕ:ð;ñ $ð ×"Ñ"Üð%Ü%0 Mð3ôô “ð 	ð ñ 	Ø×5Ñ5ØØØØØ!,Ø%Ø /Ø+EØ+Ø+Eð 6ó ˆI÷	ñ  Øñ ÜŸ™ eÓ,ˆJØ#×.Ñ.Ó0ò ;�Ø× Ñ  §¡¨zÕ:ð;à!Ÿ™¤[Ó1ˆIÜ /°
Ó ;Ðô 	�‰×"Ñ" 8¨YÀEÐ"ÔJð Ð"©=Ø×ÑØØ%Ø!Ø,ð	 õ ð ,9Ð"øôK ò 	YÜ˜}¨[¨M¸ÌÈAËÀxÐPÓQÐWXÐXûð	Yú÷F	ð 	ús$   Ä"L ÉL?Ì	L<ÌL7Ì7L<Ì?M)rj  NFNFFN)$rA   rB   rC   rü  r
  r~   rl   r  r  r  r  r   r#  r.   Úclassmethodr@  ri   rj   rÉ   Úfloatr   r   r   r¥   rž   r   rc  r`  rO   r@   rJ   r   r   r   r|  r  r  s   @rH   r  r  x  sC  ø„ ô ð ñó ðð ñ:ó ð:ð ð dò ó ðð ñ0ó ð0ò>òJòHò3ð ñ1%ó ð1%ðp Ø15Ø Ø48Ø+0Ø#Ø?CñUà�y‰yðUð ðUð ð	Uð
 ðUð # :Ñ.ðUð ðUð " .Ñ1ðUð %)ðUð ðUð %-¨T°#°s°(©^Ñ$<óUðx +/ØØÙ",£,Ø8<Ø8<Ø48Ø!&Ø %Ø+0Ø#Ø04Ø?Cñ#gà˜-Ñ(ðgð %ðgð )ð	gð
 ˜hÑ'ðgð ðgð ðgð  ðgð $Ð$4Ñ5ðgð $Ð$4Ñ5ðgð " .Ñ1ðgð ðgð ðgð %)ðgð ðgð  ˜d 3¨ 8™nÑ-ð!gð" %-¨T°#°s°(©^Ñ$<÷#grG   r  Úcompile_compute_on_modulec                 ó@   ‡ ‡‡— ˆ fd„Š G d„ dt         «      Šˆˆfd„}|S )Nc                  ó¢   •— t        j                  j                  j                  | i |¤Ž ‰�t	        | d   ‰«      r| d   j                  «        y y )Nr   )rf   rY  r�  r   r  rE  )r_   r`   r  s     €rH   Ú!fully_shard_with_compiled_computez=compiled_fsdp_test.<locals>.fully_shard_with_compiled_computeÔ  sN   ø€ Ü×Ñ×Ñ×*Ñ*¨DÐ;°FÒ;Ø$Ð,´
Ø�‰GÐ.ô1
ð �‰G�O‰OÕð1
rG   c                   ó(   — e Zd Z e«       Z e«       Zy)ú*compiled_fsdp_test.<locals>.FullyShardModeN)rA   rB   rC   r   ÚEAGERÚCOMPILED_COMPUTErF   rG   rH   ÚFullyShardModer„  Û  s   „ Ù“ˆÙ›6ÑrG   r‡  c                 ó4   •‡ — t        ‰ «      ˆˆˆ fd„«       }|S )Nc                  ó–  •— t         j                  j                  j                  }‰D �]ž  }|‰j                  k7  r t        «       st        j                  d«       Œ3t         j                  j                  j                  }t         j                  j                  j                  }t         j                  j                  «        |‰j                  k(  r|}n^|‰j                  k(  rAdt         j                  j                  _
        dt         j                  j                  _        ‰}nt        d|› �«      ‚|‰	j                   |j"                  <    ‰	| i |¤Ž t         j                  j                  «        |‰	j                   |j"                  <   |t         j                  j                  _
        |t         j                  j                  _        �Œ¡ y )Nz0Inductor on GPU needs Triton and recent GPU archTr>   z!Need to implement FullyShardMode=)rf   rY  r�  r   r…  r5   ÚwarningsÚwarnr  ÚconfigÚskip_fsdp_hooksÚ	_inductorÚcompile_threadsr½  r†  ÚNotImplementedErrorÚ__globals__rA   )
r_   r`   Úoriginal_fully_shardÚmodeÚoriginal_skip_fsdp_hooksÚoriginal_compile_threadsÚfully_shard_patchr‡  r‚  Úfuncs
          €€€rH   Úwrapperz6compiled_fsdp_test.<locals>.decorator.<locals>.wrapperà  sd  ø€ ä(-×(9Ñ(9×(>Ñ(>×(JÑ(JÐ Ø&ó R�Ø˜>×/Ñ/Ò/¼
¼Ü—M‘MÐ"TÔUØä+0¯=©=×+?Ñ+?×+OÑ+OÐ(Ü+0¯?©?×+AÑ+A×+QÑ+QÐ(Ü×!Ñ!×)Ñ)Ô+à˜>×/Ñ/Ò/Ø(<Ñ%Ø˜^×<Ñ<Ò<Ø;?”E—M‘M×(Ñ(Ô8Ø=>”E—O‘O×*Ñ*Ô:Ø(IÑ%ä-Ø;¸D¸6ÐBóð ð CT�× Ñ Ð!5×!>Ñ!>Ñ?Ù�dÐ%˜fÒ%ä×!Ñ!×)Ñ)Ô+ØBV�× Ñ Ð!5×!>Ñ!>Ñ?Ø7O”—‘×$Ñ$Ô4Ø9Q”—‘×&Ñ&Ö6ñ9RrG   r	   )r—  r˜  r‡  r‚  s   ` €€rH   Ú	decoratorz%compiled_fsdp_test.<locals>.decoratorß  s#   ù€ Ü	ˆt‹õ	Ró 
ð	Rð@ ˆrG   )r   )r  r™  r‡  r‚  s   ` @@rH   Úcompiled_fsdp_testrš  Ó  s"   ú€ ôô"œô "õ"ðH ÐrG   c                   ó&   ‡ — e Zd Zdˆ fd„Zd„ Zˆ xZS )Ú
SkipModulec                 ó\   •— t         ‰| �  «        t        j                  ddd¬«      | _        y ©Né
   Fr¡  )rÜ   rº   ri   râ   Úlinrþ  s    €rH   rº   zSkipModule.__init__  s"   ø€ Ü‰ÑÔÜ—9‘9˜R ¨%Ô0ˆ�rG   c                 ó$   — | j                  |«      S r‡   )r   r-  s     rH   rý   zSkipModule.forward  s   € Ø�x‰x˜‹{ÐrG   rc   rj  r  s   @rH   rœ  rœ    s   ø„ õ1örG   rœ  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )ÚNestedLinearc                 óú   •— t         ‰| �  «        |r:t        t        j                  ddd¬«      j                  t        «      «      | _        y t        j                  ddd¬«      j                  t        «      | _        y rž  )rÜ   rº   r!   ri   râ   r–   rˆ   Únested_linear)rS   Ú	fsdp_wraprï   s     €rH   rº   zNestedLinear.__init__  sV   ø€ Ü‰ÑÔÙÜ!%¤b§i¡i°°B¸UÔ&C×&FÑ&FÄ{Ó&SÓ!TˆDÕä!#§¡¨2¨r¸Ô!>×!AÑ!AÄ+Ó!NˆDÕrG   c                 ó$   — | j                  |«      S r‡   )r¥  r-  s     rH   rý   zNestedLinear.forward  s   € Ø×!Ñ! !Ó$Ð$rG   rj  r  s   @rH   r£  r£    s   ø„ ôOö%rG   r£  c                   ó$   ‡ — e Zd Zˆ fd„Zd„ Zˆ xZS )Ú	SkipModelc                 ó  •— t         ‰| �  «        t        j                  ddd¬«      j	                  t
        «      | _        t        «       j	                  t
        «      | _        t        t        |¬«      t
        ¬«      | _        y )NrŸ  Fr¡  )r¦  )r?  )rÜ   rº   ri   râ   r–   rˆ   Úlinearrœ  Úlinear_skipr!   r£  r¥  )rS   Údouble_nestrï   s     €rH   rº   zSkipModel.__init__  sW   ø€ Ü‰ÑÔÜ—i‘i  B¨UÔ3×6Ñ6´{ÓCˆŒÜ%›<Ÿ?™?¬;Ó7ˆÔÜ!Ü ;Ô/¼;ô
ˆÕrG   c                 ól   — | j                  |«      }| j                  |«      }| j                  |«      }|S r‡   )r«  r¬  r¥  r-  s     rH   rý   zSkipModel.forward$  s4   € Ø�K‰K˜‹NˆØ×Ñ˜QÓˆØ×Ñ˜qÓ!ˆØˆrG   rj  r  s   @rH   r©  r©    s   ø„ ô
örG   r©  )FT)FFr¹  )rF   r‡   )�Ú
contextlibr  r=  r5  rR  rŠ  Úabcr   r   r   Úcopyr   Úenumr   r   Ú	functoolsr
   Útypingr   r   r   r   r   r   Úunittestr   rf   Útorch.distributedrY  rt   Útorch.nnri   Útorch.nn.functionalr  r•  Útorch.distributed._composabler   Útorch.distributed.device_meshr   Útorch.distributed.fsdpr   r   r   rŒ   Ú$torch.distributed.fsdp._common_utilsr   Ú5torch.distributed.fsdp._fully_shard._fsdp_param_groupr   r   Ú"torch.distributed.fsdp._init_utilsr   Ú2torch.distributed.fsdp.fully_sharded_data_parallelr   r   r   Ú*torch.distributed.fsdp.sharded_grad_scalerr   Útorch.distributed.fsdp.wrapr   r    r!   Útorch.distributed.tensorr"   r#   r$   Ú!torch.distributed.tensor.parallelr%   r&   r'   r(   r)   r*   Útorch.nn.parallel.distributedr+   ro  Ú*torch.testing._internal.common_distributedr,   r-   r.   r/   Ú$torch.testing._internal.common_utilsr0   r1   r2   r3   r4   Útorch.utils._tritonr5   rù  rˆ   r3  r7   Údevice_countr:   r@   rJ   rj   rO   r  r…   r‰   r  r”   rš   r¡   r¨   r¬   r¯   r³   rµ   rË   r  r6  r9  rI  rb  rf  rl  r‹  r&  rœ  r²  Úcontextmanagerr¿  rÃ  rÇ  rÌ  rÑ  rÖ  rÚ  rß  re   rž   rô  rö  r  r`  rš  rœ  r£  r©  rF   rG   rH   ú<module>rÊ     sa  ðó Û 	Û 	Û 
Û Û ß #Ý "Ý ß Ý ß F× FÝ ã Ý  Ý ß Ð Ý 4Ý 4÷ñ õ
 ?÷õ S÷ñ õ
 Iß RÑ Rß FÑ F÷ó ÷ FÝ H÷ó ÷õ õ +ð €áØ€KØ ÐØ—:‘:×*Ñ*Ó,�LÙØ€KØ ÑÙØ€KØ ÐØ—9‘9×)Ñ)Ó+�Là€KØ ÐØ€Lô�4ô ô�Tô ô�B—I‘I˜sô ð4Ø�9‰9ðà×$Ñ$ðð óò>%ð Øñ#Ø�9‰9ð#àó#ó"òòñ"2˜2Ÿ9™9ð 2¨tó 2ð>˜2Ÿ9™9ð >°dó >ðD˜BŸI™Ið D°$ó D÷ñ ô.Q" -ô Q"ôh[J˜-ô [Jô|Ð$7ô ôD]JÐ.ô ]Jô@J
�mô J
ôZ
 ?ô 
ô.,ˆr�y‰yô ,ôJJÐ*ô JJôZ/ˆ"�)‰)ô /ô@*ˆr�}‰}ô *ôZ'�2—9‘9ô 'ð6 ×Ñð6°ò 6ó ð6ð ×Ñð9°Hò 9ó ð9ð ×Ñð* Xò *ó ð*ð Ø×Ñð.˜xò .ó ó ð.ð Ø×Ñð.˜xò .ó ó ð.ð Ø×Ñð:¨8ò :ó ó ð:ð Ø×Ñð>¸Xò >ó ó ð>ð0à!ð0ð ð0ð ð	0ð
 ó0ð. +-ñ	Tà—y‘yðTð —I‘IðTð ˜c 3˜h™ó	TôBÐ/ô ô&XÐ#ô Xñv
0°(¸4±.ó 0ôf�—‘ô ô	%�2—9‘9ô 	%ô�—	‘	õ rG   