Ë
    [^(h¢Û  ã                   óÄ  — U d dl Z d dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl	Z	d dl
Z
d dlZd dlZd dlZd dlZd dlmZ d dlmZ d dlmZ d dlmZ d dlmZmZmZ d dlmZ d dlmZmZm Z m!Z!m"Z" d d	l#m$Z$ d d
l%m&Z& d dl'Z'd dl(Z'd dl)Z'd dl*m+Z, d dl-m.Z. d dl/m0Z0 d dl1m2Z2 d dl3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z<m=Z=m>Z>m?Z? d dl@mAZAmBZBmCZC d dlDZD ejŠ                  ejŒ                  ¬«        ejŽ                  eH«      ZI G d„ de«      ZJi d eJdd«      “d eJdd«      “d eJdd«      “d eJdd«      “d eJdd «      “d! eJd"d#«      “d$ eJd%d&«      “d' eJd(d)«      “d* eJd+d,«      “d- eJd.d/«      “d0 eJd1d2«      “d3 eJd4d5«      “d6 eJd7d8«      “d9 eJd:d;«      “d< eJd=d>«      “d? eJd@dA«      “dB eJdCdD«      “dE eJdFdG«      i¥ZKe G dH„ dI«      «       ZLdJ„ ZMdK„ ZNdL„ ZOdM„ ZPdN„ ZQdO„ ZRdP„ ZSdQ„ ZTdR„ ZUdS„ ZVdT„ ZWdU„ ZXdV„ ZYdW„ ZZdX„ Z[dY„ Z\dZ„ Z]d[„ Z^d\„ Z_d]e'jÀ                  d^ead_ead`ebfda„Zce7dbdcdd ede¬f«      dddgddfdh„«       Zde;rdiZen ea ejÌ                  djdk«      «      ZedldmiZge:rdnegdo<   d–dp„Zhd`eafdq„Ziedr„ «       Zjd—dseadteadueafdv„Zkdteadwelfdx„Zmdanee	jÞ                     epdy<   d–dzeel   d`dfd{„Zqd˜d|„Zrd}Zs G d~„ de<«      Zt G d€„ d�et«      Zud‚evelewe!   f   dƒe"d„e!fd…„Zxdayd`ebfd†„Zzd‡„ Z{deeesfdˆ„Z| G d‰„ dŠe<«      Z} G d‹„ dŒe2jü                  «      Z G d�„ dŽe2jü                  «      Z€ed™d�„«       Z� G d�„ d‘e'�j                  �j                  jx                  «      Z„ G d’„ d“et«      Z… G d”„ d•e<«      Z†y)šé    N)Úcontextmanager)Ú	dataclass)Ú	timedelta)ÚEnum)ÚpartialÚreduceÚwraps)ÚStringIO)Ú
NamedTupleÚOptionalÚUnionÚAnyÚCallable)Úpatch)Ú	trace_log)Ú
DeviceType)Ú_SymmetricMemory)ÚFILE_SCHEMAÚfind_free_portÚIS_SANDCASTLEÚretry_on_connect_failuresÚskip_but_pass_in_sandcastleÚskip_but_pass_in_sandcastle_ifÚTEST_WITH_ROCMÚTEST_WITH_TSANÚTestCaseÚ	run_testsÚTEST_HPUÚTEST_XPU)Ú_install_threaded_pgÚ_uninstall_threaded_pgÚProcessLocalGroup)Úlevelc                   ó"   — e Zd ZU eed<   eed<   y)ÚTestSkipÚ	exit_codeÚmessageN)Ú__name__Ú
__module__Ú__qualname__ÚintÚ__annotations__Ústr© ó    úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/testing/_internal/common_distributed.pyr%   r%   <   s   … ØƒNØ„Lr/   r%   Úbackend_unavailableéH   z5Skipped because distributed backend is not available.Úsmall_worldsizeéI   z Skipped due to small world size.Úodd_worldsizeéW   zSkipped due to odd world size.Úno_cudaéJ   zCUDA is not available.zmulti-gpu-1éK   zNeed at least 1 CUDA devicezmulti-gpu-2éM   zNeed at least 2 CUDA deviceszmulti-gpu-3éP   zNeed at least 3 CUDA deviceszmulti-gpu-4éQ   zNeed at least 4 CUDA deviceszmulti-gpu-5éR   zNeed at least 5 CUDA deviceszmulti-gpu-6éS   zNeed at least 6 CUDA deviceszmulti-gpu-7éT   zNeed at least 7 CUDA deviceszmulti-gpu-8éU   zNeed at least 8 CUDA devicesÚnccléL   z#c10d not compiled with NCCL supportÚ
skipIfRocméN   zTest skipped for ROCmÚno_peer_accesséO   z'Test skipped because no GPU peer accessÚgenericéV   zHTest skipped at subprocess level, look at subprocess log for skip reasonÚimporterroréX   z"Test skipped due to missing importÚno_acceleratoréY   zaccelerator is not available.c                   óÀ   — e Zd Zi Zh d£ed<    e«       ed<   ddhed<   ddhed<   i Zh d£ed	<   h d£ed
<   h d£ed<   h d£ed<    e«       ed<   erdhed<   erdhed<   yy)ÚDistTestCases>   ÚmpiÚuccrA   Úallgather_coalescedr   rA   rP   zsendrecv anysourcezcpu barrier>   rP   ÚgloorA   ÚgpuÚcudaÚddpÚsubgroupÚpluginÚhcclÚhpuÚxcclÚxpuN)r(   r)   r*   Úskip_collectiveÚsetÚbackend_featurer   r   r.   r/   r0   rN   rN   [   s›   „ ð €OÚ-C€OÐ)Ñ*Ù #£€O�HÑØ-3°U¨O€OÐ(Ñ)Ø&,¨e _€O�MÑ"ð €OÚ4€O�EÑÚ5€O�FÑÚ4€O�EÑÚ"9€O�JÑÙ #£€O�HÑÙØ"( ˆ˜ÑÙØ"( ˆ˜Òð r/   rN   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )z”Skips if the world size exceeds the number of GPUs, ensuring that if the
    test is run, each rank has its own GPU via ``torch.cuda.device(rank)``.c                  ó˜  •— t         j                  j                  «       s&t        j                  t
        d   j                  «       t        t        j                  d   «      }t         j                  j                  «       |k  r)t        j                  t
        d|› �   j                  «       t        rFt         j                  j                  |k  r)t        j                  t
        d|› �   j                  «       t        rFt         j                  j                  |k  r)t        j                  t
        d|› �   j                  «        ‰| i |¤ŽS )Nr7   Ú
WORLD_SIZEú
multi-gpu-z
multi-xpu-)ÚtorchrT   Úis_availableÚsysÚexitÚ
TEST_SKIPSr&   r+   ÚosÚenvironÚdevice_countr   rY   r   r[   )ÚargsÚkwargsÚ
world_sizeÚfuncs      €r0   Úwrapperzskip_if_no_gpu.<locals>.wrapperu   sä   ø€ ä�z‰z×&Ñ&Ô(Ü�H‰H”Z 	Ñ*×4Ñ4Ô5ÜœŸ™ LÑ1Ó2ˆ
Ü�:‰:×"Ñ"Ó$ zÒ1Ü�H‰H”Z *¨Z¨LÐ 9Ñ:×DÑDÔEÝœŸ	™	×.Ñ.°Ò;Ü�H‰H”Z *¨Z¨LÐ 9Ñ:×DÑDÔEÝœŸ	™	×.Ñ.°Ò;Ü�H‰H”Z *¨Z¨LÐ 9Ñ:×DÑDÔEá�TÐ$˜VÑ$Ð$r/   ©r	   ©rn   ro   s   ` r0   Úskip_if_no_gpurr   q   s"   ø€ ô ˆ4ƒ[ó%ó ð%ð €Nr/   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )Nc                  óÊ   •— t         j                  d   dk7  rEt        t         j                  d   «      dk  r&t        j                  t
        d   j                  «        ‰| i |¤ŽS )NÚBACKENDrO   ra   é   r3   ©rh   ri   r+   re   rf   rg   r&   ©rk   rl   rn   s     €r0   ro   z(skip_if_small_worldsize.<locals>.wrapper�   sR   ø€ ä�J‰J�yÑ! UÒ*´´B·J±J¸|Ñ4LÓ0MÐPQÒ0QÜ�H‰H”ZÐ 1Ñ2×<Ñ<Ô=á�TÐ$˜VÑ$Ð$r/   rp   rq   s   ` r0   Úskip_if_small_worldsizery   Œ   ó    ø€ Ü
ˆ4ƒ[ó%ó ð%ð €Nr/   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )Nc                  óÐ   •— t         j                  d   dk7  rHt        t         j                  d   «      dz  dk(  r&t        j                  t
        d   j                  «        ‰| i |¤ŽS )Nru   rO   ra   é   é   r5   rw   rx   s     €r0   ro   z&skip_if_odd_worldsize.<locals>.wrapper˜   sW   ø€ ä�J‰J�yÑ! UÒ*´´B·J±J¸|Ñ4LÓ0MÐPQÑ0QÐUVÒ0VÜ�H‰H”Z Ñ0×:Ñ:Ô;á�TÐ$˜VÑ$Ð$r/   rp   rq   s   ` r0   Úskip_if_odd_worldsizer   —   rz   r/   c                 ó   ‡ ‡— ˆˆ fd„}|S )Nc                 ó4   •‡ — t        ‰ «      ˆˆ ˆfd„«       }|S )Nc                  ó´   •— ‰dk(  rKt         j                  j                  «       ‰k  r*t        j                  t
        d‰› �   j                  «       y  ‰| i |¤ŽS ©NrA   rb   )rc   rT   rj   re   rf   rg   r&   )rk   rl   Úbackendrn   Úns     €€€r0   ro   zCrequire_n_gpus_for_nccl_backend.<locals>.decorator.<locals>.wrapper¤   sM   ø€ à˜&Ò ¤U§Z¡Z×%<Ñ%<Ó%>ÀÒ%BÜ—‘œ j°°Ð$4Ñ5×?Ñ?Õ@á˜TÐ, VÑ,Ð,r/   rp   )rn   ro   r„   r…   s   ` €€r0   Ú	decoratorz2require_n_gpus_for_nccl_backend.<locals>.decorator£   s    ù€ Ü	ˆt‹õ	-ó 
ð	-ð ˆr/   r.   )r…   r„   r†   s   `` r0   Úrequire_n_gpus_for_nccl_backendr‡   ¢   s   ù€ õð Ðr/   c                  ó   — d„ } | S )Nc                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )Nc                  ó�   •— 	 ddl m}m}  ‰| i |¤ŽS # t        $ r) t	        j
                  t        d   j                  «       Y y w xY w)Nr   )ÚAutoModelForMaskedLMÚ
BertConfigrI   )Útransformersr‹   rŒ   ÚImportErrorre   rf   rg   r&   )rk   rl   r‹   rŒ   rn   s       €r0   ro   z?import_transformers_or_skip.<locals>.decorator.<locals>.wrapper²   sE   ø€ ð>÷ñ
 ˜TÐ, VÑ,Ð,øÜò >Ü—‘œ MÑ2×<Ñ<Ö=ð>ús   ƒ “/AÁArp   rq   s   ` r0   r†   z.import_transformers_or_skip.<locals>.decorator±   s    ø€ Ü	ˆt‹ó		>ó 
ð		>ð ˆr/   r.   )r†   s    r0   Úimport_transformers_or_skipr�   °   s   € òð Ðr/   c                 ó„   — t         j                  j                  «       xr! t         j                  j                  «       | k\  S ©N)rc   rT   rd   rj   )Úxs    r0   Úat_least_x_gpur“   Ã   s,   € Ü�:‰:×"Ñ"Ó$ÒG¬¯©×)@Ñ)@Ó)BÀaÑ)GÐGr/   c                 ó   ‡ — ˆ fd„}|S )Nc                 ó2   •‡ — t        ‰ «      ˆ ˆfd„«       }|S )Nc                  ó¢  •— t         j                  j                  «       r)t         j                  j                  «       ‰k\  r ‰| i |¤ŽS t        r)t         j
                  j                  «       ‰k\  r ‰| i |¤ŽS t        r)t         j                  j                  «       ‰k\  r ‰| i |¤ŽS t        j                  t        d‰› �   j                  «       y )Nrb   )rc   rT   rd   rj   r   rY   r   r[   re   rf   rg   r&   )rk   rl   rn   r’   s     €€r0   ro   z4skip_if_lt_x_gpu.<locals>.decorator.<locals>.wrapperÉ   s£   ø€ ä�z‰z×&Ñ&Ô(¬U¯Z©Z×-DÑ-DÓ-FÈ!Ò-KÙ˜TÐ, VÑ,Ð,ÝœEŸI™I×2Ñ2Ó4¸Ò9Ù˜TÐ, VÑ,Ð,ÝœEŸI™I×2Ñ2Ó4¸Ò9Ù˜TÐ, VÑ,Ð,Ü�H‰H”Z *¨Q¨CÐ 0Ñ1×;Ñ;Õ<r/   rp   )rn   ro   r’   s   ` €r0   r†   z#skip_if_lt_x_gpu.<locals>.decoratorÈ   s    ù€ Ü	ˆt‹ô	=ó 
ð	=ð ˆr/   r.   )r’   r†   s   ` r0   Úskip_if_lt_x_gpur—   Ç   s   ø€ ôð Ðr/   c                 ó   ‡ ‡— ˆ ˆfd„}|S )Nc                 ó4   •‡ — t        ‰ «      ˆˆ ˆfd„«       }|S )Nc                  ó   •— ‰dk7  r ‰| i |¤ŽS t         j                  j                  «       r)t         j                  j                  «       ‰k\  r ‰| i |¤ŽS t	        j
                  t        d‰› �   j                  «       y rƒ   )rc   rT   rd   rj   re   rf   rg   r&   )rk   rl   r„   rn   r’   s     €€€r0   ro   z9nccl_skip_if_lt_x_gpu.<locals>.decorator.<locals>.wrapperÛ   sm   ø€ à˜&Ò Ù˜TÐ, VÑ,Ð,Ü�z‰z×&Ñ&Ô(¬U¯Z©Z×-DÑ-DÓ-FÈ!Ò-KÙ˜TÐ, VÑ,Ð,Ü�H‰H”Z *¨Q¨CÐ 0Ñ1×;Ñ;Õ<r/   rp   )rn   ro   r„   r’   s   ` €€r0   r†   z(nccl_skip_if_lt_x_gpu.<locals>.decoratorÚ   s    ù€ Ü	ˆt‹õ	=ó 
ð	=ð ˆr/   r.   )r„   r’   r†   s   `` r0   Únccl_skip_if_lt_x_gpur›   Ù   s   ù€ õ	ð Ðr/   c                 óÈ   — | j                  «       }d|v sJ ‚d|v sJ ‚d|v sJ ‚|d   }|j                  d«      dk(  r|n|j                  d«      d   }||v sJ d|› d|› �«       ‚y )	NÚ	iterationÚ	has_errorÚerrorz
Exception raised from éÿÿÿÿr   zDid not find expected z in ddp logging data error: )Ú_get_ddp_logging_dataÚfindÚsplit)Ú	model_DDPÚ
err_substrÚddp_logging_dataÚlogging_errÚactuals        r0   Úverify_ddp_error_loggedr©   è   s¤   € à ×6Ñ6Ó8ÐØÐ*Ñ*Ð*Ð*ØÐ*Ñ*Ð*Ð*ØÐ&Ñ&Ð&Ð&Ø" 7Ñ+€Kð �?‰?Ð5Ó6¸"Ò<ñ 	à×ÑÐ8Ó9¸!Ñ<ð ð 	�+ÑðRà	 ˜xÐ'CÀKÀ=ÐQóRÙr/   c                 ó.   ‡ — t        ‰ «      ˆ fd„«       }|S )aJ  
    Convenience decorator to set/unset TORCH_NCCL_BLOCKING_WAIT flag. Note that use of
    this decorator will override the setting of TORCH_NCCL_ASYNC_ERROR_HANDLING for
    the particular test. After the test, both TORCH_NCCL_BLOCKING_WAIT and
    TORCH_NCCL_ASYNC_ERROR_HANDLING will be restored to their original values.
    c                  óÜ  •— 	 t         j                  d   }t         j                  d= 	 t         j                  d   }dt         j                  d<   	  ‰| i |¤Ž}||�|t         j                  d<   |�|t         j                  d<   S S # t        $ r d }Y Œjw xY w# t        $ r d }Y Œgw xY w# dt         j                  d<   w xY w# |�|t         j                  d<   |�|t         j                  d<   w w xY w)NÚTORCH_NCCL_ASYNC_ERROR_HANDLINGÚTORCH_NCCL_BLOCKING_WAITÚ1)rh   ri   ÚKeyError)rk   rl   Ú cached_nccl_async_error_handlingÚcached_nccl_blocking_waitÚretrn   s        €r0   ro   z(with_nccl_blocking_wait.<locals>.wrapper  s  ø€ ð	4ÜACÇÁØ1ñBÐ,ô —
‘
Ð<Ð=ð	9Ü:<¿*¹*Ø*ñ;Ð%ð 69ŒB�J‰JÐ1Ñ2ð	SÙ˜Ð' Ñ'ˆCØð 0Ð;ð 5ô —
‘
Ø5ñð )Ð4Ø9R”—
‘
Ð5Ò6ð 5øô1 ò 	4à/3Ò,ð	4ûô ò 	-Ø(,Ò%ð	-ûð 69ŒB�J‰JÐ1Ò2ûð 0Ð;ð 5ô —
‘
Ø5ñð )Ð4Ø9R”—
‘
Ð5Ò6ð 5ús@   ƒ$B ¨B Á	B> ÂBÂBÂB#Â B& Â"B#Â#B& Â&B;Â>-C+rp   rq   s   ` r0   Úwith_nccl_blocking_waitr³   ú   s%   ø€ ô ˆ4ƒ[ó Só ð SðD €Nr/   c                 ó   ‡ — ˆ fd„}|S )zK
    Runs a test for each distributed debug level specified in levels.
    c                 ó2   •‡ — t        ‰ «      ˆ ˆfd„«       }|S )Nc                  ó  •— t         j                  j                  dd «      }‰D ][  }|t         j                  d<   t        j                  «         ‰| i |¤Ž}t        j
                  «        |€ŒI|t         j                  d<   Œ] S )NÚTORCH_DISTRIBUTED_DEBUG)rh   ri   ÚgetÚc10dÚset_debug_level_from_envÚbarrier)rk   rl   Ú	old_levelr#   r²   rn   Úlevelss        €€r0   ro   z:with_dist_debug_levels.<locals>.decorator.<locals>.wrapper.  sx   ø€ äŸ
™
Ÿ™Ð'@À$ÓGˆIØò F�Ø8=”—
‘
Ð4Ñ5Ü×-Ñ-Ô/Ù˜DÐ+ FÑ+�Ü—‘”ØÑ(Ø<E”B—J‘JÐ8Ò9ðFð ˆJr/   rp   )rn   ro   r½   s   ` €r0   r†   z)with_dist_debug_levels.<locals>.decorator-  s    ù€ Ü	ˆt‹ô	ó 
ð	ð ˆr/   r.   )r½   r†   s   ` r0   Úwith_dist_debug_levelsr¾   (  s   ø€ ô
ð$ Ðr/   c                  ó@   — t        t        j                  «        d«      S )Nz+c10d was not compiled with the Gloo backend)r   r¹   Úis_gloo_availabler.   r/   r0   Úrequires_gloorÁ   B  ó!   € Ü)Ü×"Ñ"Ó$Ð$Ø5óð r/   c           	      ó  — t        j                  «       st        d«      S t        t        j
                  j                  j                  «       | k  d| › dt        j
                  j                  j                  «       › d|› �«      S )Nú+c10d was not compiled with the NCCL backendz0Requires NCCL version greater than or equal to: z	, found: z
, reason: )r¹   Úis_nccl_availabler   r   rc   rT   rA   Úversion)rÆ   Úmsgs     r0   Úrequires_nccl_versionrÈ   I  sv   € Ü×!Ñ!Ô#Ü*Ø9ó
ð 	
ô .Ü�J‰J�O‰O×#Ñ#Ó%¨Ñ/Ø>¸w¸iÀyÔQV×Q[ÑQ[×Q`ÑQ`×QhÑQhÓQjÐPkÐkuÐvyÐuzÐ{ó
ð 	
r/   c                  ó@   — t        t        j                  «        d«      S )NrÄ   )r   r¹   rÅ   r.   r/   r0   Úrequires_ncclrÊ   U  rÂ   r/   c                  ó@   — t        t        j                  «        d«      S )Nz*c10d was not compiled with the UCC backend)r   r¹   Úis_ucc_availabler.   r/   r0   Úrequires_uccrÍ   [  ó!   € Ü)Ü×!Ñ!Ó#Ð#Ø4óð r/   c                  ó@   — t        t        j                  «        d«      S )Nz*c10d was not compiled with the MPI backend)r   r¹   Úis_mpi_availabler.   r/   r0   Úrequires_mpirÑ   a  rÎ   r/   c                  ó¤   — t         j                  j                  «       xr$ t        j                  t
        j                  d«      } t        |  d«      S )Nr   z"multicast support is not available)rc   rT   rd   r   Úhas_multicast_supportr   ÚCUDAr   )rÓ   s    r0   Úrequires_multicast_supportrÕ   h  sI   € ä�
‰
×ÑÓ!ò 	GÜ×2Ñ2´:·?±?ÀAÓFð ô *Ø!Ð!Ø,óð r/   c                 ó<   ‡ — d‰ _         t        ‰ «      ˆ fd„«       }|S )zSkips a test for ROCmTc                  ón   •— t         s ‰| i |¤ŽS t        j                  t        d   j                  «       y )NrC   )r   re   rf   rg   r&   rx   s     €r0   ro   z*skip_if_rocm_multiprocess.<locals>.wrapperw  s-   ø€ åÙ˜Ð( Ñ(Ð(Ü�‰”˜LÑ)×3Ñ3Õ4r/   )Úskip_if_rocm_multiprocessr	   rq   s   ` r0   rØ   rØ   s  s(   ø€ à%)€DÔ"ä
ˆ4ƒ[ó5ó ð5ð
 €Nr/   c                  ó<   — t        t        j                  dk(  d«      S )NÚwin32z8This unit test case is not supported on Windows platform)r   re   Úplatformr.   r/   r0   Úskip_if_win32rÜ   €  s   € Ü)Ü�‰˜ÑØBóð r/   ÚdeviceÚmajorÚminorÚreturnc                 ó´   — | j                   dk7  rt        d«      ‚t        j                  j                  �yt        j
                  j                  | «      ||fk\  S )z¶
    Returns True if the device's compute capability is (major, minor) or higher.
    Error out if the device is not a CUDA device.
    Returns False if device is a RoCM device.
    rT   z3sm_is_or_later() is only supported for CUDA devicesF)ÚtypeÚ
ValueErrorrc   rÆ   ÚhiprT   Úget_device_capability)rÝ   rÞ   rß   s      r0   Úsm_is_or_higher_thanræ   ‡  sM   € ð ‡{�{�fÒÜÐNÓOÐOä‡}�}×ÑÐ$àä�:‰:×+Ñ+¨FÓ3¸¸u°~ÑEÐEr/   Ú	localhostr~   Té   )ÚminutesFc                 óÚ   — t        «       }|rEt        |t        d¬«      z  «      }t        j                  j
                  j                  | ||||«      S t        j                  | |||||¬«      S )zL
    Creates a TCP store. Retries if the chosen port is already in use.
    r~   )Úmilliseconds)Úwait_for_workersÚ	use_libuv)r   r+   r   rc   ÚclassesÚ	dist_c10dÚTCPStorer¹   )	Úaddrrm   Ú	is_masterÚtimeoutrì   Ú	jit_classrí   ÚportÚtimeout_milliseconds	            r0   Úcreate_tcp_storer÷   —  so   € ô Ó€DÙÜ! '¬IÀ1Ô,EÑ"EÓFÐÜ�}‰}×&Ñ&×/Ñ/Ø�$˜
 IÐ/Bó
ð 	
ô �}‰}Ø�$˜
 IÐ@PÐ\eô
ð 	
r/   iô  Ú!DISTRIBUTED_TESTS_DEFAULT_TIMEOUTÚ300Útest_ddp_uneven_inputsi�  éÈ   Útest_join_kwargsc                 ó¬   — t         j                  dk(  s| € t        j                  j	                  d¬«      S t        j                  j	                  | ¬«      S )NrÚ   z	127.0.0.1)Úhostname©Ú	interface)re   rÛ   r¹   ÚProcessGroupGlooÚcreate_devicerÿ   s    r0   r  r  ½  sG   € Ü
‡|�|�wÒ )Ð"3Ü×$Ñ$×2Ñ2¸KÐ2ÓHÐHä×$Ñ$×2Ñ2¸YÐ2ÓGÐGr/   c                 óZ   — t         j                  | j                  d«      d   t        «      S ©Nú.r    )ÚTIMEOUT_OVERRIDEr¸   r£   ÚTIMEOUT_DEFAULT)Útest_ids    r0   Úget_timeoutr	  Ä  s#   € Ü×Ñ §¡¨cÓ 2°2Ñ 6¼ÓHÐHr/   c               #   óN  K  — t        «       t        «       }} t        j                  t        j                  }}	 | |ct        _        t        _        t        j                  t        j                  f–— ||ct        _        t        _        y # ||ct        _        t        _        w xY w­wr‘   )r
   re   ÚstdoutÚstderr)Únew_outÚnew_errÚold_outÚold_errs       r0   Úcaptured_outputr  È  sl   è ø€ ä“z¤8£:ˆW€GÜ—z‘z¤3§:¡:ˆW€Gð2Ø!(¨'ÐŒŒ
”C”JÜ�j‰jœ#Ÿ*™*Ð$Ò$à!(¨'ÐŒŒ
”C•Jø ¨'ÐŒŒ
”C•Jüs   ‚5B%¸9B	 Á1B%Â	B"Â"B%Úrankrm   Ú
num_inputsc                 ó¸  — ddt         dt         dt         dt         fd„}dt         fd„}t        |d¬«      t        |d	¬«      t        |d
¬«      t        |d¬«      t        |d	¬«      t        |d
¬«      fD ��cg c]N  }t        |«      D �cg c]  } ||| z  |z   ||z  «      ‘Œ c}t        |«      D �cg c]  } ||||z  «      ‘Œ c}f‘ŒP c}}S c c}w c c}w c c}}w )zæ
    Generate a number of basic test cases for sparse reduction.
    These cover tensors with a varying number of sparse dimensions and a varying
    number of dense dimensions. The only reduction operation we support is sum.
    r~   r  rm   Úsparse_dimsÚ
dense_dimsc           	      óê  — t        j                  t        j                  | dz   «      d| dz   f«      }|gt        |«      D �cg c]  }d‘Œ c}z   }t        |dz
  «      D ]A  }t        j                  |t        j
                  d| dz   «      f«      }|j                  |«       ŒC t        j                  | dz   gt        |«      D �cg c]  }d‘Œ c}z   «      }t        j                  |||«      S c c}w c c}w )Nr~   r}   )	rc   ÚreshapeÚarangeÚrangeÚcatÚzerosÚappendÚonesÚsparse_coo_tensor)r  rm   r  r  ÚindicesÚ_ÚshapeÚvaluess           r0   Úgeneratez,simple_sparse_reduce_tests.<locals>.generateÚ  sÖ   € ô —-‘-¤§¡¨T°A©XÓ 6¸¸DÀ1¹H¸ÓFˆØ�¬5°Ó+<Ö= a¢Ò=Ñ=ˆÜ�{ Q‘Ó'ò 	%ˆAÜ—i‘i ¬%¯+©+°a¸À¹Ó*BÐ CÓDˆGØ�L‰L˜Õ$ð	%ô —‘˜T A™X˜J´U¸:Ó5FÖ)G°ª!Ò)GÑGÓHˆÜ×&Ñ& w°¸Ó>Ð>ùò  >ùò *Hs   Á	C+Ã	C0
c           
      ó|   — t        t        j                  t        |«      D �cg c]  } | ||«      ‘Œ c}«      S c c}w r‘   )r   ÚoperatorÚaddr  )Úfnrm   r  s      r0   Úcompute_sumz/simple_sparse_reduce_tests.<locals>.compute_sumæ  s2   € ÜÜ�L‰L¼EÀ*Ó<MÖN°D™2˜d JÕ/ÒNó
ð 	
ùÚNs   ¢9
)r  r}   é   )r  )r~   r   )r+   r   r  )r  rm   r  r$  r)  r(  Úis          r0   Úsimple_sparse_reduce_testsr,  Ó  sò   € ñ
?”sð 
?¬ð 
?¼#ð 
?Ìsó 
?ð
¤Có 
ô �H¨!Ô,Ü�H¨!Ô,Ü�H¨!Ô,Ü�H¨Ô+Ü�H¨Ô+Ü�H¨Ô+ð
÷ð ô	 ˜zÓ*öàñ �: Ñ$ qÑ(¨*°zÑ*AÕBòô @EÀZÓ?PÖQ¸!‰[˜˜Z¨*Ñ4Õ5ÒQò	
óð ùòùò Rùós$   Á5CÂCÂ CÂ/CÃCÃ
Cr„   c           
      óf  — t         j                  j                  «       }t        rt         j                  j                  «       }t
        rt         j                  j                  «       }t        |«      }d}| |kD  r|| z  }t        | «      D �ci c]  }|t        |||z  |dz   |z   «      “Œ }}|S c c}w )züMultigpu tests are designed to simulate the multi nodes with multi
    GPUs on each node. Nccl backend requires equal #GPUs in each process.
    On a single node, all visible GPUs are evenly
    divided to subsets, each process only uses a subset.
    r~   )	rc   rT   rj   r   rY   r   r[   r  Úlist)rm   r„   ÚnGPUsÚvisible_devicesÚnGPUs_per_processr+  Úrank_to_GPUs          r0   Úinit_multigpu_helperr3  ÿ  s¶   € ô �J‰J×#Ñ#Ó%€EÝÜ—	‘	×&Ñ&Ó(ˆÝÜ—	‘	×&Ñ&Ó(ˆÜ˜E“l€Oð ÐØ�EÒØ! ZÑ/Ðô �zÓ"öàð 	
Œ4� Ð$5Ñ 5¸¸Q¹ÐBSÑ8SÐTÓUÑUð€Kð ð Ðùò	s   ÂB.Útmp_dirÚinit_methodc                 ó”  — t        j                  «       at        j                  t        j
                  d<   t	        j                  t        j                  j                  t        j                  d«      «       t	        j                  t        j                  j                  t        j                  d«      «       t        j                  j                  t        j                  d«      }t	        j                  |«       | �| t        j
                  d<   y t        t        j                  j                  |d«      z   t        j
                  d<   y )NÚTEMP_DIRr»   Útest_dirÚinit_dirÚINIT_METHODÚshared_init_file)
ÚtempfileÚTemporaryDirectoryr4  Únamerh   ri   ÚmkdirÚpathÚjoinr   )r5  Úinit_dir_paths     r0   Úinitialize_temp_directoriesrC    sÀ   € ä×)Ñ)Ó+€GÜ$Ÿ\™\„B‡J�JˆzÑÜ‡H�HŒR�W‰W�\‰\œ'Ÿ,™,¨	Ó2Ô3Ü‡H�HŒR�W‰W�\‰\œ'Ÿ,™,¨
Ó3Ô4Ü—G‘G—L‘L¤§¡¨zÓ:€MÜ‡H�Hˆ]ÔàÐØ$/Œ�
‰
�=Ò!ä$/´"·'±'·,±,ØÐ-ó3
ñ %
Œ�
‰
�=Ò!r/   c                  ó:   — t         �t         j                  «        y y r‘   )r4  Úcleanupr.   r/   r0   Úcleanup_temp_dirrF  ,  s   € ÜÐÜ�‰Õð r/   é   c            	       ó<  ‡ — e Zd ZdZdZdefd„Zedefd„«       Zede	fd„«       Z
d„ Zdded	edd
fˆ fd„Zdˆ fd„Zdˆ fd„Zdefd„Zdd„Zdd„Z G d„ de«      Zede	fd„«       Zede	dededd
fd„«       Zdedd
fd„Zdd„Zdd„Zdd„Zdd„Zedefd„«       Zˆ xZS ) ÚMultiProcessTestCaser    é
   rà   c                  ó   — y)NFr.   ©Úselfs    r0   Ú_should_stop_test_suitez,MultiProcessTestCase._should_stop_test_suiteI  s   € Ør/   c                  ó   — y)NTr.   rL  s    r0   Údestroy_pg_upon_exitz)MultiProcessTestCase.destroy_pg_upon_exitQ  s   € àr/   c                 ó   — t         S r‘   ©ÚDEFAULT_WORLD_SIZErL  s    r0   rm   zMultiProcessTestCase.world_sizeU  ó   € ä!Ð!r/   c                 óV   ‡— t        ‰«      ˆfd„«       }t        j                  || «      S )Nc                 ój   •— | j                   | j                  k(  r| j                  ‰«       y  ‰«        y r‘   )r  ÚMAIN_PROCESS_RANKÚ_join_processes©rM  r(  s    €r0   ro   z1MultiProcessTestCase.join_or_run.<locals>.wrapperZ  s(   ø€ à�y‰y˜D×2Ñ2Ò2Ø×$Ñ$ RÕ(á•r/   ©r	   ÚtypesÚ
MethodType©rM  r(  ro   s    ` r0   Újoin_or_runz MultiProcessTestCase.join_or_runY  ó.   ø€ Ü	ˆr‹ó	ó 
ð	ô ×Ñ ¨Ó.Ð.r/   Úmethod_nameÚ
methodNameNc                 óò   •— |dk7  r|}t         ‰| �  |«       	 t        | |«      }t        | || j	                  |«      «       y # t
        $ r+}|dk7  rt        d| j                  › d|› �«      |‚Y d }~y d }~ww xY w©NÚrunTestzno such test method in z: ©ÚsuperÚ__init__ÚgetattrÚsetattrr^  ÚAttributeErrorrã   Ú	__class__©rM  r`  ra  r(  Úerk  s        €r0   rg  zMultiProcessTestCase.__init__g  óŠ   ø€ ð ˜Ò"Ø$ˆKÜ‰Ñ˜Ô%ð	bÜ˜˜{Ó+ˆBÜ�D˜+ t×'7Ñ'7¸Ó';Õ<øÜò 	bØ˜YÒ&ô !Ð#:¸4¿>¹>Ð:JÈ"ÈZÈLÐ!YÓZÐ`aÐaô 'ûð	búó   ™(A Á	A6Á!A1Á1A6c                 ó¸   •— t         ‰| �  «        g | _        g | _        | j                  | _        t        j                  d¬«      j                  | _	        i | _
        y )NF)Údelete)rf  ÚsetUpÚskip_return_code_checksÚ	processesrW  r  r<  ÚNamedTemporaryFiler>  Ú	file_nameÚpid_to_pipe©rM  rk  s    €r0   rr  zMultiProcessTestCase.setUpv  sH   ø€ Ü‰‰ŒØ')ˆÔ$ØˆŒØ×*Ñ*ˆŒ	Ü!×4Ñ4¸EÔB×GÑGˆŒàˆÕr/   c                 ór   •— t         ‰| �  «        | j                  D ]  }|j                  «        Œ g | _        y r‘   )rf  ÚtearDownrt  Ú	terminate)rM  Úprk  s     €r0   rz  zMultiProcessTestCase.tearDown  s3   ø€ Ü‰ÑÔØ—‘ò 	ˆAØ�K‰K�Mð	ð ˆ�r/   c                 óF   — | j                  «       j                  d«      d   S r  ©Úidr£   rL  s    r0   Ú_current_test_namez'MultiProcessTestCase._current_test_name‰  s   € à�w‰w‹y�‰˜sÓ# BÑ'Ð'r/   c                 ó  — g | _         t        t        | j                  «      «      D ]Ý  }t        j
                  j                  «       \  }} || j                  j                  dt        |«      z   || j                  «       | j                  |fdt        | dd«      i¬«      }|j                  «        t        j                  d||j                   «       || j"                  |j                   <   | j                   j%                  |«       Œß y )Nzprocess Úfake_pgF)Útargetr>  rk   rl   zStarted process %s with pid %s)rt  r  r+   rm   rc   ÚmultiprocessingÚPiperk  Ú_runr-   r€  rv  rh  ÚstartÚloggerÚinfoÚpidrw  r  )rM  Úprocr  Úparent_connÚ
child_connÚprocesss         r0   Ú_start_processesz%MultiProcessTestCase._start_processes�  sÔ   € ØˆŒÜœ#˜dŸo™oÓ.Ó/ò 	+ˆDÜ&+×&;Ñ&;×&@Ñ&@Ó&BÑ#ˆK˜ÙØ—~‘~×*Ñ*Ø¤# d£)Ñ+Ø˜D×3Ñ3Ó5°t·~±~ÀzÐRàœw t¨Y¸Ó>ðô	ˆGð �M‰MŒOÜ�K‰KÐ8¸$ÀÇÁÔLØ,7ˆD×Ñ˜WŸ[™[Ñ)Ø�N‰N×!Ñ! 'Õ*ñ	+r/   c                 óx   — t         j                  j                  d«      j                  }| j	                  |«       y )NÚspawn)rc   r„  Úget_contextÚProcessr�  )rM  r‹  s     r0   Ú_spawn_processesz%MultiProcessTestCase._spawn_processesž  s,   € Ü×$Ñ$×0Ñ0°Ó9×AÑAˆØ×Ñ˜dÕ#r/   c                   ó   — e Zd ZdZy)úMultiProcessTestCase.Eventr~   N)r(   r)   r*   ÚGET_TRACEBACKr.   r/   r0   ÚEventr–  ¢  s   „ Ø‰r/   r˜  r  c                 ó‚  — t         j                  d|«       	 t        j                  j	                  | |g«      }| |v rñ| j
                  rt         j                  d|«       y | j                  «       }t         j                  d||«       |t        j                  j                  k(  rŠt        j                  d¬«      5 }t        j                  |«       |j                  «        |j                  d«       | j!                  |j#                  «       «       t         j                  d|«       d d d «       ||v ry �Œ# 1 sw Y   ŒxY w)Nz*Starting event listener thread for rank %sz:Pipe closed for process %s, stopping event listener threadzReceived event %s on process %szr+)Úmoder   zProcess %s sent traceback)rˆ  r‰  r„  Ú
connectionÚwaitÚclosedÚrecvrI  r˜  r—  r<  ru  ÚfaulthandlerÚdump_tracebackÚflushÚseekÚsendÚread)Úparent_pipeÚsignal_piper  Úready_pipesÚeventÚtmp_files         r0   Ú_event_listenerz$MultiProcessTestCase._event_listener¥  s  € ä�‰Ð@À$ÔGØÜ)×4Ñ4×9Ñ9¸;ÈÐ:TÓUˆKà˜kÑ)à×%Ò%Ü—K‘KØTÐVZôð à#×(Ñ(Ó*�Ü—‘Ð=¸uÀdÔKàÔ0×6Ñ6×DÑDÒDä!×4Ñ4¸$Ô?ð GÀ8Ü$×3Ñ3°HÔ=à Ÿ™Ô(Ø Ÿ™ aÔ(Ø#×(Ñ(¨¯©«Ô9äŸ™Ð$?ÀÔF÷Gð ˜kÑ)Øñ5 ÷ Gð Gús   Â:A,D5Ä5D>Ú	test_namerv  c                 óT   —  | |«      }||_         ||_        |j                  ||«       y r‘   )r  rv  Úrun_test©Úclsr  r«  rv  r¥  rl   rM  s          r0   r†  zMultiProcessTestCase._runÄ  s'   € á�9‹~ˆØˆŒ	Ø"ˆŒØ�‰�i Õ-r/   c           	      ó   — t         j                  j                  d¬«      \  }}t        j                  t
        j                  ||| j                  fd¬«      }|j                  «        t        j                  dk7  r2t        j                  dk7  rt         j                  j                  d«       dt        j                  d<   	  t        | |«      «        |�|j;                  d «       |€J ‚|j=                  «        |j?                  «        | j@                  r	 tC        jD                  «        y y # t         j"                  $ r[}t$        j'                  d	| j                  |t)        |«      «       t        j*                  t,        d
   j.                  «       Y d }~ŒÃd }~wt0        $ rŒ t$        j3                  dt5        j6                  «       | j                  t
        j8                  «       |j;                  t5        j6                  «       «       t        j*                  t
        j8                  «       Y �ŒZw xY w# |�|j;                  d «       |€J ‚|j=                  «        |j?                  «        w xY w# tF        tH        f$ r Y y w xY w)NF)ÚduplexT)rƒ  rk   ÚdaemonrÚ   Údarwinr®   ÚTORCH_SHOW_CPP_STACKTRACESz4Process %s skipping test %s for following reason: %srG   z;Caught exception: 
%s exiting process %s with exit code: %s)%rc   r„  r…  Ú	threadingÚThreadrI  rª  r  r‡  re   rÛ   Ú_CÚ'_set_print_stack_traces_on_fatal_signalrh   ri   rh  ÚunittestÚSkipTestrˆ  r‰  r-   rf   rg   r&   Ú	ExceptionrŸ   Ú	tracebackÚ
format_excÚTEST_ERROR_EXIT_CODEr£  rA  ÚcloserP  r¹   Údestroy_process_groupÚAssertionErrorrã   )rM  r«  r¥  Úsignal_recv_pipeÚsignal_send_pipeÚevent_listener_threadÚses          r0   r­  zMultiProcessTestCase.run_testË  s  € ä-2×-BÑ-B×-GÑ-GÈuÐ-GÓ-UÑ*ÐÐ*Ü )× 0Ñ 0Ü'×7Ñ7ØÐ/°·±Ð;Øô!
Ðð
 	×#Ñ#Ô%Ü�<‰<˜7Ò"¤s§|¡|°xÒ'?ô �H‰H×<Ñ<¸TÔBà36Œ�
‰
Ð/Ñ0ð	 Ø$ŒG�D˜)Ó$Ô&ð   Ð+Ø ×%Ñ% dÔ+à(Ð4Ð4Ð4Ø!×&Ñ&Ô(à×ÑÔà×$Ò$ðô ×*Ñ*Õ,ð	 %øô/ × Ñ ò 	6Ü�K‰KØFÈÏ	É	ÐS\Ô^aÐbdÓ^eôô �H‰H”Z 	Ñ*×4Ñ4×5Ñ5ûÜò 	@Ü�L‰Lð0ä×$Ñ$Ó&¨¯	©	Ô3G×3\Ñ3\ôð ×ÑœY×1Ñ1Ó3Ô4Ü�H‰HÔ)×>Ñ>×?ð	@ûð  Ð+Ø ×%Ñ% dÔ+à(Ð4Ð4Ð4Ø!×&Ñ&Ô(à×ÑÕûô #¤JÐ/ò ÙðúsJ   Ã D+ ÄI+ Ä+H,Ä>AFÆH/ ÆBH,È(H/ È+H,È,H/ È/9I(É+I=É<I=c                 ó¸  — g }t        | j                  «      D ]h  \  }}|j                  �Œ| j                  |j                     }	 |j                  t        j                  j                  «       |j                  ||f«       Œj |D ]x  \  }}	 |j                  d«      rK|j                  rt        j                  d|«       Œ;|j!                  «       }t        j                  d||«       nt        j                  d|«       Œz y # t        $ r"}t        j                  d||«       Y d }~�Œd }~ww xY w# t        $ r!}t        j                  d||«       Y d }~ŒÏd }~ww xY w)NzBEncountered error while trying to get traceback for process %s: %srè   z5Pipe closed for process %s, cannot retrieve tracebackz)Process %s timed out with traceback: 

%sz6Could not retrieve traceback for timed out process: %s)Ú	enumeratert  Úexitcoderw  rŠ  r£  rI  r˜  r—  r  ÚConnectionErrorrˆ  rŸ   Úpollr�  r‰  rž  )rM  Úpipesr+  rŽ  Úpiperm  r  r¼  s           r0   Ú_get_timedout_process_tracebackz4MultiProcessTestCase._get_timedout_process_tracebackþ  s>  € ØˆÜ# D§N¡NÓ3ò 		‰JˆAˆwØ×ÑÑ'Ø×'Ñ'¨¯©Ñ4�ðØ—I‘IÔ2×8Ñ8×FÑFÔGØ—L‘L ! T Õ+ð		ð  ò 	‰JˆD�$ðà—9‘9˜Q”<Ø—{’{ÜŸ™ØSÐUYôð !à $§	¡	£�IÜ—L‘LØEÀtÈYõô —L‘LØPÐRVôøñ	øô 'ò Ü—L‘LØ\Ð^_Ðab÷ò ûðûô0 #ò Ü—‘ØXÐZ^Ð`a÷ñ ûðús6   Á<DÂ3D/Ã >D/Ä	D,Ä
D'Ä'D,Ä/	EÄ8EÅEc                 ó  — t        | j                  «       «      }t        j                  «       }d}	 	 t        | j                  «      D ]w  \  }}|j
                  t        j                  k(  sŒ$t        d|› d|j
                  › d�«       t        j                  j                  «       }|D ]  }|j                  «        Œ d} n |rn‘t        d„ | j                  D «       «      rntt        j                  «       |z
  }	|	|kD  rA| j                  «        t        d|› d�«       | j                  D ]  }|j                  «        Œ nt        j                  d	«       �Œ#t        j                  «       |z
  }
|| j                   v r| j#                  |
«       n| j%                  |
«       | j&                  j)                  «       D ]  }|j+                  «        Œ y # | j&                  j)                  «       D ]  }|j+                  «        Œ w xY w)
NFTúProcess z terminated with exit code z", terminating remaining processes.c              3   ó8   K  — | ]  }|j                   d u–— Œ y ­wr‘   )rÈ  )Ú.0r|  s     r0   ú	<genexpr>z7MultiProcessTestCase._join_processes.<locals>.<genexpr>9  s   è ø€ ÒF°!�q—z‘z¨Ô-ÑFùs   ‚zTiming out after z" seconds and killing subprocesses.gš™™™™™¹?)r	  r  ÚtimerÇ  rt  rÈ  rI  r¾  Úprintrc   r„  Úactive_childrenr{  ÚallrÍ  Úsleeprs  Ú_check_no_test_errorsÚ_check_return_codesrw  r#  r¿  )rM  r(  ró   Ú
start_timeÚsubprocess_errorr+  r|  rÕ  ÚacÚelapsedÚelapsed_timerÌ  s               r0   rX  z$MultiProcessTestCase._join_processes#  sÈ  € Ü˜dŸg™g›iÓ(ˆÜ—Y‘Y“[ˆ
Ø Ðð*	Øä'¨¯©Ó7ò ‘F�Q˜ð —z‘zÔ%9×%NÑ%NÓNÜØ& q cÐ)DÀQÇZÁZÀLÐPrÐsôô +0×*?Ñ*?×*OÑ*OÓ*Q˜Ø"1ò +˜BØŸL™L�Nð+à+/Ð(Ùðñ $ØäÑF°t·~±~ÔFÔFØäŸ)™)›+¨
Ñ2�Ø˜WÒ$Ø×8Ñ8Ô:ÜØ+¨G¨9Ð4VÐWôð "Ÿ^™^ò &˜ØŸ™�ð&àä—
‘
˜3”ñ= ô@  Ÿ9™9›;¨Ñ3ˆLà�T×1Ñ1Ñ1Ø×*Ñ*¨<Õ8à×(Ñ(¨Ô6ð ×(Ñ(×/Ñ/Ó1ò �Ø—
‘
•ñø˜×(Ñ(×/Ñ/Ó1ò �Ø—
‘
•ñús   ±9G Á+D2G Ç1G>c                 óÆ   — t        | j                  «      D ]I  \  }}|j                  €t        d|› d|› d�«      ‚| j	                  | j
                  |j                  «       ŒK y)zV
        Checks that we didn't have any errors thrown in the child processes.
        NrÏ  z timed out after ú seconds)rÇ  rt  rÈ  ÚRuntimeErrorÚassertNotEqualr¾  )rM  rÞ  r+  r|  s       r0   rØ  z*MultiProcessTestCase._check_no_test_errorsS  sh   € ô ˜dŸn™nÓ-ò 	G‰DˆAˆqØ�z‰zÐ!Ü"Ø˜q˜cÐ!2°<°.ÀÐIóð ð ×Ñ × 9Ñ 9¸1¿:¹:ÕFñ	Gr/   c                 ó8  — | j                   st        j                  d«       y| j                   d   }t        | j                   «      D ��cg c]&  \  }}|j                  t
        j                  k(  r||f‘Œ( }}}|r[d}|D ]I  \  }}| j                  |j                     j                  «       }|d|› dt
        j                  › d|› d�z  }ŒK t        |«      ‚t        | j                   «      D ]h  \  }}|j                  €t        d|› d	|› d
�«      ‚| j                  |j                  |j                  d|› d|j                  › d|j                  › �¬«       Œj t        j                  «       D ]q  }	|j                  |	j                  k(  sŒt        r1t        j!                  d| j#                  «       |	j$                  «        yt'        j(                  |	j$                  «      ‚ | j                  |j                  dd|j                  › d|j                  › �¬«       yc c}}w )z¦
        Checks that the return codes of all spawned processes match, and skips
        tests if they returned a return code indicating a skipping condition.
        z<Note: no subprocesses were spawned, test was likely skipped.Nr   Ú rÏ  z exited with error code z and exception:
ú
ú terminated or timed out after rà  zExpect process z+ exit code to match Process 0 exit code of z
, but got )rÇ   ú6Skipping %s on sandcastle for the following reason: %sz Expected zero exit code but got z
 for pid: )rt  rˆ  ÚwarningrÇ  rÈ  rI  r¾  rw  rŠ  rž  rá  ÚassertEqualrg   r#  r&   r   r‰  r  r'   r¹  rº  )
rM  rÞ  Úfirst_processr+  r|  Úerrored_processesrŸ   rŽ  Úerror_messageÚskips
             r0   rÙ  z(MultiProcessTestCase._check_return_codes^  s?  € ð �~Š~Ü�N‰NÐYÔZØàŸ™ qÑ)ˆô " $§.¡.Ó1÷
á��1Ø�z‰zÔ1×FÑFÒFð �ŠFð
Ðñ 
ñ
 ØˆEØ/ò ‘
��7à $× 0Ñ 0°·±Ñ =× BÑ BÓ D�ØØ˜q˜cÐ!9Ô:N×:cÑ:cÐ9dð e'Ø'4 o°Rð9ñ‘ðô ˜uÓ%Ð%ô ˜dŸn™nÓ-ò 		‰DˆAˆqØ�z‰zÐ!Ü"Ø˜q˜cÐ!@ÀÀÈhÐWóð ð ×ÑØ—
‘
Ø×&Ñ&Ø% a SÐ(SÐTa×TjÑTjÐSkÐkuÐvw÷  wAñ  wAð  vBð  Cð õ ð		ô ×%Ñ%Ó'ò 	:ˆDØ×%Ñ%¨¯©Ó7Ý ô
 —K‘KØPÐRV×RYÑRYÓR[Ð]a×]iÑ]iôñ ä"×+Ñ+¨D¯L©LÓ9Ð9ð	:ð 	×ÑØ×"Ñ"ØØ2°=×3IÑ3IÐ2JÈ*ÐUb×UfÑUfÐTgÐhð 	õ 	
ùóS
s   Á
+Hc                 ó    — | j                   dk(  S )Nr   ©r  rL  s    r0   rò   zMultiProcessTestCase.is_masterž  s   € à�y‰y˜A‰~Ðr/   ©rd  rd  ©rà   N) r(   r)   r*   rW  r¾  ÚboolrN  ÚpropertyrP  r+   rm   r^  r-   rg  rr  rz  r€  r�  r”  r   r˜  Ústaticmethodrª  Úclassmethodr†  r­  rÍ  rX  rØ  rÙ  rò   Ú__classcell__©rk  s   @r0   rI  rI  @  s6  ø„ ØÐð
 Ðð¨ó ð ð dò ó ðð ð"˜Cò "ó ð"ò/ñb Cð bÀð bÐUYõ bõõð( Có (ó+ó"$ô�ô ð ð¸ò ó ðð< ð.˜ð .¨ð .¸ð .ÐW[ò .ó ð.ð1 #ð 1°tó 1óf#óJ.ó`	Gó>
ð@ ð˜4ò ó ôr/   rI  c                   ó<   ‡ — e Zd Zˆ fd„Zd„ Zdefd„Zd„ Zd„ Zˆ xZ	S )ÚDistributedTestBasec                 óB   •— t         ‰| �  «        | j                  «        y r‘   ©rf  rr  r”  rx  s    €r0   rr  zDistributedTestBase.setUp©  ó   ø€ Ü‰‰ŒØ×ÑÕr/   c                 ób   — 	 t        j                  | j                  «       y # t        $ r Y y w xY wr‘   )rh   Úremoverv  ÚOSErrorrL  s    r0   rz  zDistributedTestBase.tearDown­  s)   € ð	Ü�I‰I�d—n‘nÕ%øÜò 	Ùð	ús   ‚" ¢	.­.rà   c                 ó"   — d|v ryd|v ryd|v ryy)NrT   rA   rY   rX   r[   rZ   rR   r.   )rM  rÝ   s     r0   r„   zDistributedTestBase.backend³  s$   € Ø�VÑØØ�f‰_ØØ�f‰_Øàr/   c                 ó  — t        j                  |«      j                  «       }t         j                  j	                  | j
                  |«      }t         j                  j                  | j                  |«      | j                  | j                  |¬«       d| j                  |«      v sd| j                  |«      v r)t         j                  j                  | j                  «       t         j                  j                  j                  «       S )N©r„   rm   r  ÚstorerA   rZ   )rc   Úget_device_modulerj   ÚdistributedÚ	FileStorerv  Úinit_process_groupr„   rm   r  ÚacceleratorÚset_device_indexÚdistributed_c10dÚ_get_default_group)rM  rÝ   Únum_visible_devicesr  s       r0   Ú	create_pgzDistributedTestBase.create_pg½  sÅ   € Ü#×5Ñ5°fÓ=×JÑJÓLÐÜ×!Ñ!×+Ñ+¨D¯N©NÐ<OÓPˆÜ×Ñ×,Ñ,Ø—L‘L Ó(Ø—‘Ø—‘Øð	 	-ô 	
ð �T—\‘\ &Ó)Ñ)¨V°t·|±|ÀFÓ7KÑ-KÜ×Ñ×.Ñ.¨t¯y©yÔ9Ü× Ñ ×1Ñ1×DÑDÓFÐFr/   c                 ó¢   — t        j                  |«      j                  «       }t        | j                  «      D �ci c]	  }|||z  g“Œ c}S c c}w r‘   )rc   r  rj   r  rm   )rM  rÝ   r  r+  s       r0   Úrank_to_devicez"DistributedTestBase.rank_to_deviceÊ  sG   € Ü#×5Ñ5°fÓ=×JÑJÓLÐÜ6;¸D¿O¹OÓ6LÖM°��AÐ+Ñ+Ð,Ñ,ÒMÐMùÒMs   »A)
r(   r)   r*   rr  rz  r-   r„   r  r  rö  r÷  s   @r0   rù  rù  §  s%   ø„ ô òð ó òGöNr/   rù  Úsubtest_configÚtest_fnÚtest_kwargsc                 óì  — t        |j                  «       «      }|D �cg c]  }|d   ‘Œ	 }}|D �cg c]  }|d   ‘Œ	 }}t        j                  |Ž D ]Œ  }	t	        t        ||	«      «      }
 | j                  di |
¤Ž5  t        j                  j                  «         ||i |¤|
¤Ž t        j                  j                  «        ddd«       t        j                  «        ŒŽ yc c}w c c}w # 1 sw Y   Œ*xY w)a\  
    Runs a test function given by ``test_fn`` as a subtest according to the
    configurations specified by ``subtest_config``. This amortizes the
    costly setup overhead (including process spawn and initializing the
    process group) over the subtests.

    Args:
        subtest_config (Dict[str, List[Any]]): A mapping from subtest
            keyword argument name to a list of its possible values.
        test_fn (Callable): A callable that runs the actual test.
        test_args: Positional arguments to pass to ``test_fn``.
        test_kwargs: Keyword arguments to pass to ``test_fn``.
    r   r~   Nr.   )r.  ÚitemsÚ	itertoolsÚproductÚdictÚzipÚsubTestrc   Ú_dynamoÚresetr¹   r»   )Úcls_instr  r  Ú	test_argsr  Úsubtest_config_itemsÚitemÚsubtest_config_keysÚsubtest_config_valuesr#  Úsubtest_kwargss              r0   Úrun_subtestsr#  Î  sã   € ô* 9=¸^×=QÑ=QÓ=SÓ8TÐØ:NÖ%O°$ d¨1£gÐ%OÐÐ%OØBVÖ-W¸$¨d°1«gÐ-WÐÐ-WÜ×#Ñ#Ð%:Ð;ò ˆäœcÐ"5°vÓ>Ó?ˆØˆX×ÑÑ/ Ñ/ñ 	"Ü�M‰M×ÑÔ!Ù�YÐ@ +Ð@°Ò@Ü�M‰M×ÑÔ!÷	"ô 	�‰�ñùò &PùÚ-W÷	"ð 	"ús   žC °C%Á:AC*Ã*C3	c                  ó    — t         �t         S 	 t        j                  g d¢d¬«      j                  dk(  a t         S # t        $ r
 da Y t         S w xY w)a   
    If shell command `fi_info -p efa -t FI_EP_RDM` returns exit code 0 then we assume that the machine has
    Libfabric EFA interfaces and EFA software components installed,
    see https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/efa-start.html.
    )Úfi_infoz-pÚefaz-tÚ	FI_EP_RDMF)Úcheckr   )ÚEFA_PROBE_RESULTÚ
subprocessÚrunÚ
returncodeÚFileNotFoundErrorr.   r/   r0   Úhas_efar.  ô  sZ   € ô Ð#ÜÐð!ä�N‰NÒFÈeÔT×_Ñ_ÐcdÑdð 	ô
 Ðøô ò !Ø ÑÜÐð!ús   Ž&: ºAÁAc                  ó"   — t        «       rddgS dS )a‹  
    If the machine has Libfabric EFA interfaces and EFA software components installed it may cause
    'RuntimeError: In operator() at tensorpipe/common/ibv.h:172 "": Operation not supported' if tensorpipe
    uses InfiniBand transport, so we exclude it from tensorpipe transports,
    see https://github.com/pytorch/pytorch/issues/73885 and https://github.com/pytorch/pytorch/issues/65022
    ÚshmÚuvN)r.  r.   r/   r0   Útp_transportsr2    s   € ô $œIˆE�4ˆ=Ð/¨4Ð/r/   c                 ód   ‡ ‡‡— ‰ €t        t        |‰¬«      S d„ Št        ‰ «      ˆˆ ˆfd„«       }|S )z+
    Wrapper to use with a test method
    )ró   rm   c                 óü   ‡ ‡‡‡— t        «       Št        j                  «       }ˆfd„Šˆˆˆ fd„}g }t        ‰ «      D ]=  }t	        j
                  ||‰|f¬«      }|j                  «        |j                  |«       Œ? |S )Nc                  ó>   •— ‰ t         j                  j                  k(  S r‘   ©r¹   r
  Ú_world©Úworlds   €r0   Úworld_is_validzaspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads.<locals>.world_is_valid!  ó   ø€ ØœD×1Ñ1×8Ñ8Ñ8Ð8r/   c                 ó~  •— t        j                  d| ‰|¬«       	  ‰«         ‰«       rt        j                  «        y y # t        $ rR}t        j                  j                  | t        j                  «       f«       t        j                  |«       Y d }~Œsd }~ww xY w#  ‰«       rt        j                  «        w w xY w)NÚthreaded©r„   r  rm   r  )r¹   r  ÚBaseExceptionÚMultiThreadedTestCaseÚexception_queueÚputre   Úexc_infor"   Úexception_handlerÀ  )r  Úworld_pgr  ÚexÚcallbackr:  rm   s       €€€r0   ÚworkerzYspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads.<locals>.worker$  s™   ø€ Ü×#Ñ#Ø"¨¸*ÈEõð1Ù”
ñ "Ô#Ü×.Ñ.Õ0ð $øô !ò 7ä%×5Ñ5×9Ñ9¸4ÄÇÁÃÐ:PÔQÜ!×2Ñ2°2×6Ñ6ûð7ûñ
 "Ô#Ü×.Ñ.Õ0ð $ús*   œA  Á 	BÁ	ABÂB ÂBÂB ÂB<©rƒ  rk   )r    r¹   Ú	HashStorer  rµ  r¶  r‡  r  )	rm   rG  Úglobal_storerH  Úthreadsr  Útr9  r:  s	   ``     @@r0   Ú#_run_test_method_with_multi_threadszIspawn_threads_and_init_comms.<locals>._run_test_method_with_multi_threads  sq   û€ Ü$Ó&ˆÜ—~‘~Ó'ˆô	9ö	1ð ˆÜ˜*Ó%ò 	ˆDÜ× Ñ ¨°d¸EÀ<Ð5PÔQˆAØ�G‰GŒIØ�N‰N˜1Õð	ð
 ˆr/   c                 óX  •‡ ‡‡— t         j                  j                  j                  d«       	  ‰‰ˆˆˆˆ fd„«      }t        j                  |‰«       t         j                  j                  j                  d«       y # t         j                  j                  j                  d«       w xY w)NTc                  ó   •—  ‰‰g‰ ¢­i ‰¤ŽS r‘   r.   )rk   rn   rl   rM  s   €€€€r0   ú<lambda>z?spawn_threads_and_init_comms.<locals>.wrapper.<locals>.<lambda>@  s   ø€ ÉdÐSWÐNiÐZ^ÒNiÐbhÑNi€ r/   F)rc   r·  Ú_distributed_c10dÚ_set_thread_isolation_moder@  Ú_join_threads)rM  rk   rl   rL  rN  rn   rm   s   ``` €€€r0   ro   z-spawn_threads_and_init_comms.<locals>.wrapper;  sq   û€ ô 	�‰×"Ñ"×=Ñ=¸dÔCð	IÙ9¸*ÖFiÓjˆGä!×/Ñ/°¸Ô>ä�H‰H×&Ñ&×AÑAÀ%ÕHøŒE�H‰H×&Ñ&×AÑAÀ%ÕHús   ¯%A> Á>+B))r   Úspawn_threads_and_init_commsr	   )rn   ró   rm   ro   rN  s   ` ` @r0   rU  rU    sD   ú€ ð €|ÜÜ(°'Àjô
ð 	
ò
ô< ˆ4ƒ[õIó ðIð €Nr/   c                   ó  ‡ — e Zd ZdZ ej
                  «       ZdZd„ Zdde	de	ddfˆ fd„Z
d	„ Zd
„ Zdˆ fd„Zˆ fd„Zd„ Zed„ «       Zd„ Zed„ «       Zed„ «       Zedefd„«       Zede	fd„«       Zdddœd„Zdddœd„Zˆ xZS )r@  a5  
    Test runner that runs all tests with the in-proc process group using
    multiple threads with the threaded process group.

    Each test spawns world_size threads and run the test method in each thread.

    Difference from regular MultiProcess test runner:
    Must explicitly defines SetUp and call self._spawn_threads() to run the tests.
    Cannot use setUp / tearDown (must use perThreadSetup / perThreadShutdown)
        to set up / tear down each thread when running each test.
    No global state possible
        How bad of a limitation is this?
    r    c                 óV   ‡— t        ‰«      ˆfd„«       }t        j                  || «      S )Nc                 ó€   •— | j                   | j                  k(  r| j                  | j                  ‰«       y  ‰«        y r‘   )r  ÚMAIN_THREAD_RANKrT  rL  rY  s    €r0   ro   z2MultiThreadedTestCase.join_or_run.<locals>.wrapper\  s.   ø€ à�y‰y˜D×1Ñ1Ò1Ø×"Ñ" 4§<¡<°Õ4á•r/   rZ  r]  s    ` r0   r^  z!MultiThreadedTestCase.join_or_run[  r_  r/   r`  ra  rà   Nc                 óò   •— |dk7  r|}t         ‰| �  |«       	 t        | |«      }t        | || j	                  |«      «       y # t
        $ r+}|dk7  rt        d| j                  › d|› �«      |‚Y d }~y d }~ww xY wrc  re  rl  s        €r0   rg  zMultiThreadedTestCase.__init__e  rn  ro  c                  ó   — y r‘   r.   rL  s    r0   ÚperThreadSetUpz$MultiThreadedTestCase.perThreadSetUpt  s   € àr/   c                  ó   — y r‘   r.   rL  s    r0   ÚperThreadTearDownz'MultiThreadedTestCase.perThreadTearDownx  s   € Ør/   c                 óx   •— t         ‰| �  «        | j                  | _        g | _        dt
        j                  d<   y)z‘
        setUp only set up things in the main thread, if you want to configure things
        in the spawned threads, use perThreadSetUp
        r®   r´  N)rf  rr  rY  r  rL  rh   ri   rx  s    €r0   rr  zMultiThreadedTestCase.setUp{  s1   ø€ ô
 	‰‰ŒØ×)Ñ)ˆŒ	ØˆŒà36Œ�
‰
Ð/Ò0r/   c                 ó0   •— t         ‰| �  «        g | _        y)z—
        tearDown only set up things in the main thread, if you want to configure things
        in the spawned threads, use perThreadTearDown
        N)rf  rz  rL  rx  s    €r0   rz  zMultiThreadedTestCase.tearDown†  s   ø€ ô
 	‰ÑÔØˆ�r/   c                 óò  ‡— t         j                  j                  j                  d«       | j                  }t        «       Št        j                  «       | j                  _	        ˆfd„} |«       st        d«      ‚t        | j                  «      D ]e  }t        j                  | j                  j                  ||| j                  f¬«      }|j!                  «        | j"                  j%                  |«       Œg y)zk
        class method to spawn threads and run test, use this method in the SetUp of your TestCase
        Tc                  ó>   •— ‰ t         j                  j                  k(  S r‘   r6  r8  s   €r0   r:  z<MultiThreadedTestCase._spawn_threads.<locals>.world_is_valid˜  r;  r/   zInvalid worldrI  N)rc   r·  rR  rS  r€  r    r¹   rJ  rk  rK  rá  r  rm   rµ  r¶  r†  r‡  rL  r  )rM  r«  r:  r  rM  r9  s        @r0   Ú_spawn_threadsz$MultiThreadedTestCase._spawn_threadsŽ  s¸   ø€ ô 	�‰×"Ñ"×=Ñ=¸dÔCØ×+Ñ+ˆ	ä$Ó&ˆÜ&*§n¡nÓ&6ˆ�‰Ô#ô	9ñ ÔÜ˜Ó/Ð/ä˜$Ÿ/™/Ó*ò 	#ˆDÜ× Ñ ¨¯©×(;Ñ(;À9ÈdÐTX×TcÑTcÐBdÔeˆAØ�G‰GŒIØ�L‰L×Ñ Õ"ñ	#r/   c                 ó  —  | |«      }||_         t        |d«      rWt        j                  «       |_        t
        j                  |j                  _        t
        j                  |j                  _	        |j                  |||«       y )NÚ_tls)r  Úhasattrrµ  Úlocalre  r   Ú
_precisionÚ	precisionÚ_rel_tolÚrel_tolÚrun_test_with_threaded_pg)r¯  r«  r  rm   rl   rM  s         r0   r†  zMultiThreadedTestCase._run£  sb   € á�9‹~ˆØˆŒ	ô �4˜Ô Ü!Ÿ™Ó)ˆDŒIÜ"*×"5Ñ"5ˆD�I‰IÔÜ (× 1Ñ 1ˆD�I‰IÔà×&Ñ& y°$¸
ÕCr/   c                 óð  — t        j                  d||| j                  j                  ¬«       | j	                  «        	  t        | |«      «        t        j                  «        | j                  «        y# t        $ rN}| j                  j                  |t        j                  «       f«       t        j                  |«       Y d}~Œwd}~ww xY w# t        j                  «        | j                  «        w xY w)zd
        Run the current test associated with `test_name` using the threaded process group.
        r=  r>  N)r¹   r  rk  rK  r\  rh  r?  rA  rB  re   rC  r"   rD  rÀ  r^  )rM  r«  r  rm   rF  s        r0   rl  z/MultiThreadedTestCase.run_test_with_threaded_pg³  s¾   € ô 	×ÑØ T°jÈÏÉ×HcÑHcõ	
ð 	×ÑÔð	%Ø$ŒG�D˜)Ó$Ô&ô
 ×&Ñ&Ô(Ø×"Ñ"Õ$øô ò 	3Ø× Ñ ×$Ñ$ d¬C¯L©L«NÐ%;Ô<Ü×.Ñ.¨r×2Ñ2ûð	3ûô ×&Ñ&Ô(Ø×"Ñ"Õ$ús*   ¿A5 Á5	CÁ>ACÃC ÃCÃC Ã&C5c           
      óâ  — t         }	 t        |«      D ]f  \  }}|j                  t        d|«      «       |j	                  «       sŒ2t
        j                  j                  |t        t        d|› d�«      d ff«       Œh t        j                  «        g }| j                  j                  «       sF| j                  j                  «       }|j                  |«       | j                  j                  «       sŒFt        «        t        j                   j"                  j%                  d«       | j'                  |||«       y # t        «        t        j                   j"                  j%                  d«       w xY w)Nr   zRank failed to join in under rà  F)r  rÇ  rA  ÚmaxÚis_aliver@  rA  rB  ÚTimeoutErrorr"   r  Úemptyr¸   r  r!   rc   r·  rR  rS  rÙ  )r¯  rL  r(  ró   ÚidxÚthreadÚfailed_ranksÚfailures           r0   rT  z#MultiThreadedTestCase._join_threadsÆ  s-  € ä!ˆð	IÜ(¨Ó1ò ‘��VØ—‘œC  7›OÔ,Ø—?‘?Õ$Ü)×9Ñ9×=Ñ=àä ,Ü ,Ø&CÀGÀ9ÈHÐ$Uó!"ð !%ðð	õðô ×#Ñ#Ô%ØˆLØ×)Ñ)×/Ñ/Ô1Ø×-Ñ-×1Ñ1Ó3�Ø×#Ñ# GÔ,ð ×)Ñ)×/Ñ/Õ1ô #Ô$Ü�H‰H×&Ñ&×AÑAÀ%ÔHà×Ñ ¨g°rÕ:øô #Ô$Ü�H‰H×&Ñ&×AÑAÀ%ÕHús   ˆ<D9 ÁB,D9 Ä95E.c           	      ó˜  — d}d}|D �]1  \  }}|d   }t        |t        j                  «      r;t        j	                  d||t        |«      «       |dk  sŒMt        d   j                  }Œat        |t        «      r)d|› d|› d	�}	t        j                  |	«       t        |	«      ‚t        |t        «      rEdj                  t        j                  |Ž «      }	t        j                  d
|	|«       |d|› d|	› d�z  }Œït        |t        «      s�Œt!        |j"                  «      t$        k(  s�Œ|dk  s�Œ&|j"                  }�Œ4 t'        |«      dkD  rt        |«      ‚|dkD  rqt        j)                  «       D ]Y  }
||
j                  k(  sŒt*        r#t        j	                  d||
j,                  «        y t        j                  |
j,                  «      ‚ y y )Nrä  r    r~   z3Thread %s skipping test %s for following reason: %sr   rG   zThread ræ  z	 seconds
z'Caught exception: 
%s exiting thread %sz exited with exception:
rå  rç  )Ú
isinstancer¹  rº  rˆ  r‰  r-   rg   r&   rq  rŸ   rá  r»  rA  r¼  Úformat_exceptionÚ
SystemExitrâ   Úcoder+   Úlenr#  r   r'   )r¯  ru  ró   r(  Ú	error_msgÚ	skip_coder  rC  ÚexcrÇ   rí  s              r0   rÙ  z)MultiThreadedTestCase._check_return_codesä  s§  € ð ˆ	Øˆ	Ø*ó 	)‰NˆD�(Ø˜1‘+ˆCÜ˜#œx×0Ñ0Ô1Ü—‘ØIÈ4ÐQSÔUXÐY\ÓU]ôð ˜q“=Ü *¨9Ñ 5× ?Ñ ?‘IÜ˜C¤Ô.Ø ˜vÐ%DÀWÀIÈZÐX�Ü—‘˜SÔ!Ü" 3Ó'Ð'Ü˜C¤Ô+Ø—g‘gœi×8Ñ8¸(ÐCÓD�Ü—‘Ø>ÀÀTôð Ø˜d˜VÐ#<¸S¸EÀÐDñ‘	ô ˜C¤Ö,Ü˜Ÿ™“>¤SÔ(¨Y¸¬]Ø #§¡’Ið-	)ô2 ˆy‹>˜AÒÜ˜yÓ)Ð)à�qŠ=Ü"×)Ñ)Ó+ò 	>�Ø §¡Ó.Ý$äŸ™ØTÐVXÐZ^×ZfÑZfôñ ä&×/Ñ/°·±Ó=Ð=ñ	>ð r/   c                 ó   — t         S r‘   rR  rL  s    r0   rm   z MultiThreadedTestCase.world_size  rT  r/   c                 óF   — | j                  «       j                  d«      d   S r  r~  rL  s    r0   r€  z(MultiThreadedTestCase._current_test_name  s   € ð �w‰w‹y�‰˜sÓ# BÑ'Ð'r/   r   rï  c                óJ   — | j                   |k(  r| j                  |||«       yy)zÁ
        The reason why we have this util function instead of
        self.assertEqual is all threads are sharing one CPU RNG
        so the assertion result is only reliable on rank 0
        N)r  ré  ©rM  r’   ÚyrÇ   r  s        r0   ÚassertEqualOnRankz'MultiThreadedTestCase.assertEqualOnRank  s'   € ð �9‰9˜ÒØ×Ñ˜Q  3Õ'ð r/   c                óH   — | j                   |k(  r| j                  ||«       y y r‘   )r  râ  rƒ  s        r0   ÚassertNotEqualOnRankz*MultiThreadedTestCase.assertNotEqualOnRank'  s#   € Ø�9‰9˜ÒØ×Ñ  1Õ%ð r/   rð  rñ  r‘   )r(   r)   r*   Ú__doc__ÚqueueÚQueuerA  rY  r^  r-   rg  r\  r^  rr  rz  rc  rõ  r†  rl  rT  rÙ  ró  r+   rm   r€  r…  r‡  rö  r÷  s   @r0   r@  r@  I  sñ   ø„ ñð "�e—k‘k“m€OàÐò/ñb Cð bÀð bÐUYõ bòòõ	7ôò#ð* ñDó ðDò%ð& ñ;ó ð;ð: ñ.>ó ð.>ð` ð"˜Cò "ó ð"ð ð( Cò (ó ð(ð(¸ô (ð&¸1÷ &ð &r/   r@  c                   ó–   ‡ — e Zd Zdeej
                  ej                  f   deddfˆ fd„Z	dej                  dej                  fd„Z
ˆ xZS )ÚSaveForwardInputsModuleÚforward_inputsÚcast_forward_inputsrà   Nc                 ót   •— t         ‰| �  «        t        j                  dd«      | _        || _        || _        y )Néd   )rf  rg  ÚnnÚLinearÚlr�  rŽ  ©rM  r�  rŽ  rk  s      €r0   rg  z SaveForwardInputsModule.__init__-  s2   ø€ ô
 	‰ÑÔÜ—‘˜3 Ó$ˆŒØ,ˆÔØ#6ˆÕ r/   r’   c                 óÀ   — || j                   | <   | j                  | j                  r3|j                  | j                  j                  j
                  «      «      S |«      S r‘   )r�  r“  rŽ  ÚtoÚweightÚdtype©rM  r’   s     r0   ÚforwardzSaveForwardInputsModule.forward7  sI   € Ø$%ˆ×Ñ˜DÑ!Ø�v‰v°4×3KÒ3K�a—d‘d˜4Ÿ6™6Ÿ=™=×.Ñ.Ó/ÓSÐSÐQRÓSÐSr/   ©r(   r)   r*   r  r‘  ÚModulerc   ÚTensorrò  rg  rš  rö  r÷  s   @r0   rŒ  rŒ  ,  sT   ø„ ð7à˜RŸY™Y¨¯©Ð4Ñ5ð7ð "ð7ð 
õ	7ðT˜Ÿ™ð T¨%¯,©,÷ Tr/   rŒ  c                   ó–   ‡ — e Zd Zdeej
                  ej                  f   deddfˆ fd„Z	dej                  dej                  fd„Z
ˆ xZS )ÚSaveForwardInputsModelr�  rŽ  rà   Nc                 ót   •— t         ‰| �  «        t        ||«      | _        t        ||«      | _        || _        y r‘   )rf  rg  rŒ  Úc1Úc2r�  r”  s      €r0   rg  zSaveForwardInputsModel.__init__=  s6   ø€ ô
 	‰ÑÔÜ)¨.Ð:MÓNˆŒÜ)¨.Ð:MÓNˆŒØ,ˆÕr/   r’   c                 ó`   — || j                   | <   | j                  | j                  |«      «      S r‘   )r�  r¢  r¡  r™  s     r0   rš  zSaveForwardInputsModel.forwardG  s)   € Ø$%ˆ×Ñ˜DÑ!Ø�w‰w�t—w‘w˜q“zÓ"Ð"r/   r›  r÷  s   @r0   rŸ  rŸ  <  sQ   ø„ ð-à˜RŸY™Y¨¯©Ð4Ñ5ð-ð "ð-ð 
õ	-ð#˜Ÿ™ð #¨%¯,©,÷ #r/   rŸ  c              #   óÎ  K  — |st         j                  j                  | «       dt        j                  d<   dt        j                  d<   |rp|rVt         j
                  j                  j                  j                  j                  «       }t        j                  d|| |¬«       nt        j                  d| |¬«       t         j                  j                  «        t         j                  j                  j                  j!                  «        	 d –— t         j                  j                  «        t         j                  j                  j                  j!                  «        |rt        j"                  «        y y # t         j                  j                  «        t         j                  j                  j                  j!                  «        |rt        j"                  «        w w xY w­w)	Nrç   ÚMASTER_ADDRÚ6789ÚMASTER_PORTÚfaker  rA   ©r  rm   )rc   r  r	  rh   ri   ÚtestingÚ	_internalr  r‚  Ú	FakeStorer¹   r  r  r  ÚutilsÚcountersÚclearrÀ  )r  rm   Úinit_pgr‚  r  s        r0   Ú_dynamo_dist_per_rank_initr±  K  sJ  è ø€ ñ Ü×Ñ×*Ñ*¨4Ô0Ø +„B‡J�Jˆ}ÑØ &„B‡J�Jˆ}ÑÙÙÜ—M‘M×+Ñ+×7Ñ7×?Ñ?×IÑIÓKˆEÜ×#Ñ#ØØ%ØØö	ô ×#Ñ# F°À*ÕMÜ	‡M�M×ÑÔÜ	‡M�M×Ñ× Ñ ×&Ñ&Ô(ð)Ûä�‰×ÑÔÜ�‰×Ñ×$Ñ$×*Ñ*Ô,ÙÜ×&Ñ&Õ(ð øô 	�‰×ÑÔÜ�‰×Ñ×$Ñ$×*Ñ*Ô,ÙÜ×&Ñ&Õ(ð üs    ‚D
G%ÄE9 ÄA(G%Å9A)G"Ç"G%c                   ó@   ‡ — e Zd ZdZeˆ fd„«       Zeˆ fd„«       Zˆ xZS )Ú#DynamoDistributedSingleProcTestCasez£
    Test harness for single-process dynamo distributed tests,
    initializes dist process group.

    Prefer this for simple tests, as it's easier to debug.
    c                 ó`  •— t         ‰| �  «        | j                  j                  t	        j
                  t        j                  dddœ«      «       d| _        d| j                  › �| _	        d| j                  v rd n| j                  g| _
        t        j                  d| j                  d¬	«       y )
Nrç   Ú12355)r¥  r§  r   zcuda:rT   rA   r~   r©  )rf  Ú
setUpClassÚ_exit_stackÚenter_contextr   r  rh   ri   r  rÝ   Ú
device_idsr¹   r  ©r¯  rk  s    €r0   r¶  z.DynamoDistributedSingleProcTestCase.setUpClassq  s‰   ø€ ä‰ÑÔà�‰×%Ñ%Ü�J‰JÜ—
‘
à#.Ø#*ñóô	
ð ˆŒØ˜SŸX™X˜JÐ'ˆŒ
Ø!'¨3¯:©:Ñ!5™¸C¿H¹H¸:ˆŒÜ×Ñ ¨S¯X©XÀ!ÖDr/   c                 óJ   •— t        j                  «        t        ‰| �  «        y r‘   )r¹   rÀ  rf  ÚtearDownClassrº  s    €r0   r¼  z1DynamoDistributedSingleProcTestCase.tearDownClassƒ  s   ø€ ä×"Ñ"Ô$Ü‰ÑÕr/   )r(   r)   r*   rˆ  rõ  r¶  r¼  rö  r÷  s   @r0   r³  r³  i  s2   ø„ ñð óEó ðEð" ó ó ô r/   r³  c            	       ód   ‡ — e Zd ZdZˆ fd„Zˆ fd„Zedefd„«       Ze	dede
de
dd	fd
„«       Zˆ xZS )Ú"DynamoDistributedMultiProcTestCasea   
    Use this for tests that actually run on multiple GPUs.

    Decorate tests with @skip_if_lt_x_gpu(ngpu)

    Note: MultiProcTestCase spawns processes per test and is slow.
    Prefer MultiThreadedTestCase for most tests. Perhaps use this one
    sparingly for integration tests.
    c                 óB   •— t         ‰| �  «        | j                  «        y r‘   rû  rx  s    €r0   rr  z(DynamoDistributedMultiProcTestCase.setUp“  rü  r/   c                 ó€   •— t         ‰| �  «        	 t        j                  | j                  «       y # t
        $ r Y y w xY wr‘   )rf  rz  rh   rþ  rv  rÿ  rx  s    €r0   rz  z+DynamoDistributedMultiProcTestCase.tearDown—  s5   ø€ Ü‰ÑÔð	Ü�I‰I�d—n‘nÕ%øÜò 	Ùð	ús   ‘1 ±	=¼=rà   c                 ó>   — t         j                  j                  «       S r‘   )rc   rT   rj   rL  s    r0   rm   z-DynamoDistributedMultiProcTestCase.world_sizež  s   € ä�z‰z×&Ñ&Ó(Ð(r/   r  r«  rv  Nc                 ó¢   — t        j                  t        j                  «       «        | |«      }||_        ||_        |j                  ||«       y r‘   )r   Ú
addHandlerÚloggingÚNullHandlerr  rv  r­  r®  s          r0   r†  z'DynamoDistributedMultiProcTestCase._run¢  s@   € ä×ÑœW×0Ñ0Ó2Ô3ñ �9‹~ˆØˆŒ	Ø"ˆŒØ�‰�i Õ-r/   )r(   r)   r*   rˆ  rr  rz  ró  r+   rm   rõ  r-   r†  rö  r÷  s   @r0   r¾  r¾  ‰  s^   ø„ ñô ôð ð)˜Cò )ó ð)ð ð.˜ð .¨ð .¸ð .ÐW[ò .ó ô.r/   r¾  c            	       óø   ‡ — e Zd ZU dZeed<   dZeed<   dZee	   ed<    e
d¬«      Ze
ed	<   eej                  d
e	fd„«       «       Zedd„«       Zeˆ fd„«       Zeˆ fd„«       Ze	 ddededee	   fd„«       Zˆ xZS )ÚMultiProcContinousTestr}   rm   r    r  NÚ	rdvz_fileéx   )Úsecondsró   rà   c                 ó   — t        d«      ‚)zƒ
        ProcessGroup backend str.
        To be customized by sub test classes, e.g. "nccl".
        Here we raise error.
        z/Please implement backend_str in your test class)ÚNotImplementedError)r¯  s    r0   Úbackend_strz"MultiProcContinousTest.backend_str¹  s   € ô "Ð"SÓTÐTr/   c                  ó   — y)z“
        ProcessGroup init options.
        To be customized by sub test classes, e.g. ProcessGroupNCCLOpTest
        Here we return None.
        Nr.   )r¯  Úhigh_priority_streams     r0   ÚoptszMultiProcContinousTest.optsÃ  s   € ð r/   c                 óf  •— t         ‰| �  «        d| j                  cxk  r| j                  k  s'n t	        d| j                  › d| j                  › �«      ‚| j
                  r+t        j                  | j
                  | j                  «      }nd}| j                  «       }| j                  «       }t        d|›�«       t        j                  || j                  | j                  ||| j                  ¬«       t        j                  j                  «       | _        t        d| j                  › d�«       y)	z…
        Class-scope test fixture. Run once for entire test class, before any test starts.
        Set up the process group.
        r   zBRank must be set and in the range of 0 to world_size. World size: z Rank: NzTesting backend=)r„   rm   r  r  Ú
pg_optionsró   úRank z setup complete)rf  r¶  r  rm   rá  rÈ  r¹   r  rÐ  rÍ  rÔ  r  ró   r
  r  Úpg)r¯  r  rÐ  r„   rk  s       €r0   r¶  z!MultiProcContinousTest.setUpClassÌ  sù   ø€ ô 	‰ÑÔØ�C—H‘HÔ-˜sŸ~™~Ô-ÜðØ"Ÿ~™~Ð.¨g°c·h±h°ZðAóð ð �=Š=Ü—N‘N 3§=¡=°#·.±.ÓA‰Eð ˆEØ�x‰x‹zˆØ—/‘/Ó#ˆÜÐ!˜˜
Ð#Ô$ä×ÑØØ—~‘~Ø—‘ØØØ—K‘Kõ	
ô ×&Ñ&×9Ñ9Ó;ˆŒÜ��c—h‘h�Z˜Ð/Õ0r/   c                 óò   •— t        j                  «        t        ‰| �  «        | j                  r 	 t        j                  | j                  «       t        d| j                  › d�«       y# t        $ r Y Œ%w xY w)zˆ
        Class-scope test fixture. Run once for entire test class, after all tests finish.
        Tear down the process group.
        rÓ  z teardown completeN)
r¹   rÀ  rf  r¼  rÈ  rh   rþ  rÿ  rÔ  r  rº  s    €r0   r¼  z$MultiProcContinousTest.tearDownClassì  sd   ø€ ô 	×"Ñ"Ô$Ü‰ÑÔà�=Š=ðÜ—	‘	˜#Ÿ-™-Ô(ô 	��c—h‘h�ZÐ1Ð2Õ3øô ò Ùðús   ±A* Á*	A6Á5A6c                 óB   — || _         || _        || _        t        «        y)ad  
        This is an entry point for each rank to run the tests in `MultiProcContinousTest`.
        In this entry point, we set the class variables for the test class.
        Then we run all tests.

        Note:
        - This helper only works for a subclass of `MultiProcContinousTest`.

        Example:
        - See `test_c10d_ops_nccl.py`.
        N)r  rm   rÈ  r   )r¯  r  rm   rÈ  s       r0   Úrun_rankzMultiProcContinousTest.run_rankü  s   € ð& ˆŒØ#ˆŒØ!ˆŒä�r/   )Fr‘   )r(   r)   r*   rm   r+   r,   r  rÈ  r   r-   r   ró   rõ  ÚabcÚabstractmethodrÍ  rÐ  r¶  r¼  r×  rö  r÷  s   @r0   rÇ  rÇ  ­  sÖ   ø… ð €J�Óà€Dˆ#ƒNà#€Iˆx˜‰}Ó#á"¨3Ô/€GˆYÓ/ð Ø×ÑðU˜Cò Uó ó ðUð òó ðð ó1ó ð1ð> ó4ó ð4ð ð
 $(ñ	àðð ðð ˜C‘=ò	ó ôr/   rÇ  r‘   )r~   rñ  )TF)‡rØ  rŸ  r  rÄ  r„  rh   r‰  r*  re   r<  rµ  rÓ  r¼  r[  r¹  Ú
contextlibr   Údataclassesr   Údatetimer   Úenumr   Ú	functoolsr   r   r	   Úior
   Útypingr   r   r   r   r   Úunittest.mockr   Útorch._logging._internalr   rc   Útorch._dynamo.test_caseÚtorch.cuda.ncclÚtorch.distributedr  r¹   Útorch._C._autogradr   Útorch._C._distributed_c10dr   Útorch.nnr‘  Ú$torch.testing._internal.common_utilsr   r   r   r   r   r   r   r   r   r   r   r   Ú5torch.testing._internal.distributed.multi_threaded_pgr    r!   r"   r&  ÚbasicConfigÚINFOÚ	getLoggerr(   rˆ  r%   rg   rN   rr   ry   r   r‡   r�   r“   r—   r›   r©   r³   r¾   rÁ   rÈ   rÊ   rÍ   rÑ   rÕ   rØ   rÜ   rÝ   r+   rò  ræ   r÷   r  Úgetenvr  r  r	  r  r,  r-   r3  r4  r=  r,   rC  rF  rS  rI  rù  r  r.  r#  r)  r.  r2  rU  r@  rœ  rŒ  rŸ  r±  r  Ú	test_caser³  r¾  rÇ  r.   r/   r0   ú<module>rð     sÉ  ðô Û Û Û Û Û 	Û Û Û 
Û Û Û Û Û Û Ý %Ý !Ý Ý ß ,Ñ ,Ý ß =Õ =Ý å .Û Û Û Ý  Ý )Ý 7Ý ÷÷ ÷ ó ÷ñ ó
 à €× Ñ ˜'Ÿ,™,Õ 'Ø	ˆ×	Ñ	˜8Ó	$€ôˆzô ð
Ø™8Ø
ÐCóðð ‘x Ð$FÓGð	ð
 ‘X˜bÐ"BÓCðð ‰x˜Ð4Ó5ðð ‘8˜BÐ =Ó>ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‘8˜BÐ >Ó?ðð ‰H�RÐ>Ó?ðð  ‘(˜2Ð6Ó7ð!ð" ‘h˜rÐ#LÓMð#ð$ ‰xØ
ÐVóð%ð* ‘8˜BÐ DÓEð+ð, ‘h˜rÐ#BÓCñ-€
ð4 ÷*ð *ó ð*ò*ò6òòòò&Hòò$òRò$+ò\ò4ò	
òòòòò
òðF §¡ð F°cð FÀ#ð FÈ$ó Fð  à	ØØÙ˜aÔ ØØØò
ó ð
ñ0 à�Oá˜)˜"Ÿ)™)Ð$GÈÓOÓP€OØ,¨cÐ2Ð ñ Ø+.ÐÐ'Ñ(óHðI˜Có Ið ñ2ó ð2ñ( Sð (°cð (Àsó (ðX Sð °3ó ð2 26€ˆ�(×-Ñ-Ñ	.Ó 5ñ
¨X°c©]ð 
Àdó 
ó"ð Ð ô`˜8ô `ôN%NÐ.ô %NðNà˜˜d 3™i˜Ñ(ðð ðð
 óðF Ð ð�ó ò&0ð 
�Ð3Eó5ôp`&˜Hô `&ôFT˜bŸi™iô Tô #˜RŸY™Yô #ð ò)ó ð)ô: ¨%¯-ª-×*AÒ*A×*JÑ*Jô  ô@!.Ð)=ô !.ôHf˜Xõ fr/   