Ë
    g^(hÞÜ  ã                   ó`  — U d dl Z d dlZd dlZd dlZd dlmZmZ d dlmZm	Z	m
Z
 d dlmZ d dlmZmZmZmZmZmZ d dlZd dlmZ d dlmZ d dlmZ d dlmZmZmZm Z m!Z!m"Z" d dl#m$Z$ d d	l%m&Z&m'Z'm(Z)m*Z*m+Z+m,Z,m-Z-m.Z. d d
l/m0Z0m1Z1 d dl2m3Z3 d dl4m5Z5 d dl6m7Z8 d dl9m:Z: g d¢Z;dZ<dZ=dZ>dZ?e@eA   ZBee3eej†                  eDeEeAf   ZFeeFeGeF   eHeF   eIeAdf   f   ZJeIeAeJf   ZKeGeK   ZLeIeAeeKeLf   f   ZM e@«       ZNe@e   eOd<   e j                   d„ «       ZQe	 G d„ d«      «       ZRe	 G d„ deR«      «       ZSej¨                  	 	 	 dIdejª                  deAdeAdeVdeVd eBfd!„«       ZW G d"„ d#«      ZXdJd$„ZYddd%œdejª                  d&eHej´                  j¶                  d'f   d(eVd)ee@ejª                        d*eeR   d eSfd+„Z\d,eIeAeJf   d-eMd.eSd dfd/„Z]d0eejª                  ej´                  j¶                  f   d1eAd efd2„Z^d3eIeAef   d.eSd eIeAef   fd4„Z_ ejÀ                  «       dejª                  d.eSd eIeAeJf   fd5„«       Za ejÀ                  «       dejª                  d3eIeAeJf   d.eSd e5fd6„«       Zbd7ej´                  j¶                  d dfd8„Zcd3eMd eIeAeJf   fd9„Zdd7ej´                  j¶                  d3eIeAeJf   d.eSd eMfd:„Ze ejÀ                  «       dejª                  d;eHej´                  j¶                  d'f   d.eSd eMfd<„«       Zfdejª                  d7ej´                  j¶                  d-eMd.eSd eMf
d=„Zg ejÀ                  «       dejª                  d;eHej´                  j¶                  d'f   d3eMd.eSd df
d>„«       Zhddd%œdejª                  d)ee@ejª                        d*eeR   d eIeAeJf   fd?„Ziddd%œdejª                  d;eej´                  j¶                  eej´                  j¶                     f   d)ee@ejª                        d*eeR   d eMf
d@„Zjddd%œdejª                  d;eej´                  j¶                  eej´                  j¶                     f   d)ee@ejª                        d*eeR   d eHeIeAeJf   eMf   f
dA„Zkdejª                  d3eeIejª                  eIeAeJf   f   eIeAeJf   f   d eIeAeJf   fdB„ZlddCœdejª                  d,eIeAeJf   d*eeR   d e5fdD„ZmddCœdejª                  d;eej´                  j¶                  eej´                  j¶                     f   d-eMd*eeR   d df
dE„ZnddCœdejª                  d;eej´                  j¶                  eej´                  j¶                     f   d,eIeAeJf   d-eMd*eeR   d e5fdF„ZoeddCœdejª                  d*eeR   d dfdG„«       ZpeddCœdejª                  d;eHej´                  j¶                  d'f   d*eeR   d dfdH„«       Zqy)Ké    N)Ú	GeneratorÚIterable)ÚasdictÚ	dataclassÚfield)Úchain)ÚAnyÚCallableÚcastÚno_type_checkÚOptionalÚUnion)ÚShardedTensor)Ú_broadcast_state_dictÚ_distribute_state_dictÚ_flatten_state_dictÚ_gather_state_dictÚ_offload_state_dict_to_cpuÚ_unflatten_state_dict)Ú_CHECKPOINT_PREFIX)ÚFullOptimStateDictConfigÚFullStateDictConfigÚFullyShardedDataParallelÚOptimStateDictConfigÚShardedOptimStateDictConfigÚShardedStateDictConfigÚStateDictConfigÚStateDictType)Ú._get_module_fsdp_state_if_fully_sharded_moduleÚFSDP_WRAPPED_MODULE)ÚDTensor)Ú_IncompatibleKeys)ÚDistributedDataParallel)Útree_map_only)ÚFQNS_TÚPrimitiveTypeÚ	ValueTypeÚDictValueTypeÚListDictValueTypeÚOptimizerStateTypeÚStateDictOptionsÚget_model_state_dictÚget_optimizer_state_dictÚget_state_dictÚset_model_state_dictÚset_optimizer_state_dictÚset_state_dictÚ_flat_paramÚparam_groupsÚparamsÚstater'   Ú_patched_state_dictc               #   óÌ   K  — t        j                  «       } t        j                  «        	 d –— | rt        j                  «        y y # | rt        j                  «        w w xY w­w©N)ÚgcÚ	isenabledÚdisableÚenable)Ú
is_enableds    úe/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/state_dict.pyÚ_gc_contextr?   Q   sD   è ø€ ä—‘“€JÜ‡J�J„LðÛáÜ�I‰I�Kð ø‰:Ü�I‰I�Kð üs   ‚)A$¬A °A$ÁA!Á!A$c                   ó‚   — e Zd ZU dZdZeed<   dZeed<   dZeed<   dZ	eed<   dZ
eed<   dZeed	<   dZeed
<   dZeed<   y)r+   ap  
    This dataclass specifies how get_state_dict/set_state_dict will work.

    - ``full_state_dict``: if this is set to True, all the tensors in the
      returned state_dict will be gathered. No ShardedTensor and DTensor
      will be in the returned state_dict.

    - ``cpu_offload``: offload all the tensors to cpu. To prevent CPU OOM, if
      ``full_state_dict`` is also true, then only the rank0 will get the
      state_dict and all other ranks will get empty state_dict.

    - ``ignore_frozen_params``: if the value is True, the returned state_dict
      won't contain any frozen parameters -- the ``requires_grad`` is False.
      The default value is False.

    - ``keep_submodule_prefixes`` (deprecated): when ``submodules`` is not None, this option
      indicates whether to keep the submodule prefixes from the state_dict keys.
      or example, if the submodule is ``module.pretrain`` and the full FQN of
      the parameter is ``pretrain.layer1.weight`` of the param. When this option
      is True, the parameter's key in the returned state_dict will be
      ``pretrain.layer1.weight``. If the options is False, the key will be
      ``layer1.weight``.
      Note that if ``keep_submodule_prefixes`` is False, there may be conflicted
      FQNs, hence there should be only one submodule in ``submodules``.

    - ``strict``: the ``strict`` option when ``set_state_dict`` calls
      model.load_state_dict().

    - ``broadcast_from_rank0``: when the option is True, rank0 should receive a
       full state_dict and will broadcast the tensors in the state_dict/
       optim_state_dict one by one to other ranks. Other ranks will receive
       the tensors and shard according to the local shards in the model and
       optimizer. ``full_state_dict`` must be set to True when using this option.
       This option currently only supports DTensor, not the legacy ShardedTensor.
    FÚfull_state_dictÚcpu_offloadÚignore_frozen_paramsTÚkeep_submodule_prefixesÚstrictÚbroadcast_from_rank0Úflatten_optimizer_state_dictÚ_fqn_modifiersÚdsd_fqn_modifiersN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__rA   ÚboolÚ__annotations__rB   rC   rD   rE   rF   rG   rI   Ústr© ó    r>   r+   r+   \   s_   … ñ"ðH "€O�TÓ!Ø€K�ÓØ!&Ð˜$Ó&Ø$(Ð˜TÓ(Ø€FˆDÓØ!&Ð˜$Ó&Ø).Ð  $Ó.Ø-Ð�sÔ-rR   r+   c                   óh  — e Zd ZU  ee¬«      Zeeeej                  f   ee
ej                  f   f   ed<    ee¬«      Zeeeej                  f   ee
ej                  f   f   ed<    ee¬«      Zee   ed<   dZeed<   dZeed<   ej&                  Zeed<    ee¬«      Zeej2                     ed	<   y
)Ú_StateDictInfo)Údefault_factoryÚfqn_param_mappingÚshared_params_mappingÚsubmodule_prefixesTÚhandle_modelÚhandle_optimÚfsdp_contextÚfsdp_modulesN)rJ   rK   rL   r   ÚdictrV   r   rP   ÚtorchÚTensorr%   rO   rW   ÚsetrX   rY   rN   rZ   Ú
contextlibÚnullcontextr[   r
   Úlistr\   ÚnnÚModulerQ   rR   r>   rT   rT   Œ   s×   … ñ
 	˜dÔ#ð �tØˆc�5—<‘<ÐÑ Øˆf�e—l‘lÐ"Ñ#ð	%ñó $ñ 	˜dÔ#ð ˜4Øˆc�5—<‘<ÐÑ Øˆf�e—l‘lÐ"Ñ#ð	%ñó $ñ $)¸Ô#=Ð˜˜C™Ó=Ø€L�$ÓØ€L�$ÓØ'×3Ñ3€L�(Ó3Ù$)¸$Ô$?€L�$�r—y‘y‘/Ô?rR   rT   ÚmodelÚnamerI   Úskip_ddp_prefixÚskip_compiler_prefixÚreturnc                 ó¶  — |j                  t        d«      }d|vr|hS |j                  d«      }g }| }t        |«      D �]î  \  }}	t	        |t
        «      r(|	dk(  sJ ‚|j                  }|rŒ-|j                  |	«       Œ?t	        |t        «      r |t        |«      dz
  k  rW||dz      t        k(  rHdj                  |«      }
t        |t        «      }|
r|
› d�}
|j                  D �ch c]  }|
› |› �’Œ
 c}c S t        |t        «      }|	t        k7  sŒÑ|j                  |	«       t        ||	«      }Œït	        |t        j                   j"                  j$                  «      r*|	dk(  sJ ‚|j&                  }|r�Œ4|j                  |	«       �ŒGt)        ||«      r: t        ||«      «       j+                  |	«      x}rt)        ||«      rt        ||«      }|j                  |	«       |	t,        j.                  j                  j0                  k(  r|t        |«      dz
  k7  s�ŒØt3        d«      ‚t        ||	«      }�Œñ dj                  |«      j                  t        d«      hS c c}w )aá  
    This API is used to convert the name of a parameter to the FQNs. For FSDP
    without `use_orig_params`, the name of FlatParameter can be mapped to
    multiple original parameters. As a result, the return type of this function
    is `set[str]`.

    Args:
        module (nn.Module): the root model.
        name (str): the name
        skip_ddp_prefix (bool): whether to skip DDP's `module` prefix

    Returns:
        The canonical FQNs based on the model traversal.
    Ú ú.Úmoduleé   Ú	_orig_modz-Expect `_extra_state` to be the last obj name)Úreplacer   ÚsplitÚ	enumerateÚ
isinstanceÚDDPrn   ÚappendÚFSDPÚlenÚ_FLAT_PARAMÚjoinÚgetattrÚ_fqnsr    r^   Ú_dynamoÚ
eval_frameÚOptimizedModulerp   ÚhasattrÚgetrd   ÚmodulesÚ_EXTRA_STATE_KEY_SUFFIXÚRuntimeError)rf   rg   rI   rh   ri   Ú	obj_namesÚfqn_obj_namesÚcurr_objÚiÚcurr_obj_nameÚprefixÚ
flat_paramÚfqnÚremoved_fqns                 r>   Ú	_get_fqnsrŽ   �   s4  € ð0 �<‰<Ô*¨BÓ/€DØ
�$�Øˆvˆà—
‘
˜3“€IØ€MØ€HÜ% iÓ0ó $<Ñˆˆ=Ü�h¤Ô$Ø  HÒ,Ð,Ð,Ø—‘ˆHÚ"Ø×$Ñ$ ]Õ3Ü˜¤$Ô'Ø”3�y“> AÑ%Ò%¨)°A¸±EÑ*:¼kÒ*IØŸ™ -Ó0�Ü$ X¬{Ó;�
ÙØ &˜x q˜\�FØ4>×4DÑ4DÖE¨S˜6˜( 3 %Ò(ÒEÒEÜ˜xÔ)<Ó=ˆHØÔ 3Ó3Ø×$Ñ$ ]Ô3Ü" 8¨]Ó;‘Ü˜¤%§-¡-×":Ñ":×"JÑ"JÔKØ  KÒ/Ð/Ð/Ø×)Ñ)ˆHÛ'Ø×$Ñ$ ]Ö3ô �xÐ!2Ô3Ø"F¤'¨(Ð4EÓ"FÓ"H×"LÑ"LØ!ó#ð �;ð ô ˜x¨Ô5Ü#*¨8°[Ó#A˜Ø× Ñ  Ô/Ø¤§
¡
× 1Ñ 1× IÑ IÒIØœ˜I›¨Ñ*Ô*Ü&Ð'VÓWÐWä" 8¨]Ó;’ðI$<ðL �H‰H�]Ó#×+Ñ+Ô,>ÀÓCÐDÐDùò5 Fs   Ã#Ic                   ó   — e Zd Zy)Ú_EXTRA_STATEN)rJ   rK   rL   rQ   rR   r>   r�   r�   å   s   „ ØrR   r�   c              #   ó”   ‡‡‡K  — t        «       Šdt        j                  dt        dt        fˆˆˆfd„Š ‰| d«      E d {  –—†  y 7 Œ­w)Nrn   Úcurr_fqnrj   c              3   ó´  •K  — ‰j                  | «       |r|› d�nd}| j                  «       D ]T  \  }}|‰v rŒt        | ‰«      r'| t        | ‰«      «       j	                  «       v r|d d }n|› |› �} ‰||«      E d {  –—†  ŒV t        | j                  d¬«      | j                  d¬«      «      D ]   \  }}|| j                  v rŒ|› |› �}||f–— Œ" t        | j                  dt        j                  j                  «      t        j                  j                  k7  r7|› t        j                  j                  j                  › �}|t!        «       f–— y y 7 Œ×­w)Nrm   rl   éÿÿÿÿF)ÚrecurseÚget_extra_state)ÚaddÚnamed_childrenr€   r{   Úvaluesr   Únamed_buffersÚnamed_parametersÚ_non_persistent_buffers_setÚ	__class__rd   re   r–   r‚   rn   rƒ   r�   )	rn   r’   rg   Ú	submoduleÚnew_fqnÚobjrI   r•   Úvisited_moduless	         €€€r>   r•   z+_iterate_valid_model_state.<locals>.recurseì   sr  øè ø€ Ø×Ñ˜FÔ#á%-�h�Z˜q‘>°2ˆØ%×4Ñ4Ó6ò 	3‰OˆD�)Ø˜OÑ+Øô ˜Ð 1Ô2ØÐ>œG FÐ,=Ó>Ó@×GÑGÓIÑIð # 3 B˜-‘à%˜J t fÐ-�Ù˜y¨'Ó2×2Ñ2ð	3ô Ø× Ñ ¨Ð Ó/°×1HÑ1HÐQVÐ1HÓ1Wó
ò 	‰IˆD�#ð �v×9Ñ9Ñ9ØØ!˜
 4 &Ð)ˆGØ˜3�,Óð	ô �F×$Ñ$Ð&7¼¿¹×9RÑ9RÓSÜ�y‰y×(Ñ(ò)ð "˜
¤2§:¡:×#4Ñ#4×#LÑ#LÐ"MÐNˆGØœ<›>Ð)Ó)ð	)ð 3ús   ƒA;EÁ>EÁ?CErl   )r`   rd   re   rP   r   )rf   rI   r•   r¡   s    `@@r>   Ú_iterate_valid_model_stater¢   é   s?   úè ø€ Ü&)£e€Oð *œŸ	™	ð  *¬Sð  *´Y÷  *ñD �u˜bÓ!×!Ò!ús   …;AÁ AÁA)Ú
submodulesÚoptionsÚoptims.Ú
optim_onlyr£   r¤   c                ó  — |rt        j                  dt        «       |r|st        d«      ‚|xs
 t	        «       }i }i }t        | «      D ]˜  \  }}t        |t        «      rŒt        | |«      }	|j                  |d«      }
|
�2t        t        t           ||   «      j                  |	«       ||   ||<   n|	j                  «       ||<   |	D ]  }
t        |t        «      rŒ|||
<   Œ Œš t        |j!                  «       «      D ])  \  }}|D ]  }
t        t"        j$                  |«      ||
<   Œ! Œ+ t        «       }|rat        |«      }| j'                  «       D ]C  \  }}||vrŒt        | |«      }	t)        |	«      dk(  sJ d«       ‚|j                  d„ |	D «       «       ŒE |j*                  r|j,                  st/        d«      ‚t1        j2                  | «      }|rÚ|j,                  rat5        |j6                  |j6                  ¬«      }t9        |j6                  |j6                  xs |j*                  ¬«      }t:        j<                  }n<t?        |j6                  ¬	«      }tA        |j6                  ¬	«      }t:        jB                  }tD        jF                  d
„ «       }tI        jJ                  || |||¬«      }ntD        jL                  }tO        di tQ        |«      ¤||||t        t        tR        jT                     |«      | t)        |«      dkD  dœ¤ŽS )zW
    Verify the model and options passed by the user and generates _StateDictInfo.
    z¼Getting submodules only model/optim state_dict is deprecated and will be removed in 2.5. This feature can be achieved by manually filtering out the state_dict returned from get_state_dict.z;Optimizers are not passed in but optim_only is set to True.Nro   z)Submodule FQN should only have 1 instancec              3   ó&   K  — | ]	  }|› d �–— Œ y­w)rm   NrQ   )Ú.0rŒ   s     r>   ú	<genexpr>z"_verify_options.<locals>.<genexpr>L  s   è ø€ Ò%@°C¨¨¨Q¤iÑ%@ùs   ‚z?full_state_dict must be True when broadcast_from_rank0 is True.)Úoffload_to_cpuÚ
rank0_only)r«   c              3   óú   K  — t        j                  «       5  t        j                  ddt        ¬«       t	        j
                  | |||¬«      5  d –— d d d «       d d d «       y # 1 sw Y   ŒxY w# 1 sw Y   y xY w­w)NÚignorezFSDP.state_dict_type)ÚmessageÚcategory©rn   Ústate_dict_typeÚstate_dict_configÚoptim_state_dict_config)ÚwarningsÚcatch_warningsÚfilterwarningsÚFutureWarningrw   r²   r±   s       r>   Ú$fsdp_state_dict_type_without_warningz=_verify_options.<locals>.fsdp_state_dict_type_without_warningj  sy   è ø€ ô ×(Ñ(Ó*ñ 
Ü×'Ñ'ØÐ&<Ä}õô ×)Ñ)Ø!Ø$3Ø&7Ø,Cô	ñ ó ÷÷	
ð 
÷ð ú÷	
ð 
üs4   ‚A;—6A/ÁA#ÁA/Á	A;Á#A,	Á(A/Á/A8Á4A;r±   r   )rV   rW   rX   r[   r\   rY   rZ   rQ   )+rµ   Úwarnr¸   r„   r+   r¢   rt   r�   rŽ   r�   r   r`   rP   ÚupdateÚcopyrc   Úitemsr^   r_   Únamed_modulesrx   rF   rA   Ú
ValueErrorrw   r\   r   rB   r   r   ÚFULL_STATE_DICTr   r   ÚSHARDED_STATE_DICTra   ÚcontextmanagerÚ	functoolsÚpartialrb   rT   r   rd   re   )rf   r¥   r¦   r£   r¤   rV   rW   rg   ÚparamÚfqnsrŒ   Úparam_Úfqns_rX   rn   r\   r³   r´   r²   r¹   r[   s                        r>   Ú_verify_optionsrÉ     s+  € ñ Ü�‰ðIô ô		
ñ ™&ÜØIó
ð 	
ð Ò+Ô)Ó+€Gð 	ð ð
 	ð ô 2°%Ó8ò /‰ˆˆeÜ�eœ\Ô*Øä˜ Ó%ˆØ×#Ñ# E¨4Ó0ˆØˆ?Ü””S‘Ð,¨UÑ3Ó4×;Ñ;¸DÔAØ+<¸UÑ+CÐ! %Ò(ð (,§y¡y£{Ð˜eÑ$Øò 	/ˆCÜ˜e¤\Õ2Ø).Ð! #Ò&ñ	/ð/ô  Ð3×9Ñ9Ó;Ó<ò D‰ˆ�Øò 	DˆCÜ)-¬e¯l©l¸FÓ)CÐ! #Ò&ñ	DðDô $'£5ÐÙÜ˜“_ˆ
Ø!×/Ñ/Ó1ò 	A‰LˆD�&Ø˜ZÑ'ØÜ˜U DÓ)ˆDÜ�t“9 ’>ÐNÐ#NÓN�>Ø×%Ñ%Ñ%@¸4Ô%@Õ@ð	Að ×#Ò#¨G×,CÒ,CÜØMó
ð 	
ô ×$Ñ$ UÓ+€Lñ à×"Ò"Ü 3Ø&×2Ñ2¸w×?RÑ?Rô!Ðô '?Ø&×2Ñ2Ø#×/Ñ/ÒO°7×3OÑ3Oô'Ð#ô ,×;Ñ;‰Oä 6Ø&×2Ñ2ô!Ðô 'BØ&×2Ñ2ô'Ð#ô ,×>Ñ>ˆOä	×	"Ñ	"ñ	ó 
#ð	ô$ !×(Ñ(Ø0ØØ+Ø/Ø$;ô
‰ô "×-Ñ-ˆäñ 	Ü
�‹/ð	à+Ø3Ø-Ø!Üœ$œrŸy™y™/¨<Ó8Ø#�^Ü˜&“k A‘oó	ð 	rR   Úmodel_state_dictÚoptim_state_dictÚinfoc                 ó   — |j                   D ]  }t        |«      }|�ŒJ d«       ‚ |j                  rk| si|j                  s]|j                  sQ|j
                  r|j                  s9|j                  r-|j                  s!t        dt        j                  «       ›d�«      ‚|j                  r4|s2|j
                  r|j                  s|j                  st        d|› �«      ‚| j                  «       D ]  }t        |v sŒt        |› dt        › d�«      ‚ y )Nz)Expected a fsdp_state with a fsdp module.z}The option indicates that model state_dict is required to save or load, but model state_dict is empty.rank = dist.get_rank()=rm   zgThe option indicates that model state_dict is required to save, or load but optim state_dict is empty. z
 contains z6. This can happen if the model is not the root module.)r\   r   rY   rX   rC   rB   rA   rE   rF   r„   ÚdistÚget_rankrZ   Úkeysry   )rÊ   rË   rÌ   rn   Ú
fsdp_stateÚkeys         r>   Ú_verify_state_dictrÓ   “  s'  € ð
 ×#Ñ#ò SˆÜCÀFÓKˆ
ØÑ%ÐRÐ'RÓRÐ%ðSð 	×ÒÙ Ø×'Ò'Ø×)Ò)Ø×!Ò! d×&:Ò&:Ø�KŠKØ×)Ò)äð'ä—m‘m“oÐ' qð*ó
ð 	
ð ×Òá Ø×%Ò%¨$×*>Ò*>Ø×.Ò.äð:Ø:JÐ9KðMóð ð
  ×$Ñ$Ó&ò ˆÜ˜#ÒÜØ�%�z¤+ ð /*ð *óð ñrR   r    Úapic                 ó„   — t        | |«      }|t        v r+t        j                  t        | j                  |«      | ¬«      }|S )N)Úself)r{   r6   rÃ   rÄ   r�   )r    rÔ   Úcalls      r>   Ú_state_dict_fnrØ   À  s9   € Ü�3˜Ó€DØÔ"Ñ"Ü× Ñ ¤¨¯©¸Ó!<À3ÔGˆØ€KrR   Ú
state_dictc                 óØ   — |j                   rF|j                  rt        j                  j	                  «       sdnd}t        | |j                  |¬«      S |j                  rt        | «      S | S )NrQ   )r   )rB   Ú
ranks_only)rA   rB   r^   ÚdistributedÚis_initializedr   r   )rÙ   rÌ   rÛ   s      r>   Ú_maybe_full_or_cpu_state_dictrÞ   Ç  sn   € ð ×Òð ×$Ò$¬E×,=Ñ,=×,LÑ,LÔ,Nñ àð 	ô
 "Ø D×$4Ñ$4Àô
ð 	
ð 
×	Ò	Ü)¨*Ó5Ð5àÐrR   c                 ó  — |j                   si S |j                  «       5   t        | d«      «       }d d d «       t        j	                  «       «      D ]w  }t        | |«      }t        |«      dk(  s	J ||f«       ‚t        t        |«      «      }||k7  sŒ@dt        fd„} |||«      st        d|› d|› �«      ‚|j                  |«      ||<   Œy |j                  rgi }|j	                  «       D ]P  }|j                  D ]?  }|j                  |«      sŒ|j                  r	||   ||<   Œ*|t        |«      d  }	||   ||	<   ŒA ŒR |}|j                  rI| j!                  «       D ]6  \  }}
|
j"                  rŒt        | |«      }|D ]  }|j                  |«       Œ Œ8 t        |j%                  «       «      D ]9  \  }}t'        j(                  |«      sŒ|j*                  sŒ)|j                  |«       Œ; t-        ||«      S # 1 sw Y   �ŒÆxY w)NrÙ   ro   rj   c                 ó  — t        |«      t        | «      k\  ry|j                  d«      }| j                  d«      }d}t        |«      D ]:  \  }}|||   k(  r'|dz  }|t        |«      k(  sŒ"|t        |«      dz
  k(  c S |dv rŒ: y y)NFrm   r   ro   )rn   rp   T)rx   rr   rs   )rÒ   rŒ   Ú	fqn_splitÚ	key_splitÚfqn_idxÚkey_idxÚkey_names          r>   Úverifyz%_get_model_state_dict.<locals>.verifyë  s—   € Ü�s“8œs 3›xÒ'Ø ØŸI™I c›N�	ØŸI™I c›N�	Ø�Ü)2°9Ó)=ò %Ñ%�G˜XØ 9¨WÑ#5Ò5Ø 1™˜Ø"¤c¨)£nÓ4Ø#*¬c°)«n¸qÑ.@Ñ#@Ò@Ø!Ð%<Ñ<Ø á$ð%ð rR   zAn unexpected key, z, exists. FQN is )rY   r[   rØ   rc   rÐ   rŽ   rx   ÚnextÚiterrN   r„   ÚpoprX   Ú
startswithrD   rC   r›   Úrequires_gradr½   r^   Ú	is_tensorÚis_metarÞ   )rf   rÌ   rÙ   rÒ   rÆ   rŒ   ræ   Únew_state_dictrŠ   rŸ   rÅ   Úps               r>   Ú_get_model_state_dictrð   Ù  s  € ð ×ÒØˆ	à	×	Ñ	Ó	ñ ;Ø8”^ E¨<Ó8Ó:ˆ
÷;ô �J—O‘OÓ%Ó&ò 2ˆÜ˜ Ó$ˆÜ�4‹y˜AŠ~Ð*  T˜{Ó*ˆ~Ü”4˜“:ÓˆØ�#‹:ð¤Dó ñ" ˜#˜sÔ#Ü"Ð%8¸¸Ð=NÈsÈeÐ#TÓUÐUØ(Ÿn™n¨SÓ1ˆJ�sŠOð72ð: ×ÒØ/1ˆà—?‘?Ó$ò 	>ˆCØ×1Ñ1ò >�Ø—~‘~ fÔ-ØØ×/Ò/Ø*4°S©/�N 3Ò'à!¤# f£+ -Ð0�GØ.8¸©o�N 7Ò+ñ>ð	>ð $ˆ
à× Ò Ø×0Ñ0Ó2ò 	$‰JˆC�Ø×"Ò"ØÜ˜U CÓ(ˆDØò $�Ø—‘˜sÕ#ñ$ð		$ô �z×'Ñ'Ó)Ó*ò  ‰ˆˆQÜ�?‰?˜1Õ !§)£)Ø�N‰N˜3Õð ô )¨°TÓ:Ð:÷u;ñ ;ús   ŸG5Ç5G?c           	      ó°  — |j                   r|s|j                  st        i i «      S i }t        | |j                  «      D ]«  \  }}t        | ||j                  «      }t        | ||j                  dd¬«      }t        ||«      D ]f  \  }}	|j                  rt        j                  «       dk(  r9||	k7  r4|j                  |d «      }
|
€|j                  rt        d|› d�«      ‚|
||	<   |||	<   Œh Œ­ d}|j                  s|j                  �r}t        «       }|j                  «       D ]J  \  }}t        j                   |«      sŒ|j#                  «       dkD  sŒ0|j%                  |j&                  «       ŒL t        j&                  d«      |v r&|j)                  t        j&                  d«      «       d}t+        |«      dk(  r.|j%                  t        j,                  j/                  «       «       nt+        |«      dkD  rt1        d	«      ‚|j                  r3t3        |||j                  «       |j                  |j4                  ¬
«       n(|j                  rt7        |||j                  «       ¬«       |j                  «       D ]
  \  }}|||<   Œ |j9                  «       5  t;        t         t=        | d«      ||j                  |¬«      «      cd d d «       S # 1 sw Y   y xY w)NF)rh   ri   r   zMissing key: rm   ÚmetaTro   zMultiple devices found)ÚdevicerE   rB   ©ró   Úload_state_dict)rÙ   rE   Úassign)rY   rF   r"   r¢   rI   rŽ   ÚziprÎ   rÏ   ré   rE   r„   rA   r`   r½   r^   rì   Údimr—   ró   Úremoverx   Údistributed_c10dÚ_get_pg_default_devicer¿   r   rB   r   r[   r   rØ   )rf   rÙ   rÌ   Úlocal_state_dictrÒ   ÚvaluerÆ   Úfqns_with_prefixrŒ   Úfqn_with_prefixÚ
load_valuerö   ÚdevicesÚlocal_states                 r>   Ú_load_model_state_dictr    sŽ  € ð ×Ò¡Z¸×8QÒ8QÜ   RÓ(Ð(àÐÜ0°¸×8NÑ8NÓOò 6‰
ˆˆUÜ˜  T×%;Ñ%;Ó<ˆÜ$ØØØ×"Ñ"Ø!Ø!&ô
Ðô %(¨Ð.>Ó$?ò 
	6Ñ ˆC�à×-Ò-´·±³ÀAÒ1EØ˜Ò(Ø'Ÿ^™^¨C°Ó6�
ØÐ%Ø—{’{Ü*¨]¸3¸%¸qÐ+AÓBÐBà2<�J˜Ñ/Ø05Ð˜_Ò-ñ
	6ð6ð, €FØ× Ò  D×$8Ó$8Ü“%ˆØ*×0Ñ0Ó2ò 	*‰JˆC�Ü�‰˜uÕ%¨%¯)©)«+¸«/Ø—‘˜EŸL™LÕ)ð	*ô
 �<‰<˜Ó 7Ñ*Ø�N‰Nœ5Ÿ<™<¨Ó/Ô0ØˆFÜˆw‹<˜1ÒØ�K‰Kœ×-Ñ-×DÑDÓFÕGÜ�‹\˜AÒÜÐ5Ó6Ð6à×$Ò$Ü!ØØ Ø—{‘{“}Ø—{‘{Ø ×,Ñ,öð ×!Ò!Ü" :Ð/?ÈÏÉËÕVØ 0× 6Ñ 6Ó 8ò 	*ÑˆC�Ø)ˆJ�sŠOð	*ð 
×	Ñ	Ó	ñ 
ÜÜØ4ŒN˜5Ð"3Ó4Ø%¨d¯k©kÀ&ôó
÷
÷ 
ò 
ús   Ê-KËKÚoptimc                 óh  — | j                   ry| j                  D ]  }|t           D ]  }|j                  €Œ  y Œ! | j                  D ]7  }|t           D ])  }|j                  sŒt        j                  |«      |_        Œ+ Œ9 g }| j                  D ]R  }d|v sŒ|j                  |d   «       t        |d   t
        j                  «      rt        j                  d«      nd|d<   ŒT | j                  d¬«       | j                  D ]  }d|v sŒ|j                  d«      |d<   Œ | j                  d¬«       y)zH
    Initialize optim states by calling the step() with zero grads.
    NÚlrg        )Úclosurer   T)Úset_to_none)r5   r3   Ú_PARAMSÚgradrë   r^   Ú
zeros_likerv   rt   r_   ÚtensorÚstepré   Ú	zero_grad)r  Úparam_grouprÅ   Úlrss       r>   Ú_init_optim_stater  c  sB  € ð ‡{‚{àð ×)Ñ)ò ˆØ ¤Ñ)ò 	ˆEØ�z‰zÑ%Úñ	ðð
 ×)Ñ)ò 5ˆØ ¤Ñ)ò 	5ˆEØ×"Ó"Ü"×-Ñ-¨eÓ4�•
ñ	5ð5ð €CØ×)Ñ)ò ˆØ�;ÒØ�J‰J�{ 4Ñ(Ô)ô ˜k¨$Ñ/´·±Ô>ô —‘˜SÔ!àð ˜Òðð 
‡J�J�t€JÔð ×)Ñ)ò +ˆØ�;ÒØ #§¡¨£
ˆK˜Òð+ð 
‡O�O €OÕ%rR   c           
      óÔ  — d„ }i }t        t        | t           «      j                  «       D ]D  \  }}t        t        |«      j                  «       D ]  \  }} ||«       ||t        › d|› d|› �<   Œ  ŒF t        t        | t
           «      D ]\  }|j                  t        «      }t        t        t           |«      D ]+  }|j                  «       D ]  \  }}||t
        › d|› d|› �<   Œ Œ- Œ^ |S )aI  
    This API flattens the optimizer state_dict to support optimizer resharding for
    MPMD, e.g., pipeline parallelism.

    Without the API, the original optimizer state_dict looks like:
    {
        "state": {
            "layer1.weight": {
                "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor
            },
            "layer2.weight": {
                "step": 10, "exp_avg": SomeTensor, "exp_avg_sq": SomeTensor
            },
        },
        "param_group": [
            {
                "lr": 0.0,
                "betas": (0.9, 0.95), ...,
                "params": ["layer1.weight", "layer2.weight"]
            }
        ]
    }

    With this API, the optimizer state_dict looks like:
    {
        "state.layer1.weight.step": 10,
        "state.layer2.weight.step": 10,
        "state.layer1.weight.exp_avg": SomeTensor,
        "state.layer2.weight.exp_avg": SomeTensor,
        "state.layer1.weight.exp_avg_sq": SomeTensor,
        "state.layer2.weight.exp_avg_sq": SomeTensor,
        "param_group.layer1.weight.lr" : 0.1,
        "param_group.layer2.weight.lr" : 0.1,
        "param_group.layer1.weight.betas" : (0.9, 0.95),
        "param_group.layer2.weight.betas" : (0.9, 0.95),
    }

    Note that if any of the value is a container, like the betas in the example,
    this API won't flattent it.
    c                 ó~   — t        | t        j                  t        t        f«      st        dt        | «      › d�«      ‚y )NzUFlattening optimizer state_dict only supports tensor, int, float states now. Type is rm   )rt   r^   r_   ÚintÚfloatÚNotImplementedErrorÚtype)Úvs    r>   Ú_raise_if_type_not_supportedz?_flatten_optim_state_dict.<locals>._raise_if_type_not_supported·  s>   € Ü˜!œeŸl™l¬C´Ð7Ô8Ü%ðä ›7˜) 1ð&óð ð 9rR   rm   )
r   r(   Ú_STATEr½   r)   Ú_PGré   r	  rc   rP   )	rÙ   r  ÚretrŒ   r5   Úkr  r  rÆ   s	            r>   Ú_flatten_optim_state_dictr  �  s   € òTð !#€CÜœ=¨*´VÑ*<Ó=×CÑCÓEò +‰
ˆˆUÜœ¨Ó.×4Ñ4Ó6ò 	+‰DˆAˆqÙ(¨Ô+Ø)*ˆC”6�(˜!˜C˜5  ! Ð%Ò&ñ	+ð+ô
 Ô-¨z¼#©Ó?ò ,ˆØ�‰œwÓ'ˆÜœœS™	 4Ó(ò 	,ˆCØ#×)Ñ)Ó+ò ,‘��1Ø*+�”s�e˜1˜S˜E  1 #Ð&Ò'ñ,ñ	,ð,ð
 €JrR   c                 ó€  — i }g }t         |t        |i}| j                  D �]›  }|j                  t        g i«       |t           D ]è  }|j
                  |   D ]Ô  }||j                  v r7d}	|j                  «       D ]!  }
|
t        k(  rŒt        › d|› d|
› �}||v rd}	 n nd}	|	sŒM|d   t           }t        |t        «      sJ ‚|j                  |«       |j                  sŒ‰i ||<   | j                  |   j                  «       D ]'  }|t         › d|› d|› �   t        t        ||   «      |<   Œ) ŒÖ Œê t        t        t           |d   t           «      d   }|j                  «       D ]V  }
|
t        k(  rŒ|t        › d|› d|
› �   }|
|d   vr	||d   |
<   Œ.|d   |
   |k7  sŒ:t        d|› d|
› d|› d|d   |
   › d�	«      ‚ �Œž |S )	zœ
    This API unflattens the state_dict generated by _flatten_optim_state_dict().
    See the docstring of _flatten_optim_state_dict() for more detail.
    Frm   Tr”   r   zaAll the parameters in the same parameter group should have the same saved param_group value. But z is z while other(s) is )r  r  r3   rv   r	  rV   rW   rÐ   rt   rc   rë   r5   r   r(   rP   r„   )r  rÙ   rÌ   r5   Úpg_stateÚ
return_osdr  rÅ   rŒ   Ú	in_paramsr  Úflatten_keyr4   Ú
state_nameÚfirst_param_fqnrý   s                   r>   Ú_unflatten_optim_state_dictr&  Í  s:  € ð €EØ"$€HÜ&,¨e´S¸(Ð%C€Jà×)Ñ)ó -ˆØ�‰œ "˜Ô&Ø ¤Ñ)ò 	ˆEØ×-Ñ-¨eÑ4ò �ð ˜$×4Ñ4Ñ4Ø %�IØ(×-Ñ-Ó/ò ˜Ø¤š<Ø$Ü),¨¨Q¨s¨e°1°Q°CÐ&8˜Ø&¨*Ñ4Ø(,˜IÙñð !%�Iá Øà! "™¤gÑ.�Ü! &¬$Ô/Ð/Ð/Ø—‘˜cÔ"Ø×*Ò*ØØ��c‘
Ø"'§+¡+¨eÑ"4×"9Ñ"9Ó";ò �JØBLÜ!˜( ! C 5¨¨*¨Ð6ñC”Dœ¨¨c©
Ó3°JÒ?ññ3ð	ô> œt¤C™y¨(°2©,´wÑ*?Ó@ÀÑCˆØ×!Ñ!Ó#ò 	ˆAØ”GŠ|ØØ¤#  a¨Ð'8¸¸!¸Ð=Ñ>ˆEØ˜ ™Ñ$Ø"'�˜‘˜Q’Ø˜"‘˜a‘ EÓ)Ü"ð=Ø=LÐ<MÈQÈqÈcð RØ˜Ð 3°H¸R±LÀ±OÐ3DÀAðGóð ò	ðE-ð^ ÐrR   Ú
optimizersc                 óâ  — |j                   si S t        i t        g i}|D �]†  }t        |«        t	        |d«      «       }|j
                  rÎ|j                  «       5  t        j                  | ||«      }d d d «       |sŒ_t        |t           j                  «       «      D ]9  }d|v sŒ|t           j                  |«      |t           |j                  dd«      <   Œ; |t           D ]1  }|t           D �cg c]  }|j                  dd«      ‘Œ }}||t        <   Œ3 �n/t        t        j                  d„ |j                   D «       «      «      }t#        t%        |t'        t)        |«      «      «      «      }	i }
| j+                  «       D ]I  \  }}t-        | |«      }t)        |«      dk(  sJ ‚t/        t1        |«      «      }||	vrŒ;|	|   }||
|<   ||
|<   ŒK t        |t           j                  «       «      D ])  }|
|   }|t           j                  |«      |t           |<   Œ+ |t           D ]#  }|t           D �cg c]  }|
|   ‘Œ	 c}|t        <   Œ% |s�Œ-t3        t4        |t           «      j7                  |t           «       t3        t8        |t           «      j;                  |t           «       �Œ‰ |j<                  rt3        t>        tA        |«      «      }tC        ||«      S # 1 sw Y   �ŒixY wc c}w c c}w )NrÙ   rp   z
_orig_mod.rl   c              3   ó.   K  — | ]  }|t            –— Œ y ­wr8   )r	  )r©   Úgs     r>   rª   z(_get_optim_state_dict.<locals>.<genexpr>*  s   è ø€ Ò-U¸Q¨a´­jÑ-Uùs   ‚ro   )"rZ   r  r  r  rØ   r\   r[   rw   rË   rc   rÐ   ré   rq   r	  r   Úfrom_iterabler3   r]   r÷   Úrangerx   r›   rŽ   rç   rè   r   r(   r»   r)   ÚextendrG   r*   r  rÞ   )rf   r'  rÌ   rË   r  Úosdr  r*  r4   Úparam_pid_mappingÚfqn_pid_mappingrÒ   rÅ   rÆ   rŒ   ÚpidÚgroups                    r>   Ú_get_optim_state_dictr3    sØ  € ð ×ÒØˆ	ä,2°B¼¸RÐ+@ÐØó ,HˆÜ˜%Ô Ø1Œn˜U LÓ1Ó3ˆØ×ÒØ×"Ñ"Ó$ñ ?Ü×+Ñ+¨E°5¸#Ó>�÷?ñ ØÜ˜#œf™+×*Ñ*Ó,Ó-ò R�Ø !Ò#Ø?BÄ6¹{¿¹ÈqÓ?Q�Cœ‘K §	¡	¨,¸Ó ;Ò<ðRð œ‘Xò $�Ø?@Ä¹zÖJ¸!˜!Ÿ)™) L°"Õ5ÐJ�ÐJØ#�”'’
ò$ô œ%×-Ñ-Ñ-UÀ%×BTÑBTÔ-UÓUÓVˆFÜ $¤S¨´´s¸6³{Ó1CÓ%DÓ EÐØ ˆOØ#×4Ñ4Ó6ò +‘
��UÜ  ¨Ó,�Ü˜4“y A’~Ð%�~Üœ4 ›:Ó&�ØÐ 1Ñ1ØØ'¨Ñ.�Ø'*� Ñ$Ø'*� Ò$ð+ô ˜C¤™K×,Ñ,Ó.Ó/ò 8�Ø% cÑ*�Ø#&¤v¡;§?¡?°3Ó#7�”F‘˜CÒ ð8ð œS™ò R�ØBGÌÁ.Ö!Q¸3 /°#Ó"6Ò!Q�”g’ðRñ ÙäŒ]Ð,¬VÑ4Ó5×<Ñ<¸SÄ¹[ÔIÜÔÐ 0´Ñ 5Ó6×=Ñ=¸cÄ#¹hÖGðY,Hð\ ×(Ò(ÜÜÔ 9Ð:JÓ Kó
Ðô )Ð)9¸4Ó@Ð@÷_?ñ ?üò Kùò* "Rs   ÁKÃ1K'È0K,ËK$	c           
      óÞ  — i }g }t         |t        |i}i }t        d„ t        t        |t            «      j                  «       D «       «      r|S |j                  D �]*  }|j                  t        g i«       |t           D �]%  }	|j                  |	   D �]  }
|
|j                  v rCd}t        t        |t           «      D ]&  }|
t        t        t           |t           «      v sŒ$d} n nd}|sŒZ|d   t           }t        |t        «      sJ ‚|j                  |
«       |	j                  rt        t        |t            «      |
   ||
<   t        t        |t           «      D ]D  }|
t        t        t           |t           «      v sŒ$t!        |t           «      dz
  |t#        |«      <   ŒF �Œ �Œ( t!        |t           «      dk(  s�Œdg }t        t        |t           «      D ]>  }t!        t        t        t           |t           «      «      dk(  sŒ.|j                  |«       Œ@ t!        |«      dk7  rt%        d«      ‚t!        |t           «      t!        |j                  «      k7  rt%        d«      ‚t!        |t           «      dz
  |t#        «      <   �Œ- t        t        |t           «      D ]M  }|j'                  t#        |«      d«      }|dk(  rŒ$|j)                  «       D ]  \  }}|t        k(  rŒ|||   |<   Œ ŒO |S )	að  
    Extract the corresponding optim state_dict from ``optim_state_dict`` for
    ``optim`` and return the result optim state_dict.

    Args:
        model (nn.Module): the root model.
        optim (torch.optim.Optimizer): the optimizer.
        optim_state_dict (Dict[str, ValueType]): the superset optim state_dict that
            contains the optim state_dict of ``optim``.
        info (_StateDictInfo): state dict information.

    Returns:
        The optim state_dict of ``optim``.
    c              3   ó<   K  — | ]  }t        |t        «      –— Œ y ­wr8   )rt   r  )r©   r  s     r>   rª   z*_split_optim_state_dict.<locals>.<genexpr>f  s   è ø€ ò Ø Œ
�1”c×ñùs   ‚FTr”   ro   r   zÛThere are param groups that have zero parameters. In such a case, DSD only support exactly one param group with zero parameters.But the loaded state_dict has zero or more than one param groups that have zero parameters.z`When there is a parameter group that has zero parameters, multiple optimizers are not supported.)r  r  Úallr   r(   rÐ   r3   rv   r	  rV   rW   r)   rc   rP   rt   rë   rx   Úidr¿   r�   r½   )rf   r  rË   rÌ   r5   r   r!  Ú
pg_mappingr  rÅ   rŒ   r"  Úloaded_param_groupr4   r  Úpg_idxrÒ   rý   s                     r>   Ú_split_optim_state_dictr;  L  s	  € ð* €EØ"$€HÜ&,¨e´S¸(Ð%C€JØ!#€Jä
ñ Ü$(¬Ð8HÌÑ8PÓ$Q×$VÑ$VÓ$Xôô ð  Ðà×)Ñ)ó /JˆØ�‰œ "˜Ô&Ø ¤Ñ)ó 	VˆEØ×-Ñ-¨eÑ4ó V�Ø˜$×4Ñ4Ñ4Ø %�IÜ.2Ü)Ð+;¼CÑ+@ó/ò "Ð*ð ¤$¤t¬C¡yÐ2DÄWÑ2MÓ"NÒNØ(,˜IÙ!ñ"ð !%�IÙ Øà! "™¤gÑ.�Ü! &¬$Ô/Ð/Ð/Ø—‘˜cÔ"Ø×&Ò&Ü!%¤mÐ5EÄfÑ5MÓ!NÈsÑ!S�E˜#‘JÜ*.Ü%Ð'7¼Ñ'<ó+ò VÐ&ð œd¤4¬¡9Ð.@ÄÑ.IÓJÒJÜ=@ÀÌCÁÓ=QÐTUÑ=U˜
¤2Ð&8Ó#9Ò:ò	Vò'Vð	Vô4 ˆ{œ7Ñ#Ó$¨Ô)àˆCÜ&*Ô+<Ð>NÌsÑ>SÓ&Tò 3Ð"Ü”tœD¤™IÐ'9¼'Ñ'BÓCÓDÈÓIØ—J‘JÐ1Õ2ð3ô �3‹x˜1Š}Ü ð1óð ô Ð#¤CÑ(Ó)¬S°×1CÑ1CÓ-DÒDÜ ð=óð ô 25°ZÄ±_Ó1EÈÑ1IˆJ”rÐ,Ó-Ó.ð_/Jôb Ô-Ð/?ÄÑ/DÓEò 	*ˆØ—‘¤ ;£°Ó4ˆØ�RŠ<Øà%×+Ñ+Ó-ò 	*‰JˆC�Ø”gŠ~Øà$)ˆH�VÑ˜SÒ!ñ		*ð	*ð ÐrR   c           
      óî  ‡— |j                   sy |D �]Ð  }t        |«       |r@t        |v rt        | |||«      }n+t	        |t        t        t        t        f   |«      |«      }ni }|j                  �rT| j                  «       D �]  \  }}t        | |«      }t        | |d¬«      }	||	k(  rŒ't        |«      dk(  sJ ‚|j                  «       }
|	j                  «       }|t           D ]N  }t        t        t        t        f   |«      }|t            D �cg c]  }|j#                  |
|«      ‘Œ }}||t         <   ŒP t        t$        |t           «      }t'        |j)                  «       «      D ]+  }|
|v sŒ|j                  |«      ||j#                  |
|«      <   Œ- �Œ |j+                  «       5  t-        j.                  | ||«      }d d d «       �n
|j0                  rþd|_        t3        | |f|«      }d|_        d Šˆfd„}t5        t6        j8                  ||«      }‰€J ‚t;        |«      \  }}t;        |«      \  }}|j<                  rt?        ||‰¬«       ntA        ||‰¬«       |j)                  «       D ]  }||vsŒ||v sJ ‚||   ||<   ||   ||<   Œ tC        ||«      }|t           D ]/  }t         |vsŒg t        t        t        t        f   |«      t         <   Œ1  tE        |d«      |¬«       �ŒÓ y c c}w # 1 sw Y   Œ%xY w)	NF)ri   ro   Tc                 ó‚   •— | j                  «       dkD  r*‰€| j                  Š| S ‰| j                  k7  rt        d«      ‚| S )Nr   zDevice mismatch)rø   ró   r¿   )Útró   s    €r>   Ú_devicez'_load_optim_state_dict.<locals>._deviceä  sD   ø€ Ø—5‘5“7˜Q’;à�~Ø!"§¡˜ð �ð   1§8¡8Ò+Ü(Ð):Ó;Ð;Ø�rR   rô   rõ   )rÙ   )#rZ   r  r  r;  r&  r   r]   rP   r'   r\   r›   rŽ   rx   ré   r  r	   r	  rq   r(   rc   rÐ   r[   rw   Úoptim_state_dict_to_loadrA   r3  r$   r^   r_   r   rF   r   r   r   rØ   )rf   r'  rÙ   rÌ   r  rË   Úoriginal_fqnÚ_rÆ   Úfqns_with_compilerrŒ   Úfqn_with_compilerr*  ÚvalrÒ   r4   Ú	osd_stater  rü   r?  Úflatten_osdÚosd_mappingÚflatten_local_osdÚlocal_osd_mappingÚ	optim_keyÚpgró   s                             @r>   Ú_load_optim_state_dictrM  ª  sC  ø€ ð ×ÒØàó TNˆÜ˜%Ô ÙÜ˜Ñ#Ü#:Ø˜5 *¨dó$Ñ ô $?Øœ4¤¤S¬) ^Ñ 4°jÓAÀ4ó$Ñ ð  "ÐØ×Óð $)×#9Ñ#9Ó#;ó X‘�˜aÜ  ¨Ó5�Ü%.Ø˜<¸eô&Ð"ð Ð-Ò-Øä˜4“y A’~Ð%�~Ø—h‘h“j�Ø$6×$:Ñ$:Ó$<Ð!Ø)¬#Ñ.ò *�AÜœt¤C¬ H™~¨qÓ1�CàGJÌ7Á|öØ@C˜Ÿ™ CÐ):Õ;ð�Fð ð $*�Cœ’Lð*ô !¤Ð0@ÄÑ0HÓI�	Ü˜iŸn™nÓ.Ó/ò X�AØ˜a’xØGPÇ}Á}ÐUVÓGW˜	 !§)¡)¨CÐ1BÓ"CÒDòXð%Xð, ×"Ñ"Ó$ñ Ü#'×#@Ñ#@Ø˜5Ð"2ó$Ð ÷ñ ð ×!Ò!Ø#(ˆDÔ Ü4°U¸U¸HÀdÓKÐØ#'ˆDÔ ØˆFôô œeŸl™l¨GÐ5EÓFˆAØÐ%Ð%Ð%Ü':Ð;KÓ'LÑ$ˆK˜Ü3FÐGWÓ3XÑ0ÐÐ0Ø×(Ò(Ü% kÐ3DÈVÖTä& {Ð4EÈfÕUð
 )×-Ñ-Ó/ò J�	ØÐ$5Ò5Ø$¨Ñ3Ð3Ð3Ø3>¸yÑ3IÐ% iÑ0Ø3>¸yÑ3IÐ% iÒ0ð	Jô
  5Ø!Ð#4ó Ðð '¤sÑ+ò A�Ü "Ò$Ø>@”Dœœc¤9˜nÑ-¨rÓ2´7Ò;ðAð 	1Œ�uÐ/Ó0Ð<L×MñiTNùò:÷ð ús   ÄK&
Æ#K+Ë+K4	c                ó˜   — t        «       5  t        | dd||¬«      }t        | |«      }t        |i |«       |cddd«       S # 1 sw Y   yxY w)aH  
    Return the model state_dict of ``model``.

    See ``get_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        The state_dict for ``model``.

    :rtype: typing.Dict[str, ValueType]
    rQ   F©r¦   r£   r¤   N)r?   rÉ   rð   rÓ   )rf   r£   r¤   rÌ   rÊ   s        r>   r,   r,     sV   € ô0 
‹ñ 
 ÜØØØØ!Øô
ˆô 1°¸Ó=ÐÜÐ+¨R°Ô6Ø÷
 ÷ 
 ò 
 ús   ‹+A Á A	c                óþ   — t        «       5  t        |t        j                  j                  «      r|fn
t        |«      }t        | |d||¬«      }t        | ||«      }t        i ||«       |cddd«       S # 1 sw Y   yxY w)aË  
    Return the combined state_dict for optimizers.

    See ``get_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[None, Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        The state_dict for ``optimizers``.

    :rtype: OptimizerStateType
    TrO  N)	r?   rt   r^   r  Ú	OptimizerÚtuplerÉ   r3  rÓ   )rf   r'  r£   r¤   rÌ   rË   s         r>   r-   r-   0  s„   € ô6 
‹ñ  ô ˜*¤e§k¡k×&;Ñ&;Ô<ð ‰Mä�zÓ"ð 	ô
 ØØØØ!Øô
ˆô 1°¸
ÀDÓIÐÜ˜2Ð/°Ô6Ø÷ ÷  ò  ús   ‹AA3Á3A<c                ó  — t        «       5  t        |t        j                  j                  «      r|fn
t        |«      }t        | |d||¬«      }t        | |«      }t        | ||«      }t        |||«       ||fcddd«       S # 1 sw Y   yxY w)aô  
    Return the model state_dict and optimizers state_dict.

    ``get_state_dict`` can process any module that is parallelized by PyTorch
    FSDP/fully_shard, DDP/replicate, tensor_parallel/parallelize_module, and any
    combination of these parallelisms. The main functions of ``get_state_dict``
    are: 1.) returning a model and optimizer state_dict that can be resharded
    with a different number of trainers and/or different parallelisms.
    2.) hiding the parallelism-specific state_dict APIs. Users don't have to call
    these APIs.
    3.) sanity checking the result state_dict.

    The keys of the result state dictionary are the canonical FQNs (Fully
    Qualified Names).  A canonical FQN refers to the FQN based on a parameter's
    position in an nn.Module hierarchy. More specifically, a canonical FQN to a
    parameter is the FQN returned by ``module.named_parameters()`` or
    ``module.named_buffers()`` when the module is not distributed by any
    parallelisms. Since the optimizer internally uses parameter IDs to represent
    a parameter, there will be a conversion from the parameter IDs to the
    canonical FQNs when calling this API.

    ``get_state_dict`` can also process a module that is not parallelized. In
    such a case, ``get_state_dict`` only performs one function -- converting the
    optimizer parameter IDs to the canonical FQNs.

    Example:
        >>> # xdoctest: +SKIP
        >>> import torch
        >>> from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        >>> from torch.nn.parallel import DistributedDataParallel as DDP
        >>> from torch.distributed.checkpoint.state_dict import get_state_dict

        >>> fsdp_model = FSDP(copy.deepcopy(model))
        >>> fsdp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)
        >>> ddp_model = DDP(copy.deepcopy(model))
        >>> ddp_optim = torch.optim.Adam(model.parameters(), lr=1e-3)


        >>> ddp_state_dict, ddp_optim_state_dict = get_state_dict(ddp_model, ddp_optim)
        >>> fsdp_state_dict, fsdp_optim_state_dict = get_state_dict(
        ...     fsdp_model, fsdp_optim
        ... )

        >>> # if we simply call ddp_model.state_dict() and fsdp_model.state_dict(),
        >>> # the asserts will fail.
        >>> assert ddp_state_dict == fsdp_state_dict
        >>> assert ddp_optim_state == fsdp_optim_state_dict


    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[None, Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        submodules (deprecated): Optional[set[nn.Module]]: only return the model parameters
            that belong to the submodules.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be returned. See
            `StateDictOptions` for the details.

    Returns:
        ``Tuple`` that contain model state_dict and optimizer state_dict.

    :rtype: typing.Tuple[typing.Dict[str, ValueType], OptimizerStateType]
    FrO  N)
r?   rt   r^   r  rQ  rR  rÉ   rð   r3  rÓ   )rf   r'  r£   r¤   rÌ   rÊ   rË   s          r>   r.   r.   ]  s™   € ôP 
‹ñ 2ô ˜*¤e§k¡k×&;Ñ&;Ô<ð ‰Mä�zÓ"ð 	ô
 ØØØØ!Øô
ˆô 1°¸Ó=ÐÜ0°¸
ÀDÓIÐÜÐ+Ð-=¸tÔDØÐ!1Ð1÷!2÷ 2ò 2ús   ‹A,BÂB
c           
      ó°  — |si S t        t        t        |j                  «       «      «      t        j
                  «      röt        j                  dt        «       t        t        t        j
                  t        t        t        f   f   |«      }i }|j                  «       D ]‘  \  }}| j                  «       D ]y  \  }}||k7  rŒt        | |«      }t!        |«      dk(  sJ d«       ‚t        t        |«      «      › d�}	|j#                  |j                  «       D �
�ci c]  \  }
}|	|
z   |“Œ c}}
«       Œ{ Œ“ |S t        t        t        t        f   |«      S c c}}
w )NzÑPassing model_state_dict as a ``Dict[nn.Module, Dict[str, Any]]``is deprecated and will be removed in 2.5. If you need this feature, please preprocessing the model_state_dict to achieve the same functionality.ro   z/FQNs for a submodule should only have 1 elementrm   )rt   rç   rè   rÐ   rd   re   rµ   rº   r¸   r   r]   rP   r'   r½   r¾   rŽ   rx   r»   )rf   rÙ   Úcast_state_dictrî   rž   Úsub_state_dictrg   ÚmrÆ   rŠ   Úsubfqnrý   s               r>   Ú_unflatten_model_state_dictrY  ¸  s<  € ñ Øˆ	ä”$”t˜JŸO™OÓ-Ó.Ó/´·±Ô;Ü�‰ð"ô ô	
ô œt¤B§I¡I¬t´C¼°NÑ/CÐ$CÑDÀjÓQˆØ/1ˆØ)8×)>Ñ)>Ó)@ò 
	Ñ%ˆI�~Ø ×.Ñ.Ó0ò 	‘��aØ˜	’>Øä  ¨Ó-�Ü˜4“y A’~ÐXÐ'XÓX�~Ü ¤ d£Ó,Ð-¨QÐ/�Ø×%Ñ%ØAO×AUÑAUÓAW×X±°¸�V˜f‘_ eÑ+ÓXõñ	ð
	ð Ðä”Dœœi˜Ñ(¨*Ó5Ð5ùó	 Ys   ÄE)r¤   c                ó¬   — t        | |«      }t        «       5  t        | dd|¬«      }t        |i |«       t	        | ||«      cddd«       S # 1 sw Y   yxY w)a=  Load the model state_dict.

    The counterpart of ``get_model_state_dict`` to set the state_dict to the
    model. See ``set_state_dict`` for the detail usage.

    Args:
        model (nn.Module): the nn.Module to the model.
        model_state_dict: (Dict[str, ValueType]):
           the model state_dict to load. If the key of the ``model_state_dict``
           is nn.Module, the key is a submodule of ``model`` and the value should
           be the state_dict of the submodule. When loading the state_dict,
           the prefix of the submodule will be append to the state_dict.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
            * **missing_keys** is a list of str containing the missing keys
            * **unexpected_keys** is a list of str containing the unexpected keys

    :type model_state_dict: typing.Dict[str, ValueType]
    rQ   F©r¦   r¤   N)rY  r?   rÉ   rÓ   r  )rf   rÊ   r¤   rÌ   s       r>   r/   r/   Ù  s`   € ô: .IØÐó.Ðô 
‹ñ EÜ˜u b°UÀGÔLˆäÐ+¨R°Ô6Ü% eÐ-=¸tÓD÷	E÷ Eò Eús   —)A
Á
Ac                óú   — t        «       5  t        |t        j                  j                  «      r|fn
t        |«      }t        | |d|¬«      }t        i ||«       t        | |||«       ddd«       y# 1 sw Y   yxY w)a¬  Load the optimizers state_dict.

    The counterpart of ``get_optimizer_state_dict`` to set the state_dict to the
    optimizers. See ``set_state_dict`` for the detail usage.

    WARN: ``set_optimizer_state_dict`` can only be called before ``backward()`` or after
        ``step()`` is called on the optimizers. Otherwise, the optimizer states won't be
        initialized correctly.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        optim_state_dict: OptimizerStateType:
            the optimizer state_dict to load.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        None

    :type optim_state_dict: typing.OptimizerStateType
    Tr[  N)	r?   rt   r^   r  rQ  rR  rÉ   rÓ   rM  )rf   r'  rË   r¤   rÌ   s        r>   r0   r0      sz   € ô> 
‹ñ 	Jô ˜*¤e§k¡k×&;Ñ&;Ô<ð ‰Mä�zÓ"ð 	ô
 ˜u j¸TÈ7ÔSˆä˜2Ð/°Ô6Ü˜u jÐ2BÀDÔI÷	J÷ 	Jñ 	Jús   ‹AA1Á1A:c                ó.  — t        | |«      }t        «       5  t        |t        j                  j
                  «      r|fn
t        |«      }t        | || |¬«      }t        |||«       t        | |||«       t        | ||«      cddd«       S # 1 sw Y   yxY w)aü  Load the model state_dict and optimizers state_dict.

    The counterpart of ``get_state_dict`` to set the state_dict to the model and
    optimizers.  The given ``model_state_dict`` and ``optim_state_dict`` do not
    have to be returned by ``get_state_dict`` but must meet the following
    requirements: 1) all FQNs are canonical FQNs as defined in ``get_state_dict``,
    2) if a tensor is sharded, it must be either a ShardedTensor or DTensor,
    3) optimizer state_dict cannot contain the parameter IDs; the keys should be
    the canonical FQNs.

    WARN: ``set_state_dict`` can only be called before ``backward()`` or after ``step()``
        is called on the optimizers. Otherwise, the optimizer states won't be initialized
        correctly.

    Args:
        model (nn.Module): the nn.Module to the model.
        optimizers (Union[Optimizer, Iterable[Optimizer]]):
            The optimizers that are used to optimize ``model``.
        model_state_dict: (Union[Dict[nn.Module, Dict[str, ValueType]], Dict[str, ValueType]]):
           the model state_dict to load. If the key of the ``model_state_dict``
           is nn.Module, the key is a submodule of ``model`` and the value should
           be the state_dict of the submodule. When loading the state_dict,
           the prefix of the submodule will be append to the state_dict.
        optim_state_dict: OptimizerStateType:
            the optimizer state_dict to load.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.

    Returns:
        ``NamedTuple`` with ``missing_keys`` and ``unexpected_keys`` fields:
            * **missing_keys** is a list of str containing the missing keys of the model state_dict.
            * **unexpected_keys** is a list of str containing the unexpected keys of the model state_dict.

    :type model_state_dict: typing.Dict[str, ValueType]
    :type optim_state_dict: typing.OptimizerStateType
    r[  N)rY  r?   rt   r^   r  rQ  rR  rÉ   rÓ   rM  r  )rf   r'  rÊ   rË   r¤   rÌ   s         r>   r1   r1   +  s¦   € ô\ .IØÐó.Ðô 
‹ñ Eô ˜*¤e§k¡k×&;Ñ&;Ô<ð ‰Mä�zÓ"ð 	ô
 Ø�:Ð.>Ð*>Èô
ˆô 	Ð+Ð-=¸tÔDÜ˜u jÐ2BÀDÔIÜ% eÐ-=¸tÓD÷E÷ Eò Eús   —A*BÂBc                ó$  ‡‡— t        j                  t        | |¬«      Šˆfd„}|| _        t        j                  t        | |¬«      Šdt
        t        t        f   fˆfd„}|| _        t        j                  |«       t        j                  |«       y)aó  Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model``.

    Patch the ``state_dict`` and ``load_state_dict`` attributes of ``model`` to
    be a partial function to call ``get_state_dict`` and ``set_state_dict``.

    Example:
        from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        from torch.distributed.checkpoint.state_dict import patch_model_state_dict

        model = fsdp(model)
        patch_model_state_dict(model)

    Args:
        model (nn.Module): the nn.Module to the model.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.
    Returns:
        None
    )rf   r¤   c                  ó   •—  ‰ «       S r8   rQ   ©Ú_state_dict_calls   €r>   Ústate_dict_callz0_patch_model_state_dict.<locals>.state_dict_callŽ  ó   ø€ ÙÓ!Ð!rR   rÙ   c                 ó   •—  ‰| ¬«       y )N)rÊ   rQ   ©rÙ   Ú_load_state_dict_calls    €r>   Úload_state_dict_callz5_patch_model_state_dict.<locals>.load_state_dict_call™  ó   ø€ Ù¨zÖ:rR   N)rÃ   rÄ   r,   rÙ   r/   r]   rP   r	   rõ   r6   r—   )rf   r¤   rb  rg  rf  ra  s       @@r>   Ú_patch_model_state_dictri  m  sƒ   ù€ ô6 !×(Ñ(ÜØØôÐô"ð '€EÔä%×-Ñ-ÜØØôÐð;¬¬c´3¨h©õ ;ð 1€EÔä×Ñ˜OÔ,Ü×ÑÐ0Õ1rR   c                óš  ‡‡— t        j                  t        | ||¬«      Šˆfd„}t        j                  t        | ||¬«      Šdt        t
        t        f   fˆfd„}t        j                  |«       t        j                  |«       t        |t        j                  j                  «      r|fn
t        |«      }|D ]  }||_        ||_        Œ y)a¤  Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers``.

    Patch the ``state_dict`` and ``load_state_dict`` attributes of ``optimizers`` to
    be a partial function to call ``get_state_dict`` and ``set_state_dict``.

    Note that if there are multiple optimizers, all of the optimizers will be patched.
    So users only need to call one of the state_dict() to get the full result.

    Example:
        from torch.distributed.fsdp import FullyShardedDataParallel as FSDP
        from torch.distributed.checkpoint.state_dict import patch_model_state_dict

        model = fsdp(model)
        patch_model_state_dict(model)

    Args:
        model (nn.Module): the nn.Module to the model.
        options (StateDictOptions): the options to control how
            model state_dict and optimizer state_dict should be loaded. See
            `StateDictOptions` for the details.
    Returns:
        None
    )rf   r'  r¤   c                  ó   •—  ‰ «       S r8   rQ   r`  s   €r>   rb  z4_patch_optimizer_state_dict.<locals>.state_dict_callÊ  rc  rR   rÙ   c                 ó   •—  ‰| ¬«       y )N)rË   rQ   re  s    €r>   rg  z9_patch_optimizer_state_dict.<locals>.load_state_dict_callÔ  rh  rR   N)rÃ   rÄ   r-   r0   r]   rP   r	   r6   r—   rt   r^   r  rQ  rR  rÙ   rõ   )rf   r'  r¤   rb  rg  r  rf  ra  s         @@r>   Ú_patch_optimizer_state_dictrm  ¤  sÄ   ù€ ô> !×(Ñ(Ü ØØØô	Ðô"ô &×-Ñ-Ü ØØØô	Ðð;¬¬c´3¨h©õ ;ô ×Ñ˜OÔ,Ü×ÑÐ0Ô1ô �j¤%§+¡+×"7Ñ"7Ô8ð 
‰ä�:Óð ð
 ò 5ˆØ*ˆÔØ 4ˆÕñ5rR   )rH   TT)rH   )rra   rÃ   r9   rµ   Úcollections.abcr   r   Údataclassesr   r   r   Ú	itertoolsr   Útypingr	   r
   r   r   r   r   r^   Útorch.distributedrÜ   rÎ   Útorch.nnrd   Ú'torch.distributed._shard.sharded_tensorr   Ú#torch.distributed._state_dict_utilsr   r   r   r   r   r   Ú;torch.distributed.algorithms._checkpoint.checkpoint_wrapperr   Útorch.distributed.fsdpr   r   r   rw   r   r   r   r   r   Ú$torch.distributed.fsdp._common_utilsr   r    Útorch.distributed.tensorr!   Útorch.nn.modules.moduler"   Útorch.nn.parallelr#   ru   Útorch.utils._pytreer$   Ú__all__ry   r  r	  r  r`   rP   r%   r_   r  r  r&   rc   rR  r]   r'   r(   r)   r*   r6   rO   rÂ   r?   r+   rT   Úcachere   rN   rŽ   r�   r¢   r  rQ  rÉ   rÓ   rØ   rÞ   Úno_gradrð   r  r  r  r&  r3  r;  rM  r,   r-   r.   rY  r/   r0   r1   ri  rm  rQ   rR   r>   ú<module>r€     sž  ðä Û Û 	Û ß /ß 0Ñ 0Ý ß F× Fã Ý  Ý Ý A÷÷ õ÷	÷ 	ó 	÷õ -Ý 5Ý <Ý -ò€ð" €Ø€Ø
€Ø	€à	ˆS‰€Ø�g˜}¨e¯l©l¸CÀÈÐKÑL€ØØ�4˜Ñ&¨¨mÑ(<¸dÀ3ÈÐCSÑ>TÐTñ€	ð �S˜)�^Ñ$€Ø˜Ñ'Ð Ø˜#˜u ]Ð4EÐ%EÑFÐFÑGÐ ñ &)£UÐ �S˜‘]Ó *ð ×Ññó ðð ÷,.ð ,.ó ð,.ð^ ô@Ð%ó @ó ð@ð  ‡�ð .Ø Ø!%ñDEØ�9‰9ðDEà
ðDEð ðDEð ð	DEð
 ðDEð òDEó ðDE÷N	ñ 	ó%"ðZ ,0Ø*.òØ�9‰9ðà�%—+‘+×'Ñ'¨Ð,Ñ-ðð ðð
 ˜˜RŸY™Y™Ñ(ðð Ð&Ñ'ðð óðD*Ø˜3 	˜>Ñ*ð*à(ð*ð ð*ð 
ó	*ðZ˜˜bŸi™i¨¯©×)>Ñ)>Ð>Ñ?ð Àcð Èhó ðØ�S˜#�X‘ðØ&4ðà	ˆ#ˆsˆ(�^óð$ €‡�ƒð@;Ø�9‰9ð@;Ø*ð@;à	ˆ#ˆyˆ.Ñò@;ó ð@;ðF €‡�ƒðB
Ø�9‰9ðB
à�S˜)�^Ñ$ðB
ð ðB
ð ò	B
ó ðB
ðJ'&˜UŸ[™[×2Ñ2ð '&°tó '&ðT=Ð*<ð =ÀÀcÈ9ÀnÑAUó =ð@<Ø�;‰;× Ñ ð<à�S˜)�^Ñ$ð<ð ð<ð ó	<ð~ €‡�ƒð<AØ�9‰9ð<Aà�e—k‘k×+Ñ+¨SÐ0Ñ1ð<Að ð<Að ò	<Aó ð<Að~[Ø�9‰9ð[à�;‰;× Ñ ð[ð )ð[ð ð	[ð
 ó[ð| €‡�ƒð]NØ�9‰9ð]Nà�e—k‘k×+Ñ+¨SÐ0Ñ1ð]Nð #ð]Nð ð	]Nð
 
ò]Nó ð]NðF ,0Ø*.ò	" Ø�9‰9ð" ð ˜˜RŸY™Y™Ñ(ð" ð Ð&Ñ'ð	" ð
 
ˆ#ˆyˆ.Ñó" ðR ,0Ø*.ò* Ø�9‰9ð* à�e—k‘k×+Ñ+¨X°e·k±k×6KÑ6KÑ-LÐLÑMð* ð ˜˜RŸY™Y™Ñ(ð	* ð
 Ð&Ñ'ð* ð ó* ðb ,0Ø*.òX2Ø�9‰9ðX2à�e—k‘k×+Ñ+¨X°e·k±k×6KÑ6KÑ-LÐLÑMðX2ð ˜˜RŸY™Y™Ñ(ð	X2ð
 Ð&Ñ'ðX2ð ˆ4��Y�ÑÐ!3Ð3Ñ4óX2ðv6Ø�9‰9ð6à�d˜2Ÿ9™9 d¨3°	¨>Ñ&:Ð:Ñ;¸TÀ#ÀyÀ.Ñ=QÐQÑRð6ð 
ˆ#ˆyˆ.Ñó6ðJ +/ò	$EØ�9‰9ð$Eà˜3 	˜>Ñ*ð$Eð Ð&Ñ'ð	$Eð
 ó$EðX +/ò(JØ�9‰9ð(Jà�e—k‘k×+Ñ+¨X°e·k±k×6KÑ6KÑ-LÐLÑMð(Jð )ð(Jð
 Ð&Ñ'ð(Jð 
ó(Jðb +/ò=EØ�9‰9ð=Eà�e—k‘k×+Ñ+¨X°e·k±k×6KÑ6KÑ-LÐLÑMð=Eð ˜3 	˜>Ñ*ð	=Eð
 )ð=Eð Ð&Ñ'ð=Eð ó=EðD ð +/ò12Ø�9‰9ð12ð Ð&Ñ'ð12ð 
ò	12ó ð12ðl ð
 +/ò	;5Ø�9‰9ð;5ð �e—k‘k×+Ñ+¨SÐ0Ñ1ð;5ð Ð&Ñ'ð	;5ð
 
ò;5ó ñ;5rR   