Ë
    [^(h=: ã                   ón  — d Z ddlZddlZddlZddlmZ ddlmZ ddlm	Z	m
Z
 ddlmZmZ ddlmZmZmZmZmZmZmZmZ ddlmZ dd	lmZmZ d
dlmZ g d¢ZdZde defd„Z! G d„ d«      Z"de"fd„Z# G d„ de"«      Z$ G d„ d«      Z% G d„ de"«      Z& G d„ de"«      Z' G d„ de"«      Z( G d„ d e"«      Z) G d!„ d"e"«      Z* G d#„ d$e"«      Z+ G d%„ d&e"«      Z, G d'„ d(e"«      Z- G d)„ d*e"«      Z. G d+„ d,e"«      Z/ G d-„ d.e"«      Z0 G d/„ d0e"«      Z1 G d1„ d2e"«      Z2 G d3„ d4e"«      Z3 G d5„ d6e«      Z4 G d7„ d8e"«      Z5y)9zLearning Rate Scheduler.é    N)Úbisect_right)ÚCounter)ÚIterableÚSequence)ÚpartialÚwraps)ÚAnyÚCallableÚcastÚLiteralÚOptionalÚSupportsFloatÚ	TypedDictÚUnion)Úref)ÚinfÚTensoré   )Ú	Optimizer)ÚLambdaLRÚMultiplicativeLRÚStepLRÚMultiStepLRÚ
ConstantLRÚLinearLRÚExponentialLRÚSequentialLRÚCosineAnnealingLRÚChainedSchedulerÚReduceLROnPlateauÚCyclicLRÚCosineAnnealingWarmRestartsÚ
OneCycleLRÚPolynomialLRÚLRSchedulera–  The epoch parameter in `scheduler.step()` was not necessary and is being deprecated where possible. Please use `scheduler.step()` to step the scheduler. During the deprecation, if epoch is different from None, the closed form is used instead of the new chainable form, where available. Please open an issue if you are unable to replicate your use case: https://github.com/pytorch/pytorch/issues/new/choose.ÚnameÚ	optimizerc                 ó8  — d„ }t        |t        t        f«      rTt        |«      t        |j                  «      k7  rLt        | › d| › dt        |«      › dt        |j                  «      › d�«      ‚|gt        |j                  «      z  }t        t        ||«      «      S )z<Return correctly formatted lr/momentum for each param group.c                 óF   — t        | t        «      r| j                  «       S | S ©N)Ú
isinstancer   Úclone)Ú_params    úV/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/torch/optim/lr_scheduler.pyÚ_copyz_format_param.<locals>._copy;   s   € Ü!+¨F´FÔ!;ˆv�|‰|‹~ÐGÀÐGó    z6 must have the same length as optimizer.param_groups. ú has z values, param_groups has ú.)r+   ÚlistÚtupleÚlenÚparam_groupsÚ
ValueErrorÚmap)r&   r'   Úparamr/   s       r.   Ú_format_paramr:   8   sš   € òHô �%œ$¤˜Ô'Üˆu‹:œ˜Y×3Ñ3Ó4Ò4ÜØ�&ÐNØ�&˜œc %›j˜\Ð)CÄCÈ	×H^ÑH^ÓD_ÐC`Ð`aðcóð ð
 �œ#˜i×4Ñ4Ó5Ñ5ˆä”�E˜5Ó!Ó"Ð"r0   c                   óŽ   — e Zd ZU dZdZeed<   	 ddedefd„Z	d„ Z
d„ Zd	eeef   fd
„Zdee   fd„Zdee   fd„Zddee   fd„Zy)r%   z.Adjusts the learning rate during optimization.FÚ_get_lr_called_within_stepr'   Ú
last_epochc                 ó*  ‡ — t        |t        «      s!t        t        |«      j                  › d�«      ‚|‰ _        |dk(  rI|j                  D ]9  }|d   }t        |t        «      r|j                  «       }|j                  d|«       Œ; n0t        |j                  «      D ]  \  }}d|vsŒt        d|› d�«      ‚ |j                  D �cg c]  }|d   ‘Œ	 c}‰ _        |‰ _        dt        fˆ fd„} |‰ j
                  «       ‰ j                  «        y c c}w )	Nú is not an OptimizeréÿÿÿÿÚlrÚ
initial_lrz4param 'initial_lr' is not specified in param_groups[z] when resuming an optimizerÚoptc                 ó‚   •— t        | j                  d«      r| j                  S ˆfd„} || j                  «      | _        y )NÚ_wrapped_by_lr_schedc                 ó„   •‡‡— t        ‰j                  «      Š| j                  Št        ‰«      ˆˆfd„«       }d|_        |S )Nc                  ód   •—  ‰«       }d|_          ‰j                  ||j                  «      | i |¤ŽS ©NT)Ú_opt_calledÚ__get__Ú	__class__)ÚargsÚkwargsrC   ÚfuncÚopt_refs      €€r.   ÚwrapperzYLRScheduler.__init__.<locals>.patch_track_step_called.<locals>.wrap_step.<locals>.wrapperx   s2   ø€ á!›)�CØ&*�C”OØ;˜4Ÿ<™<¨¨S¯]©]Ó;¸TÐLÀVÑLÐLr0   T)r   r'   Ú__func__r   rE   )Ústep_fnrP   rN   rO   Úselfs     @@€r.   Ú	wrap_stepzHLRScheduler.__init__.<locals>.patch_track_step_called.<locals>.wrap_stept   sC   ú€ Ü˜dŸn™nÓ-�Ø×'Ñ'�ä�t“ôMó ðMð
 04�Ô,Ø�r0   )ÚhasattrÚstep)rC   rT   rS   s     €r.   Úpatch_track_step_calledz5LRScheduler.__init__.<locals>.patch_track_step_calledo   s2   ø€ Ü�s—x‘xÐ!7Ô8à—x‘x�ôñ ! §¡Ó*ˆC�Hr0   )r+   r   Ú	TypeErrorÚtypeÚ__name__r'   r6   r   r,   Ú
setdefaultÚ	enumerateÚKeyErrorÚbase_lrsr=   Ú_initial_step)rS   r'   r=   ÚgrouprB   ÚirW   s   `      r.   Ú__init__zLRScheduler.__init__O   s$  ø€ ô ˜)¤YÔ/Üœt I›×7Ñ7Ð8Ð8LÐMÓNÐNØ"ˆŒð ˜ÒØ"×/Ñ/ò ;�Ø" 4™[�
Ü˜j¬&Ô1Ø!+×!1Ñ!1Ó!3�JØ× Ñ  ¨zÕ:ñ	;ô & i×&<Ñ&<Ó=ò ‘��5Ø uÒ,Ü"ð+Ø+,¨#Ð-IðKóð ðð .7×-CÑ-Cö&
Ø$)ˆE�,Óò&
ˆŒð %ˆŒð
	+¬õ 	+ñ( 	  §¡Ô/Ø×ÑÕùò;&
s   ÃDc                 ó2   — d| _         | j                  «        y)z*Initialize step counts and perform a step.r   N)Ú_step_countrV   ©rS   s    r.   r_   zLRScheduler._initial_step†   s   € àˆÔØ�	‰	�r0   c                 óv   — | j                   j                  «       D ��ci c]  \  }}|dk7  sŒ||“Œ c}}S c c}}w )z¤Return the state of the scheduler as a :class:`dict`.

        It contains an entry for every variable in self.__dict__ which
        is not the optimizer.
        r'   ©Ú__dict__Úitems©rS   ÚkeyÚvalues      r.   Ú
state_dictzLRScheduler.state_dict‹   s=   € ð *.¯©×)<Ñ)<Ó)>÷
Ù%˜3 À#ÈÓBTˆC�‰Jó
ð 	
ùó 
ó   ž5¬5rm   c                 ó:   — | j                   j                  |«       y)ú³Load the scheduler's state.

        Args:
            state_dict (dict): scheduler state. Should be an object returned
                from a call to :meth:`state_dict`.
        N)rh   Úupdate©rS   rm   s     r.   Úload_state_dictzLRScheduler.load_state_dict•   s   € ð 	�‰×Ñ˜ZÕ(r0   Úreturnc                 ó   — | j                   S )z8Return last computed learning rate by current scheduler.)Ú_last_lrre   s    r.   Úget_last_lrzLRScheduler.get_last_lrž   s   € à�}‰}Ðr0   c                 ó   — t         ‚)z<Compute learning rate using chainable form of the scheduler.)ÚNotImplementedErrorre   s    r.   Úget_lrzLRScheduler.get_lr¢   s   € ä!Ð!r0   NÚepochc                 óŽ  — | j                   dk(  rlt        | j                  j                  d«      st	        j
                  dt        «       n1t        | j                  dd«      st	        j
                  dt        «       | xj                   dz  c_         t        | «      5  |€&| xj                  dz  c_	        | j                  «       }ngt	        j
                  t        t        «       || _	        t        | d«      r&t        t        t           | j                  «       «      }n| j                  «       }ddd«       t!        | j                  j"                  «      D ]2  \  }}t%        |d	   t&        «      r|d	   j)                  |«       Œ.||d	<   Œ4 | j                  j"                  D �cg c]  }|d	   ‘Œ	 c}| _        y# 1 sw Y   Œ‹xY wc c}w )
úPerform a step.r   rE   a  Seems like `optimizer.step()` has been overridden after learning rate scheduler initialization. Please, make sure to call `optimizer.step()` before `lr_scheduler.step()`. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-raterI   Fax  Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rateNÚ_get_closed_form_lrrA   )rd   rU   r'   rV   ÚwarningsÚwarnÚUserWarningÚgetattrÚ_enable_get_lr_callr=   rz   ÚEPOCH_DEPRECATION_WARNINGr   r3   Úfloatr~   Úzipr6   r+   r   Úfill_rv   )rS   r{   ÚvaluesÚparam_grouprA   r`   s         r.   rV   zLRScheduler.step¦   sw  € ð ×Ñ˜qÒ Ü˜4Ÿ>™>×.Ñ.Ð0FÔGÜ—‘ð]ô  õô ˜TŸ^™^¨]¸EÔBÜ—‘ð]ô  ôð 	×Ò˜AÑÕä  Ó&ñ 
	+Øˆ}Ø—’ 1Ñ$•ØŸ™›‘ä—‘Ô7¼ÔEØ"'�”Ü˜4Ð!6Ô7Ü!¤$¤u¡+¨t×/GÑ/GÓ/IÓJ‘Fà!Ÿ[™[›]�F÷
	+ô  # 4§>¡>×#>Ñ#>ÀÓGò 	'‰OˆK˜Ü˜+ dÑ+¬VÔ4Ø˜DÑ!×'Ñ'¨Õ+à$&�˜DÒ!ð		'ð &*§^¡^×%@Ñ%@ö&
Ø!ˆE�$‹Kò&
ˆ�÷%
	+ð 
	+üò$&
s   ÂBF6Æ!GÆ6F?©r@   r*   )rZ   Ú
__module__Ú__qualname__Ú__doc__r<   ÚboolÚ__annotations__r   Úintrb   r_   rm   ÚdictÚstrr	   rs   r3   r…   rw   rz   r   rV   © r0   r.   r%   r%   J   s{   … Ù9à',Ð Ó,ð
 ñ5àð5ð ó5ònò

ð)¨$¨s°C¨x©.ó )ð˜T %™[ó ð"˜˜U™ó "ñ/
˜( 3™-ô /
r0   r%   Úlr_schedulerc                 óV   — | j                   st        j                  dt        d¬«       y y )NzTTo get the last learning rate computed by the scheduler, please use `get_last_lr()`.é   )Ú
stacklevel)r<   r   r€   r�   )r”   s    r.   Ú_warn_get_lr_called_within_stepr˜   Ø   s'   € Ø×2Ò2Ü�‰ð*äØö		
ð 3r0   c                   ó   — e Zd Zy)Ú_LRSchedulerN)rZ   r‹   rŒ   r“   r0   r.   rš   rš   ä   s   „ Ør0   rš   c                   ó$   — e Zd Zdefd„Zd„ Zd„ Zy)rƒ   Úoc                 ó   — || _         y r*   )rœ   )rS   rœ   s     r.   rb   z_enable_get_lr_call.__init__é   s	   € Øˆ�r0   c                 ó(   — d| j                   _        | S rH   ©rœ   r<   re   s    r.   Ú	__enter__z_enable_get_lr_call.__enter__ì   s   € Ø,0ˆ�‰Ô)Øˆr0   c                 ó&   — d| j                   _        y )NFrŸ   )rS   rY   rl   Ú	tracebacks       r.   Ú__exit__z_enable_get_lr_call.__exit__ð   s   € Ø,1ˆ�‰Õ)r0   N)rZ   r‹   rŒ   r%   rb   r    r£   r“   r0   r.   rƒ   rƒ   è   s   „ ð˜+ó òó2r0   rƒ   c            
       ón   ‡ — e Zd ZdZ	 d	dedeeegef   e	eegef      f   defˆ fd„Z
d„ Zd„ Zd„ Zˆ xZS )
r   a•  Sets the initial learning rate.

    The learning rate of each parameter group is set to the initial lr
    times a given function. When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        lr_lambda (function or list): A function which computes a multiplicative
            factor given an integer parameter epoch, or a list of such
            functions, one for each group in optimizer.param_groups.
        last_epoch (int): The index of last epoch. Default: -1.

    Example:
        >>> # xdoctest: +SKIP
        >>> # Assuming optimizer has two groups.
        >>> lambda1 = lambda epoch: epoch // 30
        >>> lambda2 = lambda epoch: 0.95 ** epoch
        >>> scheduler = LambdaLR(optimizer, lr_lambda=[lambda1, lambda2])
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r'   Ú	lr_lambdar=   c                 ór  •— || _         |  t        |t        «      s/t        |t        «      s|gt	        |j
                  «      z  | _        n^t	        |«      t	        |j
                  «      k7  r-t        dt	        |j
                  «      › dt	        |«      › �«      ‚t        |«      | _        t        ‰| �%  ||«       y ©Nz	Expected z lr_lambdas, but got ©
r'   r+   r3   r4   r5   r6   Ú
lr_lambdasr7   Úsuperrb   ©rS   r'   r¥   r=   rK   s       €r.   rb   zLambdaLR.__init__  ó    ø€ ð #ˆŒáÜ˜)¤TÔ*´:¸iÌÔ3OØ(˜k¬C°	×0FÑ0FÓ,GÑGˆD�Oä�9‹~¤ Y×%;Ñ%;Ó!<Ò<Ü Ø¤ I×$:Ñ$:Ó ;Ð<Ð<QÔRUÐV_ÓR`ÐQaÐbóð ô # 9›oˆDŒOÜ‰Ñ˜ JÕ/r0   c                 ó\  — | j                   j                  «       D ��ci c]  \  }}|dvr||“Œ }}}dgt        | j                  «      z  |d<   t	        | j                  «      D ]@  \  }}t        |t        j                  «      rŒ!|j                   j                  «       |d   |<   ŒB |S c c}}w )a¢  Return the state of the scheduler as a :class:`dict`.

        It contains an entry for every variable in self.__dict__ which
        is not the optimizer.
        The learning rate lambda functions will only be saved if they are callable objects
        and not if they are functions or lambdas.

        When saving or loading the scheduler, please make sure to also save or load the state of the optimizer.
        ©r'   r©   Nr©   ©	rh   ri   r5   r©   r\   r+   ÚtypesÚFunctionTypeÚcopy©rS   rk   rl   rm   ÚidxÚfns         r.   rm   zLambdaLR.state_dict   s²   € ð #Ÿm™m×1Ñ1Ó3÷
á��UØÐ5Ñ5ð �‰Jð
ˆ
ñ 
ð
 %) 6¬C°·±Ó,@Ñ#@ˆ
�<Ñ ä  §¡Ó1ò 	C‰GˆC�Ü˜b¤%×"4Ñ"4Õ5Ø02·±×0@Ñ0@Ó0B�
˜<Ñ(¨Ò-ð	Cð Ðùó
ó   žB(c                 óâ   — |j                  d«      }| j                  j                  |«       ||d<   t        |«      D ]0  \  }}|€Œ	| j                  |   j                  j                  |«       Œ2 y)a$  Load the scheduler's state.

        When saving or loading the scheduler, please make sure to also save or load the state of the optimizer.

        Args:
            state_dict (dict): scheduler state. Should be an object returned
                from a call to :meth:`state_dict`.
        r©   N©Úpoprh   rq   r\   r©   ©rS   rm   r©   r´   rµ   s        r.   rs   zLambdaLR.load_state_dict7  sj   € ð  —^‘^ LÓ1ˆ
Ø�‰×Ñ˜ZÔ(ð $.ˆ
�<Ñ ä  Ó,ò 	9‰GˆC�Ø‰~Ø—‘ Ñ$×-Ñ-×4Ñ4°RÕ8ñ	9r0   c                 ó°   — t        | «       t        | j                  | j                  «      D ��cg c]  \  }}| || j                  «      z  ‘Œ c}}S c c}}w )zCompute learning rate.)r˜   r†   r©   r^   r=   )rS   ÚlmbdaÚbase_lrs      r.   rz   zLambdaLR.get_lrJ  sM   € ä'¨Ô-ô #& d§o¡o°t·}±}Ó"E÷
á��wð ‘e˜DŸO™OÓ,Ó,ó
ð 	
ùó 
s   ¯ArŠ   ©rZ   r‹   rŒ   r�   r   r   r
   r�   r…   r3   rb   rm   rs   rz   Ú__classcell__©rK   s   @r.   r   r   ô   sb   ø„ ñð8 ñ	0àð0ð ˜ 3 %¨ ,Ñ/°°hÀ¸uÀe¸|Ñ6LÑ1MÐMÑNð0ð õ	0ò&ò.9ö&
r0   r   c            
       ón   ‡ — e Zd ZdZ	 d	dedeeegef   e	eegef      f   defˆ fd„Z
d„ Zd„ Zd„ Zˆ xZS )
r   a  Multiply the learning rate of each parameter group by the factor given in the specified function.

    When last_epoch=-1, set initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        lr_lambda (function or list): A function which computes a multiplicative
            factor given an integer parameter epoch, or a list of such
            functions, one for each group in optimizer.param_groups.
        last_epoch (int): The index of last epoch. Default: -1.

    Example:
        >>> # xdoctest: +SKIP
        >>> lmbda = lambda epoch: 0.95
        >>> scheduler = MultiplicativeLR(optimizer, lr_lambda=lmbda)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r'   r¥   r=   c                 ór  •— || _         |  t        |t        «      s/t        |t        «      s|gt	        |j
                  «      z  | _        n^t	        |«      t	        |j
                  «      k7  r-t        dt	        |j
                  «      › dt	        |«      › �«      ‚t        |«      | _        t        ‰| �%  ||«       y r§   r¨   r«   s       €r.   rb   zMultiplicativeLR.__init__j  r¬   r0   c                 ó\  — | j                   j                  «       D ��ci c]  \  }}|dvr||“Œ }}}dgt        | j                  «      z  |d<   t	        | j                  «      D ]@  \  }}t        |t        j                  «      rŒ!|j                   j                  «       |d   |<   ŒB |S c c}}w )a1  Return the state of the scheduler as a :class:`dict`.

        It contains an entry for every variable in self.__dict__ which
        is not the optimizer.
        The learning rate lambda functions will only be saved if they are callable objects
        and not if they are functions or lambdas.
        r®   Nr©   r¯   r³   s         r.   rm   zMultiplicativeLR.state_dict}  s²   € ð #Ÿm™m×1Ñ1Ó3÷
á��UØÐ5Ñ5ð �‰Jð
ˆ
ñ 
ð
 %) 6¬C°·±Ó,@Ñ#@ˆ
�<Ñ ä  §¡Ó1ò 	C‰GˆC�Ü˜b¤%×"4Ñ"4Õ5Ø02·±×0@Ñ0@Ó0B�
˜<Ñ(¨Ò-ð	Cð Ðùó
r¶   c                 óâ   — |j                  d«      }| j                  j                  |«       ||d<   t        |«      D ]0  \  }}|€Œ	| j                  |   j                  j                  |«       Œ2 y)rp   r©   Nr¸   rº   s        r.   rs   z MultiplicativeLR.load_state_dict’  sj   € ð  —^‘^ LÓ1ˆ
Ø�‰×Ñ˜ZÔ(ð $.ˆ
�<Ñ ä  Ó,ò 	9‰GˆC�Ø‰~Ø—‘ Ñ$×-Ñ-×4Ñ4°RÕ8ñ	9r0   c                 ó@  — t        | «       | j                  dkD  rSt        | j                  | j                  j
                  «      D ��cg c]  \  }}|d    || j                  «      z  ‘Œ c}}S | j                  j
                  D �cg c]  }|d   ‘Œ	 c}S c c}}w c c}w ©ú2Compute the learning rate of each parameter group.r   rA   )r˜   r=   r†   r©   r'   r6   )rS   r¼   r`   s      r.   rz   zMultiplicativeLR.get_lr£  sŠ   € ä'¨Ô-à�?‰?˜QÒô %(¨¯©¸¿¹×9TÑ9TÓ$U÷á �E˜5ð �d‘™e D§O¡OÓ4Ó4óð ð
 .2¯^©^×-HÑ-HÖI E�E˜$“KÒIÐIùóùò
 Js   Á"BÂBrŠ   r¾   rÀ   s   @r.   r   r   T  sc   ø„ ñð2 ñ	0àð0ð ˜ 3 %¨ ,Ñ/°°hÀ¸uÀe¸|Ñ6LÑ1MÐMÑNð0ð õ	0ò&ò*9ö"
Jr0   r   c            	       óF   ‡ — e Zd ZdZ	 	 d	dedededefˆ fd„Zd„ Zd„ Z	ˆ xZ
S )
r   aÝ  Decays the learning rate of each parameter group by gamma every step_size epochs.

    Notice that such decay can happen simultaneously with other changes to the learning rate
    from outside this scheduler. When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        step_size (int): Period of learning rate decay.
        gamma (float): Multiplicative factor of learning rate decay.
            Default: 0.1.
        last_epoch (int): The index of last epoch. Default: -1.

    Example:
        >>> # xdoctest: +SKIP
        >>> # Assuming optimizer uses lr = 0.05 for all groups
        >>> # lr = 0.05     if epoch < 30
        >>> # lr = 0.005    if 30 <= epoch < 60
        >>> # lr = 0.0005   if 60 <= epoch < 90
        >>> # ...
        >>> scheduler = StepLR(optimizer, step_size=30, gamma=0.1)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r'   Ú	step_sizeÚgammar=   c                 óB   •— || _         || _        t        ‰| �  ||«       y r*   )rÉ   rÊ   rª   rb   )rS   r'   rÉ   rÊ   r=   rK   s        €r.   rb   zStepLR.__init__Ë  s"   ø€ ð #ˆŒØˆŒ
Ü‰Ñ˜ JÕ/r0   c                 ó8  — t        | «       | j                  dk(  s| j                  | j                  z  dk7  r'| j                  j                  D �cg c]  }|d   ‘Œ	 c}S | j                  j                  D �cg c]  }|d   | j
                  z  ‘Œ c}S c c}w c c}w rÆ   )r˜   r=   rÉ   r'   r6   rÊ   ©rS   r`   s     r.   rz   zStepLR.get_lrÖ  sz   € ä'¨Ô-à�O‰O˜qÒ  d§o¡o¸¿¹Ñ&FÈ!Ò&KØ-1¯^©^×-HÑ-HÖI E�E˜$“KÒIÐIØ6:·n±n×6QÑ6QÖR¨U��d‘˜dŸj™jÓ(ÒRÐRùò JùÚRs   ÁBÁ6Bc                 óŽ   — | j                   D �cg c]+  }|| j                  | j                  | j                  z  z  z  ‘Œ- c}S c c}w r*   )r^   rÊ   r=   rÉ   ©rS   r½   s     r.   r~   zStepLR._get_closed_form_lrÞ  sC   € ð  Ÿ=™=ö
àð �d—j‘j T§_¡_¸¿¹Ñ%FÑGÓGò
ð 	
ùò 
s   �0A©çš™™™™™¹?r@   ©rZ   r‹   rŒ   r�   r   r�   r…   rb   rz   r~   r¿   rÀ   s   @r.   r   r   °  sF   ø„ ñð< Øñ	0àð	0ð ð	0ð ð		0ð
 õ	0òSö
r0   r   c            	       óL   ‡ — e Zd ZdZ	 	 d	dedee   dedefˆ fd„Zd„ Z	d„ Z
ˆ xZS )
r   aÿ  Decays the learning rate of each parameter group by gamma once the number of epoch reaches one of the milestones.

    Notice that such decay can happen simultaneously with other changes to the learning rate
    from outside this scheduler. When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        milestones (list): List of epoch indices. Must be increasing.
        gamma (float): Multiplicative factor of learning rate decay.
            Default: 0.1.
        last_epoch (int): The index of last epoch. Default: -1.

    Example:
        >>> # xdoctest: +SKIP
        >>> # Assuming optimizer uses lr = 0.05 for all groups
        >>> # lr = 0.05     if epoch < 30
        >>> # lr = 0.005    if 30 <= epoch < 80
        >>> # lr = 0.0005   if epoch >= 80
        >>> scheduler = MultiStepLR(optimizer, milestones=[30,80], gamma=0.1)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r'   Ú
milestonesrÊ   r=   c                 óT   •— t        |«      | _        || _        t        ‰| �  ||«       y r*   )r   rÔ   rÊ   rª   rb   )rS   r'   rÔ   rÊ   r=   rK   s        €r.   rb   zMultiStepLR.__init__ÿ  s'   ø€ ô " *Ó-ˆŒØˆŒ
Ü‰Ñ˜ JÕ/r0   c                 óF  — t        | «       | j                  | j                  vr'| j                  j                  D �cg c]  }|d   ‘Œ	 c}S | j                  j                  D �cg c].  }|d   | j
                  | j                  | j                     z  z  ‘Œ0 c}S c c}w c c}w )rÇ   rA   )r˜   r=   rÔ   r'   r6   rÊ   rÍ   s     r.   rz   zMultiStepLR.get_lr
  sŠ   € ä'¨Ô-à�?‰? $§/¡/Ñ1Ø-1¯^©^×-HÑ-HÖI E�E˜$“KÒIÐIð Ÿ™×4Ñ4ö
àð �$‰K˜$Ÿ*™*¨¯©¸¿¹Ñ(HÑHÓHò
ð 	
ùò Jùò
s   ¼BÁ#3Bc           	      óÎ   — t        | j                  j                  «       «      }| j                  D �cg c](  }|| j                  t        || j                  «      z  z  ‘Œ* c}S c c}w r*   )ÚsortedrÔ   Úelementsr^   rÊ   r   r=   )rS   rÔ   r½   s      r.   r~   zMultiStepLR._get_closed_form_lr  sX   € Ü˜DŸO™O×4Ñ4Ó6Ó7ˆ
ð  Ÿ=™=ö
àð �d—j‘j¤L°¸T¿_¹_Ó$MÑMÓMò
ð 	
ùò 
s   ²-A"rÐ   )rZ   r‹   rŒ   r�   r   r   r�   r…   rb   rz   r~   r¿   rÀ   s   @r.   r   r   å  sI   ø„ ñð: Øñ	0àð	0ð ˜S‘Mð	0ð ð		0ð
 õ	0ò	
ö
r0   r   c            	       óH   ‡ — e Zd ZdZ	 	 	 d	dedededefˆ fd„Zd„ Zd„ Z	ˆ xZ
S )
r   aö  Multiply the learning rate of each parameter group by a small constant factor.

    The multiplication is done until the number of epoch reaches a pre-defined milestone: total_iters.
    Notice that such multiplication of the small constant factor can
    happen simultaneously with other changes to the learning rate from outside this scheduler.
    When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        factor (float): The number we multiply learning rate until the milestone. Default: 1./3.
        total_iters (int): The number of steps that the scheduler multiplies the learning rate by the factor.
            Default: 5.
        last_epoch (int): The index of the last epoch. Default: -1.

    Example:
        >>> # xdoctest: +SKIP
        >>> # Assuming optimizer uses lr = 0.05 for all groups
        >>> # lr = 0.025   if epoch == 0
        >>> # lr = 0.025   if epoch == 1
        >>> # lr = 0.025   if epoch == 2
        >>> # lr = 0.025   if epoch == 3
        >>> # lr = 0.05    if epoch >= 4
        >>> scheduler = ConstantLR(optimizer, factor=0.5, total_iters=4)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r'   ÚfactorÚtotal_itersr=   c                 ól   •— |dkD  s|dk  rt        d«      ‚|| _        || _        t        ‰| �  ||«       y )Nç      ð?r   z>Constant multiplicative factor expected to be between 0 and 1.)r7   rÛ   rÜ   rª   rb   )rS   r'   rÛ   rÜ   r=   rK   s        €r.   rb   zConstantLR.__init__;  sB   ø€ ð �CŠ<˜6 Aš:ÜØPóð ð ˆŒØ&ˆÔÜ‰Ñ˜ JÕ/r0   c                 óª  — t        | «       | j                  dk(  r4| j                  j                  D �cg c]  }|d   | j                  z  ‘Œ c}S | j                  | j
                  k7  r'| j                  j                  D �cg c]  }|d   ‘Œ	 c}S | j                  j                  D �cg c]  }|d   d| j                  z  z  ‘Œ c}S c c}w c c}w c c}w )rÇ   r   rA   rÞ   )r˜   r=   r'   r6   rÛ   rÜ   rÍ   s     r.   rz   zConstantLR.get_lrK  s·   € ä'¨Ô-à�?‰?˜aÒØ;?¿>¹>×;VÑ;VÖW°%�E˜$‘K $§+¡+Ó-ÒWÐWà�?‰?˜d×.Ñ.Ò.Ø-1¯^©^×-HÑ-HÖI E�E˜$“KÒIÐIð <@¿>¹>×;VÑ;Vö
Ø27ˆE�$‰K˜3 §¡Ñ,Ó-ò
ð 	
ùò Xùò Jùò
s   ³CÂ CÂ'Cc                 ó®   — | j                   D �cg c];  }|| j                  | j                  | j                  k\  d| j                  z
  z  z   z  ‘Œ= c}S c c}w )Nr   )r^   rÛ   r=   rÜ   rÏ   s     r.   r~   zConstantLR._get_closed_form_lrY  sX   € ð  Ÿ=™=ö
ð ð Ø�{‰{˜dŸo™o°×1AÑ1AÑAÀaÈ$Ï+É+ÁoÑVÑVóXò
ð 	
ùò 
s   �A A)çUUUUUUÕ?é   r@   ©rZ   r‹   rŒ   r�   r   r…   r�   rb   rz   r~   r¿   rÀ   s   @r.   r   r     sI   ø„ ñð@  ØØñ0àð0ð ð0ð ð	0ð
 õ0ò 
ö
r0   r   c                   óN   ‡ — e Zd ZdZ	 	 	 	 d
dededededef
ˆ fd„Zd„ Zd	„ Z	ˆ xZ
S )r   aÄ  Decays the learning rate of each parameter group by linearly changing small multiplicative factor.

    The multiplication is done until the number of epoch reaches a pre-defined milestone: total_iters.
    Notice that such decay can happen simultaneously with other changes to the learning rate
    from outside this scheduler. When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        start_factor (float): The number we multiply learning rate in the first epoch.
            The multiplication factor changes towards end_factor in the following epochs.
            Default: 1./3.
        end_factor (float): The number we multiply learning rate at the end of linear changing
            process. Default: 1.0.
        total_iters (int): The number of iterations that multiplicative factor reaches to 1.
            Default: 5.
        last_epoch (int): The index of the last epoch. Default: -1.

    Example:
        >>> # xdoctest: +SKIP
        >>> # Assuming optimizer uses lr = 0.05 for all groups
        >>> # lr = 0.025    if epoch == 0
        >>> # lr = 0.03125  if epoch == 1
        >>> # lr = 0.0375   if epoch == 2
        >>> # lr = 0.04375  if epoch == 3
        >>> # lr = 0.05    if epoch >= 4
        >>> scheduler = LinearLR(optimizer, start_factor=0.5, total_iters=4)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r'   Ústart_factorÚ
end_factorrÜ   r=   c                 ó¤   •— |dkD  s|dk  rt        d«      ‚|dkD  s|dk  rt        d«      ‚|| _        || _        || _        t        ‰| �  ||«       y )NrÞ   r   zTStarting multiplicative factor expected to be greater than 0 and less or equal to 1.z<Ending multiplicative factor expected to be between 0 and 1.)r7   rå   ræ   rÜ   rª   rb   )rS   r'   rå   ræ   rÜ   r=   rK   s         €r.   rb   zLinearLR.__init__‚  sl   ø€ ð ˜#Ò °Ò!2ÜØfóð ð ˜Ò˜z¨Aš~ÜØNóð ð )ˆÔØ$ˆŒØ&ˆÔÜ‰Ñ˜ JÕ/r0   c           
      óL  — t        | «       | j                  dk(  r4| j                  j                  D �cg c]  }|d   | j                  z  ‘Œ c}S | j                  | j
                  kD  r'| j                  j                  D �cg c]  }|d   ‘Œ	 c}S | j                  j                  D �cg c]h  }|d   d| j                  | j                  z
  | j
                  | j                  z  | j                  dz
  | j                  | j                  z
  z  z   z  z   z  ‘Œj c}S c c}w c c}w c c}w ©zCompute the learning rate.r   rA   rÞ   r   )r˜   r=   r'   r6   rå   rÜ   ræ   rÍ   s     r.   rz   zLinearLR.get_lr™  s  € ä'¨Ô-à�?‰?˜aÒà=A¿^¹^×=XÑ=XöØ49��d‘˜d×/Ñ/Ó/òð ð �?‰?˜T×-Ñ-Ò-Ø-1¯^©^×-HÑ-HÖI E�E˜$“KÒIÐIð Ÿ™×4Ñ4ö
ð ð �$‰KàØ—?‘? T×%6Ñ%6Ñ6à×$Ñ$ t×'8Ñ'8Ñ8Ø—‘¨Ñ*¨t¯©À×ARÑARÑ/RÑSñTññóò
ð 	
ùòùò
 Jùò
s   ³DÂ DÂ'A-D!c           
      óê   — | j                   D �cg c]Y  }|| j                  | j                  | j                  z
  t        | j                  | j
                  «      z  | j                  z  z   z  ‘Œ[ c}S c c}w r*   )r^   rå   ræ   ÚminrÜ   r=   rÏ   s     r.   r~   zLinearLR._get_closed_form_lr²  sv   € ð  Ÿ=™=ö	
ð ð à×!Ñ!Ø—?‘? T×%6Ñ%6Ñ6Ü�d×&Ñ&¨¯©Ó8ñ9à×"Ñ"ñ#ñ#óò	
ð 		
ùò 	
s   �AA0)rá   rÞ   râ   r@   rã   rÀ   s   @r.   r   r   a  sV   ø„ ñðF &ØØØñ0àð0ð ð0ð ð	0ð
 ð0ð õ0ò.
ö2

r0   r   c                   ó@   ‡ — e Zd ZdZ	 ddededefˆ fd„Zd„ Zd„ Z	ˆ xZ
S )	r   a=  Decays the learning rate of each parameter group by gamma every epoch.

    When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        gamma (float): Multiplicative factor of learning rate decay.
        last_epoch (int): The index of last epoch. Default: -1.
    r'   rÊ   r=   c                 ó4   •— || _         t        ‰| �	  ||«       y r*   )rÊ   rª   rb   )rS   r'   rÊ   r=   rK   s       €r.   rb   zExponentialLR.__init__Ê  s   ø€ ð ˆŒ
Ü‰Ñ˜ JÕ/r0   c                 ó   — t        | «       | j                  dk(  r'| j                  j                  D �cg c]  }|d   ‘Œ	 c}S | j                  j                  D �cg c]  }|d   | j                  z  ‘Œ c}S c c}w c c}w rÆ   )r˜   r=   r'   r6   rÊ   rÍ   s     r.   rz   zExponentialLR.get_lrÓ  sf   € ä'¨Ô-à�?‰?˜aÒØ-1¯^©^×-HÑ-HÖI E�E˜$“KÒIÐIØ6:·n±n×6QÑ6QÖR¨U��d‘˜dŸj™jÓ(ÒRÐRùò JùÚRs   ³A6ÁA;c                 ót   — | j                   D �cg c]  }|| j                  | j                  z  z  ‘Œ  c}S c c}w r*   )r^   rÊ   r=   rÏ   s     r.   r~   z!ExponentialLR._get_closed_form_lrÛ  s,   € ØEIÇ]Á]ÖS¸'�˜$Ÿ*™* d§o¡oÑ5Ó5ÒSÐSùÒSs   �#5rŠ   rã   rÀ   s   @r.   r   r   ¿  s:   ø„ ñð ñ	0àð0ð ð0ð õ	0òSöTr0   r   c            	       óR   — e Zd ZdZ	 ddedee   dee   defd„Zdd„Z	d	„ Z
d
„ Zd„ Zy)r   a¿  Contains a list of schedulers expected to be called sequentially during the optimization process.

    Specifically, the schedulers will be called according to the milestone points, which should provide exact
    intervals by which each scheduler should be called at a given epoch.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        schedulers (list): List of chained schedulers.
        milestones (list): List of integers that reflects milestone points.
        last_epoch (int): The index of last epoch. Default: -1.

    Example:
        >>> # xdoctest: +SKIP
        >>> # Assuming optimizer uses lr = 1. for all groups
        >>> # lr = 0.1     if epoch == 0
        >>> # lr = 0.1     if epoch == 1
        >>> # lr = 0.9     if epoch == 2
        >>> # lr = 0.81    if epoch == 3
        >>> # lr = 0.729   if epoch == 4
        >>> scheduler1 = ConstantLR(optimizer, factor=0.1, total_iters=2)
        >>> scheduler2 = ExponentialLR(optimizer, gamma=0.9)
        >>> scheduler = SequentialLR(optimizer, schedulers=[scheduler1, scheduler2], milestones=[2])
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r'   Ú
schedulersrÔ   r=   c                 ó¼  — t        |«      dk  r"t        | j                  j                  › d�«      ‚t	        |«      D ]Ú  \  }}t        |d«      s%t        | j                  j                  › d|› d�«      ‚t        |t        «      r%t        | j                  j                  › d|› d�«      ‚||j                  k7  sŒ|t        | j                  j                  › d|j                  j                  › d|› d	|j                  › d
|j                  j                  › d�
«      ‚ t        |«      t        |«      dz
  k7  r#t        dt        |«      › dt        |«      › �«      ‚|| _
        || _        |dz   | _        || _	        | j                  j                  D ]
  }|d   |d<   Œ | j                  «        | j                  d   j                  «        |d   j!                  «       | _        y )Nr   z6 expects at least one scheduler, but got no scheduler.r'   ú
 at index ú* should have `optimizer` as its attribute.ú‹ does not support `ReduceLROnPlateau` scheduler as it requires additional kwargs to be specified when calling `step`, but got one at index ú" in the given schedulers sequence.úK expects all schedulers to belong to the same optimizer, but got scheduler r1   ú, which is different from r2   z�Sequential Schedulers expects number of schedulers provided to be one more than the number of milestone points, but got number of schedulers z- and the number of milestones to be equal to rB   rA   r   )r5   r7   rK   rZ   r\   rU   rX   r+   r    r'   Ú_schedulersÚ_milestonesr=   r6   Úrecursive_undor_   rw   rv   )rS   r'   rñ   rÔ   r=   Úscheduler_idxÚ	schedulerr`   s           r.   rb   zSequentialLR.__init__ü  s  € ô ˆz‹?˜QÒÜØ—>‘>×*Ñ*Ð+Ð+aÐbóð ô )2°*Ó(=ò 	Ñ$ˆM˜9Ü˜9 kÔ2ÜØ—~‘~×.Ñ.Ð/¨z¸-¸ÐHrÐsóð ô ˜)Ô%6Ô7Ü Ø—~‘~×.Ñ.Ð/ð 0,à,9¨?Ð:\ð^óð ð
 ˜I×/Ñ/Ó/Ü Ø—~‘~×.Ñ.Ð/ð 0%Ø%.×%8Ñ%8×%AÑ%AÐ$BÀ*È]ÈOÐ[`Ðaj×atÑatÐ`uð v/Ø/8×/BÑ/B×/KÑ/KÐ.LÈAðOóð ð	ô$ ˆz‹?œc *›o°Ñ1Ò1ÜðUÜUXÐYcÓUdÐTeð f7Ü7:¸:³Ð6GðIóð ð
 &ˆÔØ%ˆÔØ$ q™.ˆŒØ"ˆŒð —^‘^×0Ñ0ò 	.ˆEØ Ñ-ˆE�$ŠKð	.ð 	×ÑÔð 	×Ñ˜Ñ×)Ñ)Ô+à" 1™×1Ñ1Ó3ˆ�r0   Nc                 ó²   — |€| n|}t        |d«      r#|j                  D ]  }| j                  |«       Œ yt        |d«      r|xj                  dz  c_        yy)zb
        Recursively undo any step performed by the initialisation of
        schedulers.
        Nrù   r=   r   )rU   rù   rû   r=   )rS   ÚschedÚschedsÚss       r.   rû   zSequentialLR.recursive_undo1  s\   € ð
 ˜‘¨Eˆä�6˜=Ô)Ø×'Ñ'ò '�Ø×#Ñ# AÕ&ñ'ä�V˜\Ô*Ø×Ò Ñ"Öð +r0   c                 óB  — | xj                   dz  c_         t        | j                  | j                   «      }| j                  |   }|dkD  r1| j                  |dz
     | j                   k(  r|j	                  d«       n|j	                  «        |j                  «       | _        y)r}   r   r   N)r=   r   rú   rù   rV   rw   rv   )rS   r´   rý   s      r.   rV   zSequentialLR.step>  s}   € à�Š˜1Ñ�Ü˜4×+Ñ+¨T¯_©_Ó=ˆØ×$Ñ$ SÑ)ˆ	Ø�Š7�t×'Ñ'¨¨a©Ñ0°D·O±OÒCØ�N‰N˜1Õà�N‰NÔà!×-Ñ-Ó/ˆ�r0   c                 ó  — | j                   j                  «       D ��ci c]  \  }}|dvr||“Œ }}}dgt        | j                  «      z  |d<   t	        | j                  «      D ]  \  }}|j                  «       |d   |<   Œ |S c c}}w ©zÝReturn the state of the scheduler as a :class:`dict`.

        It contains an entry for every variable in self.__dict__ which
        is not the optimizer.
        The wrapped scheduler states will also be saved.
        )r'   rù   Nrù   ©rh   ri   r5   rù   r\   rm   ©rS   rk   rl   rm   r´   r  s         r.   rm   zSequentialLR.state_dictJ  ó�   € ð #Ÿm™m×1Ñ1Ó3÷
á��UØÐ6Ñ6ð �‰Jð
ˆ
ñ 
ð
 &* F¬S°×1AÑ1AÓ-BÑ$Bˆ
�=Ñ!ä × 0Ñ 0Ó1ò 	<‰FˆC�Ø-.¯\©\«^ˆJ�}Ñ% cÒ*ð	<ð Ðùó
ó   žBc                 óÈ   — |j                  d«      }| j                  j                  |«       ||d<   t        |«      D ]#  \  }}| j                  |   j                  |«       Œ% y©rp   rù   N©r¹   rh   rq   r\   rù   rs   ©rS   rm   rù   r´   r  s        r.   rs   zSequentialLR.load_state_dict]  óa   € ð !—n‘n ]Ó3ˆØ�‰×Ñ˜ZÔ(ð %0ˆ
�=Ñ!ä Ó,ò 	5‰FˆC�Ø×Ñ˜SÑ!×1Ñ1°!Õ4ñ	5r0   rŠ   r*   )rZ   r‹   rŒ   r�   r   r3   r%   r�   rb   rû   rV   rm   rs   r“   r0   r.   r   r   ß  sV   „ ñðB ñ34àð34ð ˜Ñ%ð34ð ˜‘Ið	34ð
 ó34ój#ò
0òó&5r0   r   c            	       óH   ‡ — e Zd ZdZ	 	 	 d	dedededefˆ fd„Zd„ Zd„ Z	ˆ xZ
S )
r$   a   Decays the learning rate of each parameter group using a polynomial function in the given total_iters.

    When last_epoch=-1, sets initial lr as lr.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        total_iters (int): The number of steps that the scheduler decays the learning rate. Default: 5.
        power (float): The power of the polynomial. Default: 1.0.

    Example:
        >>> # xdoctest: +SKIP("undefined vars")
        >>> # Assuming optimizer uses lr = 0.001 for all groups
        >>> # lr = 0.001     if epoch == 0
        >>> # lr = 0.00075   if epoch == 1
        >>> # lr = 0.00050   if epoch == 2
        >>> # lr = 0.00025   if epoch == 3
        >>> # lr = 0.0       if epoch >= 4
        >>> scheduler = PolynomialLR(optimizer, total_iters=4, power=1.0)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    r'   rÜ   Úpowerr=   c                 óB   •— || _         || _        t        ‰| �  ||«       y r*   )rÜ   r  rª   rb   )rS   r'   rÜ   r  r=   rK   s        €r.   rb   zPolynomialLR.__init__‡  s#   ø€ ð 'ˆÔØˆŒ
Ü‰Ñ˜ JÕ/r0   c                 ó°  — t        | «       | j                  dk(  s| j                  | j                  kD  r'| j                  j                  D �cg c]  }|d   ‘Œ	 c}S d| j                  | j                  z  z
  d| j                  dz
  | j                  z  z
  z  | j
                  z  }| j                  j                  D �cg c]
  }|d   |z  ‘Œ c}S c c}w c c}w ré   )r˜   r=   rÜ   r'   r6   r  )rS   r`   Údecay_factors      r.   rz   zPolynomialLR.get_lr’  sÀ   € ä'¨Ô-à�?‰?˜aÒ 4§?¡?°T×5EÑ5EÒ#EØ-1¯^©^×-HÑ-HÖI E�E˜$“KÒIÐIð �4—?‘? T×%5Ñ%5Ñ5Ñ5Ø�d—o‘o¨Ñ)¨T×-=Ñ-=Ñ=Ñ=ñ?à�Z‰Zñˆð 9=¿¹×8SÑ8SÖT¨u��d‘˜lÓ*ÒTÐTùò Jùò Us   ÁCÂ<Cc           	      ó¼   — | j                   D �cg c]B  }|dt        | j                  | j                  «      | j                  z  z
  | j                  z  z  ‘ŒD c}S c c}w ©NrÞ   )r^   rë   rÜ   r=   r  rÏ   s     r.   r~   z PolynomialLR._get_closed_form_lrŸ  s_   € ð  Ÿ=™=ö
ð ð	 Øœ˜T×-Ñ-¨t¯©Ó?À$×BRÑBRÑRÑRØ—:‘:ñóò
ð 	
ùò 
s   �AA)râ   rÞ   r@   rÒ   rÀ   s   @r.   r$   r$   n  sI   ø„ ñð6 ØØñ	0àð	0ð ð	0ð ð		0ð
 õ	0òUö
r0   r$   c            	       óF   ‡ — e Zd ZdZ	 	 d	dedededefˆ fd„Zd„ Zd„ Z	ˆ xZ
S )
r   a\  Set the learning rate of each parameter group using a cosine annealing schedule.

    The :math:`\eta_{max}` is set to the initial lr and
    :math:`T_{cur}` is the number of epochs since the last restart in SGDR:

    .. math::
        \begin{aligned}
            \eta_t & = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1
            + \cos\left(\frac{T_{cur}}{T_{max}}\pi\right)\right),
            & T_{cur} \neq (2k+1)T_{max}; \\
            \eta_{t+1} & = \eta_{t} + \frac{1}{2}(\eta_{max} - \eta_{min})
            \left(1 - \cos\left(\frac{1}{T_{max}}\pi\right)\right),
            & T_{cur} = (2k+1)T_{max}.
        \end{aligned}

    When last_epoch=-1, sets initial lr as lr. Notice that because the schedule
    is defined recursively, the learning rate can be simultaneously modified
    outside this scheduler by other operators. If the learning rate is set
    solely by this scheduler, the learning rate at each step becomes:

    .. math::
        \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1 +
        \cos\left(\frac{T_{cur}}{T_{max}}\pi\right)\right)

    It has been proposed in
    `SGDR: Stochastic Gradient Descent with Warm Restarts`_. Note that this only
    implements the cosine annealing part of SGDR, and not the restarts.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        T_max (int): Maximum number of iterations.
        eta_min (float): Minimum learning rate. Default: 0.
        last_epoch (int): The index of last epoch. Default: -1.

    .. _SGDR\: Stochastic Gradient Descent with Warm Restarts:
        https://arxiv.org/abs/1608.03983
    r'   ÚT_maxÚeta_minr=   c                 óB   •— || _         || _        t        ‰| �  ||«       y r*   )r  r  rª   rb   )rS   r'   r  r  r=   rK   s        €r.   rb   zCosineAnnealingLR.__init__Ñ  s"   ø€ ð ˆŒ
ØˆŒÜ‰Ñ˜ JÕ/r0   c                 ó  — t        | «       | j                  dk(  r'| j                  j                  D �cg c]  }|d   ‘Œ	 c}S | j                  dk(  rª| j                  dkD  r›t        | j                  | j                  j                  «      D ��cg c]e  \  }}| j                  || j                  z
  dt        j                  | j                  t        j                  z  | j                  z  «      z   z  dz  z   ‘Œg c}}S | j                  dz
  | j                  z
  d| j                  z  z  dk(  r‡t        | j                  | j                  j                  «      D ��cg c]Q  \  }}|d   || j                  z
  dt        j                  t        j                  | j                  z  «      z
  z  dz  z   ‘ŒS c}}S | j                  j                  D �cg c]¦  }dt        j                  t        j                  | j                  z  | j                  z  «      z   dt        j                  t        j                  | j                  dz
  z  | j                  z  «      z   z  |d   | j                  z
  z  | j                  z   ‘Œ¨ c}S c c}w c c}}w c c}}w c c}w )z3Retrieve the learning rate of each parameter group.r   rA   r   r–   )r˜   r=   r'   r6   rd   r†   r^   r  ÚmathÚcosÚpir  )rS   r`   r½   s      r.   rz   zCosineAnnealingLR.get_lrÜ  s$  € ä'¨Ô-à�?‰?˜aÒØ-1¯^©^×-HÑ-HÖI E�E˜$“KÒIÐIØ×Ñ Ò" t§¡¸Ò':ô '*¨$¯-©-¸¿¹×9TÑ9TÓ&U÷ñ
 #�G˜Uð	 —‘Ø˜TŸ\™\Ñ)Ø”t—x‘x §¡´D·G±GÑ ;¸d¿j¹jÑ HÓIÑIñKàñóóð ð �o‰o Ñ! D§J¡JÑ.°1°t·z±z±>ÑBÀaÒGô '*¨$¯-©-¸¿¹×9TÑ9TÓ&U÷ñ #�G˜Uð �d‘Ø˜TŸ\™\Ñ)¨a´$·(±(¼4¿7¹7ÀTÇZÁZÑ;OÓ2PÑ.PÑQÐTUÑUóVóð ð Ÿ™×4Ñ4ö
ð
 ð	 ”—‘œ$Ÿ'™' D§O¡OÑ3°d·j±jÑ@ÓAÑAØ”4—8‘8œDŸG™G t§¡¸Ñ':Ñ;¸d¿j¹jÑHÓIÑIñKà�T‰{˜TŸ\™\Ñ)ñ+ð �l‰lóò
ð 	
ùò Jùóùóùò

s   ³I7ÂA*I<ÅAJÇ	B+Jc           
      óü   — | j                   D �cg c]b  }| j                  || j                  z
  dt        j                  t        j                  | j
                  z  | j                  z  «      z   z  dz  z   ‘Œd c}S c c}w )Nr   r–   )r^   r  r  r  r  r=   r  rÏ   s     r.   r~   z%CosineAnnealingLR._get_closed_form_lrø  st   € ð  Ÿ=™=ö
ð
 ð	 �L‰LØ˜Ÿ™Ñ%Ø”4—8‘8œDŸG™G d§o¡oÑ5¸¿
¹
ÑBÓCÑCñEàñóò
ð 	
ùò 
s   �A'A9)ç        r@   rÒ   rÀ   s   @r.   r   r   ª  sF   ø„ ñ$ðT Øñ	0àð	0ð ð	0ð ð		0ð
 õ	0ò
ö8
r0   r   c                   óB   — e Zd ZdZ	 d	dee   dee   fd„Zd„ Z	d„ Z
d„ Zy)
r   aë  Chains a list of learning rate schedulers.

    Takes in a sequence of chainable learning rate schedulers and calls their
    step() functions consecutively in just one call to step().

    Args:
        schedulers (sequence): sequence of chained schedulers.
        optimizer (Optimizer, optional): Wrapped optimizer. Default: None.

    Example:
        >>> # xdoctest: +SKIP
        >>> # Assuming optimizer uses lr = 1. for all groups
        >>> # lr = 0.09     if epoch == 0
        >>> # lr = 0.081    if epoch == 1
        >>> # lr = 0.729    if epoch == 2
        >>> # lr = 0.6561   if epoch == 3
        >>> # lr = 0.59049  if epoch >= 4
        >>> scheduler1 = ConstantLR(optimizer, factor=0.1, total_iters=2)
        >>> scheduler2 = ExponentialLR(optimizer, gamma=0.9)
        >>> scheduler = ChainedScheduler([scheduler1, scheduler2], optimizer=optimizer)
        >>> for epoch in range(100):
        >>>     train(...)
        >>>     validate(...)
        >>>     scheduler.step()
    Nrñ   r'   c                 óò  — t        |«      dk  r"t        | j                  j                  › d�«      ‚|xs |d   j                  }t        |«      D ]Ú  \  }}t        |d«      s%t        | j                  j                  › d|› d�«      ‚t        |t        «      r%t        | j                  j                  › d|› d�«      ‚||j                  k7  sŒ|t        | j                  j                  › d	|j                  j                  › d|› d
|j                  › d|j                  j                  › d�
«      ‚ || _
        || _        | j                  d   j                  j                  D �cg c]  }|d   ‘Œ	 c}| _        y c c}w )Nr   zD expects at least one scheduler to be chained, but got no scheduler.r   r'   ró   rô   rõ   rö   r÷   r1   rø   r2   r@   rA   )r5   r7   rK   rZ   r'   r\   rU   rX   r+   r    rù   r6   rv   )rS   rñ   r'   rü   rý   r`   s         r.   rb   zChainedScheduler.__init__  s“  € ô ˆz‹?˜QÒÜØ—>‘>×*Ñ*Ð+Ð+oÐpóð ð Ò8 ¨A¡×!8Ñ!8ˆ	Ü(1°*Ó(=ò 	Ñ$ˆM˜9Ü˜9 kÔ2ÜØ—~‘~×.Ñ.Ð/¨z¸-¸ÐHrÐsóð ô ˜)Ô%6Ô7Ü Ø—~‘~×.Ñ.Ð/ð 0,à,9¨?Ð:\ð^óð ð
 ˜I×/Ñ/Ó/Ü Ø—~‘~×.Ñ.Ð/ð 0%Ø%.×%8Ñ%8×%AÑ%AÐ$BÀ*È]ÈOÐ[`Ðaj×atÑatÐ`uð v/Ø/8×/BÑ/B×/KÑ/KÐ.LÈAðOóð ð	ð" &ˆÔØ"ˆŒà%)×%5Ñ%5°bÑ%9×%CÑ%C×%PÑ%Pö
Ø!ˆE�$‹Kò
ˆ�ùò 
s   ÅE4c                 óÂ   — | j                   D ]  }|j                  «        Œ | j                   d   j                  j                  D �cg c]  }|d   ‘Œ	 c}| _        yc c}w )r}   r@   rA   N)rù   rV   r'   r6   rv   )rS   rý   r`   s      r.   rV   zChainedScheduler.step=  sZ   € à×)Ñ)ò 	ˆIØ�N‰NÕð	ð &*×%5Ñ%5°bÑ%9×%CÑ%C×%PÑ%Pö
Ø!ˆE�$‹Kò
ˆ�ùò 
s   ÁAc                 ó  — | j                   j                  «       D ��ci c]  \  }}|dvr||“Œ }}}dgt        | j                  «      z  |d<   t	        | j                  «      D ]  \  }}|j                  «       |d   |<   Œ |S c c}}w r  r  r  s         r.   rm   zChainedScheduler.state_dictE  r  r  c                 óÈ   — |j                  d«      }| j                  j                  |«       ||d<   t        |«      D ]#  \  }}| j                  |   j                  |«       Œ% yr
  r  r  s        r.   rs   z ChainedScheduler.load_state_dictX  r  r0   r*   )rZ   r‹   rŒ   r�   r   r%   r   r   rb   rV   rm   rs   r“   r0   r.   r   r     s=   „ ñð6 SWñ
Ø" ;Ñ/ð
Ø<DÀYÑ<Oó
ò@
òó&5r0   r   c                   ó¬   — e Zd ZdZ	 	 	 	 	 	 	 	 ddeded   dedededed	   d
edee	e   ef   defd„Z
d„ Zddefd„Zd„ Zed„ «       Zd„ Zd„ Zd„ Zd„ Zy)r    aà  Reduce learning rate when a metric has stopped improving.

    Models often benefit from reducing the learning rate by a factor
    of 2-10 once learning stagnates. This scheduler reads a metrics
    quantity and if no improvement is seen for a 'patience' number
    of epochs, the learning rate is reduced.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        mode (str): One of `min`, `max`. In `min` mode, lr will
            be reduced when the quantity monitored has stopped
            decreasing; in `max` mode it will be reduced when the
            quantity monitored has stopped increasing. Default: 'min'.
        factor (float): Factor by which the learning rate will be
            reduced. new_lr = lr * factor. Default: 0.1.
        patience (int): The number of allowed epochs with no improvement after
            which the learning rate will be reduced.
            For example, consider the case of having no patience (`patience = 0`).
            In the first epoch, a baseline is established and is always considered good as there's no previous baseline.
            In the second epoch, if the performance is worse than the baseline,
            we have what is considered an intolerable epoch.
            Since the count of intolerable epochs (1) is greater than the patience level (0),
            the learning rate is reduced at the end of this epoch.
            From the third epoch onwards, the learning rate continues to be reduced at the end of each epoch
            if the performance is worse than the baseline. If the performance improves or remains the same,
            the learning rate is not adjusted.
            Default: 10.
        threshold (float): Threshold for measuring the new optimum,
            to only focus on significant changes. Default: 1e-4.
        threshold_mode (str): One of `rel`, `abs`. In `rel` mode,
            dynamic_threshold = best * ( 1 + threshold ) in 'max'
            mode or best * ( 1 - threshold ) in `min` mode.
            In `abs` mode, dynamic_threshold = best + threshold in
            `max` mode or best - threshold in `min` mode. Default: 'rel'.
        cooldown (int): Number of epochs to wait before resuming
            normal operation after lr has been reduced. Default: 0.
        min_lr (float or list): A scalar or a list of scalars. A
            lower bound on the learning rate of all param groups
            or each group respectively. Default: 0.
        eps (float): Minimal decay applied to lr. If the difference
            between new and old lr is smaller than eps, the update is
            ignored. Default: 1e-8.

    Example:
        >>> # xdoctest: +SKIP
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
        >>> scheduler = ReduceLROnPlateau(optimizer, 'min')
        >>> for epoch in range(10):
        >>>     train(...)
        >>>     val_loss = validate(...)
        >>>     # Note that step should be called after validate()
        >>>     scheduler.step(val_loss)
    r'   Úmode)rë   ÚmaxrÛ   ÚpatienceÚ	thresholdÚthreshold_mode)ÚrelÚabsÚcooldownÚmin_lrÚepsc
                 ó
  — |dk\  rt        d«      ‚|| _        t        |t        «      s!t	        t        |«      j                  › d�«      ‚|| _        t        |t        t        f«      rft        |«      t        |j                  «      k7  r-t        dt        |j                  «      › dt        |«      › �«      ‚d | _        t        |«      | _        n%|| _        |gt        |j                  «      z  | _        || _        || _        d| _        || _        || _        || _        |  |  |  |	| _        d| _        | j                  j                  D �
cg c]  }
|
d   ‘Œ	 c}
| _        | j/                  |||¬«       | j1                  «        y c c}
w )	NrÞ   zFactor should be < 1.0.r?   z	expected z min_lrs, got r   rA   ©r%  r(  r)  )r7   rÛ   r+   r   rX   rY   rZ   r'   r3   r4   r5   r6   Údefault_min_lrÚmin_lrsr'  r,  Úcooldown_counterr%  r(  r)  r.  r=   rv   Ú_init_is_betterÚ_reset)rS   r'   r%  rÛ   r'  r(  r)  r,  r-  r.  r`   s              r.   rb   zReduceLROnPlateau.__init__   se  € ð �SŠ=ÜÐ6Ó7Ð7ØˆŒô ˜)¤YÔ/Üœt I›×7Ñ7Ð8Ð8LÐMÓNÐNØ"ˆŒä�fœt¤U˜mÔ,Ü�6‹{œc )×"8Ñ"8Ó9Ò9Ü Ø¤ I×$:Ñ$:Ó ;Ð<¸NÌ3ÈvË;È-ÐXóð ð #'ˆDÔÜ ›<ˆD�Là"(ˆDÔØ"˜8¤c¨)×*@Ñ*@Ó&AÑAˆDŒLà ˆŒà ˆŒØ !ˆÔØˆŒ	Ø"ˆŒØ,ˆÔÙÙÙØˆŒØˆŒØ26·.±.×2MÑ2MÖN¨˜˜t›ÒNˆŒØ×ÑØ ¸>ð 	ô 	
ð 	�‰�ùò	 Os   ÅF c                 óB   — | j                   | _        d| _        d| _        y)z2Reset num_bad_epochs counter and cooldown counter.r   N)Ú
mode_worseÚbestr3  Únum_bad_epochsre   s    r.   r5  zReduceLROnPlateau._resetÒ  s   € à—O‘OˆŒ	Ø !ˆÔØˆÕr0   NÚmetricsc                 ó>  — t        |«      }|€| j                  dz   }nt        j                  t        t
        «       || _        | j                  || j                  «      r|| _        d| _        n| xj                  dz  c_        | j                  r| xj                  dz  c_
        d| _        | j                  | j                  kD  r)| j                  |«       | j                  | _
        d| _        | j                  j                  D �cg c]  }|d   ‘Œ	 c}| _        yc c}w )r}   Nr   r   rA   )r…   r=   r   r€   r„   r�   Ú	is_betterr8  r9  Úin_cooldownr3  r'  Ú
_reduce_lrr,  r'   r6   rv   )rS   r:  r{   Úcurrentr`   s        r.   rV   zReduceLROnPlateau.stepØ  sß   € ô ˜“.ˆØˆ=Ø—O‘O aÑ'‰Eä�M‰MÔ3´[ÔAØˆŒà�>‰>˜' 4§9¡9Ô-ØˆDŒIØ"#ˆDÕà×Ò 1Ñ$Õà×ÒØ×!Ò! QÑ&Õ!Ø"#ˆDÔà×Ñ §¡Ò.Ø�O‰O˜EÔ"Ø$(§M¡MˆDÔ!Ø"#ˆDÔà26·.±.×2MÑ2MÖN¨˜˜t›ÒNˆ�ùÒNs   ÄDc                 óT  — t        | j                  j                  «      t        | j                  «      k7  r€| j                  €Bt        dt        | j                  j                  «      › dt        | j                  «      › d�«      ‚| j                  gt        | j                  j                  «      z  | _        t        | j                  j                  «      D ]Q  \  }}t        |d   «      }t        || j                  z  | j                  |   «      }||z
  | j                  kD  sŒM||d<   ŒS y )Nz/The number of param groups in the `optimizer` (z9) differs from when `ReduceLROnPlateau` was initialized (z˜), usually due to a new param group being added to the optimizer. Please modify the `min_lrs` field to match the length of the `optimizer` param groups.rA   )r5   r'   r6   r2  r1  ÚRuntimeErrorr\   r…   r&  rÛ   r.  )rS   r{   ra   r‰   Úold_lrÚnew_lrs         r.   r>  zReduceLROnPlateau._reduce_lró  s  € Üˆt�~‰~×*Ñ*Ó+¬s°4·<±<Ó/@Ò@Ø×"Ñ"Ð*Ü"ðÜ˜DŸN™N×7Ñ7Ó8Ð9ð :ä˜DŸL™LÓ)Ð*ð +7ð7óð ð !%× 3Ñ 3Ð4´s¸4¿>¹>×;VÑ;VÓ7WÑW�”ä'¨¯©×(CÑ(CÓDò 	+‰NˆAˆ{Ü˜; tÑ,Ó-ˆFÜ˜ $§+¡+Ñ-¨t¯|©|¸A©Ó?ˆFØ˜‰ §¡Ó)Ø$*�˜DÒ!ñ		+r0   c                 ó    — | j                   dkD  S )Nr   )r3  re   s    r.   r=  zReduceLROnPlateau.in_cooldown  s   € à×$Ñ$ qÑ(Ð(r0   c                 óZ  — | j                   dk(  r&| j                  dk(  rd| j                  z
  }|||z  k  S | j                   dk(  r!| j                  dk(  r||| j                  z
  k  S | j                   dk(  r&| j                  dk(  r| j                  dz   }|||z  kD  S ||| j                  z   kD  S )Nrë   r*  rÞ   r+  r&  )r%  r)  r(  )rS   Úar8  Úrel_epsilons       r.   r<  zReduceLROnPlateau.is_better  s³   € Ø�9‰9˜Ò $×"5Ñ"5¸Ò">Ø §¡Ñ.ˆKØ�t˜kÑ)Ñ)Ð)à�Y‰Y˜%Ò D×$7Ñ$7¸5Ò$@Ø�t˜dŸn™nÑ,Ñ,Ð,à�Y‰Y˜%Ò D×$7Ñ$7¸5Ò$@ØŸ.™.¨3Ñ.ˆKØ�t˜kÑ)Ñ)Ð)ð �t˜dŸn™nÑ,Ñ,Ð,r0   c                 ó¼   — |dvrt        d|z   dz   «      ‚|dvrt        d|z   dz   «      ‚|dk(  rt        | _        nt         | _        || _        || _        || _        y )N>   r&  rë   zmode z is unknown!>   r+  r*  zthreshold mode rë   )r7   r   r7  r%  r(  r)  )rS   r%  r(  r)  s       r.   r4  z!ReduceLROnPlateau._init_is_better  si   € Ø�~Ñ%Ü˜W t™^¨nÑ<Ó=Ð=Ø Ñ/ÜÐ.°Ñ?À.ÑPÓQÐQà�5Š=Ü!ˆD�Oä"˜dˆDŒOàˆŒ	Ø"ˆŒØ,ˆÕr0   c                 óv   — | j                   j                  «       D ��ci c]  \  }}|dk7  sŒ||“Œ c}}S c c}}w )Nr'   rg   rj   s      r.   rm   zReduceLROnPlateau.state_dict*  s;   € à)-¯©×)<Ñ)<Ó)>÷
Ù%˜3 À#ÈÓBTˆC�‰Jó
ð 	
ùó 
rn   c                 óž   — | j                   j                  |«       | j                  | j                  | j                  | j
                  ¬«       y)úLoad the scheduler's state.r0  N)rh   rq   r4  r%  r(  r)  rr   s     r.   rs   z!ReduceLROnPlateau.load_state_dict/  s<   € à�‰×Ñ˜ZÔ(Ø×ÑØ—‘ d§n¡nÀT×EXÑEXð 	õ 	
r0   )rë   rÑ   é
   g-Cëâ6?r*  r   r   g:Œ0âŽyE>r*   )rZ   r‹   rŒ   r�   r   r   r…   r�   r   r3   rb   r5  r   rV   r>  Úpropertyr=  r<  r4  rm   rs   r“   r0   r.   r    r    i  sØ   „ ñ4ðr ',ØØØØ05ØØ,-Øñ0àð0ð �lÑ#ð0ð ð	0ð
 ð0ð ð0ð   Ñ-ð0ð ð0ð �d˜5‘k 5Ð(Ñ)ð0ð ó0òd ñO˜Mó Oò6+ð* ñ)ó ð)ò-ò-ò
ó

r0   r    c                   ó0  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddedeeee   f   deeee   f   dede	e   de
d   d	ed
e	eegef      de
d   dedededefˆ fd„Zd„ Zdefd„Zededefd„«       Zededefd„«       Zed	ededefd„«       Zd„ Zˆ fd„Zˆ fd„Zˆ xZS )r!   aË  Sets the learning rate of each parameter group according to cyclical learning rate policy (CLR).

    The policy cycles the learning rate between two boundaries with a constant frequency,
    as detailed in the paper `Cyclical Learning Rates for Training Neural Networks`_.
    The distance between the two boundaries can be scaled on a per-iteration
    or per-cycle basis.

    Cyclical learning rate policy changes the learning rate after every batch.
    `step` should be called after a batch has been used for training.

    This class has three built-in policies, as put forth in the paper:

    * "triangular": A basic triangular cycle without amplitude scaling.
    * "triangular2": A basic triangular cycle that scales initial amplitude by half each cycle.
    * "exp_range": A cycle that scales initial amplitude by :math:`\text{gamma}^{\text{cycle iterations}}`
      at each cycle iteration.

    This implementation was adapted from the github repo: `bckenstler/CLR`_

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        base_lr (float or list): Initial learning rate which is the
            lower boundary in the cycle for each parameter group.
        max_lr (float or list): Upper learning rate boundaries in the cycle
            for each parameter group. Functionally,
            it defines the cycle amplitude (max_lr - base_lr).
            The lr at any cycle is the sum of base_lr
            and some scaling of the amplitude; therefore
            max_lr may not actually be reached depending on
            scaling function.
        step_size_up (int): Number of training iterations in the
            increasing half of a cycle. Default: 2000
        step_size_down (int): Number of training iterations in the
            decreasing half of a cycle. If step_size_down is None,
            it is set to step_size_up. Default: None
        mode (str): One of {triangular, triangular2, exp_range}.
            Values correspond to policies detailed above.
            If scale_fn is not None, this argument is ignored.
            Default: 'triangular'
        gamma (float): Constant in 'exp_range' scaling function:
            gamma**(cycle iterations)
            Default: 1.0
        scale_fn (function): Custom scaling policy defined by a single
            argument lambda function, where
            0 <= scale_fn(x) <= 1 for all x >= 0.
            If specified, then 'mode' is ignored.
            Default: None
        scale_mode (str): {'cycle', 'iterations'}.
            Defines whether scale_fn is evaluated on
            cycle number or cycle iterations (training
            iterations since start of cycle).
            Default: 'cycle'
        cycle_momentum (bool): If ``True``, momentum is cycled inversely
            to learning rate between 'base_momentum' and 'max_momentum'.
            Default: True
        base_momentum (float or list): Lower momentum boundaries in the cycle
            for each parameter group. Note that momentum is cycled inversely
            to learning rate; at the peak of a cycle, momentum is
            'base_momentum' and learning rate is 'max_lr'.
            Default: 0.8
        max_momentum (float or list): Upper momentum boundaries in the cycle
            for each parameter group. Functionally,
            it defines the cycle amplitude (max_momentum - base_momentum).
            The momentum at any cycle is the difference of max_momentum
            and some scaling of the amplitude; therefore
            base_momentum may not actually be reached depending on
            scaling function. Note that momentum is cycled inversely
            to learning rate; at the start of a cycle, momentum is 'max_momentum'
            and learning rate is 'base_lr'
            Default: 0.9
        last_epoch (int): The index of the last batch. This parameter is used when
            resuming a training job. Since `step()` should be invoked after each
            batch instead of after each epoch, this number represents the total
            number of *batches* computed, not the total number of epochs computed.
            When last_epoch=-1, the schedule is started from the beginning.
            Default: -1

    Example:
        >>> # xdoctest: +SKIP
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=0.1, momentum=0.9)
        >>> scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, base_lr=0.01, max_lr=0.1)
        >>> data_loader = torch.utils.data.DataLoader(...)
        >>> for epoch in range(10):
        >>>     for batch in data_loader:
        >>>         train_batch(...)
        >>>         scheduler.step()


    .. _Cyclical Learning Rates for Training Neural Networks: https://arxiv.org/abs/1506.01186
    .. _bckenstler/CLR: https://github.com/bckenstler/CLR
    r'   r½   Úmax_lrÚstep_size_upÚstep_size_downr%  ©Ú
triangularÚtriangular2Ú	exp_rangerÊ   Úscale_fnÚ
scale_mode)ÚcycleÚ
iterationsÚcycle_momentumÚbase_momentumÚmax_momentumr=   c                 ó^  •— t        |t        «      s!t        t        |«      j                  › d�«      ‚|| _        t        d||«      }|dk(  rmt        ||j                  «      D ]T  \  }}t        |d   t        «      r7t        |t        «      r|j                  «       n|}|d   j                  |«       ŒP||d<   ŒV t        d||«      | _        t        |«      }|�t        |«      n|}||z   | _        || j                  z  | _        |dvr|€t!        d«      ‚|| _        || _        |  || _        |	| _        | j+                  «        |
| _        |
rÌd|j.                  vrd	|j.                  vrt!        d
«      ‚d	| j
                  j.                  v | _        t        d||«      | _        t        d||«      | _        |dk(  r_t        | j4                  | j2                  |j                  «      D ]1  \  }}}| j0                  r|g|d	   dd  ¢­|d	<   n||d<   ||d<   ||d<   Œ3 t6        ‰| �q  ||«       || _        y )Nr?   r½   r@   rA   rO  rR  z$mode is invalid and scale_fn is NoneÚmomentumÚbetasúMoptimizer must support momentum or beta1 with `cycle_momentum` option enabledr[  r\  r   )r+   r   rX   rY   rZ   r'   r:   r†   r6   r   Úitemr‡   Úmax_lrsr…   Ú
total_sizeÚ
step_ratior7   r%  rÊ   Ú_scale_fn_customrW  Ú_init_scale_fnrZ  ÚdefaultsÚ	use_beta1Úbase_momentumsÚmax_momentumsrª   rb   r^   )rS   r'   r½   rO  rP  rQ  r%  rÊ   rV  rW  rZ  r[  r\  r=   r^   rA   r`   Úlr_valÚ
m_momentumÚ
b_momentumrK   s                       €r.   rb   zCyclicLR.__init__”  sJ  ø€ ô" ˜)¤YÔ/Üœt I›×7Ñ7Ð8Ð8LÐMÓNÐNØ"ˆŒä  ¨I°wÓ?ˆØ˜ÒÜ  ¨9×+AÑ+AÓBò %‘	��EÜ˜e D™k¬6Ô2Ü*4°R¼Ô*@˜RŸW™WœYÀb�FØ˜$‘K×%Ñ% fÕ-à"$�E˜$’Kð%ô % X¨y¸&ÓAˆŒä˜\Ó*ˆà%3Ð%?ŒE�.Ô!À\ð 	ð '¨Ñ7ˆŒØ&¨¯©Ñ8ˆŒàÐAÑAÀhÐFVÜÐCÓDÐDàˆŒ	ØˆŒ
áØ (ˆÔØ$ˆŒØ×ÑÔà,ˆÔÙà )×"4Ñ"4Ñ4Ø 9×#5Ñ#5Ñ5ä Øcóð ð %¨¯©×(?Ñ(?Ð?ˆDŒNÜ"/Ø ¨Mó#ˆDÔô "/¨~¸yÈ,Ó!WˆDÔØ˜RÒÜ58Ø×&Ñ&¨×(;Ñ(;¸Y×=SÑ=Só6ò 8Ñ1�J 
¨Eð —~’~Ø*4Ð)J°u¸W±~ÀaÀbÐ7IÑ)J˜˜gšà,6˜˜jÑ)Ø,6�E˜.Ñ)Ø-7�E˜/Ò*ð8ô 	‰Ñ˜ JÔ/Ø ˆ�r0   c                 ó6  — | j                   �y | j                  dk(  r| j                  | _        d| _        y | j                  dk(  r| j
                  | _        d| _        y | j                  dk(  r-t        | j                  | j                  «      | _        d| _        y y )NrS  rX  rT  rU  rY  )	re  r%  Ú_triangular_scale_fnÚ_scale_fn_refrW  Ú_triangular2_scale_fnr   Ú_exp_range_scale_fnrÊ   re   s    r.   rf  zCyclicLR._init_scale_fnã  s„   € Ø× Ñ Ð,ØØ�9‰9˜Ò$Ø!%×!:Ñ!:ˆDÔØ%ˆD�OØ�Y‰Y˜-Ò'Ø!%×!;Ñ!;ˆDÔØ%ˆD�OØ�Y‰Y˜+Ò%Ü!(¨×)AÑ)AÀ4Ç:Á:Ó!NˆDÔØ*ˆD�Oð &r0   rt   c                 ó^   — | j                   �| j                  |«      S | j                  |«      S )zGet the scaling policy.)re  rp  )rS   Úxs     r.   rV  zCyclicLR.scale_fnð  s0   € à× Ñ Ð,Ø×(Ñ(¨Ó+Ð+à×%Ñ% aÓ(Ð(r0   rt  c                  ó   — yr  r“   ©rt  s    r.   ro  zCyclicLR._triangular_scale_fn÷  s   € àr0   c                 ó   — dd| dz
  z  z  S )Nr   ç       @r“   rv  s    r.   rq  zCyclicLR._triangular2_scale_fnû  s   € à�C˜A ™E‘NÑ#Ð#r0   c                 ó   — | |z  S r*   r“   )rÊ   rt  s     r.   rr  zCyclicLR._exp_range_scale_fnÿ  s   € à�a‰xˆr0   c                 óö  — t        | «       t        j                  d| j                  | j                  z  z   «      }d| j                  | j                  z  z   |z
  }|| j
                  k  r|| j
                  z  }n|dz
  | j
                  dz
  z  }g }t        | j                  | j                  «      D ]f  \  }}||z
  |z  }| j                  dk(  r||| j                  |«      z  z   }n!||| j                  | j                  «      z  z   }|j                  |«       Œh | j                  rÔg }	t        | j                  | j                  «      D ]f  \  }
}||
z
  |z  }| j                  dk(  r||| j                  |«      z  z
  }n!||| j                  | j                  «      z  z
  }|	j                  |«       Œh t        | j                  j                   |	«      D ]&  \  }}| j"                  r|g|d   dd ¢­|d<   Œ"||d<   Œ( |S )zúCalculate the learning rate at batch index.

        This function treats `self.last_epoch` as the last batch index.

        If `self.cycle_momentum` is ``True``, this function has a side effect of
        updating the optimizer's momentum.
        r   rÞ   rX  r_  Nr^  )r˜   r  Úfloorr=   rc  rd  r†   r^   rb  rW  rV  ÚappendrZ  ri  rj  r'   r6   rh  )rS   rX  rt  Úscale_factorÚlrsr½   rO  Úbase_heightrA   Ú	momentumsr[  r\  r^  r‰   s                 r.   rz   zCyclicLR.get_lr  sù  € ô 	(¨Ô-ä—
‘
˜1˜tŸ™°·±Ñ@Ñ@ÓAˆØ�$—/‘/ D§O¡OÑ3Ñ3°eÑ;ˆØ�—‘ÒØ˜tŸ™Ñ.‰Là ™E d§o¡o¸Ñ&9Ñ:ˆLàˆÜ" 4§=¡=°$·,±,Ó?ò 	‰OˆG�VØ! GÑ+¨|Ñ;ˆKØ�‰ 'Ò)Ø˜{¨T¯]©]¸5Ó-AÑAÑA‘à˜{¨T¯]©]¸4¿?¹?Ó-KÑKÑK�Ø�J‰J�r�Nð	ð ×ÒØˆIÜ/2Ø×#Ñ# T×%7Ñ%7ó0ò 
+Ñ+�˜|ð  ,¨mÑ;¸|ÑK�Ø—?‘? gÒ-Ø+¨k¸D¿M¹MÈ%Ó<PÑ.PÑP‘Hà+¨k¸D¿M¹MØŸ™ó=ñ /ñ  �Hð × Ñ  Õ*ð
+ô *-¨T¯^©^×-HÑ-HÈ)Ó)Tò 7Ñ%�˜XØ—>’>Ø,4Ð+P°{À7Ñ7KÈAÈBÐ7OÑ+P�K Ò(à.6�K 
Ò+ð	7ð ˆ
r0   c                 óæ   •— t         ‰| �  «       }|j                  dd «       |j                  d«      }d |d<   |�7t        |t        j
                  «      s|j                  j                  «       |d<   |S )Nrp  re  )rª   rm   r¹   r+   r°   r±   rh   r²   )rS   Ústaterµ   rK   s      €r.   rm   zCyclicLR.state_dict2  sm   ø€ Ü‘Ñ"Ó$ˆð 	�	‰	�/ 4Ô(Ø�Y‰YÐ)Ó*ˆØ$(ˆÐ Ñ!Øˆ>¤*¨R´×1CÑ1CÔ"Dð )+¯©×(8Ñ(8Ó(:ˆEÐ$Ñ%àˆr0   c                 ó´   •— |j                  d«      }t        ‰| �	  |«       |�%| j                  j                  j                  |«       | j                  «        y)rK  re  N)r¹   rª   rs   re  rh   rq   rf  )rS   rm   rµ   rK   s      €r.   rs   zCyclicLR.load_state_dict@  sJ   ø€ à�^‰^Ð.Ó/ˆÜ‰Ñ 
Ô+Øˆ>Ø×!Ñ!×*Ñ*×1Ñ1°"Ô5Ø×ÑÕr0   )
iÐ  NrS  rÞ   NrX  Tgš™™™™™é?gÍÌÌÌÌÌì?r@   )rZ   r‹   rŒ   r�   r   r   r…   r3   r�   r   r   r
   rŽ   rb   rf  rV  Ústaticmethodro  rq  rr  rz   rm   rs   r¿   rÀ   s   @r.   r!   r!   7  s�  ø„ ñZðB !Ø(,ØBNØØ7;Ø5<Ø#Ø"Ø!ØñM!àðM!ð �u˜d 5™kÐ)Ñ*ðM!ð �e˜T %™[Ð(Ñ)ð	M!ð
 ðM!ð ! ™ðM!ð Ð>Ñ?ðM!ð ðM!ð ˜8 U G¨U NÑ3Ñ4ðM!ð Ð1Ñ2ðM!ð ðM!ð ðM!ð ðM!ð õM!ò^+ð)˜Uó )ð ð ð ¨%ò ó ðð ð$ ð $¨5ò $ó ð$ð ð 5ð ¨Uð °uò ó ðò-ô^÷ð r0   r!   c                   óN   ‡ — e Zd ZdZ	 	 	 d
dededededef
ˆ fd„Zd„ Zdd	„Z	ˆ xZ
S )r"   az  Set the learning rate of each parameter group using a cosine annealing schedule.

    The :math:`\eta_{max}` is set to the initial lr, :math:`T_{cur}`
    is the number of epochs since the last restart and :math:`T_{i}` is the number
    of epochs between two warm restarts in SGDR:

    .. math::
        \eta_t = \eta_{min} + \frac{1}{2}(\eta_{max} - \eta_{min})\left(1 +
        \cos\left(\frac{T_{cur}}{T_{i}}\pi\right)\right)

    When :math:`T_{cur}=T_{i}`, set :math:`\eta_t = \eta_{min}`.
    When :math:`T_{cur}=0` after restart, set :math:`\eta_t=\eta_{max}`.

    It has been proposed in
    `SGDR: Stochastic Gradient Descent with Warm Restarts`_.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        T_0 (int): Number of iterations until the first restart.
        T_mult (int, optional): A factor by which :math:`T_{i}` increases after a restart. Default: 1.
        eta_min (float, optional): Minimum learning rate. Default: 0.
        last_epoch (int, optional): The index of the last epoch. Default: -1.

    .. _SGDR\: Stochastic Gradient Descent with Warm Restarts:
        https://arxiv.org/abs/1608.03983
    r'   ÚT_0ÚT_multr  r=   c                 óX  •— |dk  st        |t        «      st        d|› �«      ‚|dk  st        |t        «      st        d|› �«      ‚t        |t        t        f«      st        d|› dt	        |«      › �«      ‚|| _        || _        || _        || _        || _	        t        ‰| �-  ||«       y )Nr   z'Expected positive integer T_0, but got r   z&Expected integer T_mult >= 1, but got z'Expected float or int eta_min, but got z	 of type )r+   r�   r7   r…   rY   r†  ÚT_ir‡  r  ÚT_currª   rb   )rS   r'   r†  r‡  r  r=   rK   s         €r.   rb   z$CosineAnnealingWarmRestarts.__init__e  s¯   ø€ ð �!Š8œ: c¬3Ô/ÜÐFÀsÀeÐLÓMÐMØ�AŠ:œZ¨´Ô4ÜÐEÀfÀXÐNÓOÐOÜ˜'¤E¬3 <Ô0ÜØ9¸'¸À)ÌDÐQXËMÈ?Ð[óð ð ˆŒØˆŒØˆŒØˆŒØˆŒ
Ü‰Ñ˜ JÕ/r0   c           
      ó  — t        | «       | j                  D �cg c]b  }| j                  || j                  z
  dt        j                  t        j
                  | j                  z  | j                  z  «      z   z  dz  z   ‘Œd c}S c c}w )z"Compute the initial learning rate.r   r–   )r˜   r^   r  r  r  r  rŠ  r‰  rÏ   s     r.   rz   z"CosineAnnealingWarmRestarts.get_lr|  s{   € ä'¨Ô-ð  Ÿ=™=ö
ð
 ð	 �L‰LØ˜Ÿ™Ñ%Ø”4—8‘8œDŸG™G d§j¡jÑ0°4·8±8Ñ;Ó<Ñ<ñ>àñóò
ð 	
ùò 
s   šA'Bc                 ór  — |€| j                   dk  rd}|€{| j                   dz   }| j                  dz   | _        | j                  | j                  k\  �r>| j                  | j                  z
  | _        | j                  | j                  z  | _        �n |dk  rt	        d|› �«      ‚|| j
                  k\  rÆ| j                  dk(  r|| j
                  z  | _        nºt        t        j                  || j
                  z  | j                  dz
  z  dz   | j                  «      «      }|| j
                  | j                  |z  dz
  z  | j                  dz
  z  z
  | _        | j
                  | j                  |z  z  | _        n| j
                  | _        || _        t        j                  |«      | _         t        | «      5  t        | j                  j                  | j                  «       «      D ]
  \  }}||d<   Œ 	 ddd«       | j                  j                  D �cg c]  }|d   ‘Œ	 c}| _        y# 1 sw Y   Œ6xY wc c}w )aX  Step could be called after every batch update.

        Example:
            >>> # xdoctest: +SKIP("Undefined vars")
            >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
            >>> iters = len(dataloader)
            >>> for epoch in range(20):
            >>>     for i, sample in enumerate(dataloader):
            >>>         inputs, labels = sample['inputs'], sample['labels']
            >>>         optimizer.zero_grad()
            >>>         outputs = net(inputs)
            >>>         loss = criterion(outputs, labels)
            >>>         loss.backward()
            >>>         optimizer.step()
            >>>         scheduler.step(epoch + i / iters)

        This function can be called in an interleaved way.

        Example:
            >>> # xdoctest: +SKIP("Undefined vars")
            >>> scheduler = CosineAnnealingWarmRestarts(optimizer, T_0, T_mult)
            >>> for epoch in range(20):
            >>>     scheduler.step()
            >>> scheduler.step(26)
            >>> scheduler.step() # scheduler.step(27), instead of scheduler(20)
        Nr   r   z%Expected non-negative epoch, but got rA   )r=   rŠ  r‰  r‡  r7   r†  r�   r  Úlogr{  rƒ   r†   r'   r6   rz   rv   )rS   r{   Únr‰   rA   r`   s         r.   rV   z CosineAnnealingWarmRestarts.stepˆ  sÚ  € ð6 ˆ=˜TŸ_™_¨qÒ0ØˆEàˆ=Ø—O‘O aÑ'ˆEØŸ™ a™ˆDŒJØ�z‰z˜TŸX™XÓ%Ø!ŸZ™Z¨$¯(©(Ñ2�”
ØŸ8™8 d§k¡kÑ1�–à�qŠyÜ Ð#HÈÈÐ!PÓQÐQØ˜Ÿ™Ò Ø—;‘; !Ò#Ø!&¨¯©Ñ!1�D•JäÜŸ™Ø" T§X¡XÑ-°·±¸q±ÑAÀAÑEÈÏÉóó�Að
 "'¨¯©°T·[±[À!±^ÀaÑ5GÑ)HØŸ™ a™ñ*ñ "�D”Jð  $Ÿx™x¨$¯+©+¸!Ñ*<Ñ<�D•HàŸ8™8�”Ø"�”
ÜŸ*™* UÓ+ˆŒä  Ó&ñ 	'Ü#& t§~¡~×'BÑ'BÀDÇKÁKÃMÓ#Rò '‘�˜RØ$&�˜DÒ!ñ'÷	'ð 37·.±.×2MÑ2MÖN¨˜˜t›ÒNˆ�÷		'ð 	'üò Os   Æ6<H(ÈH4È(H1)r   r  r@   r*   )rZ   r‹   rŒ   r�   r   r�   r…   rb   rz   rV   r¿   rÀ   s   @r.   r"   r"   I  sS   ø„ ñð> ØØñ0àð0ð ð0ð ð	0ð
 ð0ð õ0ò.

÷=Or0   r"   c                   ó@   — e Zd ZU eed<   eed<   eed<   eed<   eed<   y)Ú_SchedulePhaseÚend_stepÚstart_lrÚend_lrÚstart_momentumÚend_momentumN)rZ   r‹   rŒ   r…   r�   r’   r“   r0   r.   r�  r�  È  s   … ØƒOØƒMØƒKØÓØÔr0   r�  c                   óê   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 ddedeeee   f   dee	   dee	   dee	   dede
d	   d
edeeee   f   deeee   f   dededede	fˆ fd„Zd„ Zed„ «       Zed„ «       Zd„ Zˆ xZS )r#   a|  Sets the learning rate of each parameter group according to the 1cycle learning rate policy.

    The 1cycle policy anneals the learning rate from an initial learning rate to some maximum
    learning rate and then from that maximum learning rate to some minimum learning rate much
    lower than the initial learning rate.
    This policy was initially described in the paper `Super-Convergence:
    Very Fast Training of Neural Networks Using Large Learning Rates`_.

    The 1cycle learning rate policy changes the learning rate after every batch.
    `step` should be called after a batch has been used for training.

    This scheduler is not chainable.

    Note also that the total number of steps in the cycle can be determined in one
    of two ways (listed in order of precedence):

    #. A value for total_steps is explicitly provided.
    #. A number of epochs (epochs) and a number of steps per epoch
       (steps_per_epoch) are provided.
       In this case, the number of total steps is inferred by
       total_steps = epochs * steps_per_epoch

    You must either provide a value for total_steps or provide a value for both
    epochs and steps_per_epoch.

    The default behaviour of this scheduler follows the fastai implementation of 1cycle, which
    claims that "unpublished work has shown even better results by using only two phases". To
    mimic the behaviour of the original paper instead, set ``three_phase=True``.

    Args:
        optimizer (Optimizer): Wrapped optimizer.
        max_lr (float or list): Upper learning rate boundaries in the cycle
            for each parameter group.
        total_steps (int): The total number of steps in the cycle. Note that
            if a value is not provided here, then it must be inferred by providing
            a value for epochs and steps_per_epoch.
            Default: None
        epochs (int): The number of epochs to train for. This is used along
            with steps_per_epoch in order to infer the total number of steps in the cycle
            if a value for total_steps is not provided.
            Default: None
        steps_per_epoch (int): The number of steps per epoch to train for. This is
            used along with epochs in order to infer the total number of steps in the
            cycle if a value for total_steps is not provided.
            Default: None
        pct_start (float): The percentage of the cycle (in number of steps) spent
            increasing the learning rate.
            Default: 0.3
        anneal_strategy (str): {'cos', 'linear'}
            Specifies the annealing strategy: "cos" for cosine annealing, "linear" for
            linear annealing.
            Default: 'cos'
        cycle_momentum (bool): If ``True``, momentum is cycled inversely
            to learning rate between 'base_momentum' and 'max_momentum'.
            Default: True
        base_momentum (float or list): Lower momentum boundaries in the cycle
            for each parameter group. Note that momentum is cycled inversely
            to learning rate; at the peak of a cycle, momentum is
            'base_momentum' and learning rate is 'max_lr'.
            Default: 0.85
        max_momentum (float or list): Upper momentum boundaries in the cycle
            for each parameter group. Functionally,
            it defines the cycle amplitude (max_momentum - base_momentum).
            Note that momentum is cycled inversely
            to learning rate; at the start of a cycle, momentum is 'max_momentum'
            and learning rate is 'base_lr'
            Default: 0.95
        div_factor (float): Determines the initial learning rate via
            initial_lr = max_lr/div_factor
            Default: 25
        final_div_factor (float): Determines the minimum learning rate via
            min_lr = initial_lr/final_div_factor
            Default: 1e4
        three_phase (bool): If ``True``, use a third phase of the schedule to annihilate the
            learning rate according to 'final_div_factor' instead of modifying the second
            phase (the first two phases will be symmetrical about the step indicated by
            'pct_start').
        last_epoch (int): The index of the last batch. This parameter is used when
            resuming a training job. Since `step()` should be invoked after each
            batch instead of after each epoch, this number represents the total
            number of *batches* computed, not the total number of epochs computed.
            When last_epoch=-1, the schedule is started from the beginning.
            Default: -1

    Example:
        >>> # xdoctest: +SKIP
        >>> data_loader = torch.utils.data.DataLoader(...)
        >>> optimizer = torch.optim.SGD(model.parameters(), lr=1e-4, momentum=0.9)
        >>> scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=0.01, steps_per_epoch=len(data_loader), epochs=10)
        >>> for epoch in range(10):
        >>>     for batch in data_loader:
        >>>         train_batch(...)
        >>>         optimizer.step()
        >>>         scheduler.step()


    .. _Super-Convergence\: Very Fast Training of Neural Networks Using Large Learning Rates:
        https://arxiv.org/abs/1708.07120
    r'   rO  Útotal_stepsÚepochsÚsteps_per_epochÚ	pct_startÚanneal_strategy©r  ÚlinearrZ  r[  r\  Ú
div_factorÚfinal_div_factorÚthree_phaser=   c                 óÚ  •— t        |t        «      s!t        t        |«      j                  › d�«      ‚|| _        |�+|dk  st        |t        «      st        d|› �«      ‚|| _        n`|�S|�Qt        |t        «      r|dk  rt        d|› �«      ‚t        |t        «      r|dk  rt        d|› �«      ‚||z  | _        nt        d«      ‚|  |r_t        || j                  z  «      dz
  dd	d
ddœt        d|z  | j                  z  «      dz
  d	ddd
dœ| j                  dz
  ddd
d
dœg| _
        n;t        || j                  z  «      dz
  dd	d
ddœ| j                  dz
  d	ddd
dœg| _
        |dk  s|dkD  st        |t        «      st        d|› �«      ‚|dvrt        d|› �«      ‚|| _        t        d	| j
                  |«      }|dk(  rEt        | j
                  j                  «      D ]#  \  }}||   |z  |d<   ||   |d	<   |d   |z  |d<   Œ% || _        | j                  rÂd| j
                  j                   vr#d| j
                  j                   vrt        d«      ‚d| j
                  j                   v | _        t        d
||
«      }t        d||	«      }|dk(  rKt%        |||j                  «      D ]1  \  }}}| j"                  r|g|d   dd  ¢­|d<   n||d<   ||d
<   ||d<   Œ3 t&        ‰| �Q  ||«       y )Nr?   r   z/Expected positive integer total_steps, but got z*Expected positive integer epochs, but got z3Expected positive integer steps_per_epoch, but got zBYou must define either total_steps OR (epochs AND steps_per_epoch)r   rB   rO  r\  r[  )r‘  r’  r“  r”  r•  r–   r-  z2Expected float between 0 and 1 pct_start, but got rœ  z>anneal_strategy must be one of 'cos' or 'linear', instead got r@   r^  r_  r`  )r+   r   rX   rY   rZ   r'   r�   r7   r—  r…   Ú_schedule_phasesÚ_anneal_func_typer:   r\   r6   rZ  rg  rh  r†   rª   rb   )rS   r'   rO  r—  r˜  r™  rš  r›  rZ  r[  r\  rž  rŸ  r   r=   rb  r´   r`   rj  ri  rl  rm  rK   s                         €r.   rb   zOneCycleLR.__init__5  s—  ø€ ô$ ˜)¤YÔ/Üœt I›×7Ñ7Ð8Ð8LÐMÓNÐNØ"ˆŒð Ð"Ø˜aÒ¤z°+¼sÔ'CÜ ØEÀkÀ]ÐSóð ð  +ˆDÕØÐ OÐ$?Ü˜f¤cÔ*¨f¸ªkÜ Ð#MÈfÈXÐ!VÓWÐWÜ˜o¬sÔ3°È!Ò7KÜ ØIÈ/ÐIZÐ[óð ð  &¨Ñ7ˆDÕäØTóð ñ 	Ùô !& i°$×2BÑ2BÑ&BÓ CÀaÑ GØ ,Ø&Ø&4Ø$3ñô !& a¨)¡m°d×6FÑ6FÑ&FÓ GÈ!Ñ KØ (Ø*Ø&5Ø$2ñð !%× 0Ñ 0°1Ñ 4Ø ,Ø&Ø&4Ø$2ñð%ˆDÕ!ô4 !& i°$×2BÑ2BÑ&BÓ CÀaÑ GØ ,Ø&Ø&4Ø$3ñð !%× 0Ñ 0°1Ñ 4Ø (Ø&Ø&5Ø$2ñð%ˆDÔ!ð$ �qŠ=˜I¨šM´¸IÄuÔ1MÜØDÀYÀKÐPóð ð
 Ð"3Ñ3ÜØPÐQ`ÐPaÐbóð ð &5ˆDÔ"ô   ¨$¯.©.¸&ÓAˆØ˜ÒÜ'¨¯©×(CÑ(CÓDò I‘
��UØ&-¨c¡l°ZÑ&?��lÑ#Ø")¨#¡,��h‘Ø"'¨Ñ"5Ð8HÑ"H��h’ðIð -ˆÔØ×Òà $§.¡.×"9Ñ"9Ñ9Ø 4§>¡>×#:Ñ#:Ñ:ä Øcóð ð %¨¯©×(?Ñ(?Ð?ˆDŒNÜ)¨.¸)À\ÓRˆMÜ*¨?¸IÀ}ÓUˆNØ˜RÒÜ58Ø! >°9×3IÑ3Ió6ò 8Ñ1�J 
¨Eð —~’~Ø*4Ð)J°u¸W±~ÀaÀbÐ7IÑ)J˜˜gšà,6˜˜jÑ)Ø,6�E˜.Ñ)Ø-7�E˜/Ò*ð8ô 	‰Ñ˜ JÕ/r0   c                 óò   — t        | d«      rZ| j                  dk(  r | j                  |i |¤ŽS | j                  dk(  r | j                  |i |¤ŽS t	        d| j                  › �«      ‚ | j
                  |i |¤ŽS )Nr£  r  r�  zUnknown _anneal_func_type: )rU   r£  Ú_annealing_cosÚ_annealing_linearr7   Úanneal_func)rS   rL   rM   s      r.   Ú_anneal_funczOneCycleLR._anneal_funcº  s�   € Ü�4Ð,Ô-Ø×%Ñ%¨Ò.Ø*�t×*Ñ*¨DÐ;°FÑ;Ð;Ø×'Ñ'¨8Ò3Ø-�t×-Ñ-¨tÐ>°vÑ>Ð>ä Ð#>¸t×?UÑ?UÐ>VÐ!WÓXÐXð $�4×#Ñ# TÐ4¨VÑ4Ð4r0   c                 óp   — t        j                  t         j                  |z  «      dz   }|| |z
  dz  |z  z   S )z@Cosine anneal from `start` to `end` as pct goes from 0.0 to 1.0.r   rx  )r  r  r  )ÚstartÚendÚpctÚcos_outs       r.   r¥  zOneCycleLR._annealing_cosÆ  s8   € ô —(‘(œ4Ÿ7™7 S™=Ó)¨AÑ-ˆØ�e˜c‘k SÑ(¨7Ñ2Ñ2Ð2r0   c                 ó   — || z
  |z  | z   S )zBLinearly anneal from `start` to `end` as pct goes from 0.0 to 1.0.r“   )rª  r«  r¬  s      r.   r¦  zOneCycleLR._annealing_linearÌ  s   € ð �e‘˜sÑ" UÑ*Ð*r0   c                 ó€  — t        | «       g }| j                  }|| j                  kD  rt        d|› d| j                  › �«      ‚| j                  j
                  D ]á  }d}t        | j                  «      D ]†  \  }}|d   }||k  s|t        | j                  «      dz
  k(  rW||z
  ||z
  z  }| j                  ||d      ||d      |«      }	| j                  r| j                  ||d      ||d	      |«      }
 n|d   }Œˆ |j                  	«       | j                  sŒÁ| j                  r
g|d
   dd ¢­|d
<   ŒÝ
|d<   Œã |S )rÇ   zTried to step z/ times. The specified number of total steps is r  r‘  r   r’  r“  r”  r•  r_  Nr^  )r˜   r=   r—  r7   r'   r6   r\   r¢  r5   r¨  rZ  r|  rh  )rS   r~  Ústep_numr`   Ú
start_stepra   Úphaser‘  r¬  Úcomputed_lrÚcomputed_momentums              r.   rz   zOneCycleLR.get_lrÑ  s�  € ä'¨Ô-àˆØ—?‘?ˆà�d×&Ñ&Ò&ÜØ   
Ð*YÐZ^×ZjÑZjÐYkÐlóð ð —^‘^×0Ñ0ò 	*ˆEØˆJÜ% d×&;Ñ&;Ó<ò /‘��5Ø  Ñ,�Ø˜xÒ'¨1´°D×4IÑ4IÓ0JÈQÑ0NÒ+NØ# jÑ0°XÀ
Ñ5JÑK�CØ"&×"3Ñ"3Ø˜e JÑ/Ñ0°%¸¸h¹Ñ2HÈ#ó#�Kð ×*Ò*Ø,0×,=Ñ,=Ø! %Ð(8Ñ"9Ñ:Ø! %¨Ñ"7Ñ8Øó-Ð)ñ
 Ø" :Ñ.‘
ð/ð  �J‰J�{Ô#Ø×"Ó"Ø—>’>Ø&7Ð%M¸%À¹.ÈÈÐ:LÑ%M�E˜'’Nð *ð Ø"òð/	*ð6 ˆ
r0   )NNNg333333Ó?r  Tg333333ë?gffffffî?g      9@g     ˆÃ@Fr@   )rZ   r‹   rŒ   r�   r   r   r…   r3   r   r�   r   rŽ   rb   r¨  r„  r¥  r¦  rz   r¿   rÀ   s   @r.   r#   r#   Ð  s<  ø„ ñbðP &*Ø $Ø)-ØØ49Ø#Ø37Ø26Ø Ø"%Ø!ØñC0àðC0ð �e˜T %™[Ð(Ñ)ðC0ð ˜c‘]ð	C0ð
 ˜‘ðC0ð " #™ðC0ð ðC0ð ! Ñ1ðC0ð ðC0ð ˜U D¨¡KÐ/Ñ0ðC0ð ˜E 4¨¡;Ð.Ñ/ðC0ð ðC0ð  ðC0ð ðC0ð õC0òJ
5ð ñ3ó ð3ð
 ñ+ó ð+ö'r0   r#   )6r�   r  r°   r   Úbisectr   Úcollectionsr   Úcollections.abcr   r   Ú	functoolsr   r   Útypingr	   r
   r   r   r   r   r   r   Úweakrefr   Útorchr   r   r'   r   Ú__all__r„   r’   r:   r%   r˜   rš   rƒ   r   r   r   r   r   r   r   r   r$   r   r   r    r!   r"   r�  r#   r“   r0   r.   ú<module>r½     st  ðá Û Û Û Ý Ý ß .ß $÷	÷ 	ó 	õ ç å  ò€ð(<ð ð#˜ð #¨	ó #÷$K
ñ K
ð\
°+ó 
ô	�;ô 	÷	2ñ 	2ô]
ˆ{ô ]
ô@YJ�{ô YJôx2
ˆ[ô 2
ôj5
�+ô 5
ôpA
�ô A
ôH[
ˆ{ô [
ô|T�Kô Tô@L5�;ô L5ô^9
�;ô 9
ôxU
˜ô U
ôpd5�{ô d5ôNK
˜ô K
ô\Oˆ{ô Oôd|O +ô |Oô~�Yô ôh�õ hr0   