[^(hQJ%UddlZddlZddlZddlZddlmZddlmZddlm Z m Z ddl Z ddl m Z ddl mZddlmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZddlm Z m!Z!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'm(Z(dd l)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/m0Z0dd l1m2Z2Gd dZ3GddeZ4GddZ5GddZ6Gdde,Z7dZ8ddZ9dZ:ddZ;dZddZ?dZ@dd ZAd!ZBdd"ZCd#ZDdd$ZEd%ZFdd&ZGd'ZHdd(ZId)ZJdd*ZKd+ZLdd,ZMd-ZNdd.ZOd/ZPdd0ZQd1ZRdd2ZSd3ZTd4e eUe jfd5eUfd6ZW dd5eXe3fd8ZYe6ee9e:d9d:e*e.d;ddd^d_d`d:dadddbdcddgfe*e(e je'dedefe je'dedefiddddddddddd4S#X;O O   S#X   Or2r(ceZdZdZdZdZy)OptimizerErrorEnumz;Enumerates when an error is raised when testing optimizers.rN)r7r8r9r:CONSTRUCTION_ERROR STEP_ERRORr?r2r0rArAIsEJr2rAc<eZdZdZgdZej edddZy)ErrorOptimizerInputz An OptimizerInput that will cause the optimizer to throw an error when constructed. Includes the type and string of the resulting error. optimizer_error_inputerror_on error_type error_regexr6)rIrJrKc<||_||_||_||_yr.rG)r/rHrIrJrKs r0r1zErrorOptimizerInput.__init__Xs"&;"  $&r2N) r7r8r9r:r;rArC RuntimeErrorr1r?r2r0rFrFPs% SI $66 'r2rFceZdZdZddgfdddddigfdddddd d dd d ed eed fdeed fdedededededededeed ffdZdZ e dZ y ) OptimizerInfoz,Optimizer information to be used in testing.ct|ddSN? gamma step_sizeropts r0zOptimizerInfo.xF3cR@r2ct|Sr.rrXs r0rZzOptimizerInfo.y -c2r2foreachdifferentiable)r`ramaximize capturableFTr?N)scheduler_inputssupported_implsnot_og_supported_flagssupports_sparsehas_capturable_argonly_supports_sparse_gradsmetadata_for_sparsesupports_complexstep_requires_closuresupports_param_groupssupports_multiple_devicesskips decoratorsoptim_error_inputs_funcsupports_fused_on optim_clsre.rfrgrhrirkrlrmrnrrc||_||_||_||_||_||_||_| |_||_| |_ | |_ | |_ | |_ g|r|ng|r|ng|_ ||_||_yr.)rsoptim_inputs_funcrdrerfrgrhrjrirkrlrmrnrprqrr)r/rsrurdrerfrgrhrirjrkrlrmrnrorprqrrs r0r1zOptimizerInfo.__init__isj#!2 0.&<#."4#6 *D' 0%:"%:")B& &jB e" (?$!2r2cg}|jD]U}t|tr2|j|||||s)|j |jE|j |W|Sr.)rp isinstancer is_activeextendappend)r/ test_class test_namedevicedtype param_kwargsresult decorators r0get_decoratorszOptimizerInfo.get_decoratorssd )I)\2&& 65,MM)"6"67 i( ) r2c.|jjSr.)rsr7r4s r0namezOptimizerInfo.names~~&&&r2) r7r8r9r:rtupler>boolr1rpropertyrr?r2r0rOrOfs 6"A2  ,I3 !&#(+0 H!%&+&**. $-/gG3G3*sCx+G30!&c3h1G3>?G3B!CG3F%)GG3RSG3V $WG3Z $[G3^$(_G3f!c?gG3R ''r2rOceZdZdZddZdZy)optimszGDecorator for specifying a list of optimizers over which to run a test.Ncdt||_|||_ytjg|_yr.)r<optim_info_listtorchfloat32dtypes)r/optim_info_iterablers r0r1zoptims.__init__s*#$78 !' 2f  r2c #K| tdtj|j|jD]}\}}|j }||d} t jfd}t j|j|jj|j|} |||| fy#t$r#} td|d|j d| d} ~ wwxYww)NzThe @optims decorator is only intended to be used in a device-specific context; use it with instantiate_device_type_tests() instead of instantiate_parametrized_tests()) optim_infor~c|i|Sr.r?)argsr+tests r0 test_wrapperz.optims._parametrize_test..test_wrappers000r2zFailed to instantiate z for module !)rM itertoolsproductrrr functoolswrapspartialrr7 device_type Exceptionprint) r/r generic_cls device_clsrr~r|rr decorator_fnexs ` r0_parametrize_testzoptims._parametrize_tests  3  "+!2!243G3G!U  J#I+5uEL &1'1 )00--((MM**   $Y lKK- . ,YK|JOOCTTUV  s+AC+A&B<9C+< C(C##C((C+r.)r7r8r9r:r1rr?r2r0rrsQH$r2rctt|dk(r(ttjd||}ttjd||}t t |idt dt t ||gidtd t t d |id |igid td t t dttjd d gdtdt t d|f|gidtdt t |gddd d|fgigidtdgSgS)NcpurBr}r~zinvalid param typer)zPparams argument given to the optimizer should be an iterable of Tensors or dictsrJrKz.a param group cannot have duplicate parametersz/.*a parameter group with duplicate parameters.*r*z@duplicate parameters should not occur across param groups eitherz7some parameters appear in more than one parameter groupMbP?)lrzTensor lr must be 1-elementweightz1all optimizer params should be with/without nameszOall optimizer params should be with/without names. Some param names are missing{Gz?)r*rz8all optimizer param groups should be with/without names.zkall optimizer param groups should be with/without names. cannot add param group with names to the optimizer) _get_device_typerrrandnrFr( TypeError UserWarning ValueErrorr=tensor)r}r~ sample_param sample_param2s r0get_error_inputs_for_all_optimsrsq5( QvU!KL !%++ae"LM '- %n  (,7I 'M  %|4x6NO[ &U  5<<#?@6 &9  %|4mDL &m  $0>>!X}$=#>?S &E ]; ; z r2c ltdddidtdddddtdtjdddd g}tdid tdd d id tdddidtdddidtdddddtdddddgt|dk(r|zSgzS)NrcTr)皙? weight_decayrczcapturable with weight decayrrrcTensor lr with capturabledefaultrrnon-default lrrnonzero weight_decayrbrrbmaximize, weight_decaygffffff?rR)rhorrcudar(rrrr}r~cuda_supported_configss r0optim_inputs_func_adadeltarTsdL$+?lS$'t</ ,,u-TB,   d2I>dD$<>NO 5$'t4' $'T: !4(* :3 # r2c t||}t|dk(rtjdd|tj}tj ||_|ttdtddtd ttdtd d td ttdtddtd tt|gtdtdtjgz }|S)Nrr)gKHr)epszepsilon1 should be >= 0r)r)rz invalid dz#Clipping threshold d should be >= 1皙?)rzinvalid beta2_decayzbeta2_decay should be <= 0z#does not support complex parametersz-Adafactor does not support complex parameters)rJrKrI)rrrrand complex64 rand_likegradrFr(r=rrMrArD)r}r~r complex_params r0!optim_error_inputs_func_adafactorrs265AL5( 1aeooN "__];  N32 &5  #;$ &A  C0. &8  )?6> (K+66 9& & N r2c tdidtdddidtdddddtdd did tdddd d tddd dddtdd tjdidgS)Nrr)rrrTrrbrr)initial_accumulator_valuerr?)rlr_decayrrr Tensor lrr(rrrs r0optim_inputs_func_adagradrsd2I> 5  (+=  "&6f%=%G !R# Q$%5V$<$E 2% O *    &E #&ELL  & Lr2ct||}t|dk(r[|ttdt dddt dttdt dd d t d ttdt t jd ddt dttdt ddt jdfdt dttdt dt jddfdt dttdt dt jdt jdfddt dgz }t|dk(r_t jd||}|tt|gddddtdtt|gddddtdgz }|S)Nrr?rrrbeta1 should be between 0 and 1r)&Invalid beta parameter at index 0: 1.0rrrweight_decay should > 0Invalid weight_decay value: -1rT)rr`z7lr as Tensor doesn't work with foreach & not capturablezElr as a Tensor is not supported for capturable=False and foreach=TruerRrz0betas must be either both floats or both Tensors)rrr`zMbetas\[0\] as a Tensor is not supported for capturable=False and foreach=Truerr?r)r`fusedz/`fused` and `foreach` cannot be `True` together)rraz)`fused` does not support `differentiable`) rrrFr(r=rrremptyrM)r}r~r sample_tensors r0optim_error_inputs_func_adamrLs265AL5( 4z:: &D  4b92 &<  5<<#6ER &c  4U\\$5G/HIK &N  4 S0A4/HIK &N  $||C0%,,t2DE $ j&l ]; ; x6) BvUC  )?'+d;J (M  )?%)TBD (G   ( r2c tdddidtddddddtddddddtddd ddd tdtjd dd dd d g}tdidtdddidtdddidtdddidtdddddgt|dk(r|zSgzS)NrcTr)rRrrbrcz"capturable, maximize, weight_decayrcapturable, maximizeFcapturable, weight_decayrrrrbrcz#capturable, weight_decay, tensor LRrrrrrrrbrrrrrs r0optim_inputs_func_adamaxrs-dL$+?lS$'TN5 $%4tL' $'U$O+ ll5) #!"  7 #< d2I>dD#;=MN 5dGS>@STdD$<>NOdD#;TBdJ+=JO 5    r2c tdidtdddidtddtjdidtddd idtdd d id gS) Nrr)rrrrrtolerance_gradư>line_search_fn strong_wolferrs r0optim_inputs_func_lbfgsr-svd2I>dD$<>NOu||E': ;+  !14 8?O  $n5  r2ct||}|Sr.)rrs r0optim_error_inputs_func_lbfgsr?s265AL r2c tdddidtddddddtddddddd tdtjd ddddd d g}tdid tdd d idtdddidtdddidtdddddtddddddtdddddgt|dk(r|zSgzS)NrcTr)rRg~jtx?)rmomentum_decayrcr )rrdecoupled_weight_decayrcz"decoupled_weight_decay, capturabler)rrrrrcrrrrznon-zero momentum_decayrr)rrzweight_decay, momentum_decay)rrrrrrbrrrs r0optim_inputs_func_nadamrDs`dL$+?lS$'4tT+  #"&*."  6  ll5) #"&*." 6 #< d2I>dD$<>NO$d+*    $'4@/  #"&*. *  $'T: ;" D$4F#;v#E E" O"ODLNE" O"Or2c t||}t|dk(rU|ttdt dddt dttdt dd d t d gz }|S) Nrrrrrr)rrgɿ)rrzmomentum_decay should > 0z"Invalid momentum_decay value: -0.2rrs r0optim_error_inputs_func_nadamr s265AL5( 4z:: &D  4=4 &@   ( r2c tdddidtdddddtddddddtdtjd dddd d g}tdid tdd didtdddidtdddidtdddddtdddddgt|dk(r|zSgzS)NrcTr)r)rcrr)rcrrz0capturable, weight_decay, decoupled_weight_decayr)rrcrrz;capturable, weight_decay, decoupled_weight_decay, tensor LRrrgMb`?rrrznon-default epsrr)rrrrrbrrrs r0optim_inputs_func_radamr"s/dL$+?lS" #,   " #*. D  ll5)" #*.  O '> d2I>dD$<>NOdE4=?PQ 5dD$<>NO 5rznon-default step_sizesrbrrrs r0optim_inputs_func_rpropr1;sdL$+?lS,,u-TB,  d2I>dD$<>NO 4;M   +.) dJ+=JO $4F#;v#E  O OLN O Or2c t||}t|dk(r-|ttdt dddt dgz }|S) Nrr)rr)rr/z0 < eta1 < 1 < eta2r)zInvalid eta values: 1.0, 0.5rrrs r0optim_error_inputs_func_rpropr3TsZ265AL5( 4j9. &:    r2c@tdidtdddidtddtjdidtddd id tdd d id tdd dddtdd d ddtdd d ddtdd dd ddg S)Nrr)rrrrz tensor lrrrznon-zero weight_decayr'rRrTrrb)r' dampeningr5)r'rzweight_decay w/ momentum)r'nesterovrr6rrs r0optim_inputs_func_sgdr7esd2I>dD$<>NOu||E': ;+   5t 3C  u||E': ;+  dJ+=JO  r2c,t||}t|dk(r|ttdt dddt dttt jd t j|| gid t d ttd t jd t j|| gigidt d ttt jdd |t jgidt dgz }|S)Nrrrrrr)rrr)layoutr}r~zdense params requiredz+SparseAdam requires dense parameter tensorsr*z%dense params required in param_groupsrrzcomplex not supportedz.SparseAdam does not support complex parameters) rrrFr(r=rrzeros sparse_coorrrs r0"optim_error_inputs_func_sparseadamr@s+265AL5( 4z:: &D   e&6&6vU 0&I  % % $%+0+;+;+1*/ !"'  @ &I% ( !JJq!F%//RS0 &L W4 4 j r2r}returnct|tjrt|j}t|tsJ|j ddS)N:r)rwrr}r>typesplit)r}s r0rrsB&%,,'V[[! fc "" " << Q r2r?c tdDsJdj}tfdjD}g}|D]}t |j }t |dk7r4|D]} d|| < |jtd||jn|j||D]>} t |} d| | <|jtd| |jd | @|S) a~ Return a list of all configs for a given optimizer as a list of OptimizerInputs, including configs that have supported global cliquey kwargs (foreach, fused, differentiable) based on optim_info.supported_impls. The configs (optim_inputs) returned by optim_info.optim_inputs_func(...) intentionally do NOT include global cliquey kwargs to give flexibility to tests. For example, testing correctness between toggling foreach on and off is now trivial. That said, we sometimes want to test for all possible configs on an optimizer including all supported flags, so this helper returns all optim inputs. c3$K|]}|dv yw))r`rraNr?).0xs r0 zD_get_optim_inputs_including_global_cliquey_kwargs..s89 33sz?skip must be a subset of ['foreach', 'fused', 'differentiable']c3K|]@}|vr:tjvs|dk7rttvs|dk7r|Byw)rr`N)rrrr&)rHrIr}rskips r0rJzD_get_optim_inputs_including_global_cliquey_kwargs..sQ  D= f %)E)E Eg V $(N(P PI~  sAA rFNr)Tz & ) allrurrerr+lenrzr(r,) r}r~rrL optim_inputsreall_optim_inputs optim_input base_kwargsflag new_kwargss ` `` r01_get_optim_inputs_including_global_cliquey_kwargsrUs8 =A IHI //7L ++  O# {112  1 $' *$) D! *  # #d;[EUEUV   # #K 0$ D!+.J#Jt   # # Kgavt>c>txr|dtjk(SNr~r%rfloat64ras r0rZrZK)>*57Ou}}4r2)rbgiUMu>gkNuϵ>TestCudaOptims+test_grad_scaling_autocast_fused_optimizersc|d Srwr?ras r0rZrZnryr2zTErrors w/ Global state changed, see https://github.com/pytorch/pytorch/issues/116028) rurdrqrerhrfrrrproc|d Srwr?ras r0rZrZryr2z2Uses too much memory, even for H100, surprisingly.c>txr|dtjk(Srrras r0rZrZrr2c|d Srwr?ras r0rZrZryr2)rurqrerfrrrhrprogiUMu>gh㈵>z7ASGD internally changes the weights even with zero gradg9̗?zDoes not support param groupsz!LBFGS doesn't support multidevice*test_forloop_goes_right_direction_multigpu6test_param_group_with_lrscheduler_goes_right_directionc6tjdk(xr|dS)Ndarwinr_)sysplatformras r0rZrZgs)A**=)r2)rurqrerlrmrnroc|d Srwr?ras r0rZrZwryr2z8Errors, https://github.com/pytorch/pytorch/issues/117150gv !>gw$}>c|d Srwr?ras r0rZrZryr2gMb@?rc|d Srwr?ras r0rZrZryr2ct|ddSrQrWrXs r0rZrZs3"=r2c t|dddS)Nrrrr end_factorrrrXs r0rZrZHccqr2ct|ddSrQrWrXs r0rZrZr[r2c t|dddS)Nrg333333?rrrrXs r0rZrZrr2ct|ddS)NrrSrTrWrXs r0rZrZ!sF3dbAr2ct|dS)Nrr|r}rXs r0rZrZ"s M#T:r2ct|Sr.r]rXs r0rZrZ#r^r2ct|ddSrrrXs r0rZrZ%rr2ct|ddSrrrXs r0rZrZ&rr2ct|ddSrQrWrXs r0rZrZ(r[r2ct|Sr.r]rXs r0rZrZ)r^r2ga2U0*s?)rrbr'r6rct|ddS)Nrqi,rTrWrXs r0rZrZ>s7cBr2gy&1|?) rurdrqrerfrgrjrrrorg{Gz?z0SparseAdam is not yet supported on the XPU stack)msgz,cannot call to_sparse on p.grad, see #117184test_tensor_lrz8SparseAdam does not support dense gradients, see #116507 test_state_dict_with_cuda_params)rurqrerirjrkrooptim_dbc*eZdZdZddZdZdZdZy) TensorTrackera0 A utility to track tensor clones in a list, with the expectation of popping them later (in order) to make fair comparisons between two multi-step computation. The intended use case is usually when comparing two supposed equal computations, such as an optimizer step that each individually consists of multiple steps, where numerical deviation could multiply. The goal is to be able to compare and align numbers at every milestone so as to minimize numerical discrepancies, and so when the test fails, it is likely a real problem. Nc(|i}||_g|_yr.)assert_eq_kwargstensors)r/rs r0r1zTensorTracker.__init__s  #!  0 r2cr|jj|jjy)z@ Add a detach().clone()'d version of the tensor N)rrzdetachclone)r/rs r0addzTensorTracker.adds% FMMO1134r2c|jt|jdd|jjd}|j t |t dt||j||fi|jtj5|j|dddy#1swYyxYw)z Pop the first element in the tensor tracker, assert equality between the popped tensor and the input tensor, and then set the input tensor to have the same values as the popped tensor (with copy_). rzno tensors to popz type(ref)=N) assertGreaterrNrpop assertTruerwrrD assertEqualrrno_gradcopy_)r/ tensor_to_settestcaserefs r0 pop_check_setzTensorTracker.pop_check_sets s4<<0!5HIllq!JsF3 S |_E]CI43H3HI ]]_ %    $ % % %s B;;Cc2t|jdk(S)Nr)rNrr4s r0 all_poppedzTensorTracker.all_poppeds4<< A%%r2r.)r7r8r9r:r1rrrr?r2r0rrs 5 %&r2rr.)NN)r?)drrrunittestcopyrenumrtypingrrrrtorch.nnr torch.optimr r r r r rrrrrrrrrrtorch.optim.lr_schedulerrrrrrr*torch.testing._internal.common_device_typerr2torch.testing._internal.common_methods_invocationsr $torch.testing._internal.common_utilsr!r"r#r$r%torch.utils._foreach_utilsr&r(rArFrOrrrrrrrrrrrrrr rrrrrr r"r$r(r,r1r3r7r9r;r@r>r}rr<rUrrLbfloat16rrrexpectedFailurer__annotations__rr?r2r0rs  "NKNOO,'',Z'Z'z/ /fA|OB"4,^<&IXUp.Ob"17+O\"$ @OF60Of48Ov"O2"D" 9x U3 #45 # %'6 .6v4 @5 !-0";  !f#!   ! s!'!%(/"  !I#2  ?& -\5 A$+  FG.""I   !"JK"0"   !"JK"7"   !"JK"+  !"JK".  !"JK"*"   !"JK")  !"JK"&"   !"JK"0"   !"JK"*  !"JK";"   !"LM";  !"JK"/"   !"JK"2"    FG."!   !"JK"0!   !"JK"7!   !"JK"*!   !"JK"&!   !"JK"0!   !"JK";!   !"JK"/!   !"JK"2!  CG O`3 ?> #q 9FB    "D(A s4'@ #,  $ "3A!   !-0";  !f#!   !I#2  ) MBF 0 6 7 G HF9  :2 D D D E@2  $!=> 1 !5==#62O"PQ",5    "D(A s4'@ #,  " s7'C != 1# J "3A!   !j#;   !f#!   !I#2  - IbF2 >5 "3A!   !-0";  !f#!    RS"+  !I#2  3  'P 1 => 1 !5==#62O"PQ",5   !D(A s4'@ #, " ! s!%!((!= 5& P "3A!   !j#;   !f#!   !I#2  - kSh 0 <5 !j#;   !f#!   ! sT'B #2  !I#2    M#2  ?& -\ 1 ="#"' "0!   !!'!%*#!   =>"&   =>"0   AB"<   =>"H  ((."*  S0 9t 1 =5 "3A!   !j#;   !f#!   !N#*   !I#2  ;$ +X 1 =5 !j#;   !f#!   ! sV'D #.  !I#2  3  'P3 ?5 "3A!   !-0";  !f#!   ! sD'A #)/  !I#2  =% ,Z 1 =5 "3A!   !-0";  !f#!   !I#2  ) "F / = >  A B:2  D D D E@2 ) 2!<> !! !  C C   !j#;   !f#!   ! sE'B #)/  !I#2  3  kVn6 B#'"D\2. "!  PQ  !"PQ"&  !"PQ"    N#0   !"PQ"*  !"PQ"3  !"PQ"<  !"PQ"H  !"PQ"2  !"PQ"7 g8 AIF!$} FR(&(&r2