T^(hʏndZddlZddlmZddlmZddlmZmZm Z m Z ddl Z ddl Z ddl m Z ddlmZdd lmZdd lmZmZmZmZmZmZmZmZd d lmZej:eZd Z dZ!da"dZ#Gdde jHjJZ&d)dZ'd)dZ(Gdde jRZ*Gdde jRZ+Gdde jRZ,GddeZ-eGddeZ.eGddeZ/d Z0d!Z1ed"e0Gd#d$e-Z2ed%e0Gd&d'e-eZ3gd(Z4y)*zPyTorch RWKV model.N) dataclass)Path)ListOptionalTupleUnion)nn)GenerationMixin)PreTrainedModel) ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_bitsandbytes_availableis_ninja_availableis_torch_cuda_availablelogging) RwkvConfigzRWKV/rwkv-4-169m-pilercddlm}ttj j j j dz dz }dDcgc]}||z  }}t t j|k(rytjd|ddd d d d d d|g}|d||tjtjk(|a|t _ycc}w)Nr)loadkernelsrwkv)z wkv_op.cppz wkv_cuda.cuzwkv_cuda_bf16.cuz2Loading CUDA kernel for RWKV at context length of .z -res-usagez--maxrregcount 60z--use_fast_mathz-O3z -Xptxas -O3z--extra-device-vectorizationz-DTmax=wkv_)namesourcesverboseextra_cuda_cflags) torch.utils.cpp_extensionrr__file__resolveparentrwkv_cuda_kernelmax_seq_lengthloggerinfor get_verbosityDEBUG)context_length load_kernel kernel_folderfcuda_kernel_filesflagss d/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/rwkv/modeling_rwkv.pyload_wkv_cuda_kernelr23s=N**,33::AAIMPVVM4efq*ff#(8(G(G>(Y KKD^DTTUVW  & .!" E#N# $!&&(GMM9  '5#/gs Cc0eZdZeddZeddZy)RwkvLinearAttentionNc |j\}}} |tjkDr td|dtjd|| zt | dzdk7rtd|d| dt | dd |j |_|jjd k7sK|jjd k7s2|jjd k7s|jjd k7r td tj|jj }|j tjk(r0|j}|j}|j}|j}|j}|j}tj|tj } |s||Vtj || d tj"|jtj}|dddddfxxdzcc<nBtj$|D cgc]} | j'dc} dj}|j tj(k(rtj*} ntj,} | ||||| |nI|j tj(k(rtj.ntj0} | ||||| |j3||||| |4tj4|d dD cgc]} | j7d}} | j9|j |fScc} wcc} w)NzCannot process a batch with z+ tokens at the same time, use a maximum of z with this model. rzThe product of batch size (z) and hidden size (z") needs to be a round multiple of rcudazUCalling the CUDA kernel for wkv attention requires all tensors to be on CUDA devices. memory_formatr )dtypedevicer9籡*G)dim)sizer%r& ValueErrorminr: input_dtyper;typetorchexpfloat contiguousfloat16 empty_likecontiguous_formatzerosfloat32cat unsqueezebfloat16forward_with_state_bf16forward_with_state forward_bf16forwardsave_for_backwardchunksqueezeto) ctx time_decay time_firstkeyvaluestate return_state batch_sizeseq_len hidden_sizeoutputs forward_funcs r1rSzRwkvLinearAttention.forwardTs+.88:( G[ %44 4.wi7b#2233DF   #c+r&: :a ?-j\9L[MZ";34A7  ))    " "f ,  %%/zz&(||  F*tu uii 0 0 2 = = ?@@ 99 %#))+J))+CKKME**, nn  "!!#U5L5LM 5,} --::"'"9"9 aAg$& 5"Aa1;;q>"AqITTVyyENN*/GG /BB ZeVU K>$A%..u}} '' %BYBYZ   E4K4KL""58O8OP %-- '~~'H:E:W(66]m]v]v          !      OOK ( OOK ( HH[ ! JJ{ #     NFN)__name__ __module__ __qualname__ staticmethodrSrhrpr1r4r4Ss)<1<1|% % rpr4c|j\}}}tj|}|tj|dddftj} tj|dddftj} tj|dddftjdz } n|\} } } tj| }t |D]} |dd| fj } |dd| f}tj| | |z}tj| |z }tj| |z|z }|| z||zz}|| z|z}||z j|j|dd| f<tj| |z| }tj| |z|z }tj| |z }|| z||zz} || z|z} |}  |s|| | | g}||fS)Nr)r:r=) r?rD zeros_likerLrErangerFmaximumrWr:)rYrZr[r\r]r^_ seq_lengthrb num_state den_state max_state current_index current_key current_valuemax_for_outpute1e2 numerator denominator max_for_states r1rwkv_linear_attention_cpursxxzAz1   c "F }$$SAYemmD $$SAYemmD $$SAYemmDtK */' 9i ))J''Jz*" !]*+113 a./ y+ 2JK YYy>1 2 YY{Z/.@ ANR-%77 9nr) $- $;#?#? #Mq-  i*& YY{]2 3NR-%77 NR' ! %"(u(Iy1 5=rpctd||||fD}|jddk(}t|s|rt||||||Stj ||||||S)Nc3NK|]}|jjdk7yw)r7N)r;rC).0ts r1 z(rwkv_linear_attention..sXa!((--6)Xs#%rr]r^)anyr?r%rr4apply)rYrZr[r\r]r^no_cuda one_tokens r1rwkv_linear_attentionrsmXJ CQV3WXXG q I7i(ZeSXgstt"((ZeUT`aarpc0eZdZdfd ZddZddZxZS)RwkvSelfAttentioncrt|||_tduxrtj|j k(}t r"tr|s t|j ||_ |j}|j |jn|}||_tj t#j$||_tj t#j$||_tj t#j$dd||_tj t#j$dd||_tj t#j$dd||_tj0d|_tj4||d|_tj4||d|_tj4||d|_tj4||d|_y#t$rtjdYwxYw)Nz9Could not load the custom CUDA kernel for RWKV attention.rrrrFbias)super__init__configr%r&r+rrr2 Exceptionr'r(layer_idraattention_hidden_sizer ParameterrDemptyrYrZ time_mix_keytime_mix_valuetime_mix_receptance ZeroPad2d time_shiftLinearr[r\ receptancerb)selfrr kernel_loadedrar __class__s r1rzRwkvSelfAttention.__init__s  (4q9I9X9X\b\q\q9q  $;$=m Y$V%:%:;! (( ,2,H,H,TF ( (Ze &;",,u{{3H'IJ,,u{{3H'IJLLQ;)GH ll5;;q![+IJ#%<< Aq+0N#O ,,}599[*?eLYY{,AN ))K1FUSii 5{O ) Y WX YsHH65H6cp|jddk(r||ddddd|jf}n3|j|}| |ddddd|jf|dddf<||jz|d|jz zz}||jz|d|jz zz}||j z|d|j z zz}|j |}|j|}tj|j|}| |dddf|ddddd|jf<||||fSNrrr) r?rrrrrr[r\rDsigmoidr)rhiddenr]shiftedr[r\rs r1extract_key_valuez#RwkvSelfAttention.extract_key_values< ;;q>Q 5#4Ahq!T]]23Goof-G %aAt}})< =1 t(((7a$:K:K6K+LL,,,w!d>Q>Q:Q/RRd666AH`H`D`9aa hhsm 5!]]4??:#>?  ,21b5ME!HQ4==( )3u,,rpcj||\}}}}|tfd|ddDnd}tjj||||\}}|T|d|dddddj f<|d|dddddj f<|d|dddddj f<j ||z|fS) Nr]c3JK|]}|ddddjfywrrr)rrcrs r1rz,RwkvSelfAttention.forward..(s!FqAaDMM12Fs #r<rrrr )rtuplerrYrZrrb) rrr] use_cacherr[r\ layer_staters ` r1rSzRwkvSelfAttention.forward&s(,(>(>vU(>(S% CJOJ[eFE!"IFFae 1 OO OO  "  k  ",7NE!HQ4==( ),7NE!HQ4==( ),7NE!HQ4==( ){{:,-u44rprrrrq)rsrtrurrrS __classcell__rs@r1rrsP<-&5rprc(eZdZdfd ZddZxZS)RwkvFeedForwardcBt|||_||_|j}|j |j nd|jz}t jd|_t jtjdd||_ t jtjdd||_ t j||d|_t j||d|_t j||d|_y)NrrrFr)rrrrraintermediate_sizer rrrrDrrrrr[rr\)rrrrarrs r1rzRwkvFeedForward.__init__;s    (( (.(@(@(LF $ $RSV\VhVhRh ,,}5LLQ;)GH#%<< Aq+0N#O 99[*;%H))K5IYY0+EJ rpcz|jddk(r||ddddd|jf}n3|j|}| |ddddd|jf|dddf<||jz|d|jz zz}||jz|d|jz zz}t j t j|j|}|j|}t j|j|}| |dddf|ddddd|jf<||z|fSr) r?rrrrrDsquarerelur[r\rr)rrr]rr[rr\s r1rSzRwkvFeedForward.forwardLs) ;;q>Q 5#4Ahq!T]]23Goof-G %aAt}})< =1 t(((7a$:K:K6K+LLd666AH`H`D`9aa ll5::dhhsm45 3]]4??:#>?  ,21b5ME!HQ4==( )E!5((rprrrrsrtrurrSrrs@r1rr:sK")rprc&eZdZfdZddZxZS) RwkvBlockct|||_||_|dk(r0t j |j |j|_t j |j |j|_ t j |j |j|_ t|||_ t|||_y)Nr)eps)rrrrr LayerNormralayer_norm_epsilonpre_lnln1ln2r attentionr feed_forward)rrrrs r1rzRwkvBlock.__init__as    q=,,v'9'9v?X?XYDK<< 2 28Q8QR<< 2 28Q8QR*68<+FH=rpc|jdk(r|j|}|j|j|||\}}||z}|j |j ||\}}||z}||f}|r||fz }|S|dz }|S)Nr)r]rrrr)rrrrrr)rrr]routput_attentionsrroutputss r1rSzRwkvBlock.forwardos ==A [[(F>>$((6*:%S\>] 5)#"//0@/N e,&5/   | #G w Grp)NFFrrs@r1rr`s  >rprc4eZdZdZeZdZdgZddgZdZ dZ dZ y) RwkvPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. rrrYrZTc t|trv|j}|jj}|jj }|j }||dz z }d||z z }tjt|Dcgc]}||z  c}|jj|jj} | ddddf} t|D cgc]} dd| |dz z dd|zzzzz} } tj| |jj|jj} tjt|Dcgc] }|dzd zdz c}|jj|jjd z} tj5| |j_tj"|jt%j&d z| z|j_tj(| ||j_tj(| |d |zz|j*_tj(| d |z|j,_dddyt|t.r |j}|jj}|jj }d||z z }tjt|Dcgc]}||z  c}|jj|jj} | ddddf} tj5tj(| ||j_tj(| ||j,_dddyycc}wcc} wcc}w#1swYyxYwcc}w#1swYyxYw) zInitialize the weights.rg?r:r;Ngffffff?g?r g?g333333?) isinstancerrrnum_hidden_layersrarrDtensorrzrr:r;rYrZno_graddata ones_likemathlogpowrrr) rmodulerrrar ratio_0_to_1ratio_1_to_almost0i time_weighth decay_speedzigzags r1 _init_weightsz!RwkvPreTrainedModel._init_weightss] f/ 0H & ? ?  --33K$*$@$@ !#'81'<=L!$3D(D!E ,,*/ *<=Q[=))//**11K &dD!m4K45Q!4q89sS L c c>  ] ]s2 N&N+N0.CN55 OA O5N>ON) rsrtru__doc__r config_classbase_model_prefix_no_split_modules_keep_in_fp32_modulessupports_gradient_checkpointing _is_statefulrrwrpr1rrs8 L$ )<8&*#L7]rprceZdZUdZdZeejed<dZ ee ejed<dZ ee ejdfed<dZ ee ejdfed<y) RwkvOutputa Class for the RWKV model outputs. Args: last_hidden_state (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`): Sequence of hidden-states at the output of the last layer of the model. state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nlast_hidden_stater]. hidden_states attentions)rsrtrurrrrD FloatTensor__annotations__r]rrrrrwrpr1rrsw,6:x 1 129/3E8D**+ ,3=AM8E%"3"3S"89:A:>Ju00#567>rprceZdZUdZdZeejed<dZ eejed<dZ ee ejed<dZ ee ejdfed<dZee ejdfed<y) RwkvCausalLMOutputa| Base class for causal language model (or autoregressive) outputs. Args: loss (`torch.FloatTensor` of shape `(1,)`, *optional*, returned when `labels` is provided): Language modeling loss (for next-token prediction). logits (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`): Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax). state (list of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`): The state of the model at the last time step. Can be used in a forward method with the next `input_ids` to avoid providing the old `input_ids`. hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`): Tuple of `torch.FloatTensor` (one for the output of the embeddings, if the model has an embedding layer, + one for the output of each layer) of shape `(batch_size, sequence_length, hidden_size)`. Hidden-states of the model at the output of each layer plus the optional initial embedding outputs. attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`): Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length, sequence_length)`. Attentions weights after the attention softmax, used to compute the weighted average in the self-attention heads. Nlosslogitsr].rr)rsrtrurrrrDrrrr]rrrrrwrpr1rrs0)-D(5$$ %,*.FHU&& './3E8D**+ ,3=AM8E%"3"3S"89:A:>Ju00#567>rpra> This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`RwkvConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.LongTensor` of shape `(batch_size, input_ids_length)`): `input_ids_length` = `sequence_length` if `past_key_values` is `None` else `past_key_values[0][0].shape[-2]` (`sequence_length` of input past key value states). Indices of input sequence tokens in the vocabulary. If `past_key_values` is used, only `input_ids` that do not have their past calculated should be passed as `input_ids`. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`torch.LongTensor` of shape `(batch_size, input_ids_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. This is currently not used by `RwkvModel`, but will be supported in the future. [What are attention masks?](../glossary#attention-mask) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. state (tuple of five `torch.FloatTensor` of shape `(batch_size, hidden_size, num_hidden_layers)`, *optional*): If passed along, the model uses the previous state in all the blocks (which will give the output for the `input_ids` provided as if the model add `state_input_ids + input_ids` as context). use_cache (`bool`, *optional*): If set to `True`, the last state is returned and can be used to quickly generate the next logits. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. z^The bare RWKV Model transformer outputting raw hidden-states without any specific head on top.c6eZdZfdZdZdZeeee e e  dde e jde e jde e jde ee jd e ed e ed e ed e ed eee ffdZdZdZxZS) RwkvModelc t||tj|j|j |_tjt|jDcgc]}t||c}|_ tj|j |_ d|_d|_|j!ycc}w)NrF)rrr Embedding vocab_sizera embeddings ModuleListrzrrblocksrln_outlayers_are_rescaledgradient_checkpointing post_init)rridxrs r1rzRwkvModel.__init__Hs  ,,v'8'8&:L:LMmmPUV\VnVnPo$pYv%D$pq ll6#5#56 #( &+# %qs&C c|jSrrrrs r1get_input_embeddingszRwkvModel.get_input_embeddingsVs rpc||_yrrrrnew_embeddingss r1set_input_embeddingszRwkvModel.set_input_embeddingsYs (rp checkpoint output_typer input_idsattention_mask inputs_embedsr]rroutput_hidden_states return_dictreturnc ||n|jj}||n|jj}||n#|js|jjnd}||n|jj }|t jd|j|jk(r|j| | td| | td||j|}|r||jd|jj|jjf} tdD cgc]A} t!j"| | dkr |j$nt j&|j(dC}} |d xxd zcc<|j*r%|jr|rt jd d}|} |rd nd} |rd nd} t-|j.D]\}}|j*r0|jr$|j1|j2| |||\} }}n|| ||| \} }}|jr=|jj4dkDr$|dz|jj4zdk(r| dz } |r| | fz} |s| |fz} |j7| } |r| | fz} |st9d| || | fDSt;| || | Scc} w)NFz<`attention_mask` was passed, but it is unused in this model.zDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrrrrgꌠ9Y>)FzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...rw)r]rrr<c3&K|] }|| ywrrrw)rxs r1rz$RwkvModel.forward..stqfgfsts)rr]rr)rrrtrainingruse_return_dictr' warning_oncer _rescale_layersr@rr?rarrzrDrKr:rLr;r  enumerater_gradient_checkpointing_func__call__ rescale_everyrrr)rrrrr]rrrrshaperrall_self_attentionsall_hidden_statesr blockrs r1rSzRwkvModel.forward\s"2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IZ^ZgZgT[[=R=Rmr %0%cd d  =#8TU U   OOI6M "''*DKK,C,CT[[EbEbcE q  a-"5"5U]][h[o[oE !H H  & &4==##p" % $5b4"6BD#DKK0 JJC**t}}373T3TNNM5)EV40 uj49!)Wh40 uj ((KK--11W 9 99Q> - 1 #$58H$H! &9ZM&I#+ J. M2  1]4D D t]E;LNa$btt t++*   es5AK c |j|j k(ry|jjdkDrt j 5t |jD]\}}|jr|jjjjdt||jjzz|jjjjdt||jjzzt|jjjdr|jjjj j#dt||jjzz|jjjj j#dt||jjzzt|jjjdrN|j%|jj||j%|jj||jjjj#dt||jjzz|jjjj#dt||jjzz ddd|j |_y#1swYxYw)Nrr<SCB quant_state)r r"rr)rDrr&rrrbweightmul_intrr\hasattrr/div_ _bnb_4bit_dequantize_and_rescale)rblock_idr-s r1r%zRwkvModel._rescale_layerss`  # #DMM(9 :  ;; $ $q ( r'0'=rOHe}}..55::1HPTP[P[PiPiDi@j;jk**0077<>qCTXT_T_TmTmHmDn?no!..44;;@@c(VZVaVaVoVoJoFpApqr r"(,}}#4 # r rs KL77Mcts tdddl}|jj |j j |j j}|jdt||jjzz|jj|jddj|j}t!|d|y) z Perform the dequantization and rescaling of the weights of a given layer. After that operation the layer will be quantized again. z/Please install bitsandbytes to use this method.rNr<cpuF) requires_gradr1)r ImportError bitsandbytes functionaldequantize_4bitr1rr0r5r3rr)r Params4bitrWr;setattr)r target_layerr7bnbdequant_weights quant_weights r1r6z*RwkvModel._bnb_4bit_dequantize_and_rescales )*OP P"..889L9L9Q9QS_SfSfSrSrsQ#h$++2K2K&K"LLMvv((););E)BRW(X[[\k\r\rs  h 5rp)NNNNNNNN)rsrtrurrrrRWKV_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOCrrD LongTensorrrboolrrrSr%r6rrs@r1rrCs )++@A&$15595937$(,0/3&*Y E,,-Y !!1!12Y   1 12 Y U../0 Y D> Y $D>Y 'tnY d^Y  uj !Y B Y v506rprz The RWKV Model transformer with a language modeling head on top (linear layer with weights tied to the input embeddings). cXeZdZdgZfdZdZdZddZee e e e e  ddeej deej d eej"d eeej"d eej d eed eedeedeedeee ffdZxZS)RwkvForCausalLMz head.weightct||t||_t j |j |jd|_|jy)NFr) rrrrr rrarheadr )rrrs r1rzRwkvForCausalLM.__init__sH  f% IIf00&2C2C%P  rpc|jSrrrMrs r1get_output_embeddingsz%RwkvForCausalLM.get_output_embeddingss yyrpc||_yrrrOrs r1set_output_embeddingsz%RwkvForCausalLM.set_output_embeddingss " rpc h||dddfjd}||d|i}nd|i}||d<||d<|S)Nrrrr]r)rN)rrr]rrkwargs model_inputss r1prepare_inputs_for_generationz-RwkvForCausalLM.prepare_inputs_for_generations]  !!R%(2226I  $+];L'3L % W$- [!rprrrrr]labelsrrrrrc t| | n|jj} |j||||||| } | d} |j| } d}|*|j| |fd|jj i| }| s| f| ddz}||f|zS|St || | j| j| jS)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]` N)rr]rrrrrrr)rrr]rr) rr#rrM loss_functionrrr]rr)rrrrr]rWrrrrrT rwkv_outputsrrrrbs r1rSzRwkvForCausalLM.forwards2&1%5 $D>5 'tn5 d^5  u(( )5 B 5 rprK)rKrrrq)5rr dataclassesrpathlibrtypingrrrrrDtorch.utils.checkpointr generationr modeling_utilsr utilsr rrrrrrrconfiguration_rwkvr get_loggerrsr'rFrGr%r2autogradFunctionr4rrModulerrrrrrRWKV_START_DOCSTRINGrErrK__all__rwrpr1rjs  !// )-   +   H %-5@g %..11g T)XbC5 C5L#)bii#)L DD]/D]N ?? ?: ?? ?@ (Vda6#a6 a6H ] )?] ] @ Brp