S^(h, $ddlZddlZddlmZmZmZddlZddlZddlm Z ddl m Z ddl m Z ddlmZddlmZdd lmZmZdd lmZmZmZmZmZmZdd lmZdd lmZm Z m!Z!m"Z"m#Z#d dl$m%Z%erddlm&Z&e#jNe(Z)dZ*dZ+Gdde jXZ-Gdde jXZ.Gdde jXZ/Gdde jXZ0Gdde jXZ1Gdde jXZ2Gdde jXZ3Gd d!e3Z4Gd"d#e3Z5Gd$d%e jXZ6e3e5e4d&Z7Gd'd(e jXZ8Gd)d*e jXZ9Gd+d,e jXZ:Gd-d.e jXZ;Gd/d0eZ< dRd1ee=e=fd2e>d3e=d4eej~d5e=d6ejf d7ZAgd8ZBd9ZCd:ZDeZEe d;eCGd<d=e<ZFd>ZGd?ZHd@ZIe dAeCGdBdCe<ZJe dDeCGdEdFe<ZKe dGeCGdHdIe<ZLGdJdKe jXZMGdLdMe jXZNe dNeCGdOdPe<ZOgdQZPy)SN)OptionalTupleUnion)nn)CrossEntropyLoss)ACT2FN)is_deepspeed_zero3_enabled)is_fsdp_managed_module)!flash_attn_supports_top_left_maskis_flash_attn_available)BaseModelOutputCausalLMOutputSequenceClassifierOutputTokenClassifierOutputWav2Vec2BaseModelOutput XVectorOutput)PreTrainedModel)add_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardis_peft_availablelogging)Data2VecAudioConfig)_flash_attention_forwardz!facebook/data2vec-audio-base-960hrc&eZdZdfd ZdZxZS)Data2VecAudioConvLayerct||dkDr|j|dz nd|_|j||_t j |j|j|j||j||j|_ t j|jd|_ t|j|_y)Nrr) kernel_sizestridebiasTelementwise_affine)super__init__conv_dim in_conv_dim out_conv_dimrConv1d conv_kernel conv_stride conv_biasconv LayerNorm layer_normr feat_extract_activation activationselfconfiglayer_id __class__s r/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/data2vec/modeling_data2vec_audio.pyr&zData2VecAudioConvLayer.__init__5s !#Car9cV|jdkDr|ddddd|j f}|SNr)rLr>s r8r@zData2VecAudioPadLayer.forwardTs6    ")!Q0F43F3F2F0F*FGMr9rBrGs@r8rIrIOsKr9rIc$eZdZfdZdZxZS) Data2VecAudioPositionalConvLayerczt|tj|j|j|j |j dz|j |_t|j |_ t|j|_ tj|jd|_y)NrK)r paddinggroupsFr#)r%r&rr* hidden_sizeconv_pos_kernel_sizenum_conv_pos_embedding_groupsr.rIrSr r1r2r/r0r4r5r7s r8r&z)Data2VecAudioPositionalConvLayer.__init__[s II      33//1477   -V-H-HI  !?!?@,,v'9'9eTr9c|j|}|j|}|jdd}|j|}|jdd}|j |}|SNrrK)r.rSr=r0r2r>s r8r@z(Data2VecAudioPositionalConvLayer.forwardjsd -0  ]3 %//15  6 %//15  6 r9rBrGs@r8rQrQZs Ur9rQc$eZdZfdZdZxZS)$Data2VecAudioPositionalConvEmbeddingct|tjt |j Dcgc] }t |c}|_ycc}wN)r%r&r ModuleListrangerMrQlayersr4r5_r7s r8r&z-Data2VecAudioPositionalConvEmbedding.__init__vs@ mm?DVEcEc?d e! -f 5 e  esAc|jdd}|jD] }||} |jdd}|SrZ)r=ra)r4r?layers r8r@z,Data2VecAudioPositionalConvEmbedding.forward|sI%//15 [[ 1E!-0M 1%//15 r9rBrGs@r8r\r\us  r9r\c.eZdZdZfdZdZdZxZS)Data2VecAudioFeatureEncoderz.Construct the features from raw audio waveformc t|tjt |j Dcgc]}t ||c}|_d|_d|_ ycc}w)N)r6FT) r%r&rr_r`num_feat_extract_layersr conv_layersgradient_checkpointing_requires_grad)r4r5ir7s r8r&z$Data2VecAudioFeatureEncoder.__init__sX ==AFvGeGeAf gA #FQ 7 g ',#" hsA%cJ|jD] }d|_ d|_yNF) parameters requires_gradrlr4params r8_freeze_parametersz.Data2VecAudioFeatureEncoder._freeze_parameterss(__& (E"'E  (#r9c |dddf}|jr|jrd|_|jD]K}|jr5|jr)|jr|j |j |}D||}M|S)NT)rltrainingrqrjrk_gradient_checkpointing_func__call__)r4 input_valuesr? conv_layers r8r@z#Data2VecAudioFeatureEncoder.forwards$QW-    4==*.M '** :J""t'B'Bt}} $ A A''!! !+= 9  :r9)rCrDrE__doc__r&rtr@rFrGs@r8rgrgs8#$ r9rgc$eZdZfdZdZxZS)Data2VecAudioFeatureProjectionc4t|tj|jd|j |_tj|jd|j|_ tj|j|_ y)Nr<eps) r%r&rr/r'layer_norm_epsr0LinearrU projectionDropoutfeat_proj_dropoutdropoutrXs r8r&z'Data2VecAudioFeatureProjection.__init__sf ,,vr':@U@UV))FOOB$79K9KLzz&":":; r9cp|j|}|j|}|j|}||fSr^)r0rr)r4r?norm_hidden_statess r8r@z&Data2VecAudioFeatureProjection.forwards:!__];(:;  ]3 000r9rBrGs@r8r}r}s < 1r9r}ceZdZdZ ddededededededeeffd Z d e jd ed efd Z dde jdee jdee e jdee jdee jdede e jee jee e jffdZxZS)Data2VecAudioAttentionz=Multi-headed attention from 'Attention Is All You Need' paper embed_dim num_headsr is_decoderr" is_causalr5c t|||_||_||_||z|_||_|j |z|jk7rtd|jd|d|j dz|_||_ ||_ tj||||_ tj||||_tj||||_tj||||_y)Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g)r")r%r&rrrhead_dimr5 ValueErrorscalingrrrrk_projv_projq_projout_proj) r4rrrrr"rr5r7s r8r&zData2VecAudioAttention.__init__s "" !Y.  MMI %$.. 8MdnnM]$YKr3 }}d* $"ii 94@ ii 94@ ii 94@  )YTB r9tensorseq_lenbszc|j|||j|jjddj SrZ)viewrrr= contiguousr4rrrs r8_shapezData2VecAudioAttention._shapes7{{3GQQRSUVWbbddr9r?key_value_statespast_key_valueattention_masklayer_head_maskoutput_attentionsreturnc |du}|j\}} } |j||jz} |r0|.|djd|jdk(r |d} |d} n |rE|j |j |d|} |j |j |d|} n|}|j |j |d|} |j |j |d|} tj|d| gd} tj|d| gd} nD|j |j |d|} |j |j |d|} |jr| | f}||jzd|jf}|j | | |j|} | j|} | j|} | jd}tj| | jdd}|j||jz| |fk7r/t!d||jz| |fd|j|{|j|d| |fk7r#t!d |d| |fd|j|j||j| ||z}|j||jz| |}t"j$j'|d}||j|jfk7r*t!d |jfd|j|jdddd|j||j| |z}|j||jz| |}|r?|j||j| |}|j||jz| |}nd}t"j$j)||j(|j* }tj|| }|j||jz| |jfk7r9t!d ||jz| |jfd|j|j||j| |j}|jdd}|j|| |j,}|j/|}|||fS) #Input shape: Batch x Time x ChannelNrrKrr<dimz$Attention weights should be of size , but is z!Attention mask should be of size z/Head mask for a single layer should be of size )prv `attn_output` should be of size )sizerrshaperrrtorchcatrrrrreshapebmmr=rr functionalsoftmaxrrvrr)r4r?rrrrris_cross_attentionrtgt_lenrc query_states key_states value_states proj_shapesrc_len attn_weightsattn_weights_reshaped attn_probs attn_outputs r8r@zData2VecAudioAttention.forwards.T9',,.Wa{{=1DLL@ *q!''*.>.D.DQ.GG(*J)!,L T[[1A%BBLJ;;t{{3C'Db#NL  'T[[%?SIJ;;t{{='A2sKLN1$5z#BJJ 99nQ&7%FANLT[[%?SIJ;;t{{='A2sKL ??),7NDNN*B > Ct{{<#>CCZP 'Z''4 +|++Z8 //!$yyz/C/CAq/IJ    3#7'"J J6dnn8LgW^7_6`a %%'(*   %""$a'(BB 7a'8R7SS\]k]p]p]r\st(,,S$..'7SVddL',,S4>>-A7GTL}},,\r,B  &##%$..):: Et~~FWEXY',,./1+//2q!<|?P?PQTVZVdVdfmov?wwL',,S4>>-A7GTL  %1$5$5c4>>7T[$\ !055cDNN6JGU\]L$( !]]**<4<AAr9)FTFNNNNNF)rCrDrEr{intfloatboolrrr&rTensorrrr@rFrGs@r8rrsNG  04CCC C  C  CC,-C>eU\\eCece 488<1526"'vB||vB#5<<0vB!u||!45 vB !. vB "%,,/ vB vB u||Xell3XeELL>Q5RR SvBr9rcVeZdZdZfdZdej dedefdZ ddej de ej d e e ej d e ej d e ej d e d e ej e ej e e ej ffdZ xZ S)Data2VecAudioFlashAttention2aV Data2VecAudio flash attention module. This module inherits from `Data2VecAudioAttention` as the weights of the module stays untouched. The only required change would be on the forward pass where it needs to correctly call the public API of flash attention and deal with padding tokens in case the input contains any of them. cBt||i|t|_yr^)r%r&r _flash_attn_uses_top_left_mask)r4argskwargsr7s r8r&z%Data2VecAudioFlashAttention2.__init__[s# $)&) /P.Q+r9rrrcR|j|||j|jSr^)rrrrs r8_reshapez%Data2VecAudioFlashAttention2._reshapecs{{3GGr9r?rrrrrrc |r td|du}|j\}} } |j|j|d|} |rP|N|djd|jdk(r,|dj dd} |dj dd} n*|rE|j|j |d|} |j|j|d|} n||j|j |d|} |j|j|d|} tj|dj dd| gd} tj|dj dd| gd} nD|j|j |d|} |j|j|d|} |jr$| j dd| j ddf}| jd}|||djdz }| j}|tjk(rtjrtj}nMt|j dr|j j"}n |jj$j}t&j)d |d | j+|} | j+|} | j+|} t-| | | || |j.r |j0nd |j2|j4 }|j7|| d}|j9|}|sd}||fS) NzIData2VecAudioFlashAttention2 attention does not support output_attentionsr<rrKrrr;_pre_quantization_dtypezThe input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in .r)rruse_top_left_mask)rrrrrr=rrrrrdtypefloat32is_autocast_enabledget_autocast_gpu_dtypehasattrr5rweightlogger warning_oncetorrvrrrrr)r4r?rrrrrrrq_lenrcrrr kv_seq_len input_dtype target_dtyperrs r8r@z$Data2VecAudioFlashAttention2.forwardfs0 hi i.T9%**, UA}}T[[%?SI *q!''*.>.D.DQ.GG(*44Q:J)!,66q!$  (??<8L#|4J'??<8L.     $(MMDLLsnn"AA  "))#ub9 mmK0  LL.88r9r)rCrDrEr{r&rrrrrrrr@rFrGs@r8rrTs RHu||HcHH 488<1526"'i9||i9#5<<0i9!u||!45 i9 !. i9 "%,,/ i9 i9 u||Xell3XeELL>Q5RR Si9r9rc$eZdZ d dejdeejdeeejdeejdeejdedeejeejeeejfffd ZxZ S) Data2VecAudioSdpaAttentionr?rrrrrrcz|s|*tjdt| ||||||S|du}|j \}} } |j |} |r0|.|dj d|j dk(r |d} |d} n |rE|j|j|d|} |j|j|d|} n|}|j|j|d|} |j|j|d|} tj|d| gd} tj|d| gd} nD|j|j|d|} |j|j|d|} |jr| | f}|j| | |} |jr || dkDrd nd }tjjj!| | | ||j"r |j$nd | }|j ||j&| |j(fk7r7t+d ||j&| |j(fd|j |j-dd}|j/|| |j0}|j3|}|d|fS)rNaData2VecAudioModel is using Data2VecAudioSdpaAttention, but `torch.nn.functional.scaled_dot_product_attention` does not support `output_attentions=True` or `layer_head_mask` not None. Falling back to the manual attention implementation, but specifying the manual implementation will be required from Transformers version v5.0.0 onwards. This warning can be removed using the argument `attn_implementation="eager"` when loading the model.)rrrrrrrKrr<rTFr) attn_mask dropout_prrr)rrr%r@rrrrrrrrrrrrscaled_dot_product_attentionrvrrrrr=rrr)r4r?rrrrrrrrrcrrrrrr7s r8r@z"Data2VecAudioSdpaAttention.forwards  ;   l 7?!1-- /"3 # .T9',,.Wa{{=1 *q!''*.>.D.DQ.GG(*J)!,L T[[1A%BBLJ;;t{{3C'Db#NL  'T[[%?SIJ;;t{{='A2sKLN1$5z#BJJ 99nQ&7%FANLT[[%?SIJ;;t{{='A2sKL ??),7N{{<#> !NN~/E'TU+D[` hh))FF   $&*mmdll G     #t~~w !N N2CRVR_R_3`2ab$$&')  "++Aq1 "))#wG mmK0 D.00r9r) rCrDrErrrrrr@rFrGs@r8rrs488<1526"'f1||f1#5<<0f1!u||!45 f1 !. f1 "%,,/ f1 f1 u||Xell3XeELL>Q5RR Sf1f1r9rc$eZdZfdZdZxZS)Data2VecAudioFeedForwardct|tj|j|_tj |j|j|_ t|jtrt|j|_n|j|_tj |j|j|_tj|j |_yr^)r%r&rractivation_dropoutintermediate_dropoutrrUintermediate_sizeintermediate_dense isinstance hidden_actstrr intermediate_act_fn output_densehidden_dropoutoutput_dropoutrXs r8r&z!Data2VecAudioFeedForward.__init__=s $&JJv/H/H$I!"$))F,>,>@X@X"Y f'' -'-f.?.?'@D $'-'8'8D $IIf&>&>@R@RS jj)>)>?r9c|j|}|j|}|j|}|j|}|j |}|Sr^)rrrrrr>s r8r@z Data2VecAudioFeedForward.forwardJsX// > 00? 11-@ ))-8 ++M: r9rBrGs@r8rr<s @r9r)eagersdpaflash_attention_2c&eZdZfdZddZxZS)Data2VecAudioEncoderLayerct|t|j|j|j |j d|_tj|j|_ tj|j|j|_t||_tj|j|j|_y)NF)rrrrr)r%r& DATA2VEC_AUDIO_ATTENTION_CLASSES_attn_implementationrUnum_attention_headsattention_dropout attentionrrrrr/rr0r feed_forwardfinal_layer_normrXs r8r&z"Data2VecAudioEncoderLayer.__init__\s 9&:U:UV((00,,  zz&"7"78 ,,v'9'9v?T?TU4V< " V-?-?VEZEZ [r9c|}|j|||\}}}|j|}||z}|j|}||j|z}|j |}|f}|r||fz }|S)Nrr)rrr0rr)r4r?rr attn_residualrrcoutputss r8r@z!Data2VecAudioEncoderLayer.forwardjs% )- .L]*8* & |Q ]3 % 5  6 %(9(9-(HH --m<  "    &Gr9rorBrGs@r8rr[s \r9rc reZdZfdZ ddej deejdededef dZ xZ S) Data2VecAudioEncoderct|||_t||_t j |j|j|_ t j|j|_ t jt|jDcgc] }t!|c}|_d|_|j&dk(|_ycc}w)NrFr)r%r&r5r\pos_conv_embedrr/rUrr0rrrr_r`num_hidden_layersrrarkr_use_flash_attention_2rbs r8r&zData2VecAudioEncoder.__init__s  B6J,,v'9'9v?T?TUzz&"7"78 mmPUV\VnVnPo$p1%>v%F$pq &+#&,&A&AEX&X#%qs!Cr?rroutput_hidden_states return_dictc4|rdnd}|rdnd}||jdjdd|jd}d||<|jr |d|vr|nd}nd|ddddddfj |j z }|t j|j jz}|j|jdd|jd|jd}|j|} || z}|j|}|j|}txs t|} |jD]} |r||fz}t j g} |j"r| |j$j&krdnd } | r| rG|j(r+|j"r|j+| j,|||}n | ||| }|d}| rd }|s|dfz}|r||fz}|st/d |||fDSt1||| S)Nr<rrKr?rTFrNNc3&K|] }|| ywr^r ).0vs r8 z/Data2VecAudioEncoder.forward..smq_`_lms)last_hidden_stater? attentions) unsqueezerepeatrr rrrfinfominexpandrr0rr r rarandrvr5 layerdroprkrwrxtupler)r4r?rrr r all_hidden_statesall_self_attentionsexpand_attention_maskposition_embeddings synced_gpusredropout_probabilityskip_the_layer layer_outputss r8r@zData2VecAudioEncoder.forwards[#7BD$5b4  %$2$<$Q>Q2R2V2V!V!/!6!6"((+Q0D0DR0H.J^J^_aJb"#11-@%(;;  6  ]3 02R6LT6R [[ PE#$58H$H!#(**R. %)]]8KdkkNcNc8cTjoN![..4==$($E$E%&) %M%*%nXi%M!.a 0  ,  &9]1=Ms r8r@z!Data2VecAudioAdapterLayer.forwards/ -0  ))-Q)? r9rBrGs@r8r(r(s  r9r(c$eZdZfdZdZxZS)Data2VecAudioAdapterct|jjk7rTt j jj|_t jj|_ndx|_|_t jfdtjD|_ j|_ y)Nc34K|]}tywr^)r()rrcr5s r8rz0Data2VecAudioAdapter.__init__..s#p!$=f$E#ps)r%r&r*rUrrprojr/proj_layer_normr_r`num_adapter_layersrarrXs `r8r&zData2VecAudioAdapter.__init__s   $ $(:(: : &"4"4f6O6OPDI#%<<0I0I#JD /3 3DI,mm#puU[UnUnOo#pp ))r9ch|j.|j"|j|}|j|}|jdd}|jD]D}tj j }|j r||jkDs=||}F|jdd}|SrZ)r3r4r=ranprandomrvr)r4r?relayerdrop_probs r8r@zData2VecAudioAdapter.forwards 99 T%9%9%E IIm4M 00?M%//15 [[ 5EYY--/N==^dnn%D %m 4  5 &//15 r9rBrGs@r8r0r0s  *r9r0ceZdZdZeZdZdZdZdZ dZ dZ d de e jefdeefd Z d d ed e jfd Zy)Data2VecAudioPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. data2vec_audioryTct|trtjd|jj z }t jj|jj| |t jj|jj| |yt|tr5t jj|jjdyt|t jrm|jjj!d|j"j$|j%|jjj'yyt|t j(t j*frc|j$|jjj'|j&|jjj-dyyt|t j.rt jj1|j|jjtj|j2|j4|j6dzz }t jj|j| |yyy)zInitialize the weightsr)abrr)meanstdNr)rr}mathsqrtr in_featuresrinituniform_rr"rQ constant_r.rdatanormal_r5initializer_rangezero_r/ GroupNormfill_r*kaiming_normal_rT in_channelsr )r4moduleks r8 _init_weightsz*Data2VecAudioPreTrainedModel._init_weightss f< = !f//;;;tj||z |ddzS)Nfloor) rounding_moder)rdiv input_lengthr r!s r8_conv_out_lengthzWData2VecAudioPreTrainedModel._get_feat_extract_output_lengths.._conv_out_length3s"99\K7wWZ[[ [r9r)r5rTzipr+r,r`r5r,)r4rSrTr\r r!rcs r8 _get_feat_extract_output_lengthsz=Data2VecAudioPreTrainedModel._get_feat_extract_output_lengths*s2=1Ddkk--+  \ $'t{{'>'> @W@W#X Q K,]KPM Q 4;;99: _ 04;;C]C] ^  _r9feature_vector_lengthrc|jddddf}|j||}|jtj}|j d}tj ||f|j|j}d|tj|j d|j|dz f<|jdgjdjdgj}|S)Nr<rrTr)rdevicer)rb) cumsumr^rrlongrzerosrrbarangeflipr)r4r_rrTnon_padded_lengthsoutput_lengths batch_sizes r8"_get_feature_vector_attention_maskz?Data2VecAudioPreTrainedModel._get_feature_vector_attention_maskAs ,22r2:1b5A>>?Q_j>k'**5::6#))!,  . /~7K7KTbTiTi uv ^%9%9!%<^EZEZ[]kno]opq',,bT299"=BBB4HMMOr9r^)rCrDrEr{r config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_supports_flash_attn_2_supports_sdparRrr LongTensorrrrr^rkr r9r8r;r;s 'L($O&*#!N94Z^"5#3#3S#89HPQU0Y]%(:?:J:Jr9r;r mask_prob mask_lengthr min_masksrc|\}dkr tdkDrtdddtjjdj fd}|-|j j djnt|Dcgc]}c}}tj|ft } g} |} | d k(r| S|D]} || } tjjtj| dz z | d }t|d k(rdz }n|d }tj|tj| | z tj |zg}| j#|tj$| } tj&| dddddf|| f} | j)|| z} tjddddf}tj&||| fj)|| z}| |z} | j+dz kDrdz | | dz kD<tj,| | dd| Scc}w) af Computes random mask spans for a given shape. Used to implement [SpecAugment: A Simple Data Augmentation Method for ASR](https://arxiv.org/abs/1904.08779). Note that this method is not optimized to run on TPU and should be run on CPU as part of the preprocessing during training. Args: shape: The shape for which to compute masks. This should be of a tuple of size 2 where the first element is the batch size and the second element is the length of the axis to span. mask_prob: The percentage of the whole axis (between 0 and 1) which will be masked. The number of independently generated mask spans of length `mask_length` is computed by `mask_prob*shape[1]/mask_length`. Note that due to overlaps, `mask_prob` is an upper bound and the actual percentage will be smaller. mask_length: size of the mask min_masks: minimum number of masked spans attention_mask: A (right-padded) attention mask which independently shortens the feature axis of each batch dimension. rz&`mask_length` has to be bigger than 0.zO`mask_length` has to be smaller than `sequence_length`, but got `mask_length`: z and `sequence_length`: `ct|zz z}t|}|zkDrz}|dz z |krt|dz z d}|S)z;Given input length, compute how many spans should be maskedrr)rmax)r[num_masked_spanepsilonrtrsrusequence_lengths r8compute_num_masked_spanz6_compute_mask_indices..compute_num_masked_span|soi,6DwNOoy9 [ (? :- `attention_mask` should be passed if the corresponding processor has `config.return_attention_mask == True`, which is the case for all pre-trained Data2Vec Audio models. Be aware that that even with `attention_mask`, zero-padded inputs will have slightly different outputs compared to non-padded inputs because there are more than one convolutional layer in the positional encodings. For a more detailed explanation, see [here](https://github.com/huggingface/transformers/issues/25621#issuecomment-1713759349). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zgThe bare Data2VecAudio Model transformer outputting raw hidden-states without any specific head on top.c\eZdZdeffd ZdZ ddejdeejdeejfdZ e e e eeede  dd eej$deej$deejd eed eed eedeeeffdZxZS)Data2VecAudioModelr5ct||||_t||_t ||_|jdkDs|jdkDrEtjtj|jj|_t!||_|j$r t'|nd|_|j+y)Nr)r%r&r5rgfeature_extractorr}feature_projectionmask_time_probmask_feature_probr ParameterrrrUrFmasked_spec_embedrencoderrTr0adapter post_initrXs r8r&zData2VecAudioModel.__init__s   !}G[G[chcmcm n /3/E/E/H/HI\I\/]M+ , ;; ( (1 ,#8[)++77 KK;;++<< $ $)<<0D]MaMainisis#t #74#@#G#GO]_#` 23M. /r9audio) checkpoint output_typerlmodalityexpected_outputryrr r rcH||n|jj}||n|jj}||n|jj}|j |}|j dd}|!|j |jd|d}|j|\}}|j|||}|j|||||} | d}|j|j|}|s ||f| ddzSt||| j| jS) NrrKFra)rrrrr r r)rextract_featuresr?r)r5rr use_return_dictrr=rkrrrrrData2VecAudioBaseModelOutputr?r) r4ryrrrr r rr?encoder_outputss r8r@zData2VecAudioModel.forwardTsb"2C1N-TXT_T_TqTq$8$D $++JjJj &1%uEN+/*A*ABR*S' '00 ->~1 ,, )/!5# ' (* << # LL7M!#34qr7JJ J++-)77&11   r9rNNNNN)rCrDrErr&rr FloatTensorrrrrrDATA2VEC_AUDIO_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr_CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPErrrrr@rFrGs@r8rr s 2"4:>59 ,((,$E$5$56,!!1!12 ,\++JK&0$. 269=,0/3&*2 u||,2 !.2 $E$5$56 2 $D> 2 'tn 2 d^2  u22 32 L2 r9rrKz['MISTER QUILTER IS THE APOSTLE OF THE MIDDLE CLASSES AND WE ARE GLAD TO WELCOME HIS GOSPEL'g̼P@zkData2VecAudio Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).ceZdZfdZdZdZeeee e e e e  d deej deej deedeed eed eej d eee ffd ZxZS)Data2VecAudioForCTCct||t||_t j |j |_|jtd|jdt|dr|jr |jn |j}t j||j|_|j#y)NzYou are trying to instantiate z with a configuration that does not define the vocabulary size of the language model head. Please instantiate the model as follows: `Data2VecAudioForCTC.from_pretrained(..., vocab_size=vocab_size)`. or define `vocab_size` of your model's configuration.rT)r%r&rr<rr final_dropoutr vocab_sizerr7rrTr*rUrlm_headr)r4r5r*r7s r8r&zData2VecAudioForCTC.__init__s  08zz&"6"67    $00@AHH *1)GFL^L^F % %djdvdv yy!3V5F5FG  r9cXtjdt|jyrThe method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5. Please use the equivalent `freeze_feature_encoder` method instead.Nwarningswarn FutureWarningrrs r8freeze_feature_extractorz,Data2VecAudioForCTC.freeze_feature_extractor'   Q  ##%r9cL|jjjyrr<rrtrs r8rz*Data2VecAudioForCTC.freeze_feature_encoder --@@Br9)rrrlr expected_lossryrrr r labelsrc ||n|jj}|I|j|jjk\r"t d|jj|j |||||}|d}|j |}|j|} d} |b||n$tj|tj}|j|jdjtj} |dk\} | jd} |j| }tj j#| dtj$j'dd}tj(j*j-d 5tj j/||| | |jj0|jj2|jj4 } ddd|s| f|t6dz}| | f|zS|St9| | |j:|j< S#1swYExYw) a labels (`torch.LongTensor` of shape `(batch_size, target_length)`, *optional*): Labels for connectionist temporal classification. Note that `target_length` has to be smaller or equal to the sequence length of the output logits. Indices are selected in `[-100, 0, ..., config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size - 1]`. Nz$Label values must be <= vocab_size: rrrr<)rrrF)enabled)blank reduction zero_infinitylosslogitsr?r)r5rryrrr<rrr ones_likerdr^rr masked_selectrr log_softmaxrr=backendscudnnflagsctc_loss pad_token_idctc_loss_reductionctc_zero_infinity_HIDDEN_STATES_START_POSITIONrr?r)r4ryrrr r rrr?rrrS labels_masktarget_lengthsflattened_targets log_probsoutputs r8r@zData2VecAudioForCTC.forwards)0&1% D 'tn D d^ D &D  un$ %D LD r9rz Data2VecAudio Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like SUPERB Keyword Spotting. ceZdZfdZdZdZdZeee e e e d dde ejde ejd e ed e ed e ed e ejd eee ffdZxZS)&Data2VecAudioForSequenceClassificationct||t|dr|jr t dt ||_|jdz}|jr0tjtj||z |_ tj|j|j |_tj|j |j$|_|j)y)NrTzdSequence classification does not support the use of Data2VecAudio adapters (config.add_adapter=True)r)r%r&rrTrrr<ruse_weighted_layer_sumrrrr layer_weightsrrUclassifier_proj_size projector num_labels classifierrr4r5 num_layersr7s r8r&z/Data2VecAudioForSequenceClassification.__init__s  6= )f.@.@v 18--1  ( (!#ejj.Dz.Q!RD 6#5#5v7R7RS))F$?$?ARARS r9cXtjdt|jy)z Calling this function will disable the gradient computation for the feature encoder so that its parameters will not be updated during training. rNrrs r8rz?Data2VecAudioForSequenceClassification.freeze_feature_extractor-rr9cL|jjjyrrrs r8rz=Data2VecAudioForSequenceClassification.freeze_feature_encoder9rr9cP|jjD] }d|_ yz Calling this function will disable the gradient computation for the base model so that its parameters will not be updated during training. Only the classification head will be updated. FNr<rprqrrs r8freeze_base_modelz8Data2VecAudioForSequenceClassification.freeze_base_model@* ((335 (E"'E  (r9rrrrlrryrrr r rrc<||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|}||jd} n|j|j d|} | j#dj%dd|j d} d || <|jd| jdjddz } |j'| } d}|Ft)}|| jd|jj*|jd}|s| f|tdz}||f|zS|St-|| |j.|j0 S)  labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). NTrrrr<rrKrr)r5rrr<rrstackrrrrrrrr@rkrrrrrrrr?r)r4ryrrr r rrr? norm_weights pooled_output padding_maskexpand_padding_maskrrloss_fctrs r8r@z.Data2VecAudioForSequenceClassification.forwardHs!.&1%->r1a-HHMMRSMTM#AJM}5  !)..1.5MBB=CVCVWXCY[ijL"."8"8"<"C"CAq-J]J]^_J`"a 25M.. /)--!-4|7G7GA7G7N7S7STVXY7ZZM/  ')HFKKDKK,B,BCV[[QS_UDY)F)G!HHF)-)9TGf$ Ev E'!//))   r9r)rCrDrEr&rrrrrrrrrrrrrrrr@rFrGs@r8rrs" &C(++JK&,$ 26,0/3&*)-< u||,< !.< $D> < 'tn < d^ < &<  u.. /< L< r9rzi Data2VecAudio Model with a frame classification head on top for tasks like Speaker Diarization. ceZdZfdZdZdZdZeee e e e d dde ejde ejd e ejd e ed e ed e ed eee ffdZxZS)(Data2VecAudioForAudioFrameClassificationct||t|dr|jr t dt ||_|jdz}|jr0tjtj||z |_ tj|j|j |_|j |_|j%y)NrTzgAudio frame classification does not support the use of Data2VecAudio adapters (config.add_adapter=True)r)r%r&rrTrrr<rrrrrrrrrUrr init_weightsrs r8r&z1Data2VecAudioForAudioFrameClassification.__init__s  6= )f.@.@y 18--1  ( (!#ejj.Dz.Q!RD ))F$6$68I8IJ ++ r9cXtjdt|jyrrrs r8rzAData2VecAudioForAudioFrameClassification.freeze_feature_extractorrr9cL|jjjyrrrs r8rz?Data2VecAudioForAudioFrameClassification.freeze_feature_encoderrr9cP|jjD] }d|_ yrrrrs r8rz:Data2VecAudioForAudioFrameClassification.freeze_base_modelrr9rrryrrrr r rc ||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|} d} |\t} | | jd|jt j |jd|jd} |s| f|tdz} | St#| | |j$|j& S) rNTrrrr<r)axisr)r5rrr<rrrrrrrrrrrrargmaxrr?r)r4ryrrrr r rr?rrrrrs r8r@z0Data2VecAudioForAudioFrameClassification.forwardsj.&1%->r1a-HHMMRSMTM#AJM/  ')HFKKDOOtuDY)F)G!HHFM$!//))   r9r)rCrDrEr&rrrrrrrrrrrrrrrr@rFrGs@r8r r s &C(++JK&)$ 26)-,0/3&*3 u||,3 !.3 & 3 $D> 3 'tn 3 d^3  u++ ,3 L3 r9r c&eZdZdfd ZdZxZS) AMSoftmaxLossctt| ||_||_||_t jtj||d|_ t j|_ y)NT)rq) r%rr&scalemarginrrrrrandnrrr)r4 input_dimrrrr7s r8r&zAMSoftmaxLoss.__init__sS mT+-  $ll5;;y*#EUYZ '') r9c|j}tjj|jd}tjj|d}t j ||}||jz }tjj||j}|jt j|j||z}|j||}|S)Nrrr)flattenrr normalizerrmmrone_hotrrwhererr) r4r?rr cos_thetapsionehotrrs r8r@zAMSoftmaxLoss.forwards!((!(< // 1/E HH]F3 $++%&&vt?ekk&++-iHHyy( r9)g>@g?rBrGs@r8rrs * r9rcXeZdZdfd Zdej dej fdZxZS) TDNNLayerct||dkDr|j|dz n|j||_|j||_|j ||_|j||_tj|j|j z|j|_ tj|_ y)Nrr)r%r&tdnn_dimr(r) tdnn_kernelr tdnn_dilationdilationrrkernelReLUr2r3s r8r&zTDNNLayer.__init__s +BFD]D]!^))F$=$=v?X?XY&v'@'@&BSBST Rs>FcXtjdt|jyrrrs r8rz0Data2VecAudioForXVector.freeze_feature_extractorNrr9cL|jjjyrrrs r8rz.Data2VecAudioForXVector.freeze_feature_encoderZrr9cP|jjD] }d|_ yrrrrs r8rz)Data2VecAudioForXVector.freeze_base_modelarr9rScVd}|jjD] }|||d}|S)z? Computes the output length of the TDNN layers c||z |zdzS)Nrr rZs r8r\zJData2VecAudioForXVector._get_tdnn_output_lengths.._conv_out_lengthns!;.69A= =r9r)r5r&)r4rSr\r s r8_get_tdnn_output_lengthsz0Data2VecAudioForXVector._get_tdnn_output_lengthsis:  >  ;;22 LK,]KKM Lr9rrryrrr r rrc||n|jj}|jjrdn|}|j|||||}|jjrr|t}t j |d}tjj|jd} || jdddzjd}n|d}|j|}|jD] } | |} |%|jd} |j!d} n|j#|jd} |j%| }g} g} t'|D]U\}}| j)||d|fjd| j)||d|fj!dWt j | } t j | } t j*| | gd}|j-|}|j/|}d}||j1||}|s||f|tdz}||f|zS|St3||||j4|j6S) rNTrrrr<r)rr embeddingsr?r)r5rrr<rrrrrrrrrrr2r@rAr^r; enumeraterrrrr4rr?r)r4ryrrr r rrr?r tdnn_layer mean_features std_featuresfeat_extract_output_lengthstdnn_output_lengthsrmlengthstatistic_poolingoutput_embeddingsrrrs r8r@zData2VecAudioForXVector.forwardxs.&1%->r1a-HHMMRSMTM#AJM}5 )) 6J&}5M 6  !)..1.5M(,,,3L*.*O*OP^PbPbghPbPi*j '"&"?"?@["\ ML&':; J 6$$]1gvg:%>%C%C%C%JK##M!WfW*$=$A$Aa$A$HI J"KK 6M ;;|4L!II}l&CL 223DE!23  >>&&1D/07;X;Y3ZZF)-)9TGf$ Ev E(!//))   r9r)rCrDrEr&rrrrrrrrr;rrrrrrrrrrr@rFrGs@r8r0r04s& &C( eE I 'tn I d^ I &I  um# $I LI r9r0)r rrr0rr;rO)QrBrtypingrrrnumpyr7rrtorch.nnr activationsr integrations.deepspeedr integrations.fsdpr modeling_flash_attention_utilsr r modeling_outputsrrrrrrmodeling_utilsrutilsrrrrrconfiguration_data2vec_audiorr get_loggerrCrrrModulerrIrQr\rgr}rrrrrrrr(r0r;rrrrndarrayrrDATA2VEC_AUDIO_START_DOCSTRINGrrrrrrrrr rr#r0__all__r r9r8rWs* )) %!@7h.>J   H %:(RYY6BIIryy6 299  ")) F 1RYY 1[BRYY[B|{9#9{9|g1!7g1Tryy2$ &5$   FR 299R j $299>O?Ol26 t c?tttU--. t  t ZZ tn'"$"#H 7m"A 5A A H!"uu"u 6u u p# p -Ip p f#  f /Kf  f RBII. @#  N :N  N b r9