Ë
    T^(hýw  ã                   óz  — d dl Z d dlmZmZmZ d dlZd dlmZ d dlmc m	Z
 ddlmZ ddlmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZmZmZmZ d	d
lmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& ddl'm(Z(  ejR                  e*«      Z+dZ,dZ-g d¢Z.dZ/dZ0dZ1d d gZ2dZ3dZ4 G d„ de%«      Z5 G d„ de«      Z6 G d„ dejn                  «      Z8 G d„ de«      Z9 G d„ dejn                  «      Z: G d„ d ejn                  «      Z; G d!„ d"ejn                  «      Z< G d#„ d$ejn                  «      Z= G d%„ d&ejn                  «      Z> G d'„ d(ee&«      Z?d)Z@d*ZAeZB ed+e@«       G d,„ d-e$«      «       ZC ed.e@«       G d/„ d0e!«      «       ZD ed1e@«       G d2„ d3e"«      «       ZE ed4e@«       G d5„ d6e «      «       ZF ed7e@«       G d8„ d9e#«      «       ZGg d:¢ZHy);é    N)ÚOptionalÚTupleÚUnioné   )Úis_deepspeed_zero3_enabled)Úis_fsdp_managed_module)ÚBaseModelOutputÚCausalLMOutputÚSequenceClassifierOutputÚTokenClassifierOutputÚWav2Vec2BaseModelOutputÚXVectorOutput)ÚPreTrainedModel)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingé   )	ÚWav2Vec2FeatureProjectionÚWav2Vec2FeedForwardÚ#Wav2Vec2ForAudioFrameClassificationÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2ForXVectorÚWav2Vec2ModelÚWav2Vec2PositionalConvEmbeddingÚWav2Vec2PreTrainedModelé   )ÚWavLMConfigr   z1patrickvonplaten/wavlm-libri-clean-100h-base-plus)r   i$  i   zZ'mister quilter is the aposle of the middle classes and we are glad to welcome his gospel'g…ëQ¸)@zmicrosoft/wavlm-base-plus-sdzmicrosoft/wavlm-base-plus-svg
×£p=
ï?c                   ó   — e Zd Zy)ÚWavLMPositionalConvEmbeddingN©Ú__name__Ú
__module__Ú__qualname__© ó    úe/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/wavlm/modular_wavlm.pyr!   r!   3   ó   „ Ør'   r!   c                   ó   — e Zd Zy)ÚWavLMFeatureProjectionNr"   r&   r'   r(   r+   r+   7   r)   r'   r+   c                   ó  ‡ — e Zd ZdZ	 	 	 	 ddedededededefˆ fd„Z	 	 	 	 dd	ej                  d
e
ej                     de
ej                     dedeej                  e
ej                     e
eej                        f   f
d„Zd	ej                  d
eej                  ej                   f   dej                  dedej                  ej                  ff
d„Zdededej                  fd„Zdej                  dej                  fd„Zˆ xZS )ÚWavLMAttentionz=Multi-headed attention from 'Attention Is All You Need' paperÚ	embed_dimÚ	num_headsÚdropoutÚnum_bucketsÚmax_distanceÚhas_relative_position_biasc                 ó  •— t         ‰| �  «        || _        || _        || _        ||z  | _        | j
                  |z  | j                  k7  rt        d| j                  › d|› d�«      ‚| j
                  dz  | _        t        j                  ||«      | _
        t        j                  ||«      | _        t        j                  ||«      | _        t        j                  ||«      | _        || _        || _        t        j                   t#        j$                  d| j                  dd«      «      | _        t        j                  | j
                  d«      | _        |r0t        j*                  | j                  | j                  «      | _        y y )Nz;embed_dim must be divisible by num_heads (got `embed_dim`: z and `num_heads`: z).g      à¿r   é   )ÚsuperÚ__init__r.   r/   r0   Úhead_dimÚ
ValueErrorÚscalingÚnnÚLinearÚk_projÚv_projÚq_projÚout_projr1   r2   Ú	ParameterÚtorchÚonesÚgru_rel_pos_constÚgru_rel_pos_linearÚ	EmbeddingÚrel_attn_embed)Úselfr.   r/   r0   r1   r2   r3   Ú	__class__s          €r(   r7   zWavLMAttention.__init__>   s7  ø€ ô 	‰ÑÔØ"ˆŒØ"ˆŒØˆŒØ! YÑ.ˆŒà�M‰M˜IÑ%¨$¯.©.Ò8ÜØMÈdÏnÉnÐM]Ø$ Y K¨rð3óð ð —}‘} dÑ*ˆŒä—i‘i 	¨9Ó5ˆŒÜ—i‘i 	¨9Ó5ˆŒÜ—i‘i 	¨9Ó5ˆŒÜŸ	™	 )¨YÓ7ˆŒà&ˆÔØ(ˆÔä!#§¡¬e¯j©j¸¸D¿N¹NÈAÈqÓ.QÓ!RˆÔÜ"$§)¡)¨D¯M©M¸1Ó"=ˆÔá%Ü"$§,¡,¨t×/?Ñ/?ÀÇÁÓ"PˆDÕð &r'   Úhidden_statesÚattention_maskÚposition_biasÚoutput_attentionsÚreturnc                 ó   — |j                  «       \  }}}|€S| j                  ||«      }|j                  d«      j                  |ddd«      j	                  || j
                  z  ||«      }|j	                  |j                  dd | j
                  dfz   «      }	|	j                  dddd«      }	| j                  |	«      }
|
j	                  |	j                  dd dz   «      j                  d«      }
t        j                  |
«      j                  dd¬«      \  }}||| j                  z  d	z
  z  d
z   }|j	                  || j
                  z  dd«      |z  }|j	                  d||f«      }| j                  ||||«      \  }}|||fS )z'Attention layer with relative attentionNr   r   éÿÿÿÿr   r   )r   é   ©Údimç      ð?g       @)ÚsizeÚcompute_biasÚ	unsqueezeÚrepeatÚviewr/   ÚshapeÚpermuterE   ÚsumrB   ÚsigmoidÚchunkrD   Útorch_multi_head_self_attention)rH   rJ   rK   rL   rM   ÚindexÚbszÚtgt_lenÚ_Úgated_hidden_statesÚrelative_position_projÚgate_aÚgate_bÚgate_outputÚgated_position_biasÚattn_outputÚattn_weightss                    r(   ÚforwardzWavLMAttention.forwardb   s­  € ð (×,Ñ,Ó.‰ˆˆW�að Ð Ø ×-Ñ-¨g°wÓ?ˆMà×'Ñ'¨Ó*×1Ñ1°#°q¸!¸QÓ?×DÑDÀSÈ4Ï>É>ÑEYÐ[bÐdkÓlð ð ,×0Ñ0°×1DÑ1DÀSÀbÐ1IÈTÏ^É^Ð]_ÐL`Ñ1`ÓaÐØ1×9Ñ9¸!¸QÀÀ1ÓEÐð "&×!8Ñ!8Ð9LÓ!MÐØ!7×!<Ñ!<Ð=P×=VÑ=VÐWZÐXZÐ=[Ð^dÑ=dÓ!e×!iÑ!iÐjlÓ!mÐô Ÿ™Ð'=Ó>×DÑDÀQÈBÐDÓO‰ˆ�Ø ¨×)?Ñ)?Ñ ?À#Ñ EÑFÈÑLˆð *×.Ñ.¨s°T·^±^Ñ/CÀRÈÓKÈmÑ[ÐØ1×6Ñ6¸¸GÀWÐ7MÓNÐà$(×$HÑ$HØ˜>Ð+>Ð@Qó%
Ñ!ˆ�\ð ˜L¨-Ð7Ð7r'   ri   c                 óX  — |j                  dd«      x}x}}|�|j                  d«      nd}dx}	}
d}t        j                  |||| j                  | j
                  t        j                  dg«      t        j                  | j                  j                  | j                  j                  | j                  j                  f«      |	|
|| j                  | j                  j                  | j                  j                  | j                   |||d| j                  j                  | j                  j                  | j                  j                  ¬«      \  }}|j                  dd«      }|�C|dd…df   j#                  |j$                  dd | j
                  fz   |j$                  dd z   «      }||fS )zCsimple wrapper around torch's multi_head_attention_forward functionr   r   NFT)Úuse_separate_proj_weightÚq_proj_weightÚk_proj_weightÚv_proj_weight)Ú	transposeÚneÚFÚmulti_head_attention_forwardr.   r/   rB   ÚemptyÚcatr?   Úbiasr=   r>   r0   r@   ÚweightÚtrainingÚbroadcast_torZ   )rH   rJ   rK   ri   rM   ÚqueryÚkeyÚvalueÚkey_padding_maskÚbias_kÚbias_vÚadd_zero_attnrj   rk   s                 r(   r_   z.WavLMAttention.torch_multi_head_self_attention‹   s�  € ð ,×5Ñ5°a¸Ó;Ð;ˆÐ;��eØ3AÐ3M˜>×,Ñ,¨QÔ/ÐSWÐð Ðˆ�Øˆô %&×$BÑ$BØØØØ�N‰NØ�N‰NÜ�K‰K˜˜ÓÜ�I‰I�t—{‘{×'Ñ'¨¯©×)9Ñ)9¸4¿;¹;×;KÑ;KÐLÓMØØØØ�L‰LØ�M‰M× Ñ Ø�M‰M×ÑØ�M‰MØØØØ%)ØŸ+™+×,Ñ,ØŸ+™+×,Ñ,ØŸ+™+×,Ñ,ô+%
Ñ!ˆ�\ð2 "×+Ñ+¨A¨qÓ1ˆàÐ#ð (ª¨4¨Ñ0×=Ñ=Ø×"Ñ" 2 AÐ&¨$¯.©.Ð):Ñ:¸\×=OÑ=OÐPQÐPRÐ=SÑSóˆLð ˜LÐ(Ð(r'   Úquery_lengthÚ
key_lengthc                 óˆ  — t        j                  |t         j                  ¬«      d d …d f   }t        j                  |t         j                  ¬«      d d d …f   }||z
  }| j                  |«      }|j	                  | j
                  j                  j                  «      }| j                  |«      }|j                  g d¢«      }|S )N)Údtype)r   r   r   )	rB   ÚarangeÚlongÚ_relative_positions_bucketÚtorG   ry   Údevicer[   )rH   rƒ   r„   Úcontext_positionÚmemory_positionÚrelative_positionÚrelative_position_bucketÚvaluess           r(   rV   zWavLMAttention.compute_biasÂ   s¢   € Ü Ÿ<™<¨¼E¿J¹JÔGÊÈ4ÈÑPÐÜŸ,™, z¼¿¹ÔDÀTÊ1ÀWÑMˆØ+Ð.>Ñ>ÐØ#'×#BÑ#BÐCTÓ#UÐ Ø#;×#>Ñ#>¸t×?RÑ?R×?YÑ?Y×?`Ñ?`Ó#aÐ Ø×$Ñ$Ð%=Ó>ˆØ—‘¢	Ó*ˆØˆr'   Úrelative_positionsc                 ó$  — | j                   dz  }|dkD  j                  t        j                  «      |z  }t        j                  |«      }|dz  }||k  }t        j
                  |j                  «       |z  «      }|t        j
                  | j                  |z  «      z  }|||z
  z  }||z   j                  t        j                  «      }t        j                  |t        j                  ||dz
  «      «      }|t        j                  |||«      z  }|S )Nr   r   r   )r1   rŠ   rB   rˆ   ÚabsÚlogÚfloatÚmathr2   ÚminÚ	full_likeÚwhere)rH   r‘   r1   Úrelative_bucketsÚ	max_exactÚis_smallÚrelative_positions_if_largeÚrelative_position_if_larges           r(   r‰   z)WavLMAttention._relative_positions_bucketÌ   s  € Ø×&Ñ&¨!Ñ+ˆà.°Ñ2×6Ñ6´u·z±zÓBÀ[ÑPÐÜ"ŸY™YÐ'9Ó:Ðà 1Ñ$ˆ	Ø%¨	Ñ1ˆä&+§i¡iÐ0B×0HÑ0HÓ0JÈYÑ0VÓ&WÐ#Ø&AÄDÇHÁHÈT×M^ÑM^ÐajÑMjÓDkÑ&kÐ#Ø&AÀ[ÐS\ÑE\Ñ&]Ð#Ø&/Ð2MÑ&M×%QÑ%QÔRW×R\ÑR\Ó%]Ð"Ü%*§Y¡YØ&¬¯©Ð8RÐT_ÐbcÑTcÓ(dó&
Ð"ð 	œEŸK™K¨Ð2DÐF`ÓaÑaÐØÐr'   )ç        i@  i   T©NNFr   )r#   r$   r%   Ú__doc__Úintr•   Úboolr7   rB   ÚTensorr   r   rl   ÚFloatTensorr   Ú
LongTensorÚ
BoolTensorr_   rV   r‰   Ú__classcell__©rI   s   @r(   r-   r-   ;   s…  ø„ ÙGð ØØØ+/ñ"Qàð"Qð ð"Qð ð	"Qð
 ð"Qð ð"Qð %)õ"QðN 26Ø04Ø"'Øñ'8à—|‘|ð'8ð ! §¡Ñ.ð'8ð   §¡Ñ-ð	'8ð
  ð'8ð 
ˆu�|‰|˜X e§l¡lÑ3°X¸eÀEÇLÁLÑ>QÑ5RÐRÑ	Só'8ðR5)à×(Ñ(ð5)ð ˜e×.Ñ.°×0@Ñ0@Ð@ÑAð5)ð #×.Ñ.ð	5)ð
  ð5)ð ×
Ñ
˜U×.Ñ.Ð	/ó5)ðn¨ð ¸#ð À%×BSÑBSó ð ¸U×=NÑ=Nð  ÐSX×SdÑSd÷  r'   r-   c                   ó   — e Zd Zy)ÚWavLMFeedForwardNr"   r&   r'   r(   r«   r«   á   r)   r'   r«   c                   ó2   ‡ — e Zd Zddedefˆ fd„Zdd„Zˆ xZS )ÚWavLMEncoderLayerÚconfigr3   c                 óÚ  •— t         ‰| �  «        t        |j                  |j                  |j
                  |j                  |j                  |¬«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t!        |«      | _        t        j                  |j                  |j                  ¬«      | _        y ©N)r.   r/   r0   r1   r2   r3   ©Úeps©r6   r7   r-   Úhidden_sizeÚnum_attention_headsÚattention_dropoutr1   Úmax_bucket_distanceÚ	attentionr;   ÚDropoutÚhidden_dropoutr0   Ú	LayerNormÚlayer_norm_epsÚ
layer_normr«   Úfeed_forwardÚfinal_layer_norm©rH   r®   r3   rI   s      €r(   r7   zWavLMEncoderLayer.__init__æ   ó¬   ø€ Ü‰ÑÔÜ'Ø×(Ñ(Ø×0Ñ0Ø×,Ñ,Ø×*Ñ*Ø×3Ñ3Ø'Aô
ˆŒô —z‘z &×"7Ñ"7Ó8ˆŒÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ,¨VÓ4ˆÔÜ "§¡¨V×-?Ñ-?ÀV×EZÑEZÔ [ˆÕr'   c                 óî   — |}| j                  |||||¬«      \  }}}| j                  |«      }||z   }| j                  |«      }|| j                  |«      z   }| j	                  |«      }||f}|r||fz  }|S )N©rK   rL   rM   r`   )r¸   r0   r½   r¾   r¿   )	rH   rJ   rK   rL   rM   r`   Úattn_residualrk   Úoutputss	            r(   rl   zWavLMEncoderLayer.forwardõ   sœ   € Ø%ˆØ59·^±^ØØ)Ø'Ø/Øð 6Dó 6
Ñ2ˆ�| ]ð Ÿ™ ]Ó3ˆØ%¨Ñ5ˆàŸ™¨Ó6ˆà%¨×(9Ñ(9¸-Ó(HÑHˆØ×-Ñ-¨mÓ<ˆà  -Ð0ˆáØ˜�Ñ&ˆGàˆr'   ©Tr    ©r#   r$   r%   r   r£   r7   rl   r¨   r©   s   @r(   r­   r­   å   s   ø„ ñ\˜{ð \Èõ \÷r'   r­   c                   ó2   ‡ — e Zd Zddedefˆ fd„Zdd„Zˆ xZS )Ú WavLMEncoderLayerStableLayerNormr®   r3   c                 óÚ  •— t         ‰| �  «        t        |j                  |j                  |j
                  |j                  |j                  |¬«      | _        t        j                  |j                  «      | _        t        j                  |j                  |j                  ¬«      | _        t!        |«      | _        t        j                  |j                  |j                  ¬«      | _        y r°   r³   rÀ   s      €r(   r7   z)WavLMEncoderLayerStableLayerNorm.__init__  rÁ   r'   c                 óè   — |}| j                  |«      }| j                  ||||¬«      \  }}}| j                  |«      }||z   }|| j                  | j	                  |«      «      z   }||f}|r||fz  }|S )N)rK   rL   rM   )r½   r¸   r0   r¾   r¿   )rH   rJ   rK   rL   rM   rÄ   rk   rÅ   s           r(   rl   z(WavLMEncoderLayerStableLayerNorm.forward  s•   € Ø%ˆØŸ™¨Ó6ˆØ59·^±^ØØ)Ø'Ø/ð	 6Dó 6
Ñ2ˆ�| ]ð Ÿ™ ]Ó3ˆØ%¨Ñ5ˆØ%¨×(9Ñ(9¸$×:OÑ:OÐP]Ó:^Ó(_Ñ_ˆà  -Ð0ˆáØ˜�Ñ&ˆGàˆr'   rÆ   )NNFrÇ   r©   s   @r(   rÉ   rÉ     s   ø„ ñ\˜{ð \Èõ \÷r'   rÉ   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚWavLMEncoderc           
      ó¢  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        ||dk(  ¬«      ‘Œ c}«      | _        d| _        y c c}w ©Nr±   r   )r3   F)r6   r7   r®   r!   Úpos_conv_embedr;   r»   r´   r¼   r½   r¹   rº   r0   Ú
ModuleListÚrangeÚnum_hidden_layersr­   ÚlayersÚgradient_checkpointing©rH   r®   ÚirI   s      €r(   r7   zWavLMEncoder.__init__4  s�   ø€ Ü‰ÑÔØˆŒÜ:¸6ÓBˆÔÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mÜUZÐ[a×[sÑ[sÓUtÖuÐPQÔ˜vÀ1ÈÁ6ÖKÒuó
ˆŒð ',ˆÕ#ùò vó   Â!Cc                 ó  — |rdnd }|rdnd }|�5|j                  d«      j                  dd|j                  d   «      }d|| <   | j                  |«      }	||	z   }| j	                  |«      }| j                  |«      }t        «       xs t        | «      }
d }t        | j                  «      D ]±  \  }}|r||fz   }t        j                  g «      }| j                  xr  |dkD  xr || j                  j                  k  }|r|
rM| j                  r,| j                  r | j!                  |j"                  ||||«      }n ||||||¬«      }|d d \  }}|rd}|sŒ©|d   fz   }Œ³ |r||fz   }|st%        d„ |||fD «       «      S t'        |||¬	«      S )
Nr&   rP   r   r   r   rÃ   ©NNNc              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­w©Nr&   ©Ú.0Úvs     r(   ú	<genexpr>z'WavLMEncoder.forward.<locals>.<genexpr>~  ó   è ø€ Òm˜qÐ_`Ñ_lœÑmùó   ‚Š©Úlast_hidden_staterJ   Ú
attentions)rW   rX   rZ   rÐ   r½   r0   r   r   Ú	enumeraterÔ   rB   Úrandrz   r®   Ú	layerdroprÕ   Ú_gradient_checkpointing_funcÚ__call__Útupler	   ©rH   rJ   rK   rM   Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚexpand_attention_maskÚposition_embeddingsÚsynced_gpusrL   r×   ÚlayerÚdropout_probabilityÚskip_the_layerÚlayer_outputss                    r(   rl   zWavLMEncoder.forward?  sÜ  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ø45ˆMÐ0Ð0Ñ1à"×1Ñ1°-Ó@ÐØ%Ð(;Ñ;ˆØŸ™¨Ó6ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6RˆØˆä! $§+¡+Ó.ò !	P‰HˆAˆuÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]Òf¨q°1©uÒfÐ:MÐPT×P[ÑP[×PeÑPeÑ:eˆNÙ!¡[à×.Ò.°4·=²=Ø$(×$EÑ$EØŸ™Ø%Ø&Ø%Ø)ó%‘Mñ %*Ø%Ø'5Ø&3Ø*;Øô%�Mð 0=¸R¸aÐ/@Ñ,�˜}áØ 2�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ðC!	PñF  Ø 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ø+Ø*ô
ð 	
r'   ©NFFT©r#   r$   r%   r7   rl   r¨   r©   s   @r(   rÍ   rÍ   3  s   ø„ ô	,ð ØØ"Ø÷D
r'   rÍ   c                   ó.   ‡ — e Zd Zˆ fd„Z	 	 	 	 dd„Zˆ xZS )ÚWavLMEncoderStableLayerNormc           
      ó¢  •— t         ‰| �  «        || _        t        |«      | _        t        j                  |j                  |j                  ¬«      | _	        t        j                  |j                  «      | _        t        j                  t        |j                  «      D �cg c]  }t!        ||dk(  ¬«      ‘Œ c}«      | _        d| _        y c c}w rÏ   )r6   r7   r®   r!   rÐ   r;   r»   r´   r¼   r½   r¹   rº   r0   rÑ   rÒ   rÓ   rÉ   rÔ   rÕ   rÖ   s      €r(   r7   z$WavLMEncoderStableLayerNorm.__init__‡  s¦   ø€ Ü‰ÑÔØˆŒÜ:¸6ÓBˆÔÜŸ,™, v×'9Ñ'9¸v×?TÑ?TÔUˆŒÜ—z‘z &×"7Ñ"7Ó8ˆŒÜ—m‘mô ˜v×7Ñ7Ó8öàô 1°ÐUVÐZ[ÑU[Ö]òó
ˆŒð ',ˆÕ#ùòrØ   c                 ó  — |rdnd }|rdnd }|�5|j                  d«      j                  dd|j                  d   «      }d|| <   | j                  |«      }	||	z   }| j	                  |«      }t        «       xs t        | «      }
d }t        | j                  «      D ]°  \  }}|r||fz   }t        j                  g «      }| j                  xr  |dkD  xr || j                  j                  k  }|r|
rL| j                  r,| j                  r | j                  |j                   ||||«      }n |||||¬«      }|d d \  }}|rd}|sŒ¨|d   fz   }Œ² | j#                  |«      }|r||fz   }|st%        d„ |||fD «       «      S t'        |||¬	«      S )
Nr&   rP   r   r   r   )rK   rM   rL   rÚ   c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wrÜ   r&   rÝ   s     r(   rà   z6WavLMEncoderStableLayerNorm.forward.<locals>.<genexpr>Ô  rá   râ   rã   )rW   rX   rZ   rÐ   r0   r   r   ræ   rÔ   rB   rç   rz   r®   rè   rÕ   ré   rê   r½   rë   r	   rì   s                    r(   rl   z#WavLMEncoderStableLayerNorm.forward•  sÛ  € ñ #7™B¸DÐÙ$5™b¸4ÐàÐ%à$2×$<Ñ$<¸RÓ$@×$GÑ$GÈÈ1Èm×NaÑNaÐbcÑNdÓ$eÐ!Ø45ˆMÐ0Ð0Ñ1à"×1Ñ1°-Ó@ÐØ%Ð(;Ñ;ˆØŸ™ ]Ó3ˆä0Ó2ÒRÔ6LÈTÓ6RˆØˆä! $§+¡+Ó.ò  	P‰HˆAˆuÙ#Ø$5¸Ð8HÑ$HÐ!ô #(§*¡*¨R£.Ðà!Ÿ]™]Òf¨q°1©uÒfÐ:MÐPT×P[ÑP[×PeÑPeÑ:eˆNÙ!¡[ð ×.Ò.°4·=²=Ø$(×$EÑ$EØŸ™Ø%Ø&Ø%Ø)ó%‘Mñ %*Ø%Ø'5Ø*;Ø&3ô	%�Mð 0=¸R¸aÐ/@Ñ,�˜}áØ 2�â Ø&9¸]È1Ñ=MÐ<OÑ&OÑ#ðA 	PðD Ÿ™¨Ó6ˆáØ 1°]Ð4DÑ DÐáÜÑm ]Ð4EÐGZÐ$[ÔmÓmÐmÜØ+Ð;LÐYlô
ð 	
r'   rø   rù   r©   s   @r(   rû   rû   †  s   ø„ ô,ð" ØØ"Ø÷B
r'   rû   c                   ó8   ‡ — e Zd ZdZˆ fd„Zed„ «       Zd„ Zˆ xZS )ÚWavLMGumbelVectorQuantizerz¬
    Vector quantization using gumbel softmax. See [CATEGORICAL REPARAMETERIZATION WITH
    GUMBEL-SOFTMAX](https://arxiv.org/pdf/1611.01144.pdf) for more information.
    c                 ó0  •— t         ‰| �  «        |j                  | _        |j                  | _        |j                  | j                  z  dk7  r&t        d|j                  › d| j                  › d�«      ‚t        j                  t        j                  d| j                  | j
                  z  |j                  | j                  z  «      «      | _        t        j                  |j                  d   | j                  | j
                  z  «      | _        d| _        y )Nr   z`config.codevector_dim z5 must be divisible by `config.num_codevector_groups` z for concatenation.r   rP   r   )r6   r7   Únum_codevector_groupsÚ
num_groupsÚnum_codevectors_per_groupÚnum_varsÚcodevector_dimr9   r;   rA   rB   r¥   Úcodevectorsr<   Úconv_dimÚweight_projÚtemperature)rH   r®   rI   s     €r(   r7   z#WavLMGumbelVectorQuantizer.__init__à  sî   ø€ Ü‰ÑÔØ ×6Ñ6ˆŒØ×8Ñ8ˆŒà× Ñ  4§?¡?Ñ2°aÒ7ÜØ)¨&×*?Ñ*?Ð)@ð A6Ø6:·o±oÐ5Fð G%ð%óð ô Ÿ<™<Ü×Ñ˜a §¡°4·=±=Ñ!@À&×BWÑBWÐ[_×[jÑ[jÑBjÓkó
ˆÔô Ÿ9™9 V§_¡_°RÑ%8¸$¿/¹/ÈDÏMÉMÑ:YÓZˆÔð ˆÕr'   c           	      óÎ   — | j                  d¬«      }t        j                  t        j                  |t        j                  |dz   «      z  d¬«       «      j                  «       }|S )Nr   rR   gH¯¼šò×z>rP   )ÚmeanrB   Úexpr\   r”   )ÚprobsÚmarginal_probsÚ
perplexitys      r(   Ú_compute_perplexityz.WavLMGumbelVectorQuantizer._compute_perplexityõ  sR   € àŸ™¨˜Ó*ˆÜ—Y‘Y¤§	¡	¨.¼5¿9¹9À^ÐVZÑEZÓ;[Ñ*[ÐacÔ dÐdÓe×iÑiÓkˆ
ØÐr'   c                 óâ  — |j                   \  }}}| j                  |«      }|j                  ||z  | j                  z  d«      }| j                  r t
        j                  j                  |j                  «       | j                  d¬«      }|j                  |«      }t        j                  |j                  ||z  | j                  d«      j                  «       d¬«      }| j                  |«      }n}|j                  d¬«      } |j                  |j                   Ž j!                  d|j                  dd«      d«      }|j                  ||z  | j                  d«      }| j                  |«      }|j                  ||z  d«      }|j#                  d«      | j$                  z  }	|	j                  ||z  | j                  | j&                  d«      }
|
j)                  d«      j                  ||d«      }
|
|fS )NrP   T)ÚtauÚhardrR   r   rT   éþÿÿÿ)rZ   r	  rY   r  rz   r;   Ú
functionalÚgumbel_softmaxr•   r
  Útype_asrB   Úsoftmaxr  ÚargmaxÚ	new_zerosÚscatter_rW   r  r  r\   )rH   rJ   Ú
batch_sizeÚsequence_lengthr´   Úcodevector_probsÚcodevector_soft_distr  Úcodevector_idxÚcodevectors_per_groupr  s              r(   rl   z"WavLMGumbelVectorQuantizer.forwardû  sÝ  € Ø3@×3FÑ3FÑ0ˆ
�O [ð ×(Ñ(¨Ó7ˆØ%×*Ñ*¨:¸Ñ+GÈ$Ï/É/Ñ+YÐ[]Ó^ˆà�=Š=ä!Ÿ}™}×;Ñ;¸M×<OÑ<OÓ<QÐW[×WgÑWgÐnrÐ;ÓsÐØ/×7Ñ7¸ÓFÐô $)§=¡=Ø×"Ñ" :°Ñ#?ÀÇÁÐRTÓU×[Ñ[Ó]Ðceô$Ð ð ×1Ñ1Ð2FÓG‰Jð +×1Ñ1°bÐ1Ó9ˆNØ6˜}×6Ñ6¸×8KÑ8KÐL×UÑUØ�N×'Ñ'¨¨AÓ.°ó Ðð  0×4Ñ4°ZÀ/Ñ5QÐSW×SbÑSbÐdfÓgÐà×1Ñ1Ð2BÓCˆJà+×0Ñ0°¸oÑ1MÈrÓRÐà 0× :Ñ :¸2Ó >À×AQÑAQÑ QÐØ+×0Ñ0°¸oÑ1MÈtÏÉÐ`d×`mÑ`mÐoqÓrˆØ!—o‘o bÓ)×.Ñ.¨z¸?ÈBÓOˆà˜JÐ&Ð&r'   )	r#   r$   r%   r¡   r7   Ústaticmethodr  rl   r¨   r©   s   @r(   r   r   Ú  s&   ø„ ñô
ð* ñó ðö
"'r'   r   c                   ó@   — e Zd ZdZeZdZdZdZdZ	dZ
d„ Zd„ Zd„ Zd	„ Zy
)ÚWavLMPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚwavlmÚinput_valuesTFc           
      óz  — t        |t        «      r‰|j                  j                  j                  j                  dd¬«       |j                  j                  j                  j                  «        t        j                  j                  |j                  «       yt        |t        «      r²t        j                  j                  |j                  j                  ddt        j                  d|j                  j                   d   |j                  j"                  z  z  «      z  ¬«       t        j                  j%                  |j                  j                  d«       yt        |t&        «      r›t        j                  d|j(                  j*                  z  «      }t        j                  j                  |j(                  j                  | |¬«       t        j                  j                  |j(                  j                  | |¬«       yt        |t        j,                  «      rm|j                  j                  j                  d| j.                  j0                  ¬«       |j                  �%|j                  j                  j                  «        yyt        |t        j2                  t        j4                  f«      rJ|j                  j                  j                  «        |j                  j                  j7                  d«       yt        |t        j8                  «      r t        j                  j;                  |j                  «       |j                  �jt        j                  |j<                  |j"                  |j                   d   z  z  «      }t        j                  j                  |j                  | |¬«       yyy)	zInitialize the weightsrŸ   r   )r  Ústdr   r   )ÚaÚbNrT   )Ú
isinstancer   r	  ry   ÚdataÚnormal_rx   Úzero_r;   ÚinitÚuniform_r  r!   Úconvr–   ÚsqrtÚkernel_sizeÚin_channelsÚ	constant_r+   Ú
projectionÚin_featuresr<   r®   Úinitializer_ranger»   Ú	GroupNormÚfill_ÚConv1dÚkaiming_normal_Úgroups)rH   ÚmoduleÚks      r(   Ú_init_weightsz"WavLMPreTrainedModel._init_weights-  sŠ  € ô �fÔ8Ô9Ø×Ñ×%Ñ%×*Ñ*×2Ñ2¸ÀÐ2ÔCØ×Ñ×#Ñ#×(Ñ(×.Ñ.Ô0Ü�G‰G×Ñ˜V×/Ñ/Õ0Ü˜Ô <Ô=Ü�G‰G�O‰OØ—‘×"Ñ"ØØœŸ	™	 ! v§{¡{×'>Ñ'>¸qÑ'AÀFÇKÁK×D[ÑD[Ñ'[Ñ"\Ó]Ñ]ð ô ô
 �G‰G×Ñ˜fŸk™k×.Ñ.°Õ2Ü˜Ô 6Ô7Ü—	‘	˜!˜f×/Ñ/×;Ñ;Ñ;Ó<ˆAÜ�G‰G×Ñ˜V×.Ñ.×5Ñ5¸!¸¸qÐÔAÜ�G‰G×Ñ˜V×.Ñ.×3Ñ3¸°r¸QÐÕ?Ü˜¤§	¡	Ô*Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSà�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡¬r¯|©|Ð <Ô=Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜¤§	¡	Ô*Ü�G‰G×#Ñ# F§M¡MÔ2à�{‰{Ð&Ü—I‘I˜fŸm™m¨v×/AÑ/AÀF×DVÑDVÐWXÑDYÑ/YÑZÓ[�Ü—‘× Ñ  §¡°°°aÐ Õ8ð 'ð +r'   c                 ó   — t        d«      ‚©NzNot needed for WavLM©ÚAttributeError©rH   s    r(   Ú_get_adaptersz"WavLMPreTrainedModel._get_adaptersN  ó   € ÜÐ3Ó4Ð4r'   c                 ó   — t        d«      ‚rC  rD  rF  s    r(   Úinit_adapter_layersz(WavLMPreTrainedModel.init_adapter_layersQ  rH  r'   c                 ó   — t        d«      ‚rC  rD  rF  s    r(   Úload_adapterz!WavLMPreTrainedModel.load_adapterT  rH  r'   N)r#   r$   r%   r¡   r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_supports_flash_attn_2Ú_supports_sdparA  rG  rJ  rL  r&   r'   r(   r%  r%     s?   „ ñð
 €LØÐØ$€OØ&*Ð#Ø"ÐØ€Nò9òB5ò5ó5r'   r%  a‰  
    WavLM was proposed in [WavLM: Unified Speech Representation Learning with Labeled and Unlabeled
    Data](https://arxiv.org/abs/2110.13900) by Sanyuan Chen, Chengyi Wang, Zhengyang Chen, Yu Wu, Shujie Liu, Zhuo
    Chen, Jinyu Li, Naoyuki Kanda, Takuya Yoshioka, Xiong Xiao, Jian Wu, Long Zhou, Shuo Ren, Yanmin Qian, Yao Qian,
    Jian Wu, Michael Zeng, Xiangzhan Yu, Furu Wei.

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving etc.).

    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`WavLMConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aI  
    Args:
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install
            soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and
            conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details.
        attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
            1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            <Tip warning={true}>

            `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
            True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should
            **not** be passed to avoid degraded performance when doing batched inference. For such models
            `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these
            models also yield slightly different results depending on whether `input_values` is padded or not.

            </Tip>

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
z_The bare WavLM Model transformer outputting raw hidden-states without any specific head on top.c                   óT   ‡ — e Zd Z ee«       eeeede	¬«      ˆ fd„«       «       Z
ˆ xZS )Ú
WavLMModelÚaudio©Ú
checkpointÚoutput_typerM  ÚmodalityÚexpected_outputc                 ó"   •— t        ‰| �  di |¤ŽS ©Nr&   ©r6   rl   ©rH   Úsuper_kwargsrI   s     €r(   rl   zWavLMModel.forward—  s   ø€ ô ‰w‰Ñ. Ñ.Ð.r'   )r#   r$   r%   r   ÚWAVLM_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCÚWavLMBaseModelOutputÚ_CONFIG_FOR_DOCÚ_EXPECTED_OUTPUT_SHAPErl   r¨   r©   s   @r(   rT  rT  ’  s:   ø„ ñ
 +Ð+AÓBÙØ&Ø(Ø$ØØ.ôó/óó Cô/r'   rT  zcWavLM Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).c                   óT   ‡ — e Zd Z ee«       eeeee	e
¬«      ˆ fd„«       «       Zˆ xZS )ÚWavLMForCTC)rW  rX  rM  rZ  Úexpected_lossc                 ó$   •— t        ‰| �  di |¤Ž y r\  r]  r^  s     €r(   rl   zWavLMForCTC.forward¨  ó   ø€ ô 	‰‰Ñ'˜,Ó'r'   )r#   r$   r%   r   r`  r   ra  r
   rc  Ú_CTC_EXPECTED_OUTPUTÚ_CTC_EXPECTED_LOSSrl   r¨   r©   s   @r(   rf  rf  £  s:   ø„ ñ
 +Ð+AÓBÙØ&Ø"Ø$Ø,Ø(ôó(óó Cô(r'   rf  z”
    WavLM Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   óR   ‡ — e Zd Z ee«       eeeed¬«      ˆ fd„«       «       Z	ˆ xZ
S )ÚWavLMForSequenceClassificationrU  )rW  rX  rM  rY  c                 ó$   •— t        ‰| �  di |¤Ž y r\  r]  r^  s     €r(   rl   z&WavLMForSequenceClassification.forward¼  s   ø€ ô 	‰‰Ñ'˜,Ó'r'   )r#   r$   r%   r   r`  r   ra  r   rc  rl   r¨   r©   s   @r(   rm  rm  ´  s7   ø„ ñ +Ð+AÓBÙØ&Ø,Ø$Øô	ó(óó Cô(r'   rm  za
    WavLM Model with a frame classification head on top for tasks like Speaker Diarization.
    c                   óT   ‡ — e Zd Z ee«       eeeede	¬«      ˆ fd„«       «       Z
ˆ xZS )Ú WavLMForAudioFrameClassificationrU  rV  c                 ó$   •— t        ‰| �  di |¤Ž y r\  r]  r^  s     €r(   rl   z(WavLMForAudioFrameClassification.forwardÎ  ri  r'   )r#   r$   r%   r   r`  r   Ú_FRAME_CLASS_CHECKPOINTr   rc  Ú_FRAME_EXPECTED_OUTPUTrl   r¨   r©   s   @r(   rp  rp  Ç  s:   ø„ ñ +Ð+AÓBÙØ*Ø)Ø$ØØ.ôó(óó Cô(r'   rp  zi
    WavLM Model with an XVector feature extraction head on top for tasks like Speaker Verification.
    c                   óV   ‡ — e Zd Z	  ee«       eeeede	¬«      ˆ fd„«       «       Z
ˆ xZS )ÚWavLMForXVectorrU  rV  c                 ó$   •— t        ‰| �  di |¤Ž y r\  r]  r^  s     €r(   rl   zWavLMForXVector.forwardã  ri  r'   )r#   r$   r%   r   r`  r   Ú_XVECTOR_CHECKPOINTr   rc  Ú_XVECTOR_EXPECTED_OUTPUTrl   r¨   r©   s   @r(   ru  ru  Ú  s=   ø„ ð 	á*Ð+AÓBÙØ&Ø!Ø$ØØ0ôó(óó Cô(r'   ru  )rp  rf  rm  ru  rT  r%  )Ir–   Útypingr   r   r   rB   Útorch.nnr;   Útorch.nn.functionalr  rt   Úintegrations.deepspeedr   Úintegrations.fsdpr   Úmodeling_outputsr	   r
   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   r   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   r   r   Úconfiguration_wavlmr   Ú
get_loggerr#   Úloggerrc  ra  rd  rj  rk  rr  rs  rw  rx  r!   r+   ÚModuler-   r«   r­   rÉ   rÍ   rû   r   r%  ÚWAVLM_START_DOCSTRINGr`  rb  rT  rf  rm  rp  ru  Ú__all__r&   r'   r(   ú<module>rˆ     s(  ðÛ ß )Ñ )ã Ý ß Ð å @Ý 7÷÷ õ .ß uÓ u÷
÷ 
õ 
õ -ð 
ˆ×	Ñ	˜HÓ	%€à€àIÐ Ú&Ð àsÐ ØÐ à8Ð Ø˜Q˜Ð à4Ð ØÐ ô	Ð#Bô 	ô	Ð6ô 	ôc �R—Y‘Yô c ôL	Ð*ô 	ô&˜Ÿ	™	ô &ôR" r§y¡yô "ôJP
�2—9‘9ô P
ôfQ
 "§)¡)ô Q
ôhC' §¡ô C'ôL55˜?Ð,Cô 55ðpÐ ð&"Ð ðH /Ð ñ ØeØóô
/�ó 
/ó	ð
/ñ ØmØóô
(�.ó 
(ó	ð
(ñ ðð óô	(Ð%Fó 	(óð	(ñ ðð ó	ô
(Ð'Jó 
(óð
(ñ ðð ó	ô(Ð(ó (óð(ò�r'   