Ë
    T^(h¦_  ã                   óŠ  — d dl Z d dlZd dlmZ d dlmZmZmZ d dlZd dl	m
Z
 ddlmZmZmZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZmZmZmZmZmZmZm Z m!Z! d
dl"m#Z#  ejH                  e%«      Z&dZ'dZ(dZ)g d¢Z*dZ+dZ,e G d„ de«      «       Z- G d„ de!«      Z. G d„ de«      Z/ G d„ de«      Z0 G d„ de«      Z1 G d„ de«      Z2 G d„ de«      Z3 G d„ d e«      Z4d!Z5d"Z6eZ7 ed#e5«       G d$„ d%e4e «      «       Z8 ed&e5«       G d'„ d(e4«      «       Z9 ed)e5d*«       G d+„ d,e«      «       Z: ed-e5«       G d.„ d/e«      «       Z;g d0¢Z<y)1é    N)Ú	dataclass)ÚOptionalÚTupleÚUnioné   )ÚCausalLMOutputÚModelOutputÚSequenceClassifierOutputÚWav2Vec2BaseModelOutput)ÚPreTrainedModel)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚloggingÚreplace_return_docstringsé   )	ÚWav2Vec2EncoderÚWav2Vec2EncoderStableLayerNormÚWav2Vec2FeatureEncoderÚWav2Vec2FeatureProjectionÚWav2Vec2ForCTCÚ!Wav2Vec2ForSequenceClassificationÚWav2Vec2GumbelVectorQuantizerÚWav2Vec2ModelÚWav2Vec2PositionalConvEmbeddingé   )ÚUniSpeechConfigr   z/patrickvonplaten/unispeech-large-1500h-cv-timit)r   i$  i   zW'mister quilter is the apposl of the midle classes and weare glad to welcom his gosepl'gìQ¸…+1@c                   ó  — e Zd ZU dZdZeej                     ed<   dZ	eej                     ed<   dZ
eej                     ed<   dZeej                     ed<   dZeeej                        ed<   dZeeej                        ed<   y)	ÚUniSpeechForPreTrainingOutputaL  
    Output type of [`UniSpeechForPreTrainingOutput`], with potential hidden states and attentions.

    Args:
        loss (*optional*, returned when model is in train mode, `torch.FloatTensor` of shape `(1,)`):
            Total loss as the sum of the contrastive loss (L_m) and the diversity loss (L_d) as stated in the [official
            paper](https://arxiv.org/pdf/2006.11477.pdf) . (classification) loss.
        projected_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
            Hidden-states of the model projected to *config.proj_codevector_dim* that can be used to predict the masked
            projected quantized states.
        projected_quantized_states (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.proj_codevector_dim)`):
            Quantized extracted feature vectors projected to *config.proj_codevector_dim* representing the positive
            target vectors for contrastive loss.
        hidden_states (`tuple(torch.FloatTensor)`, *optional*, returned when `output_hidden_states=True` is passed or when `config.output_hidden_states=True`):
            Tuple of `torch.FloatTensor` (one for the output of the embeddings + one for the output of each layer) of
            shape `(batch_size, sequence_length, hidden_size)`.

            Hidden-states of the model at the output of each layer plus the initial embedding outputs.
        attentions (`tuple(torch.FloatTensor)`, *optional*, returned when `output_attentions=True` is passed or when `config.output_attentions=True`):
            Tuple of `torch.FloatTensor` (one for each layer) of shape `(batch_size, num_heads, sequence_length,
            sequence_length)`.

            Attentions weights after the attention softmax, used to compute the weighted average in the self-attention
            heads.
    NÚlossÚprojected_statesÚprojected_quantized_statesÚcodevector_perplexityÚhidden_statesÚ
attentions)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r    r   ÚtorchÚFloatTensorÚ__annotations__r!   r"   r#   r$   r   r%   © ó    úm/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/unispeech/modular_unispeech.pyr   r   1   s”   … ñð4 )-€Dˆ(�5×$Ñ$Ñ
%Ó,Ø48Ð�h˜u×0Ñ0Ñ1Ó8Ø>BÐ ¨×):Ñ):Ñ ;ÓBØ9=Ð˜8 E×$5Ñ$5Ñ6Ó=Ø8<€M�8˜E %×"3Ñ"3Ñ4Ñ5Ó<Ø59€J�˜˜u×0Ñ0Ñ1Ñ2Ô9r.   r   c                   ó   — e Zd Zy)Ú UniSpeechPositionalConvEmbeddingN©r&   r'   r(   r-   r.   r/   r1   r1   U   ó   „ Ør.   r1   c                   ó   — e Zd Zy)ÚUniSpeechFeatureEncoderNr2   r-   r.   r/   r5   r5   Y   r3   r.   r5   c                   ó   — e Zd Zy)ÚUniSpeechFeatureProjectionNr2   r-   r.   r/   r7   r7   ]   r3   r.   r7   c                   ó   — e Zd Zy)ÚUniSpeechEncoderNr2   r-   r.   r/   r9   r9   a   r3   r.   r9   c                   ó   — e Zd Zy)ÚUniSpeechEncoderStableLayerNormNr2   r-   r.   r/   r;   r;   e   r3   r.   r;   c                   ó"   — e Zd Zed„ «       Zd„ Zy)ÚUniSpeechGumbelVectorQuantizerc           	      óÎ   — | j                  d¬«      }t        j                  t        j                  |t        j                  |dz   «      z  d¬«       «      j                  «       }|S )Nr   ©ÚdimgH¯¼šò×z>éÿÿÿÿ)Úmeanr*   ÚexpÚsumÚlog)ÚprobsÚmarginal_probsÚ
perplexitys      r/   Ú_compute_perplexityz2UniSpeechGumbelVectorQuantizer._compute_perplexityj   sR   € àŸ™¨˜Ó*ˆÜ—Y‘Y¤§	¡	¨.¼5¿9¹9À^ÐVZÑEZÓ;[Ñ*[ÐacÔ dÐdÓe×iÑiÓkˆ
ØÐr.   c                 óÞ  — |j                   \  }}}| j                  |«      }|j                  ||z  | j                  z  d«      }| j                  ržt
        j                  j                  |j                  «       | j                  d¬«      j                  |«      }t        j                  |j                  ||z  | j                  d«      j                  «       d¬«      }| j                  |«      }n}|j                  d¬«      } |j                  |j                   Ž j!                  d|j                  dd«      d«      }|j                  ||z  | j                  d«      }| j                  |«      }|j                  ||z  d«      }|j#                  d«      | j$                  z  }	|	j                  ||z  | j                  | j&                  d«      }
|
j)                  d«      j                  ||d«      }
|
|fS )NrA   T)ÚtauÚhardr?   r   ç      ð?éþÿÿÿ)ÚshapeÚweight_projÚviewÚ
num_groupsÚtrainingÚnnÚ
functionalÚgumbel_softmaxÚfloatÚtemperatureÚtype_asr*   ÚsoftmaxrI   ÚargmaxÚ	new_zerosÚscatter_Ú	unsqueezeÚcodevectorsÚnum_varsrD   )Úselfr$   Ú
batch_sizeÚsequence_lengthÚhidden_sizeÚcodevector_probsÚcodevector_soft_distrH   Úcodevector_idxÚcodevectors_per_groupr_   s              r/   Úforwardz&UniSpeechGumbelVectorQuantizer.forwardp   sÛ  € Ø3@×3FÑ3FÑ0ˆ
�O [ð ×(Ñ(¨Ó7ˆØ%×*Ñ*¨:¸Ñ+GÈ$Ï/É/Ñ+YÐ[]Ó^ˆà�=Š=ä!Ÿ}™}×;Ñ;Ø×#Ñ#Ó%¨4×+;Ñ+;À$ð  <ó  ç‰g�mÓ$ð ô
 $)§=¡=Ø×"Ñ" :°Ñ#?ÀÇÁÐRTÓU×[Ñ[Ó]Ðceô$Ð ð ×1Ñ1Ð2FÓG‰Jð +×1Ñ1°bÐ1Ó9ˆNØ6˜}×6Ñ6¸×8KÑ8KÐL×UÑUØ�N×'Ñ'¨¨AÓ.°ó Ðð  0×4Ñ4°ZÀ/Ñ5QÐSW×SbÑSbÐdfÓgÐà×1Ñ1Ð2BÓCˆJà+×0Ñ0°¸oÑ1MÈrÓRÐà 0× :Ñ :¸2Ó >À×AQÑAQÑ QÐØ+×0Ñ0°¸oÑ1MÈtÏÉÐ`d×`mÑ`mÐoqÓrˆØ!—o‘o bÓ)×.Ñ.¨z¸?ÈBÓOˆà˜JÐ&Ð&r.   N)r&   r'   r(   ÚstaticmethodrI   ri   r-   r.   r/   r=   r=   i   s   „ Øñó ðó
#'r.   r=   c                   ó|   — e Zd ZdZeZdZdZdZdZ	dZ
d„ Zdeej                  ef   fd„Zded	ej                  fd
„Zy)ÚUniSpeechPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    Ú	unispeechÚinput_valuesTc           
      óz  — t        |t        «      r‰|j                  j                  j                  j                  dd¬«       |j                  j                  j                  j                  «        t        j                  j                  |j                  «       yt        |t        «      r²t        j                  j                  |j                  j                  ddt        j                  d|j                  j                   d   |j                  j"                  z  z  «      z  ¬«       t        j                  j%                  |j                  j                  d«       yt        |t&        «      r›t        j                  d|j(                  j*                  z  «      }t        j                  j                  |j(                  j                  | |¬«       t        j                  j                  |j(                  j                  | |¬«       yt        |t        j,                  «      rm|j                  j                  j                  d| j.                  j0                  ¬«       |j                  �%|j                  j                  j                  «        yyt        |t        j2                  t        j4                  f«      rJ|j                  j                  j                  «        |j                  j                  j7                  d«       yt        |t        j8                  «      r t        j                  j;                  |j                  «       |j                  �jt        j                  |j<                  |j"                  |j                   d   z  z  «      }t        j                  j                  |j                  | |¬«       yyy)	zInitialize the weightsç        r   )rB   Ústdr   r   )ÚaÚbNrM   )Ú
isinstancer=   rP   ÚweightÚdataÚnormal_ÚbiasÚzero_rT   ÚinitÚuniform_r_   r1   ÚconvÚmathÚsqrtÚkernel_sizeÚin_channelsÚ	constant_r7   Ú
projectionÚin_featuresÚLinearÚconfigÚinitializer_rangeÚ	LayerNormÚ	GroupNormÚfill_ÚConv1dÚkaiming_normal_Úgroups)ra   ÚmoduleÚks      r/   Ú_init_weightsz&UniSpeechPreTrainedModel._init_weights£   sŠ  € ô �fÔ<Ô=Ø×Ñ×%Ñ%×*Ñ*×2Ñ2¸ÀÐ2ÔCØ×Ñ×#Ñ#×(Ñ(×.Ñ.Ô0Ü�G‰G×Ñ˜V×/Ñ/Õ0Ü˜Ô @ÔAÜ�G‰G�O‰OØ—‘×"Ñ"ØØœŸ	™	 ! v§{¡{×'>Ñ'>¸qÑ'AÀFÇKÁK×D[ÑD[Ñ'[Ñ"\Ó]Ñ]ð ô ô
 �G‰G×Ñ˜fŸk™k×.Ñ.°Õ2Ü˜Ô :Ô;Ü—	‘	˜!˜f×/Ñ/×;Ñ;Ñ;Ó<ˆAÜ�G‰G×Ñ˜V×.Ñ.×5Ñ5¸!¸¸qÐÔAÜ�G‰G×Ñ˜V×.Ñ.×3Ñ3¸°r¸QÐÕ?Ü˜¤§	¡	Ô*Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSà�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡¬r¯|©|Ð <Ô=Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)Ü˜¤§	¡	Ô*Ü�G‰G×#Ñ# F§M¡MÔ2à�{‰{Ð&Ü—I‘I˜fŸm™m¨v×/AÑ/AÀF×DVÑDVÐWXÑDYÑ/YÑZÓ[�Ü—‘× Ñ  §¡°°°aÐ Õ8ð 'ð +r.   Úinput_lengthsc                 ó˜   — d„ }t        | j                  j                  | j                  j                  «      D ]  \  }} ||||«      }Œ |S )zH
        Computes the output length of the convolutional layers
        c                 ó>   — t        j                  | |z
  |d¬«      dz   S )NÚfloor)Úrounding_moder   )r*   Údiv)Úinput_lengthr   Ústrides      r/   Ú_conv_out_lengthzSUniSpeechPreTrainedModel._get_feat_extract_output_lengths.<locals>._conv_out_lengthÉ   s"   € ô —9‘9˜\¨KÑ7¸ÈwÔWÐZ[Ñ[Ð[r.   )Úzipr…   Úconv_kernelÚconv_stride)ra   r�   r˜   r   r—   s        r/   Ú _get_feat_extract_output_lengthsz9UniSpeechPreTrainedModel._get_feat_extract_output_lengthsÄ   sQ   € ò
	\ô
 $' t§{¡{×'>Ñ'>ÀÇÁ×@WÑ@WÓ#Xò 	QÑˆK˜Ù,¨]¸KÈÓP‰Mð	Qð Ðr.   Úfeature_vector_lengthÚattention_maskc                 óø  — |j                  d¬«      d d …df   }| j                  |«      j                  t        j                  «      }|j
                  d   }t        j                  ||f|j                  |j                  ¬«      }d|t        j                  |j
                  d   |j                  ¬«      |dz
  f<   |j                  dg«      j                  d«      j                  dg«      j                  «       }|S )NrA   r?   r   )ÚdtypeÚdevicer   )r¡   )Úcumsumrœ   Útor*   ÚlongrO   Úzerosr    r¡   ÚarangeÚflipÚbool)ra   r�   rž   Únon_padded_lengthsÚoutput_lengthsrb   s         r/   Ú"_get_feature_vector_attention_maskz;UniSpeechPreTrainedModel._get_feature_vector_attention_maskÓ   së   € ð ,×2Ñ2°rÐ2Ó:º1¸b¸5ÑAÐØ×>Ñ>Ð?QÓR×UÑUÔV[×V`ÑV`ÓaˆØ#×)Ñ)¨!Ñ,ˆ
äŸ™ØÐ.Ð/°~×7KÑ7KÐTb×TiÑTiô
ˆð uvˆœŸ™ ^×%9Ñ%9¸!Ñ%<À^×EZÑEZÔ[Ð]kÐnoÑ]oÐpÑqØ'×,Ñ,¨b¨TÓ2×9Ñ9¸"Ó=×BÑBÀBÀ4ÓH×MÑMÓOˆØÐr.   N)r&   r'   r(   r)   r   Úconfig_classÚbase_model_prefixÚmain_input_nameÚsupports_gradient_checkpointingÚ_supports_flash_attn_2Ú_supports_sdpar�   r   r*   Ú
LongTensorÚintrœ   r«   r-   r.   r/   rl   rl   –   sg   „ ñð
 #€LØ#ÐØ$€OØ&*Ð#Ø!ÐØ€Nò9ðB¸eÀE×DTÑDTÐVYÐDYÑ>Zó ðÈð Ð]b×]mÑ]mô r.   rl   a  
    UniSpeech was proposed in [UniSpeech: Unified Speech Representation Learning with Labeled and Unlabeled
    Data](https://arxiv.org/abs/2101.07597) by Chengyi Wang, Yu Wu, Yao Qian, Kenichi Kumatani, Shujie Liu, Furu Wei,
    Michael Zeng, Xuedong Huang.

    This model inherits from [`PreTrainedModel`]. Check the superclass documentation for the generic methods the
    library implements for all its model (such as downloading or saving etc.).

    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`UniSpeechConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aI  
    Args:
        input_values (`torch.FloatTensor` of shape `(batch_size, sequence_length)`):
            Float values of input raw speech waveform. Values can be obtained by loading a `.flac` or `.wav` audio file
            into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install
            soundfile`). To prepare the array into `input_values`, the [`AutoProcessor`] should be used for padding and
            conversion into a tensor of type `torch.FloatTensor`. See [`Wav2Vec2Processor.__call__`] for details.
        attention_mask (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Mask to avoid performing convolution and attention on padding token indices. Mask values selected in `[0,
            1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)

            <Tip warning={true}>

            `attention_mask` should only be passed if the corresponding processor has `config.return_attention_mask ==
            True`. For all models whose processor has `config.return_attention_mask == False`, `attention_mask` should
            **not** be passed to avoid degraded performance when doing batched inference. For such models
            `input_values` should simply be padded with 0 and passed without `attention_mask`. Be aware that these
            models also yield slightly different results depending on whether `input_values` is padded or not.

            </Tip>

        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
zcThe bare UniSpeech Model transformer outputting raw hidden-states without any specific head on top.c                   óô   — e Zd Zdefd„Zd„ Zd„ Z ee«       e	e
eede¬«      	 	 	 	 	 ddeej                      d	eej                      d
eej"                     dee   dee   dee   deeef   fd„«       «       Zy)ÚUniSpeechModelr…   c                 ó¼  — t         j                  |«       || _        t        |«      | _        t        |«      | _        |j                  dkD  s|j                  dkD  rEt        j                  t        j                  |j                  «      j                  «       «      | _        |j                   rt#        |«      | _        nt'        |«      | _        | j)                  «        y )Nrp   )rl   Ú__init__r…   r5   Úfeature_extractorr7   Úfeature_projectionÚmask_time_probÚmask_feature_probrT   Ú	Parameterr*   ÚTensorrd   r{   Úmasked_spec_embedÚdo_stable_layer_normr;   Úencoderr9   Ú	post_init)ra   r…   s     r/   r·   zUniSpeechModel.__init__"  s¢   € Ü ×)Ñ)¨&Ô1ØˆŒÜ!8¸Ó!@ˆÔÜ"<¸VÓ"DˆÔà× Ñ  3Ò&¨&×*BÑ*BÀSÒ*HÜ%'§\¡\´%·,±,¸v×?QÑ?QÓ2R×2[Ñ2[Ó2]Ó%^ˆDÔ"à×&Ò&Ü:¸6ÓBˆD�Lä+¨FÓ3ˆDŒLð 	�‰Õr.   c                 ó   — t        d«      ‚©NzNot needed for UniSpeech©ÚAttributeError©ra   s    r/   Úfreeze_feature_extractorz'UniSpeechModel.freeze_feature_extractor3  ó   € ÜÐ7Ó8Ð8r.   c                 ó   — t        d«      ‚rÃ   rÄ   rÆ   s    r/   Úfreeze_feature_encoderz%UniSpeechModel.freeze_feature_encoder6  rÈ   r.   Úaudio)Ú
checkpointÚoutput_typer¬   ÚmodalityÚexpected_outputNrn   rž   Úmask_time_indicesÚoutput_attentionsÚoutput_hidden_statesÚreturn_dictÚreturnc                 ó
  — |�|n| j                   j                  }|�|n| j                   j                  }|�|n| j                   j                  }| j	                  |«      }|j                  dd«      }|�| j                  |j                  d   |«      }| j                  |«      \  }}| j                  |||¬«      }| j                  |||||¬«      }	|	d   }|s
||f|	dd  z   S t        |||	j                  |	j                  ¬«      S )Nr   r   )rÐ   rž   ©rž   rÑ   rÒ   rÓ   r   )Úlast_hidden_stateÚextract_featuresr$   r%   )r…   rÑ   rÒ   Úuse_return_dictr¸   Ú	transposer«   rO   r¹   Ú_mask_hidden_statesrÀ   ÚUniSpeechBaseModelOutputr$   r%   )
ra   rn   rž   rÐ   rÑ   rÒ   rÓ   rØ   r$   Úencoder_outputss
             r/   ri   zUniSpeechModel.forward9  s@  € ð" 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà×1Ñ1°,Ó?ÐØ+×5Ñ5°a¸Ó;ÐàÐ%à!×DÑDÐEU×E[ÑE[Ð\]ÑE^Ð`nÓoˆNà*.×*AÑ*AÐBRÓ*SÑ'ˆÐ'Ø×0Ñ0ØÐ->È~ð 1ó 
ˆð Ÿ,™,ØØ)Ø/Ø!5Ø#ð 'ó 
ˆð (¨Ñ*ˆáØ!Ð#3Ð4°ÀqÀrÐ7JÑJÐJä'Ø+Ø-Ø)×7Ñ7Ø&×1Ñ1ô	
ð 	
r.   )NNNNN)r&   r'   r(   r   r·   rÇ   rÊ   r   ÚUNISPEECH_INPUTS_DOCSTRINGr   Ú_CHECKPOINT_FOR_DOCrÜ   Ú_CONFIG_FOR_DOCÚ_EXPECTED_OUTPUT_SHAPEr   r*   r½   r+   r¨   r   r   ri   r-   r.   r/   rµ   rµ     sØ   „ ð
˜ó ò"9ò9ñ +Ð+EÓFÙØ&Ø,Ø$ØØ.ôð 26Ø9=Ø,0Ø/3Ø&*ñ-
à˜uŸ|™|Ñ,ð-
ð ! §¡Ñ.ð-
ð $ E×$5Ñ$5Ñ6ð	-
ð
 $ D™>ð-
ð ' t™nð-
ð ˜d‘^ð-
ð 
ˆuÐ.Ð.Ñ	/ò-
óó Gñ-
r.   rµ   zPUniSpeech Model with a vector-quantization module and ctc loss for pre-training.c                   óH  ‡ — e Zd Zdefˆ fd„Zdefd„Zd„ Zd„ Ze		 dde
j                  de
j                  d	e
j                  defd
„«       Z ee«       eee¬«      	 	 	 	 ddee
j&                     dee
j&                     dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚUniSpeechForPreTrainingr…   c                 ó.  •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  «      | _        t        |«      | _	        t	        j                  |j                  |j                  «      | _        t	        j                  |j                  |j                  «      | _        t	        j                  |j                  |j                   «      | _        t	        j
                  |j$                  «      | _        | j)                  «        y )N)Úsuperr·   rµ   rm   rT   ÚDropoutÚfeat_quantizer_dropoutÚdropout_featuresr=   Ú	quantizerr„   Úcodevector_dimÚproj_codevector_dimÚ	project_qrd   Úproject_hidÚnum_ctc_classesÚctc_projÚfinal_dropoutÚdropoutrÁ   )ra   r…   Ú	__class__s     €r/   r·   z UniSpeechForPreTraining.__init__u  s¼   ø€ Ü‰Ñ˜Ô Ü'¨Ó/ˆŒÜ "§
¡
¨6×+HÑ+HÓ IˆÔä7¸Ó?ˆŒÜŸ™ 6×#8Ñ#8¸&×:TÑ:TÓUˆŒÜŸ9™9 V×%?Ñ%?À×ASÑASÓTˆÔäŸ	™	 &×"4Ñ"4°f×6LÑ6LÓMˆŒÜ—z‘z &×"6Ñ"6Ó7ˆŒð 	�‰Õr.   rX   c                 ó&   — || j                   _        y)zb
        Set the Gumbel softmax temperature to a given value. Only necessary for training
        N)ré   rX   )ra   rX   s     r/   Úset_gumbel_temperaturez.UniSpeechForPreTraining.set_gumbel_temperature„  s   € ð &1ˆ�‰Õ"r.   c                 óX   — t        j                  dt        «       | j                  «        y)z©
        Calling this function will disable the gradient computation for the feature encoder so that its parameters will
        not be updated during training.
        zžThe method `freeze_feature_extractor` is deprecated and will be removed in Transformers v5. Please use the equivalent `freeze_feature_encoder` method instead.N)ÚwarningsÚwarnÚFutureWarningrÊ   rÆ   s    r/   rÇ   z0UniSpeechForPreTraining.freeze_feature_extractorŠ  s'   € ô
 	�‰ðQäô	
ð
 	×#Ñ#Õ%r.   c                 óL   — | j                   j                  j                  «        y)z¨
        Calling this function will disable the gradient computation for the feature encoder so that its parameter will
        not be updated during training.
        N)rm   r¸   Ú_freeze_parametersrÆ   s    r/   rÊ   z.UniSpeechForPreTraining.freeze_feature_encoder–  s   € ð
 	�‰×(Ñ(×;Ñ;Õ=r.   Útarget_featuresÚnegative_featuresÚpredicted_featuresc                 óÌ   — t        j                  | |gd¬«      } t        j                  |j                  «       | j                  «       d¬«      }|j	                  | «      }||z  }|S )zé
        Compute logits for contrastive loss based using cosine similarity as the distance measure between
        `[positive_feature, negative_features]` and `[predicted_features]`. Additionally, temperature can be applied.
        r   r?   rA   )r*   ÚcatÚcosine_similarityrW   rY   )rû   rü   rý   rX   Úlogitss        r/   Úcompute_contrastive_logitsz2UniSpeechForPreTraining.compute_contrastive_logits�  sa   € ô  Ÿ)™) _Ð6GÐ$HÈaÔPˆä×(Ñ(Ð);×)AÑ)AÓ)CÀ_×EZÑEZÓE\ÐbdÔeˆØ—‘ Ó0ˆð ˜+Ñ%ˆØˆr.   )rÍ   r¬   rn   rž   rÑ   rÒ   rÓ   rÔ   c                 óþ  — |�|n| j                   j                  }| j                  |||||¬«      }|d   }| j                  |d   «      }| j	                  |«      \  }	}
| j                  |	j                  | j
                  j                  j                  «      «      }	| j                  |	«      }	t        j                  |j                  d«      |j                  d«      «      j                  | j                   j                  «      }|j                  dd«      }t        j                   |«      j#                  «       j                  |j$                  «      }|j                  dd«      }|j'                  d«      }|j)                  |d«      |	j)                  | d«      z   }| j+                  |«      }| j-                  |«      }d}|s|�|||	|
f|dd z   S ||	|
f|dd z   S t/        |||	|
|j0                  |j2                  ¬«      S )	aô  
        mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict
            masked extracted features in *config.proj_codevector_dim* space.
        sampled_negative_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length, num_negatives)`, *optional*):
            Indices indicating which quantized target vectors are used as negative sampled vectors in contrastive loss.
            Required input for pre-training.

        Returns:

        Example:

        ```python
        >>> import torch
        >>> from transformers import AutoFeatureExtractor, UniSpeechForPreTraining

        >>> feature_extractor = AutoFeatureExtractor.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> model = UniSpeechForPreTraining.from_pretrained("microsoft/unispeech-large-1500h-cv")
        >>> # TODO: Add full pretraining example
        ```NrÖ   r   r   rA   rp   r   )r    r!   r"   r#   r$   r%   )r…   rÙ   rm   rè   ré   rì   r£   ru   r    rí   r*   ÚemptyÚsizer‰   Úreplace_probrÚ   Ú	bernoullir¨   r¡   r^   Úmasked_fillrñ   rï   r   r$   r%   )ra   rn   rž   rÑ   rÒ   rÓ   ÚoutputsÚtransformer_featuresrØ   Úquantized_featuresr#   Úprob_replace_matrixÚsampled_replace_matrixr  r    s                  r/   ri   zUniSpeechForPreTraining.forward±  s  € ð> &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà—.‘.ØØ)Ø/Ø!5Ø#ð !ó 
ˆð  ' q™zÐð  ×0Ñ0°¸±Ó<ÐØ48·N±NÐCSÓ4TÑ1ÐÐ1ð "Ÿ^™^Ð,>×,AÑ,AÀ$Ç.Á.×BWÑBW×B]ÑB]Ó,^Ó_ÐØ!×-Ñ-Ð.@ÓAÐä#Ÿk™kÐ*>×*CÑ*CÀAÓ*FÐH\×HaÑHaÐbcÓHdÓe×kÑkØ�K‰K×$Ñ$ó
Ðð 2×;Ñ;¸A¸qÓAÐÜ!&§¡Ð1DÓ!E×!JÑ!JÓ!L×!OÑ!OÐPd×PkÑPkÓ!lÐØ!7×!AÑ!AÀ!ÀQÓ!GÐØ!7×!AÑ!AÀ"Ó!EÐØ%×1Ñ1Ð2HÈ#ÓNØ×*Ñ*Ð,BÐ+BÀCÓHñ
ˆð
 —‘˜fÓ%ˆØ—‘˜vÓ&ˆð ˆÙØÐØÐ2Ð4FÐH]Ð^ÐahÐijÐikÐalÑlÐlØ(Ð*<Ð>SÐTÐW^Ð_`Ð_aÐWbÑbÐbä,ØØ1Ø'9Ø"7Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r.   )r   )NNNN)r&   r'   r(   r   r·   r³   rô   rÇ   rÊ   rj   r*   r+   r  r   rÞ   r   r   rà   r   r½   r¨   r   r   ri   Ú__classcell__©rò   s   @r/   rã   rã   q  s  ø„ ð˜õ ð1°#ó 1ò
&ò>ð ð
 ñ	Ø×*Ñ*ðà ×,Ñ,ðð "×-Ñ-ðð ò	ó ðñ& +Ð+EÓFÙÐ+HÐWfÔgð 26Ø,0Ø/3Ø&*ñM
à˜uŸ|™|Ñ,ðM
ð ! §¡Ñ.ðM
ð $ D™>ð	M
ð
 ' t™nðM
ð ˜d‘^ðM
ð 
ˆuÐ3Ð3Ñ	4òM
ó hó GôM
r.   rã   zgUniSpeech Model with a `language modeling` head on top for Connectionist Temporal Classification (CTC).a/  
        target_lang (`str`, *optional*):
            Language id of adapter weights. Adapter weights are stored in the format adapter.<lang>.safetensors or
            adapter.<lang>.bin. Only relevant when using an instance of [`UniSpeechForCTC`] with adapters. Uses 'eng'
            by default.
    c                   óT   ‡ — e Zd Z ee«       eeeee	e
¬«      ˆ fd„«       «       Zˆ xZS )ÚUniSpeechForCTC)rÌ   rÍ   r¬   rÏ   Úexpected_lossc                 ó$   •— t        ‰| �  di |¤Ž y ©Nr-   ©rå   ri   ©ra   Úsuper_kwargsrò   s     €r/   ri   zUniSpeechForCTC.forward  s   ø€ ô 	‰‰Ñ'˜,Ó'r.   )r&   r'   r(   r   rÞ   r   rß   r   rà   Ú_CTC_EXPECTED_OUTPUTÚ_CTC_EXPECTED_LOSSri   r  r  s   @r/   r  r    s:   ø„ ñ +Ð+EÓFÙØ&Ø"Ø$Ø,Ø(ôó(óó Gô(r.   r  z˜
    UniSpeech Model with a sequence classification head on top (a linear layer over the pooled output) for tasks like
    SUPERB Keyword Spotting.
    c                   óR   ‡ — e Zd Z ee«       eeeed¬«      ˆ fd„«       «       Z	ˆ xZ
S )Ú"UniSpeechForSequenceClassificationrË   )rÌ   rÍ   r¬   rÎ   c                 ó$   •— t        ‰| �  di |¤Ž y r  r  r  s     €r/   ri   z*UniSpeechForSequenceClassification.forward"  s   ø€ ô 	‰‰Ñ'˜,Ó'r.   )r&   r'   r(   r   rÞ   r   rß   r
   rà   ri   r  r  s   @r/   r  r    s7   ø„ ñ +Ð+EÓFÙØ&Ø,Ø$Øô	ó(óó Gô(r.   r  )r  rã   r  rµ   rl   )=r}   rö   Údataclassesr   Útypingr   r   r   r*   Útorch.nnrT   Úmodeling_outputsr   r	   r
   r   Úmodeling_utilsr   Úutilsr   r   r   r   r   Úwav2vec2.modeling_wav2vec2r   r   r   r   r   r   r   r   r   Úconfiguration_unispeechr   Ú
get_loggerr&   ÚloggerÚ_HIDDEN_STATES_START_POSITIONrà   rß   rá   r  r  r   r1   r5   r7   r9   r;   r=   rl   ÚUNISPEECH_START_DOCSTRINGrÞ   rÜ   rµ   rã   r  r  Ú__all__r-   r.   r/   ú<module>r*     sÉ  ðÛ Û Ý !ß )Ñ )ã Ý ç nÓ nÝ -÷õ ÷
÷ 
õ 
õ 5ð 
ˆ×	Ñ	˜HÓ	%€ð !"Ð ð $€ð HÐ Ú'Ð ð qÐ ØÐ ð ô : Kó  :ó ð :ôF	Ð'Fô 	ô	Ð4ô 	ô	Ð!:ô 	ô	�ô 	ô	Ð&Dô 	ô*'Ð%Bô *'ôZJ˜ô JðZÐ ð$"Ð ðJ 3Ð ñ ØiØóôM
Ð-¨}ó M
ó	ðM
ñ` ØZÐ\uóôL
Ð6ó L
óðL
ñ^ ØqØðó	ô
(�nó 
(ó	ð
(ñ ðð óô	(Ð)Jó 	(óð	(ò�r.   