T^(h}KdZddlmZddlZddlZddlmZmZmZm Z m Z ddl Z ddl ZddlmZddlmZddlmZdd lmZmZmZmZdd lmZmZmZmZmZm Z dd l!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(m)Z)d dl*m+Z+d dl,m-Z-m.Z.e(j^e0Z1dZ2dZ3ejhfd/dZ5d0dZ6d1d2dZ7d3d4dZ8GddejrjtZ;GddejrjtZ<GddejrjtZ=GddejrjtZ>GddeZ?d Z@d!ZAeGd"d#ejrjtZBeGd$d%ejrjtZCe&d&e@eGd'd(ejrjtZDe&d&e@Gd)d*e?ZEe&d+e@Gd,d-e?eZFgd.ZGy)5zTensorFlow Whisper model.) annotationsN)DictListOptionalTupleUnion)get_tf_activation)GenerationConfig)TFLogitsProcessorList)TFBaseModelOutput+TFBaseModelOutputWithPastAndCrossAttentionsTFSeq2SeqLMOutputTFSeq2SeqModelOutput)TFCausalLanguageModelingLossTFModelInputTypeTFPreTrainedModelkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings) WhisperConfig)TASK_IDSTO_LANGUAGE_CODErgחc:|\}}|dzdk7rtd|dtjd|dzdz z }tj| tj |dztj z}tjtj |tj dtj|d z}tjtjtj|tj|gd |S) z*Returns sinusoids for positional embeddingrzVNumber of channels has to be divisible by 2 for sinusoidal positional embeddings, got z channels.i'rdtype)r)rr&axis) ValueErrormathlogtfexprangefloat32reshapecastconcatsincos)shaper%lengthchannelslog_timescale_incrementinv_timescales scaled_times m/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/whisper/modeling_tf_whisper.pysinusoidal_embedding_initr<9sFH!|qdemdnnx y  #hhuoQ1BCVV44rxxA UWU_U_7``aN**RXXfBJJ?IBJJWegnLooK 77299bff[1266+3FGaPRW XXc tj||j}tj||j}tjt |ddftj ||j}tj ||ddddfgd}tj|dk(tjt |tj ||j|}tjj|tjd|j}tj|g5tj|}ddd|S#1swY|SxYw)Nrrr&ir$) r,r1r%fillrconvert_to_tensorr2where debuggingassert_greater_equalconstantcontrol_dependenciesidentity) input_ids pad_token_iddecoder_start_token_id start_tokensshifted_input_ids assert_gte0s r;shift_tokens_rightrMGs877<9LWW%;Y__M77 I q !1%r';'; ::r=ct|d}||n|}tjd}tj||j}tj |ddddddfdd|df}||z t zS)z_ Expands attention_mask from `[bsz, seq_len]` to `[bsz, 1, tgt_seq_len, src_seq_len]`. rN?r$)rr,rDr1r%rSrQ)rXrWsrc_lenone_cst expanded_masks r; _expand_maskr`rsxq!G ,g'Gkk#G 774w}} -DGGDD$!12Q7A4FGM m #~ 55r=cBeZdZ d dfd ZfdZddZxZS)TFWhisperPositionalEmbeddingc t|di|||_||_||_t j j||_y)N) super__init__ num_positions embedding_dim padding_idxr initializersgetembedding_initializer)selfrgrhrirlkwargs __class__s r;rfz%TFWhisperPositionalEmbedding.__init__sG "6"**&%*%7%7%;%; zz$++lGS"I:V ZZ J7 zz, ; Z(+yyztL  !! | $ 4>> !7G 46dnn8LgW^7_6`a|,-/ "   % LL % %>*a'*7a'8R7ST">235 &  WW^<;M;MNN::lS$..'SZ4[\_mmL::lS4>>5I7T[4\]L%l<  & LL % %?+ Et~~EWX"?346 & ::o}E sDNNGWEIL::lS4>>5I7T[4\]L\\,\B ii L9  !! { # 4>> !7DMM :2CRVR_R_3`2ab{+,. " ll JJ{S$..'4==$Q RT` jjsGY.GH mmK0 "$**\CQXZa;b"c L.88r=c|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYAxYw#1swYxYw#1swYxYw#1swYyxYw)NTrrrr) builtgetattrr, name_scoperrrrvrrrrrmrws r;rvzTFWhisperAttention.build7s ::  44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 @ !!4t~~">? @ 44 ( 4t{{//0 @ !!4t~~">? @ 4T * 6t}}112 B ##T4$@A B B 7 @ @ @ @ @ @ B Bs0 )F32)G)G )G3F=G  GG!)rOFT) rrrrrfloatrboolrr)r tf.TensorrrrVr)NNNNF)rrrtf.Tensor | NonerzTuple[Tuple[tf.Tensor]] | NonerrrrrOptional[bool]returnz"Tuple[tf.Tensor, tf.Tensor | None]N) rrr__doc__rfrr}rvrrs@r;rrsG   VVV V  V  V:m.29=+/,0#(t9 t9+t97 t9 ) t9 * t9!t9 ,t9lBr=rcBeZdZdfd Z d ddZddZxZS)TFWhisperEncoderLayerc t|d i||j|_t |j|j |j d|_tjjdd|_ tjj|j|_ t|j|_tjj|j"|_tjj%|j&d|_tjj%|jd|_tjjdd |_||_y) N self_attn)rrrh㈵>self_attn_layer_normepsilonrrfc1rrfc2final_layer_normrd)rerfd_modelrrencoder_attention_headsattention_dropoutrrrLayerNormalizationrrrr activation_function activation_fnactivation_dropoutrencoder_ffn_dimrrrconfigrmrrnros r;rfzTFWhisperEncoderLayer.__init__Ks "6"+ NNF::FD\D\cn %*LL$C$CDWm$C$n!||++FNN; .v/I/IJ"',,"6"6v7P7P"Q<<%%f&<&<5%I<<%%dnn5%A % ? ?Se ? f r=c |}|j|}|j||||\}}}tjj t |t |dt |dt ||j ||}||z}|}|j|}|j|j|}|j||}|j|}|j ||}||z}||fS)a Args: hidden_states (`tf.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`tf.Tensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size `(encoder_attention_heads,)` )rrrrz&Self attn modified the shape of query z to rr) rrr,rBrrrrrrrr)rmrrrrresidualself_attn_weights_s r;r}zTFWhisperEncoderLayer.callZs!11-@ .2nn')+ /=/ + (! !! } % x  N))//tT^^0LM N 4 % 1txx}}- =dDNN;< = 4 % 1txx}}- JdDKK,G,GHI J 4+T 2 >t4499: J%%++T4,HI J J ? + + N N = = J J J Js< H%)H$ )H133H=$)I H!$H.1H:=I Irr)F)rrrrrrrrrrrrrfr}rvrrs@r;rrJs;  qv%0&%08A%0T]%0im%0NJr=rcZeZdZdfd Z d ddZddZxZS)TFWhisperDecoderLayerc t|di||j|_t |j|j |j dd|_tjj|j|_ t|j|_tjj|j|_tjj!dd|_t |j|j |j dd|_tjj!dd |_tjj)|j*d |_tjj)|jd |_tjj!dd |_||_y)NrT)rrrrrrrrr encoder_attn)rrrrencoder_attn_layer_normrrrrrd)rerfrrrdecoder_attention_headsrrrrrrr rrrrrrrrdecoder_ffn_dimrrrrrs r;rfzTFWhisperDecoderLayer.__init__sa "6"+nn44,,  ||++FNN; .v/I/IJ"',,"6"6v7P7P"Q$)LL$C$CDWm$C$n!. NN  * *,,  (-||'F'FtZs'F't$<<%%f&<&<5%I<<%%dnn5%A % ? ?Se ? f r=c <|} |j|}||ddnd} |j|| |||\}} } |j||}| |z}d} d}|T|} |j|}||ddnd}|j ||||||\}}} |j||}| |z}| | z} |} |j |}|j |j|}|j||}|j|}|j||}| |z}|| || fS)a Args: hidden_states (`tf.Tensor`): input to the layer of shape `(batch, seq_len, embed_dim)` attention_mask (`tf.Tensor`): attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. encoder_hidden_states (`tf.Tensor`): cross attention input to the layer of shape `(batch, seq_len, embed_dim)` encoder_attention_mask (`tf.Tensor`): encoder attention mask of size `(batch, 1, tgt_len, src_len)` where padding elements are indicated by very large negative values. layer_head_mask (`tf.Tensor`): mask for attention heads in a given layer of size `(decoder_attention_heads,)` cross_attn_layer_head_mask (`tf.Tensor`): mask for heads of the cross-attention module. `(decoder_attention_heads,)` past_key_value (`Tuple(tf.Tensor)`): cached past key and value projection states Nr#)rrrrrr)rrrrrr) rrrrrrrrrr)rmrrencoder_hidden_statesencoder_attention_maskrcross_attn_layer_head_maskrrrself_attn_past_key_valuerpresent_key_valuecross_attn_present_key_valuecross_attn_weightscross_attn_past_key_values r;r}zTFWhisperDecoderLayer.calls4!11-@ :H9S>"1#5Y] >Bnn'3)+ ?M? ; (*; ]X F  =0 (,$! ,$H 88GM@N?Yrs(;_c %NRN_N_+!65 :8! O`O KM-/K!LLLJM$}4M!24P P !--m< **488M+BC // /Q /  ]X F  =0        r=cb|jryd|_t|ddMtj|jj 5|jj ddddt|ddZtj|jj 5|jj dd|jgdddt|ddMtj|jj 5|jj ddddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dddtj|jj 5|jj dd|jjgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYsxYw#1swYxYw#1swYxYw#1swYrxYw#1swYxYw#1swYxYw#1swYyxYw) NTrrrrrrr)rrr,rrrrrvrrrrrrrrrrs r;rvzTFWhisperDecoderLayer.builds ::  4d + 7t~~223 +$$T* + 4/ 6 Bt88==> N))//tT^^0LM N 4 . :t00556 .!!''- . 42D 9 Et;;@@A Q,,22D$3OP Q 4 % 1txx}}- =dDNN;< = 4 % 1txx}}- JdDKK,G,GHI J 4+T 2 >t4499: J%%++T4,HI J J ?# + + N N . . Q Q = = J J J JsT K%)K% K2&)K? )L 43L%)L%K"%K/2K<?L  LL"%L.r)NNNNNNF)rrrrrrrrrrrzTuple[tf.Tensor] | Nonerz4Tuple[tf.Tensor, tf.Tensor, Tuple[Tuple[tf.Tensor]]]rrrs@r;rrsz@,02637,07;26P )P  0 P !1 P * P %5P 0P  >P dJr=rcBeZdZeZdZdZddZeddZ edZ y) TFWhisperPreTrainedModelmodelinput_featuresc|dz dzdz}|S)zH Computes the output length of the convolutional layers rr#rd)rm input_lengthss r; _get_feat_extract_output_lengthsz9TFWhisperPreTrainedModel._get_feat_extract_output_lengths's'*q014 r=c*|jtjjd|jj |jj dzdz gtjdtjddggtjiS)z| Dummy inputs to build the network. Returns: `Dict[str, tf.Tensor]`: The dummy inputs. rr#r$decoder_input_idsr ) main_input_namer,randomuniformr num_mel_binsmax_source_positionsr/rDrzrms r; dummy_inputsz%TFWhisperPreTrainedModel.dummy_inputs/s|  "))"3"3DKK,,dkk.N.NQR.RUV.VW_a_i_i#4# q!fXRXX!F   r=ctjd|jjdftjdtjdtj dtjdtj ddS)Nrrr~rdecoder_attention_mask)rrr)r, TensorSpecrrr/rzrs r;input_signaturez(TFWhisperPreTrainedModel.input_signature>sa!mmT4;;3K3KT,RTVT^T^euv!#|RXXL_!`&(mmL"((Qi&j  r=N)rrrr)rzDict[str, tf.Tensor]) rrrr config_classbase_model_prefixrrpropertyrrrdr=r;rr"s? L&O      r=raI This model inherits from [`TFPreTrainedModel`]. Check the superclass documentation for the generic methods the library implements for all its model (such as downloading or saving, resizing the input embeddings, pruning heads etc.) This model is also a [keras.Model](https://www.tensorflow.org/api_docs/python/tf/keras/Model) subclass. Use it as a regular TF 2.0 Keras Model and refer to the TF 2.0 documentation for all matter related to general usage and behavior. Parameters: config ([`WhisperConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~TFPreTrainedModel.from_pretrained`] method to load the model weights. ap Args: input_features (`tf.Tensor` of shape `(batch_size, feature_size, sequence_length)`): Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a tensor of type `tf.Tensor`. See [`~WhisperFeatureExtractor.__call__`] decoder_input_ids (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Indices of decoder input sequence tokens in the vocabulary. Indices can be obtained using [`SpeechToTextTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are decoder input IDs?](../glossary#decoder-input-ids) SpeechToText uses the `eos_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values` is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`). decoder_attention_mask (`tf.Tensor` of shape `(batch_size, target_sequence_length)`, *optional*): Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also be used by default. If you want to change padding behavior, you should read [`modeling_whisper._prepare_decoder_attention_mask`] and modify to your needs. See diagram 1 in [the paper](https://arxiv.org/abs/1910.13461) for more information on the default strategy. head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in the encoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. decoder_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in the decoder. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the cross-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. encoder_outputs (`tuple(tuple(tf.Tensor)`, *optional*): Tuple consists of (`last_hidden_state`, *optional*: `hidden_states`, *optional*: `attentions`) `last_hidden_state` of shape `(batch_size, sequence_length, hidden_size)`, *optional*) is a sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. past_key_values (`tuple(tuple(tf.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(tf.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. decoder_inputs_embeds (`tf.Tensor` of shape `(batch_size, target_sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `decoder_input_ids` you can choose to directly pass an embedded representation. If `past_key_values` is used, optionally only the last `decoder_inputs_embeds` have to be input (see `past_key_values`). This is useful if you want more control over how to convert `decoder_input_ids` indices into associated vectors than the model's internal embedding lookup matrix. use_cache (`bool`, *optional*): If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding (see `past_key_values`). output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. cLeZdZeZ dfd Ze ddZddZxZ S)TFWhisperEncoderc t|di|||_|j|_|j |_|j|_|j|_ |j|_ |jrtj|j nd|_tj j#|j dddd|_tj j#|j dddd|_t)|j|j t*d |_d |j,_t1|j2Dcgc]}t5|d | c}|_tj j7dd|_tj j;|j<|_ycc}w)Nr\r rvalidconv1) kernel_sizestridespaddingrrr#conv2embed_positions)rgrhrlrrFlayers.rr layer_normrrd)rerfrencoder_layerdrop layerdroprrrrHrirscale_embeddingr*sqrt embed_scalerrConv1Dr rrbr<rrtr.encoder_layersrrrrrrmrrniros r;rfzTFWhisperEncoder.__init__sj "6" 11"//!..$*$?$?!8>8N8N499T^^4TW\\((QPQ[bip(q \\((QPQ[bip(q ;33..";"   */&Z_`f`u`uZvwUV4VGA3-Pw,,99$\9Z||++FNN; xsG c ||n|jj}||n|jj}||n|jj}t j |d}t j |ddgddgddgg}tjj|j|}t j |ddgddgddgg}tjj|j|}t j |d}|jt jd|jftj}||z} |j!| | } |rd nd} |rd nd} |gtj"j%t'|dt)|j*d t)|j*d t'|dd t-|j*D]T\} } |r| | fz} t/j0dd}|r||j2kr6| | d||| nd|\} }|sO| |fz } V|j5| } |r| | fz} |st7d| | | fDSt9| | | S)a" Args: input_features (`tf.Tensor` of shape `(batch_size, feature_size, sequence_length)`): Float values of fbank features extracted from the raw speech waveform. Raw speech waveform can be obtained by loading a `.flac` or `.wav` audio file into an array of type `List[float]` or a `numpy.ndarray`, *e.g.* via the soundfile library (`pip install soundfile`). To prepare the array into `input_features`, the [`AutoFeatureExtractor`] should be used for extracting the fbank features, padding and conversion into a tensor of type `tf.Tensor`. See [`~WhisperFeatureExtractor.__call__`] head_mask (`tf.Tensor` of shape `(encoder_layers, encoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. N)rr#r)permrr)rrr#r$)rGrrdz&The head_mask should be specified for  layers, but it is for .r)rrc3&K|] }|| ywrrd.0vs r; z(TFWhisperEncoder.call..'seqWXWdeslast_hidden_stater attentions)routput_attentionsoutput_hidden_statesuse_return_dictr,rpadr activationsgelur rrrRrrzrrBrrlenr enumeraterrrrtupler )rmr head_maskr)r* return_dictr inputs_embeds embed_posrencoder_statesall_attentionsidx encoder_layerdropout_probabilityattns r;r}zTFWhisperEncoder.callsD2C1N-TXT_T_TqTq$8$D $++JjJj &1%>A"/3<3H3d! # M4!4')! *$ 6 +}.>>Ne]NN$See e +>Vd  r=c|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|ddMtj|jj 5|jj ddddt|dddtj|jj 5|jj dd|jjgdddt|ddK|jD];}tj|j 5|j dddd=yy#1swYxYw#1swY;xYw#1swYxYw#1swYxYw#1swY{xYw)NTr rrrr)rrr,rr rrrvrrrrrrrrrmrwlayers r;rvzTFWhisperEncoder.build,s ::  4$ ' 3tzz/ B   $d.?.?!@A B 4$ ' 3tzz/ ?   $dnn!=> ? 4*D 1 =t33889 1$$**40 1 4t , 8t334 I%%tT4;;3F3F&GH I 4)4 0 <,, &]]5::.&KK%&& & = B B ? ? 1 1 I I&&s< )H2)HH!33H-)H9HH!H*-H69I r)NNNNNFr) rrrrrrfrr}rvrrs@r;r r sB L<8!\ \ |&r=r cleZdZeZ dfd ZdZdZdZe ddZ d dZ xZ S) TFWhisperDecoderc ft|d i|||_tjj |j |_|j|_|j|_ |j|_ |j|_ |jrtj|j nd|_tjj%|j&|j tj(j+|jj,d|_t1|j|j d|_t5|j6Dcgc]}t9|d|c}|_tjj;dd |_ycc}w) Nr\)stddev embed_tokens) input_dim output_dimembeddings_initializerrrrrrrrrrd)rerfrrrrrdecoder_layerdroprrHrimax_target_positionsrrr*rrr Embedding vocab_sizerjTruncatedNormalinit_stdrCrbrr.decoder_layersrrrrs r;rfzTFWhisperDecoder.__init__LsO "6" ||++FNN; 11!..$*$?$?!$*$?$?!8>8N8N499V^^4TW!LL22''~~#(#5#5#E#ET[[MaMa#E#b 3  <  % %v~~zBTFWhisperDecoder._prepare_decoder_attention_mask..qs%kJ`ar=cNttjzfS)NrW)r`r,rP) batch_sizerUrsr;rXzBTFWhisperDecoder._prepare_decoder_attention_mask..rs#L*g@V6V)W!Xbijr=r&rZ)r,condr*greaterr`)rmrrwrUcombined_attention_maskexpanded_attn_maskr[rs `` @@r;_prepare_decoder_attention_maskz0TFWhisperDecoder._prepare_decoder_attention_maskjsy*!nk!n G"$'' GGOOGQ ' a j#   %!-nkRTo!V &=&E"K]`wKw $'&r=c | | n|jj} | | n|jj} | | n|jj} | | n|jj} | | t d|1t j|}t j|d|df}n&|t j|dd}n t d|t j|dddnd}|1t||jj|j|}|j|||}||n|d}|j||}||z}|j|| }| rd nd}| rd nd}| r|d nd}| rd nd}d |fd |ffD]r\}}| t jj!t#|dt%|j&d |dt%|j&dt#|ddtt)|j&D]\}}| r||fz }t+j,dd}| r||j.kr6|||nd}|||||||nd|||nd|| }|d}| r ||dfz }| so||dfz }|{||dfz }|j1|}| r||fz }| r|nd}| st3d|||||fDSt5|||||S)a" Args: input_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`): Indices of input sequence tokens in the vocabulary. Padding will be ignored by default should you provide it. Indices can be obtained using [`WhisperTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) position_ids (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Indices of positions of each decoder input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. encoder_hidden_states (`tf.Tensor` of shape `(batch_size, encoder_sequence_length, hidden_size)`, *optional*): Sequence of hidden-states at the output of the last layer of the encoder. Used in the cross-attention of the decoder. head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. cross_attn_head_mask (`tf.Tensor` of shape `(decoder_layers, decoder_attention_heads)`, *optional*): Mask to nullify selected heads of the attention modules in encoder to avoid performing cross-attention on hidden heads. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. past_key_values (`tuple(tuple(tf.Tensor))`, *optional*, returned when `use_cache=True` is passed or when `config.use_cache=True`): Tuple of `tuple(tf.Tensor)` of length `config.n_layers`, with each tuple having 2 tensors of shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`) and 2 additional tensors of shape `(batch_size, num_heads, encoder_sequence_length, embed_size_per_head)`. Contains pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention blocks) that can be used (see `past_key_values` input) to speed up sequential decoding. If `past_key_values` are used, the user can optionally input only the last `decoder_input_ids` (those that don't have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `decoder_input_ids` of shape `(batch_size, sequence_length)`. inputs_embeds (`tf.Tensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. NzTYou cannot specify both decoder_input_ids and decoder_inputs_embeds at the same timer&zEYou have to specify either decoder_input_ids or decoder_inputs_embedsrr#)rr&rWrrdr2cross_attn_head_maskzThe z should be specified for rr rr)rrrrrrr c3$K|]}|| ywrrdr"s r;r%z(TFWhisperDecoder.call..#s=s)r'past_key_valuesrr(cross_attentions)rr)r* use_cacher+r)r,r5r0rrCrDr`rrrBrrr/rMr0rrrrr1r)rmrGr position_idsrr2rbrdr4rfr)r*r3rrwrUfilled_past_positions positionsrall_hidden_statesall_self_attnsall_cross_attentionsnext_decoder_cacheattn_mask_name attn_maskr8 decoder_layerr:r layer_outputs next_caches r;r}zTFWhisperDecoder.call}sZ2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKKst t  "((9-K 9r;r?.CDI  &((=1#26Kde eHWGb/!*>A5D5P_S1VZN)-&;3<3H3dI]Ii,@,Eos-!M*!,M"}Q'7&99" =#3"55(4(]1-=,??(9 @< 6  -!1 1 +4'$ '5FXlm  ;+&+%1   r=c|jryd|_t|ddMtj|jj 5|jj ddddt|ddMtj|jj 5|jj ddddt|dddtj|jj 5|jj dd|jjgdddt|ddK|jD];}tj|j 5|j dddd=yy#1swY.xYw#1swYxYw#1swY{xYw#1swYnxYw)NTrCrrrM) rrr,rrCrrrvrrrrrMr=s r;rvzTFWhisperDecoder.build0su ::  4 . :t00556 .!!''- . 4*D 1 =t33889 1$$**40 1 4t , 8t334 I%%tT4;;3F3F&GH I 4)4 0 <,, &]]5::.&KK%&& & = . . 1 1 I I&&s0 F%F ?3F,5F8F F),F58G r) NNNNNNNNNNNNFr) rrrrrrfrPrTr`rr}rvrrs@r;r@r@Bsg L[0!"'&"!!p p d&r=r@zUThe bare Whisper Model outputting raw hidden-states without any specific head on top.ceZdZeZd fd ZdZdZdZdZ e e e e ee d dZd dZxZS) TFWhisperMainLayerc zt|di|||_t|d|_t |d|_y)Nencoderrdecoderrd)rerfrr rwr@rxrs r;rfzTFWhisperMainLayer.__init__Ks6 "6" 'Y? 'Y? r=c.|jjSrrxrCrs r;rPz'TFWhisperMainLayer.get_input_embeddingsQs||(((r=c&||j_yrrzrRs r;rTz'TFWhisperMainLayer.set_input_embeddingsTs$) !r=c|jSr)rwrs r; get_encoderzTFWhisperMainLayer.get_encoderW ||r=c|jSr)rxrs r; get_decoderzTFWhisperMainLayer.get_decoderZr~r= output_typerc| | n|jj} | | n|jj} | | n|jj} ||n|jj}||j ||| | ||}nI|rGt |ts7t|dt|dkDr|dndt|dkDr|dnd}|j||||d||| | | | | || }|s||zSt|j|j|j|j|j|j|j|jS)  Returns: Example: ```python >>> import tensorflow as tf >>> from transformers import TFWhisperModel, AutoFeatureExtractor >>> from datasets import load_dataset >>> model = TFWhisperModel.from_pretrained("openai/whisper-base") >>> feature_extractor = AutoFeatureExtractor.from_pretrained("openai/whisper-base") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> inputs = feature_extractor(ds[0]["audio"]["array"], return_tensors="tf") >>> input_features = inputs.input_features >>> decoder_input_ids = tf.convert_to_tensor([[1, 1]]) * model.config.decoder_start_token_id >>> last_hidden_state = model(input_features, decoder_input_ids=decoder_input_ids).last_hidden_state >>> list(last_hidden_state.shape) [1, 2, 512] ```N)r2r)r*r3rrrr#r&) rGrrgrr2rbrdr4rfr)r*r3rr'rddecoder_hidden_statesdecoder_attentionsreencoder_last_hidden_staterencoder_attentions)rr)r*rfr+rw isinstancer r/rxrr'rdrr(re)rmrrrdecoder_position_idsr2decoder_head_maskrbencoder_outputsrddecoder_inputs_embedsrfr)r*r3rdecoder_outputss r;r}zTFWhisperMainLayer.call]sR2C1N-TXT_T_TqTq$8$D $++JjJj "+!6IDKK /0""""++CD+?o^37;?@D>B37;?>BPTPTOS$(,0/3&*!7/797!> 7 < 7 1 797<7N7N7 M7"7*7-7$7 !7" 7#7_E7r &'r=rz^The Whisper Model with a language modeling head. Can be used for automatic speech recognition.c`eZdZdZgdZdgZdfd ZdZdZdZ dZ dfd Z e e eee e d dd Z d dfd Zd Z ddZddZxZS)!TFWhisperForConditionalGenerationr)zencoder.versionzdecoder.versionproj_out.weightrc Jt||fi|t|d|_yrrrs r;rfz*TFWhisperForConditionalGeneration.__init__Ls# *6*'W= r=c6|jjSr)rr}rs r;r}z-TFWhisperForConditionalGeneration.get_encoderPzz%%''r=c6|jjSr)rrrs r;rz-TFWhisperForConditionalGeneration.get_decoderSrr=c"|jSr)rPrs r;get_output_embeddingsz7TFWhisperForConditionalGeneration.get_output_embeddingsVs((**r=c&|j|yr)rTrRs r;set_output_embeddingsz7TFWhisperForConditionalGeneration.set_output_embeddingsYs !!%(r=c&t||}|Sr)reresize_token_embeddings)rmnew_num_tokensnew_embeddingsros r;rz9TFWhisperForConditionalGeneration.resize_token_embeddings\s8Hr=rc`||n|jj}| 9|7| 5t| |jj|jj}|j ||||||||| | | | |||}|d}t j||jjd}| dn|j| |}|s|f|ddz}||f|zS|St|||j|j|j|j|j |j"|j$ S)aE labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the language modeling loss. Indices should either be in `[0, ..., config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`. Returns: Example: ```python >>> import tensorflow as tf >>> from transformers import AutoProcessor, TFWhisperForConditionalGeneration >>> from datasets import load_dataset >>> processor = AutoProcessor.from_pretrained("openai/whisper-tiny.en") >>> model = TFWhisperForConditionalGeneration.from_pretrained("openai/whisper-tiny.en") >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> inputs = processor(ds[0]["audio"]["array"], return_tensors="tf") >>> input_features = inputs.input_features >>> generated_ids = model.generate(input_features=input_features) >>> transcription = processor.batch_decode(generated_ids, skip_special_tokens=True)[0] >>> transcription ' Mr. Quilter is the apostle of the middle classes, and we are glad to welcome his gospel.' ```N)rrrrr2rrbrdrrfr)r*r3rrTrr) losslogitsrdrrrerrr)rr+rMrHrIrr,rrweightshf_compute_lossrrdrrrerrr)rmrrrrr2rrbrrdrlabelsrfr)r*r3rrdecoder_last_hidden_state lm_logitsrrs r;r}z&TFWhisperForConditionalGeneration.call`sZf&1% Most generation-controlling parameters are set in `generation_config` which, if not passed, will be set to the model's default generation configuration. You can override any `generation_config` by passing the corresponding parameters to generate, e.g. `.generate(inputs, num_beams=4, do_sample=True)`. For an overview of generation strategies and code examples, check out the [following guide](../generation_strategies). Parameters: inputs (`tf.Tensor` of varying shape depending on the modality, *optional*): The sequence used as a prompt for the generation or as model inputs to the encoder. If unset the method initializes it with `bos_token_id` and a batch size of 1. For decoder-only models `inputs` should of in the format of `input_ids`. For encoder-decoder models *inputs* can represent any of `input_ids`, `input_values`, `input_features`, or `pixel_values`. generation_config (`~generation.GenerationConfig`, *optional*): The generation configuration to be used as base parametrization for the generation call. `**kwargs` passed to generate matching the attributes of `generation_config` will override them. If `generation_config` is not provided, the default will be used, which had the following loading priority: 1) from the `generation_config.json` model file, if it exists; 2) from the model configuration. Please note that unspecified parameters will inherit [`~generation.GenerationConfig`]'s default values, whose documentation should be checked to parameterize generation. logits_processor (`LogitsProcessorList`, *optional*): Custom logits processors that complement the default logits processors built from arguments and generation config. If a logit processor is passed that is already created with the arguments or a generation config an error is thrown. This feature is intended for advanced users. seed (`List[int]`, *optional*): Random seed to control sampling, containing two integers, used when `do_sample` is `True`. See the `seed` argument from stateless functions in `tf.random`. return_timestamps (`bool`, *optional*): Whether to return the timestamps with the text. This enables the `TFWhisperTimestampsLogitsProcessor`. task (`str`, *optional*): Task to use for generation, either "translate" or "transcribe". The `model.config.forced_decoder_ids` will be updated accordingly. language (`str`, *optional*): Language token to use for generation, can be either in the form of `<|en|>`, `en` or `english`. You can find all the possible language tokens in the `model.generation_config.lang_to_id` dictionary. is_multilingual (`bool`, *optional*): Whether or not the model is multilingual. prompt_ids (`tf.Tensor`, *optional*): Rank-1 tensor of token IDs created by passing text to [`~WhisperProcessor.get_prompt_ids`] that is provided as a prompt to each chunk. This can be used to provide or "prompt-engineer" a context for transcription, e.g. custom vocabularies or proper nouns to make it more likely to predict those words correctly. It cannot be used in conjunction with `decoder_start_token_id` as it overwrites this value. return_token_timestamps (`bool`, *optional*): Whether to return token-level timestamps with the text. This can be used with or without the `return_timestamps` option. To get word-level timestamps, use the tokenizer to group the tokens into words. kwargs (`Dict[str, Any]`, *optional*): Ad hoc parametrization of `generate_config` and/or additional model-specific kwargs that will be forwarded to the `forward` function of the model. If the model is an encoder-decoder model, encoder specific kwargs should not be prefixed and decoder specific kwargs should be prefixed with *decoder_*. Return: [`~utils.ModelOutput`] or `tf.Tensor`: A [`~utils.ModelOutput`] (if `return_dict_in_generate=True` or when `config.return_dict_in_generate=True`) or a `tf.Tensor`. If the model is *not* an encoder-decoder model (`model.config.is_encoder_decoder=False`), the possible [`~utils.ModelOutput`] types are: - [`~generation.TFGreedySearchDecoderOnlyOutput`], - [`~generation.TFSampleDecoderOnlyOutput`], - [`~generation.TFBeamSearchDecoderOnlyOutput`], - [`~generation.TFBeamSampleDecoderOnlyOutput`] If the model is an encoder-decoder model (`model.config.is_encoder_decoder=True`), the possible [`~utils.ModelOutput`] types are: - [`~generation.TFGreedySearchEncoderDecoderOutput`], - [`~generation.TFSampleEncoderDecoderOutput`], - [`~generation.TFBeamSearchEncoderDecoderOutput`], - [`~generation.TFBeamSampleEncoderDecoderOutput`] Nno_timestamps_token_idadYou are trying to return timestamps, but the generation config is not properly set. Make sure to initialize the generation config with the correct attributes that are needed such as `no_timestamps_token_id`. For more details on how to generate the approtiate config, refer to https://github.com/huggingface/transformers/issues/21878#issuecomment-1451902363Fforced_decoder_idslanguagez<|z|>r#zUnsupported language: z. Language should be one of: r z is not supported by this specific model as it is not in the `generation_config.lang_to_id`.(You should just add it to the generation config)r)rNtaskzThe `z3`task is not supported. The task should be one of `` task_to_id transcriber&rrIzfWhen specifying `prompt_ids`, you cannot also specify `decoder_start_token_id` as it gets overwritten.max_new_tokens max_lengthzQ`TFWhisperForConditionalGeneration` doesn't support returning the timestamps yet.Tr)return_dict_in_generate translatez@Token-level timestamps may not be reliable for task 'translate'.alignment_headszModel generation config has no `alignment_heads`, token-level timestamps not available. See https://gist.github.com/hollance/42e32852f24243b748ae6bc1f985b13a on how to add this property to the generation config.token_timestamps)"generation_confighasattrr)return_timestampslowerrrrrrk lang_to_idkeysr!valuesr/listappendr rrtolistrupdatepoprrIr0rloggerwarningregenerate_extract_token_timestampsr)rminputsrlogits_processorseedrrris_multilingual prompt_idsreturn_token_timestampsrnrlanguage_tokenis_language_coder8rItext_prompt_idsspecified_max_lengthdefault_max_lengthnon_prompt_max_lengthnon_prompt_forced_decoder_ids_ranktokenrankrros r;rz*TFWhisperForConditionalGeneration.generatesz  $ $ 6 6   (,.FG k 3D  /27  /  ~~'H)1  &  %)  "! 4;; 4 5$++:X:X:d!%!?!?  D**,@ A&&99E!%!7!7!J!J !',@$!G   x38J8RWaWm!# (*5$--1B1M1M1R1R1TT%6%?%?N&//3C3H3H3JJ')*:;L;U;U*V)WWY%ZN&//3C3J3J3LL')*;*D*D)ER%HN'*+<+E+E'F!'K$$01B1K1K0LM?OD!1!8!8!:;UYZjZoZoZqUrsstv"):)E)EE$)*+LL#))1.?.J.J>.Z*[\")))4(&1$))X5&--q2C2N2NO`OeOe2f.gh$ 1 6 677jksjttuv*L9"))1.?.J.J<.X*YZ(*BCL]LoLo7I(,Q/!3q"))30A0X0X*YZ  )3E  0  !zz23? |$**,J7A 4 "_.t{{/E/E.E.JQ.N.PQO MM35KL M$*::.>#E#gT`bfIg !2!A!A!aEVEaEa $8$N>2Hr]a>#bcdfhfici#j  (#21#5a#8#>#>q#A #%88,=,C,CA,F#G !/CEVE\E\]#..!2"&<$8  r=c|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYwrrrs r;rvz'TFWhisperForConditionalGeneration.buildrrr)rrrzkeras.layers.Embedding)NNNNNNNNNNNNNNNF)"rrrrrrrrr2rrrrbrrrrdrrrrrrfrr)rr*rr3rrrrz*Union[Tuple[tf.Tensor], TFSeq2SeqLMOutput]) NNNNNNNNNN)rOptional[tf.Tensor]rzOptional[GenerationConfig]rzOptional[TFLogitsProcessorList]rzOptional[List[int]]rrr Optional[str]rrrrrr)NNNNNr)rrrr_keys_to_ignore_on_load_missing_keys_to_ignore_on_saverfr}rrrrrrrrrrr}rrrrvrrs@r;rr=s   '# >((+)++CD+B37;?>BPTPTOS04$(,0/3&*#] /] 9] !> ] < ] 1 ] 9] <] N] N]  M] .] "] *] -] $!] "#] $ 4%] \E] B'+8<<@$(,0""&*.*. $a#a6a: a " a * aa a(a(aF ,# @'r=r)rrr)rr)rGrrHrrIrr)rTztf.TensorShaperUrr)rXrrWr)Hr __future__rr*rtypingrrrrrnumpynp tensorflowr,activations_tfr generation.configuration_utilsr generation.tf_logits_processr modeling_tf_outputsr rrrmodeling_tf_utilsrrrrrrtf_utilsrrrutilsrrrrconfiguration_whisperrtokenization_whisperr r! get_loggerrrrrQr/r<rMrZr`rLayerrbrrrrWHISPER_START_DOCSTRINGrr r@rurr__all__rdr=r;rs " 55/>A SRtt0<   H %!,.:: Y2;$ 665<<#5#56