T^(h.dZddlZddlZddlZddlmZddlmZmZm Z m Z ddl Z ddl ZddlmZmZddlmZdd lmZmZd d d d dZddiZdZej4eZdZidddddddddddddd d!d"d#d$d%d&d'd(d)d*d+d,d-d.d/d0d1d2d3d4id5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVidWdXdYdZd[d\d]d^d_d`dadbdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxidydzd{d|d}d~ddddddddddddddddddddddddddddidddddddddddddddddddddddddddddddddddddddddddddddddd̜Ziej?Dcic]\}}|| c}}dd)d+dddddEdEddddϜ Z ddgZ!Gd҄deZ"dԄZ#ddՄZ$dքZ% ddee&dee'de'de'fdۄZ(dee&fd܄Z)dee&fd݄Z*dބZ+dgZ,ycc}}w)z!Tokenization classes for Whisper.N) lru_cache)ListOptionalTupleUnion) AddedTokenPreTrainedTokenizer)logging)BasicTextNormalizerEnglishTextNormalizerz vocab.jsonztokenizer.jsonz merges.txtznormalizer.json) vocab_filetokenizer_file merges_filenormalizer_filezopenai/whisper-baseic tttdtddztttdtddzztttdtddzz}|dd}d }td D]1}||vs|j||jd |z|dz }3|Dcgc] }t |}}t t ||Scc}w) a8 Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control characters the bpe code barfs on. The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup tables between utf-8 bytes and unicode strings. !~r ¡¬®ÿNr)listrangeordappendchrdictzip)bscsnbs n/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/whisper/tokenization_whisper.pybytes_to_unicoder'-s U3s8SX\ *+d5TCIPQM3R.SSVZ[`adeiajloptluxyly[zV{{ AB A 4[ B; IIaL IIdQh  FA   Q#a& B  B  s C4cbt}|d}|ddD]}|j||f|}|S)z Return set of symbol pairs in a word. Word is represented as tuple of symbols (symbols being variable-length strings). rr N)setadd)wordpairs prev_charchars r& get_pairsr/IsF EEQIQR 9d#$  Lenenglishzhchinesedegermanesspanishrurussiankokoreanfrfrenchjajapanesept portuguesetrturkishplpolishcacatalannldutchararabicsvswedishititalianid indonesianhihindififinnishvi vietnamesehehebrewuk ukrainianelgreekmsmalayr#czechroromaniandadanishhu hungariantatamilno norwegianththaiururduhrcroatianbg bulgarianlt lithuanianlalatinmimaoriml malayalamcywelshskslovaktetelugufapersianlvlatvianbnbengalisrserbianaz azerbaijanisl slovenianknkannadaetestonianmk macedonianbrbretoneubasqueis icelandichyarmeniannenepalimn mongolianr"bosniankkkazakhsqalbanianswswahiliglgalicianmrmarathipapunjabisisinhalakmkhmersnshonayoyorubasosomaliaf afrikaansococcitankageorgianbe belarusiantgtajiksdsindhigugujaratiamamharicyiyiddishlolaouzuzbekfofaroesehtzhaitian creolepspashtotkturkmennnnynorskmtmaltesesanskrit luxembourgishmyanmartibetantagalogmalagasyassamesetatarhawaiianlingalahausabashkirjavanese sundanese cantonese)salbmybotlmgastthawlnhabajwsuyuerr) burmese valencianflemishhaitian letzeburgeschpushtopanjabi moldavianmoldovan sinhalese castilianmandarin translate transcribecfeZdZdZeZddgZ d7fd Zede fdZ dZ dZ d8d e ed e ed e efd Zedee fd Zd9dee fdZ d:dee de ee dedee ffd ZdZdZdZdZd;dZdZed;dZ ddZ d;defdZ!dZ" d?ded e ed!ed"e#d#ed$ed%ed&edeffd' Z$ d@d(e%e ee fded$ed%ed&edef d)Z&d*Z'd9d+ed,e ede(efd-Z)d;d.Z*dAd/Z+d0Z,dBd1efd2Z-d(ee d3e d4e fd5Z.ed6Z/xZ0S)CWhisperTokenizeraa Construct a Whisper tokenizer. This tokenizer inherits from [`PreTrainedTokenizer`] which contains some of the main methods. Users should refer to the superclass for more information regarding such methods. Args: vocab_file (`str`): Path to the vocabulary file. merges_file (`str`): Path to the merges file. normalizer_file (`str`, *optional*): Path to the normalizer_file file. errors (`str`, *optional*, defaults to `"replace"`): Paradigm to follow when decoding bytes to UTF-8. See [bytes.decode](https://docs.python.org/3/library/stdtypes.html#bytes.decode) for more information. unk_token (`str`, *optional*, defaults to `"<|endoftext|>"`): The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this token instead. bos_token (`str`, *optional*, defaults to `"<|endoftext|>"`): The beginning of sequence token. The `decoder_start_token_id` is used to set the first token as `"<|startoftranscript|>"` when generating. eos_token (`str`, *optional*, defaults to `"<|endoftext|>"`): The end of sequence token. pad_token (`str`, *optional*): The token used for padding, for example when batching sequences of different lengths. add_prefix_space (`bool`, *optional*, defaults to `False`): Whether or not to add an initial space to the input. This allows to treat the leading word just as any other word. language (`str`, *optional*): The language of the transcription text. The corresponding language id token is appended to the start of the sequence for multilingual speech recognition and speech translation tasks, e.g. for Spanish the token `"<|es|>"` is appended to the start of sequence. This should be used for multilingual fine-tuning only. task (`str`, *optional*): Task identifier to append at the start of sequence (if any). This should be used for mulitlingual fine-tuning, with `"transcribe"` for speech recognition and `"translate"` for speech translation. predict_timestamps (`bool`, *optional*, defaults to `False`): Whether to omit the `<|notimestamps|>` token at the start of the sequence. input_idsattention_maskc >t|trt|ddddn|}t|trt|ddddn|}t|trt|ddddn|}t|trt|ddddn|}t|d5}t j ||_ddd|j jDcic]\}}|| c}}|_||_ t|_ |jjDcic]\}}|| c}}|_ t|d5}|jjddd}dddDcgc]}t|j}}t!t#|t%t'||_i|_| |_|1t|d5}t j ||_dddnd|_t1j2d |_t1j2d |_| |_t;|xd |||||| d | | |_| |_ y#1swYxYwcc}}wcc}}w#1swYxYwcc}w#1swYxYw) NFT)lstriprstrip normalizedspecialutf-8encoding r zJ's|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+z<\|(\d+\.\d+)\|>)errors unk_token bos_token eos_token pad_tokenadd_prefix_space)! isinstancestrr openjsonloadencoderitemsdecoderrr' byte_encoder byte_decoderreadsplittupler r!rlen bpe_rankscacherenglish_spelling_normalizerrecompilepat timestamp_patlanguagesuper__init__taskpredict_timestamps)selfrrrrrrrrrr.r1r2kwargs vocab_handlekv merges_handle bpe_mergesmerge __class__s r&r0zWhisperTokenizer.__init__s$)S) yuX\ ] )S) yuX\ ] )S) yuX\ ] )S) yuX\ ]  *w / 3<99\2DL 3)-););)=>A1>  ,..2.?.?.E.E.GHdaQTH + 0 @M&++-33D9!B?J @8BCueEKKM*C Cc*eC O.DEF 0  &o8 KL3799\3J0 K K04D ,::opZZ(;<    -    "4G 3 3>I @ @C  K Ks6I( I5 I;#J2 JJ(I2J Jreturnc,t|jSN)r&r)r3s r& vocab_sizezWhisperTokenizer.vocab_sizeHs4<<  r0ct|jDcic]}|j||}}|j|j|Scc}wr>)rr?convert_ids_to_tokensupdateadded_tokens_encoder)r3ivocabs r& get_vocabzWhisperTokenizer.get_vocabLsK;@;QRa++A.1RR T../ SsAc|jvrj|St|}t|}|s|S t|fd}|jvrn|\}}g}d}|t |kr |j ||} |j||| | }|||k(r6|t |dz kr%||dz|k(r|j||z|dz }n|j|||dz }|t |krt|}|}t |dk(rn t|}dj|}|j|<|S#t$r|j||dYiwxYw)NcNjj|tdS)Ninf)r'getfloat)pairr3s r&z&WhisperTokenizer.bpe..\s1C1CD%PU,1Wr0keyrr  ) r(r%r/minr'r&indexextend ValueErrorrjoin) r3tokenr+r,bigramfirstsecondnew_wordrDjs ` r&bpezWhisperTokenizer.bpeRs DJJ ::e$ $U|$L$WXFT^^+"ME6HAc$i- 5!,A OOD1I.A7e#CIM(9d1q5kV>SOOEFN3FAOODG,FAc$i- XHD4yA~!$9:xx~  5 +"OODH-s1E E)(E)r.r1r2c||n |j|_||n |j|_|||_y|j|_y)a Override the prefix tokens appended to the start of the label sequence. This method can be used standalone to update the prefix tokens as required when fine-tuning. Example: ```python >>> # instantiate the tokenizer and set the prefix token to Spanish >>> tokenizer = WhisperTokenizer.from_pretrained("openai/whisper-tiny", language="spanish") >>> # now switch the prefix token from Spanish to French >>> tokenizer.set_prefix_tokens(language="french") ``` Args: language (`str`, *optional*, defaults to `None`): The language of the transcription text. task (`str`, *optional*, defaults to `None`): Task identifier to append at the start of sequence (if any). predict_timestamps (`bool`, *optional*, defaults to `None`): Whether to omit the `<|notimestamps|>` token at the start of the sequence. N)r.r1r2)r3r.r1r2s r&set_prefix_tokensz"WhisperTokenizer.set_prefix_tokens|sD,%-$8dmm  ,D$)) 8J8V"4\`\s\sr0c |jd}|jd}|jd}|jd}ttj}|j|jj |_|jt vrt |j}n|jt jvr |j}nnt|jdk(}td|jd|rtt jntt jd|j1|jtvrtd |jd t|g}|j&|j|d z|jz|j"|j|jd k(r|n||js|j||S) N<|startoftranscript|>z <|translate|>z<|transcribe|><|notimestamps|>rPzUnsupported language: z. Language should be one of: .zUnsupported task: z. Task should be in: r r)convert_tokens_to_idsr% LANGUAGESkeysr.lowerTO_LANGUAGE_CODEvaluesr&rUrr1TASK_IDSrrSr2) r3 bos_token_idtranslate_token_idtranscribe_token_idnotimestamps_token_idlangs language_idis_language_code bos_sequences r& prefix_tokenszWhisperTokenizer.prefix_tokenss112IJ !77H"889IJ $ : :;M Ninn&' == $ MM//1DM}} 00.t}}= "2"9"9";;"mm #&t}}#5#:  ,T]]O<;K-4467QUVfVkVkVmQnoopr 99 yy( #5dii[@UV^U_!`aa$~ == $    q 05;;{3K K L 99   tyyL7P 3Vh i&&    5 6r0c||j|z|jgzS|j|z|z|jgzS)z=Build model inputs from a sequence by appending eos_token_id.)rs eos_token_id)r3 token_ids_0 token_ids_1s r& build_inputs_with_special_tokensz1WhisperTokenizer.build_inputs_with_special_tokenssK  %% 3t7H7H6II I!!K/+=ARAR@SSSr0rvrwalready_has_special_tokensc|rt|||dSdgt|jz}dg}||dgt|zz|zS|dgt|zzdgt|zz|zS)a Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding special tokens using the tokenizer `prepare_for_model` method. Args: token_ids_0 (`List[int]`): List of IDs. token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs. already_has_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not the token list is already formatted with special tokens for the model. Returns: `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token. T)rvrwryr r)r/get_special_tokens_maskr&rs)r3rvrwry prefix_ones suffix_onesr;s r&r{z(WhisperTokenizer.get_special_tokens_masks& &72'[]a3 cC 2 233 c  1#K(8"89KG GqcC $445!s;?O9OPS^^^r0cg}tjj|D]a}djfd|j dD}|j dj |jdDc|S)zTokenize a string.c3<K|]}j|ywr>)r!).0r%r3s r& z-WhisperTokenizer._tokenize..s!)*!!!$sr c3 K|]}|ywr>r)r bpe_tokens r&rz-WhisperTokenizer._tokenize..sTIiT rQ)r*findallr,rVencoderTr]r$)r3text bpe_tokensrWs` r& _tokenizezWhisperTokenizer._tokenizes} ZZ$/ UEGG.3ll7.CE   T%9N9Ns9ST T  U r0c|jj||jj|jS)z0Converts a token (str) in an id using the vocab.)rrJr)r3rWs r&_convert_token_to_idz%WhisperTokenizer._convert_token_to_ids,||t||'7'7'GHHr0c:|jj|dS)z Converts an index (integer) in a token (str) using the vocab. Whisper's base tokenizer always decodes OOV tokens as "", thus we do not use the `unk_token` here. r)r rJ)r3rSs r&_convert_id_to_tokenz%WhisperTokenizer._convert_id_to_tokens ||r**r0cNtjd|j|S)NzThe private method `_normalize` is deprecated and will be removed in v5 of Transformers.You can normalize an input string using the Whisper English normalizer using the `normalize` method.)warningswarn normalize)r3rs r& _normalizezWhisperTokenizer._normalizes%  s ~~d##r0cRtjd|j||S)NzThe private method `_basic_normalize` is deprecated and will be removed in v5 of Transformers.You can normalize an input string using the Whisper basic normalizer using the `basic_normalize` method.remove_diacritics)rrbasic_normalize)r3rrs r&_basic_normalizez!WhisperTokenizer._basic_normalizes-  w ##D". rr rPNz<|z.2fz|>skip_special_tokensr)all_special_ids enumeraterKrrrdecoderV)r3 token_idsrtime_precision segment_sizetimestamp_beginoutputscur_max_timestampprev_segments_lenpenultimate_timestamprDrW timestamplast_was_single_endingss r&_decode_with_timestampsz(WhisperTokenizer._decode_with_timestampssr..r2Q6$ #!), *HAu'!5?#:n"LM 00-.!V.!!a%(O;c !a%@PTc@c9*.)^l-JJ),A))-BB)")#2,(9%$-!Y1B%BC#HKLr" ""5)- *0kr efAs#AQL_)` `  www s,Dcg}dtt|vr1t|dr%t|jr|j }t j |}|jddkDr#t|jdkDr td|jddz}||k\}t j|dd|ddzddz}|jddk(r|jdkrgSt j|dddz|vr2t j|t j|dddz}t j|dd}d} d} |D]} ||| } t| dkDr| dj|z } | dj|z }| | kr-|dk\xr||dz |k\xr ||dz |k\ }|r| |z } n| | z } |} |j| } |j!| }|j#|}|j|| |z| |zz||z| |zzfd | }|S) a Compute offsets for a given tokenized input Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Can be obtained using the `__call__` method. time_precision (`float`, *optional*, defaults to 0.02): The time ratio to convert from token to time. segment_size (`int`, *optional*, defaults to 1500): The number of features in the input mel spectrogram. torchcpurr z)Can only process a single input at a timerNrPrr)rtypehasattrcallablernparrayshaper&rUrwheresumritem_preprocess_token_ids_decode_filter_timestamp_ids)r3rrroffsetsrtimestamp_tokens consecutive last_slicerr current_slice sliced_tokensstart_timestamp_positionend_timestamp_positionis_single_endingrs r&_compute_offsetsz!WhisperTokenizer._compute_offsets?s c$y/* * 50IhW`WdWdNe! IHHY' ??1  !c)//&:Q&>HI I..r2Q6$7hh/47G7KKLQORSS   Q 1 $)9)=)=)?1)DI XX& ' *2 . 2+ E))K:J1KA1Nr1RUV1VWKXX./215 ( 'M%j?M=!A%+8+;+@+@+B_+T()6r):)?)?)AO)S&+.??'1Q(!*q.1_DuS]`aSaIbfuIu<$()\9))->>)$:!!% : := I ||M211$7 $4~EHY\jHjj2^CFWZhFhh&'J? 'Br0ch|jtdDcgc] }d||zz c}Scc}w)a Compute the timestamp token ids for a given precision and save to least-recently used (LRU) cache. Args: time_precision (`float`, *optional*, defaults to 0.02): The time ratio to convert from token to time. iz<|%.2f|>)rdr)r3rrDs r& timestamp_idszWhisperTokenizer.timestamp_idss5))X]^fXg*hSTJ!n:L,M*hii*hs/rct|r5|jd}|jd}|j|||}|S)a Pre-process the token ids for decoding by removing the prompt tokens ids and timestamp token ids. Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Typically, obtained using the `__call__` method of the tokenizer. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens from the token ids. If `True`, the prompt token ids will be removed. <|startofprev|>ra)rd _strip_prompt)r3rrprompt_token_iddecoder_start_token_ids r&rz&WhisperTokenizer._preprocess_token_idssE "889JKO%)%?%?@W%X "**9oG]^Ir0cDtj|jd|S)Nr)r*subr-)r3rs r&rz&WhisperTokenizer._filter_timestamp_idssvvd(("i88r0clean_up_tokenization_spacesoutput_offsetsrdecode_with_timestampsrrrc |j||} t|| f||||| d| } |r|j| ||} n|j | } |r|j ||} | | dS| S)a Converts a sequence of ids in a string, using the tokenizer and vocabulary with options to remove special tokens and clean up tokenization spaces. Similar to doing `self.convert_tokens_to_string(self.convert_ids_to_tokens(token_ids))`. Args: token_ids (`Union[int, List[int], np.ndarray, torch.Tensor, tf.Tensor]`): List of tokenized input ids. Can be obtained using the `__call__` method. skip_special_tokens (`bool`, *optional*, defaults to `False`): Whether or not to remove special tokens in the decoding. Will remove the previous tokens (pre-prompt) if present. clean_up_tokenization_spaces (`bool`, *optional*): Whether or not to clean up the tokenization spaces. If `None`, will default to `self.clean_up_tokenization_spaces` (available in the `tokenizer_config`). output_offsets (`bool`, *optional*, defaults to `False`): Whether or not to output the offsets of the tokens. This should only be set if the model predicted timestamps. If there are previous tokens (pre-prompt) to decode, they will only appear in the decoded text if they contain timestamp tokens. time_precision (`float`, *optional*, defaults to 0.02): The time ratio to convert from token to time. decode_with_timestamps (`bool`, *optional*, defaults to `False`): Whether or not to decode with timestamps included in the raw text. normalize (`bool`, *optional*, defaults to `False`): Whether or not to apply the English text normalizer to the decoded text. Only applicable when the target text is in English. Otherwise, the basic text normalizer should be applied. basic_normalize (`bool`, *optional*, defaults to `False`): Whether or not to apply the Basic text normalizer to the decoded text. Applicable to multilingual target text. remove_diacritics (`bool`, *optional*, defaults to `False`): Whether or not to remove diacritics when applying the Basic text normalizer. Removing diacritics may destroy information in the decoded text, hence it should be used with caution. kwargs (additional keyword arguments, *optional*): Will be passed to the underlying model specific decode method. Returns: `str`: The decoded sentence. r)rrrrr)rr)r)rr)rr/rrrr)r3rrrrrrrrrr4 filtered_idsrrr;s r&rzWhisperTokenizer.decodesd11  32 w~  3)E+/    "//^Qd0D--d3D ++In+UG W5 5 r0rc |jdd|_|j||}g}g} |D]h} |r| |jvr| |jvr6| r"|j |j | g} |j | X| j | j| r |j |j | dj|} |r|j| } | S|r|j| |} | S| S)Nuse_source_tokenizerFrrr) pop_decode_use_source_tokenizerrArrCrconvert_tokens_to_stringrVrr) r3rrrrrr4filtered_tokens sub_textscurrent_sub_textrWr clean_texts r&rzWhisperTokenizer._decodes-3JJ7Mu,U)44YTg4h  $ /E"u0D0D'D111#$$T%B%BCS%TU')$  ' ''. /    T::;KL Mwwy! -J  --dFW-XJ Kr0cdj|}t|Dcgc]}|j|c}jd|j}|Scc}w)z:Converts a sequence of tokens (string) in a single string.rr )r)rV bytearrayr"rr)r3tokensrcs r&rz)WhisperTokenizer.convert_tokens_to_stringsPwwv=1$++A.=>EEgVZVaVaEb >sAsave_directoryfilename_prefixc xtjj|stj d|dytjj ||r|dzndt dz}tjj ||r|dzndt dz}tjj ||r|dzndt dz}t|dd 5}|jtj|jd d d dzdddd}t|dd 5}|jdt|jjdD]M\} } || k7rtjd|d| }|jdj | dz|dz }O ddd|j Lt|dd 5}|jtj|j d d d dzddd|||fS#1swYxYw#1swYsxYw#1swY'xYw)NzVocabulary path (z) should be a directory-rrrrwr rrPTF)indent sort_keys ensure_asciirrz#version: 0.2 c |dS)Nr r)kvs r&rMz2WhisperTokenizer.save_vocabulary..5s Y[\]Y^r0rNzSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!rQr )ospathisdirloggererrorrVVOCAB_FILES_NAMESrwriterdumpsrsortedr'rwarningr)) r3rrr merge_filerfrSwriterr token_indexs r&save_vocabularyz WhisperTokenizer.save_vocabulary!s!ww}}^, LL,^,<.gs$ \qqDDXDXYZD[?[ \s""zJEncountered text in the prompt corresponding to disallowed special token: rc) tensor_type)stripnextrArUconvert_to_tensors)r3rreturn_tensorsbatch_encodingprompt_text_idsspecial_token_idrWs` r&get_prompt_idszWhisperTokenizer.get_prompt_idsas/tzz|1CX]^)5ab9 \O \^bc  '../?@Eijoippqrs s))n)Ek**r0rrct|ts|j|}|s|S|d|k(}|r||vr||j|dSgS|S)Nr)rr_convert_to_listrS)r3rrr has_prompts r&rzWhisperTokenizer._strip_promptosc)T*--i8I q\_4 %2 1G!H!JKK r0cvt|dr\dtt|vr|jj }nMdtt|vr7|j }n&dtt|vr|j }t |tjr|j }|S)Nnumpyr tensorflowjaxlib) rrrrrtolistrrndarray)rs r&rz!WhisperTokenizer._convert_to_lists 9g &#d9o..%MMO113 T)_!55%OO- T)_- -!((*I i ,!((*Ir0) Nreplace <|endoftext|>r#r#NFNNF)NNNr>)NF)F)F{Gz?)r$r%)r$)FNFr$FFFF)FFFF)NNT)r)1__name__ __module__ __qualname____doc__rvocab_files_namesmodel_input_namesr0propertyintr?rFr]rrboolr_rrsrxr{rrrrrr staticmethodrrrrrrrrKrrrrrrrrr rrr __classcell__)r;s@r&rrs&P*$&67 !!!  H5T!C!! (Vptt  t4I+$O   W[( ( TBHjjD$9 %*7;$ $', %"'L"L'/tn L  L  L!%LLL L Lb%* %"' 'd3i('"' '  '  ' 'T $7c$7HSM$7]bcf]g$7N "  +3 +tCy3`c$  r0rc )d))fd}g}|}d}|jddz} g} g} d} d} t|j}|jd}|jd}t|D]\}}|d d j }|j |||}|d k(r|d d j }d}| }d |vrF|d \}}}||z}||z } |r||z | z}|r't |D]}|| k\s | || z |z| krn|}g}g}t|D]\}}||vr|j|g}|dd}tj|d}|_)rU|)k7rP|sN| j|t| } |j| }!|!|d<|j|g} g}|}||d<|)|| k\r|| z |z|z}"t|"d}"|r||k\rd} | s| r||krd} |dd  |"|dd <|"|dd k(r|"|dd<| j||d k(r| j|t| | \} }#|j| }!|!|d<|d k(rt|| |#)||d<|j|g} g}g} g}|}v|j||d k(st||zd}$|dzt|krt||dz|zd}%nd}%|j|$|%fd |vr|z z }|r+| j||d k(s| j|td| Dr|}g} g}g} g}| rd|rt j#dt| | \} }#|j| }!|!|d<|d k(rt|| |#)||d<|j|dj%d|D}&|s|rr|D];}|s|j'dnt)|d|d<|r+|j'd=|d k(r%g}'|D]}|'j+|dd|'i}(|&|(fSd|i}(|&|(fSi}(|&|(fS)z Internal method meant to only be used by asr pipeline. Handles all the little quirks specific to whisper to handle the various options not allowed in other seq2seq models NcddgddS)Nr)r.rrr) last_languagesr& new_chunkz_decode_asr..new_chunks)t bQQr0rrbr Frrarrr+token_timestampsstriderPrrr.Trwordsc3 K|]}|ywr>r)rps r&rz_decode_asr..Fs1Aa1rzWhisper did not predict an ending timestamp, which can happen if audio is cut off in the middle of a word. Also make sure WhisperTimeStampLogitsProcessor was used during generation.rc3&K|] }|d yw)rNr)rchunks r&rz_decode_asr..`s:%f :schunks)rdr)rrr rreversedrrerJr_find_longest_common_sequenceround_collate_word_timestampsr&anyrrrVrr%rT)* tokenizerr rr rr4r<r; time_offsetrprevious_tokensprevious_token_timestampsskipright_stride_startrrrchunk_idoutputrr5last_timestampfirst_timestamp chunk_len stride_left stride_rightrWcurrent_tokenscurrent_token_timestampsrDrr.resolved_tokens resolved_texttimeresolved_token_timestamps start_timeend_time full_text new_chunksoptionalr3s* @r&r r s$MRF KEK556HIAMOO " D)334O556GHO&<<=TU%m4U*&8$Q'..0 ++IH^_  &%&89!<CCE  ) v 39(3C 0I{L ; &K!*\!9  "-">"P%i0 /E/+6!&!8N JM_ _!). /#% "),[ LHAu ' ''0Abz$==t4'%])BK\'..~>*G*X(1(8(8(I (5f  e,+-)+ ) (0E*%$,M/)/>AKOT1~!e~&=  Do%/2I D;'*2,0E+&q)u[1!4404k*1-'..~>,65<<=UVEb+-FFB)B)2(8(8(I (5f ,6-E )? 2 F*)001IJ111KE ON(* %') $kU*n  NN]  6S 66 22"((9 %f  &5?,E}VeE'N  e:6::IO &E$ +&%*5+=%>k"" *%  &  &J 2!!%.1 2 *-H h !&)H h  h r0c |d}t|}g}rdg}t|ddD]w\}d}||ddf}t|} td|| zD]} | dz } td|| z t ||| z| z } t j || } td| |z t | | }t j ||t| tk7r tdr#tfdt| D}nt j| k(}|| z | z}|dkDs||kDs|}| |f}|\} }| zdz}|zdz}|j|d|||d}t|}sYjd|dz|dz|j||StdkDrj||fS|gfS)Nrr rg@ziThere is a bug within whisper `decode_asr` function, please report it. Dropping to prevent bad inference.c3fK|](\}}||k(r|zdz|zkrd*yw)r Nr) ridxelem left_startleft_token_timestamp_sequenceright right_startseq_idxtoken_timestamp_sequencess r&rz0_find_longest_common_sequence..sS!Tc *9*s:JK4Wq[A+PSBSTU s.1rP) r&rrmaxrRrr RuntimeErrorrrT) sequencesrc left_sequence left_lengthtotal_sequencetotal_token_timestamp_sequenceright_sequencemax_ max_indices right_lengthrDeps left_stopleft right_stopmatchesmatchingleft_mid right_midr^r_r`rarbs ` @@@@@r&r>r>vsvaLMm$KN (A!(D%)+&#,Yqr]#;[_"KA6 <>* q+ 45( OAg+C Q a0JK|)Ca)GHI88M*Y?@Da[1K\1-JHH^K CDE4yCJ&")%.t_&&/{S(H{x$)9k:N Q( OTrrmr4rr@r)r._split_tokens_on_unicode_split_tokens_on_spaces_merge_punctuations)rBrr.r~rr7 word_tokensrzs r&rxrxso%%QQ,DYPV,W){M,CIv,V){M{M;OQde +} ,,r0c|j|d}d}g}g}g}g}g}d} t|D]\} } |j| |j| |j|d} || vs|| | j|z|k(sZ|j| |j||j|g}g}| t | z } |||fS)zlCombine tokens into words by splitting at any position where the tokens are decoded as valid unicode points.T)ru�r)rrrrSr&) rBr decoded_fullreplacement_charr7rrzrOcurrent_indicesunicode_offset token_idxrWdecodeds r&rrs##F4#HL EKMNON%f-+ 5e$y)"">$"O G +NW]];K-LLMQaa LL !   ~ .   1N O c'l *N+ +} ,,r0ct||\}}}g}g}g}t|||D]\}} } | d|jk\} |jd} |j dv} | s| s| st |dk(r4|j ||j | |j | |d|z|d<|dj| |dj| |||fS)zLCombine tokens into words by splitting at whitespace and punctuation tokens.rrQz !"#$%&'()*+,-./:;<=>?@[\]^_`{|}~r)rr!ru startswithrr&rrT)rBrsubwordssubword_tokens_listsubword_indices_listr7rrzsubwordsubword_tokenssubword_indicesr  with_space punctuations r&rr7s:RS\^d:e7H!#7 EKM47BUWk4l 60 #y'='=='', mmo)MM jK3u:? LL !   ~ .   1b G+E"I O " "> 2 "  $ $_ 5 6 +} ,,r0ct|dz }t|dz }|dk\rp||jdrO||j|vr:||||z||<||||z||<||||z||<d||<g||<g||<n|}|dz}|dk\rpd}d}|t|krq||jdsG|||vr@||xx||z cc<||xx||z cc<||xx||z cc<d||<g||<g||<n|}|dz }|t|krq|Dcgc]}|s| c}|dd|Dcgc]}|s| c}|dd|D cgc]} | s|  c} |ddycc}wcc}wcc} w)z1Merges punctuation tokens with neighboring words.rPr rrQrN)r&rrendswith) r7rr| prependedappendedrDr\r+rWr\s r&rrOs E QA E QA q& 8  s #a(8I(EQx%(*E!Hq F1I-F1I gaj0GAJE!HF1IGAJA Q q& A A c%j.Qx  %%(h*> !Ha H 1I "I AJ'!* $JE!HF1IGAJA Q c%j."'/$/E!H$*45e4F1I!(0#C#0GAJ040s$E!E/E7EE" E"r>)Nu "'“¡¿([{-u"'.。,，!！?？:：”)]}、)-r)rrr functoolsrtypingrrrrrrregexr*tokenization_utilsr r utilsr english_normalizerr rrMAX_MODEL_INPUT_SIZESr' get_loggerr&rr/rerrhrjrr r>r@r-rrxrrr__all__)coder.s00r&rs^( //AJ&( 3 0   H % e )e )e  (e  ) e   ) e   ( e  (e  *e  ,e  )e  (e  )e  'e  (e  )e   )!e " ,#e $ '%e & )'e ( ,)e * (+e , +-e . '/e 0 '1e 2 '3e 4 *5e 6 (7e 8 +9e : ';e < +=e > &?e @ &Ae B *Ce D +Ee F ,Ge H 'Ie J 'Ke L +Me N 'Oe P (Qe R (Se T )Ue V )We X )Ye Z )[e \ -]e ^ +_e ` )ae b *ce d ,ee f (ge h (ie j +ke l *me n (oe p +qe r )se t (ue v *we x )ye z *{e | )}e ~ )e @ )Ae B 'Ce D 'Ee F (Ge H (Ie J +Ke L )Me N *Oe P ,Qe R 'Se T (Ue V *We X )Ye Z )[e \ %]e ^ '_e ` )ae b ce d (ee f )ge h )ie j )ke l               Ie  P,5OO,=>.$x~>  &| *| ~bJu"p &# 0A - I-sm- -  -4-S ->-tCy-0#1L  k%?s G: