Ë
    T^(hf3  ã                   ó¤   — d dl Z d dlmZ d dlmZmZmZmZmZ d dl	Z
ddlmZmZ ddlmZ dZdd	iZ ej$                  e«      Z G d
„ de«      ZdgZy)é    N)Úcopyfile)ÚAnyÚDictÚListÚOptionalÚTupleé   )Ú
AddedTokenÚPreTrainedTokenizer)Úloggingu   â–�Ú
vocab_filezspiece.modelc            
       ó:  ‡ — e Zd ZdZeZddgZ	 	 	 	 	 	 	 	 ddeee	e
f      ddfˆ fd„Zedefd„«       Zdee	ef   fd	„Zd
„ Zd„ Zde	dee	   fd„Zde	defd„Zdede	fd„Zd„ Zdd„Zd„ Z	 ddedee   dedee   fd„Zd dee   fd„Zd de	dee	   dee	   fd„Zˆ xZS )!ÚPegasusTokenizera  
    Construct a PEGASUS tokenizer. Based on [SentencePiece](https://github.com/google/sentencepiece).

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a *.spm* extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding, for example when batching sequences of different lengths.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.

            <Tip>

            When building a sequence using special tokens, this is not the token that is used for the end of sequence.
            The token used is the `sep_token`.

            </Tip>

        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        mask_token (`str`, *optional*, defaults to `"<mask_2>"`):
            The token used for masking single token values. This is the token used when training this model with masked
            language modeling (MLM). This is the token that the PEGASUS encoder will try to predict during pretraining.
            It corresponds to *[MASK2]* in [PEGASUS: Pre-training with Extracted Gap-sentences for Abstractive
            Summarization](https://arxiv.org/pdf/1912.08777.pdf).
        mask_token_sent (`str`, *optional*, defaults to `"<mask_1>"`):
            The token used for masking whole target sentences. This is the token used when training this model with gap
            sentences generation (GSG). This is the sentence that the PEGASUS decoder will try to predict during
            pretraining. It corresponds to *[MASK1]* in [PEGASUS: Pre-training with Extracted Gap-sentences for
            Abstractive Summarization](https://arxiv.org/pdf/1912.08777.pdf).
        additional_special_tokens (`List[str]`, *optional*):
            Additional special tokens used by the tokenizer. If no additional_special_tokens are provided <mask_2> and
            <unk_2, ..., unk_102> are used as additional special tokens corresponding to the [original PEGASUS
            tokenizer](https://github.com/google-research/pegasus/blob/939830367bcf411193d2b5eca2f2f90f3f9260ca/pegasus/ops/pretrain_parsing_ops.cc#L66)
            that uses the tokens 2 - 104 only for pretraining
        sp_model_kwargs (`dict`, *optional*):
            Will be passed to the `SentencePieceProcessor.__init__()` method. The [Python wrapper for
            SentencePiece](https://github.com/google/sentencepiece/tree/master/python) can be used, among other things,
            to set:

            - `enable_sampling`: Enable subword regularization.
            - `nbest_size`: Sampling parameters for unigram. Invalid for BPE-Dropout.

              - `nbest_size = {0,1}`: No sampling is performed.
              - `nbest_size > 1`: samples from the nbest_size results.
              - `nbest_size < 0`: assuming that nbest_size is infinite and samples from the all hypothesis (lattice)
                using forward-filtering-and-backward-sampling algorithm.

            - `alpha`: Smoothing parameter for unigram sampling, and dropout probability of merge operations for
              BPE-dropout.
    Ú	input_idsÚattention_maskNÚsp_model_kwargsÚreturnc
                 ó|  •— || _         |�®t        |t        «      s't        dt	        t        «      › dt	        |«      › �«      ‚||vr|�|g|z   n|}|t        t        |«      | j                   dz
  «      D �cg c]  }d|› d�‘Œ
 c}z  }t        t        |«      «      t        |«      k7  rt        d|› d�«      ‚|}n4g }|�|gng }|t        d| j                   «      D �cg c]  }d|› d�‘Œ
 c}z  }|	€i n|	| _	        || _
        || _        t        j                  di | j                  ¤Ž| _        | j                  j                  |«       t!        t#        |«      d	¬
«      t!        t#        |«      d	¬
«      dœ}| j                  �)t!        |d	¬
«      |d<   t!        t#        |«      d	¬
«      |d<   t        d| j                   «      D ]  }t!        d|› d�d	¬
«      |t        |«      <   Œ! |
j%                  di «      | _        | j&                  j)                  |«       t+        ‰| �X  d|||||||| j                  dœ|
¤Ž y c c}w c c}w )Nz,additional_special_tokens should be of type z	, but is é   z<unk_ú>z�Please make sure that the provided additional_special_tokens do not contain an incorrectly shifted list of <unk_x> tokens. Found ú.é   T)Úspecial)r   r   r	   Úadded_tokens_decoder)Ú	eos_tokenÚ	unk_tokenÚ
mask_tokenÚ	pad_tokenÚmask_token_sentÚoffsetÚadditional_special_tokensr   © )r    Ú
isinstanceÚlistÚ	TypeErrorÚtypeÚrangeÚlenÚsetÚ
ValueErrorr   r   r   ÚspmÚSentencePieceProcessorÚsp_modelÚLoadr
   ÚstrÚpopÚ_added_tokens_decoderÚupdateÚsuperÚ__init__)Úselfr   r   r   r   r   r   r!   r    r   ÚkwargsÚ"additional_special_tokens_extendedÚir1   Ú	__class__s                 €ún/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/pegasus/tokenization_pegasus.pyr4   zPegasusTokenizer.__init___   s‘  ø€ ð ˆŒØ$Ð0ÜÐ7¼Ô>ÜØBÄ4ÌÃ:À,ð OÜÐ6Ó7Ð8ð:óð ð #Ð*CÑCÈÐHcð "Ð"Ð%>Ò>à.ð /ð /Ü&+¬CÐ0RÓ,SÐUY×U`ÑU`ÐcdÑUdÓ&eö3Ø!"�%˜�s˜!’ò3ñ Ð.ô ”3Ð9Ó:Ó;¼sÐCeÓ?fÒfÜ ð>Ø>`Ð=aÐabðdóð ð )KÑ%à13Ð.Ø=LÐ=X¨Ñ(9Ð^`Ð%Ø%ÄÀaÈÏÉÓ@UÖ)V¸1¨E°!°°Aª,Ò)VÑVÐ%à%4Ð%<™rÀ/ˆÔØ.ˆÔØ$ˆŒÜ×2Ñ2ÑJ°T×5IÑ5IÑJˆŒØ�‰×Ñ˜:Ô&ô œ#˜i›.°$Ô7Üœ#˜i›.°$Ô7ñ!
Ðð
 ×ÑÐ+Ü'1°/È4Ô'PÐ! !Ñ$Ü'1´#°j³/È4Ô'PÐ! !Ñ$ä�q˜$Ÿ+™+Ó&ò 	gˆAÜ@JÈUÐSTÐRUÐUVÈ<ÐaeÔ@fÐ!¤#Ð&;Ó"<Ò=ð	gð &,§Z¡ZÐ0FÈÓ%KˆÔ"Ø×"Ñ"×)Ñ)Ð*?Ô@ä‰Ñð 
	
ØØØ!ØØ+ØØ&?Ø ×0Ñ0ñ
	
ð ó
	
ùòK3ùò *Ws   Á5H4ÃH9c                 óF   — t        | j                  «      | j                  z   S ©N)r(   r-   r    )r5   s    r:   Ú
vocab_sizezPegasusTokenizer.vocab_sizeª   s   € ä�4—=‘=Ó! D§K¡KÑ/Ð/ó    c                 óª   — t        | j                  «      D �ci c]  }| j                  |«      |“Œ }}|j                  | j                  «       |S c c}w r<   )r'   r=   Úconvert_ids_to_tokensr2   Úadded_tokens_encoder)r5   r8   Úvocabs      r:   Ú	get_vocabzPegasusTokenizer.get_vocab®   sK   € Ü;@ÀÇÁÓ;QÖR°a�×+Ñ+¨AÓ.°Ñ1ÐRˆÐRØ�‰�T×.Ñ.Ô/Øˆùò Ss   ˜Ac                 óD   — | j                   j                  «       }d |d<   |S )Nr-   )Ú__dict__Úcopy)r5   Ústates     r:   Ú__getstate__zPegasusTokenizer.__getstate__³   s#   € Ø—‘×"Ñ"Ó$ˆØ ˆˆjÑØˆr>   c                 óÊ   — || _         t        | d«      si | _        t        j                  di | j                  ¤Ž| _        | j
                  j                  | j                  «       y )Nr   r"   )rE   Úhasattrr   r+   r,   r-   r.   r   )r5   Úds     r:   Ú__setstate__zPegasusTokenizer.__setstate__¸   sO   € ØˆŒô �tÐ.Ô/Ø#%ˆDÔ ä×2Ñ2ÑJ°T×5IÑ5IÑJˆŒØ�‰×Ñ˜4Ÿ?™?Õ+r>   Útextc                 óD   — | j                   j                  |t        ¬«      S )zPTake as input a string and return a list of strings (tokens) for words/sub-words)Úout_type)r-   Úencoder/   )r5   rM   s     r:   Ú	_tokenizezPegasusTokenizer._tokenizeÂ   s   € à�}‰}×#Ñ# D´3Ð#Ó7Ð7r>   Útokenc                 óV   — | j                   j                  |«      }|| j                  z   S )z0Converts a token (str) to an id using the vocab.)r-   Úpiece_to_idr    )r5   rR   Úsp_ids      r:   Ú_convert_token_to_idz%PegasusTokenizer._convert_token_to_idÆ   s%   € à—‘×)Ñ)¨%Ó0ˆØ�t—{‘{Ñ"Ð"r>   Úindexc                 óª   — || j                   k  r| j                  j                  |«      S | j                  j                  || j                   z
  «      }|S )z=Converts an index (integer) to a token (str) using the vocab.)r    r-   Ú	IdToPiece)r5   rW   rR   s      r:   Ú_convert_id_to_tokenz%PegasusTokenizer._convert_id_to_tokenË   sE   € à�4—;‘;ÒØ—=‘=×*Ñ*¨5Ó1Ð1Ø—‘×'Ñ'¨°·±Ñ(;Ó<ˆØˆr>   c                 óú   — g }d}|D ]E  }|| j                   v r$|| j                  j                  |«      |z   z  }g }Œ5|j                  |«       ŒG || j                  j                  |«      z  }|j	                  «       S )z:Converts a sequence of tokens (string) in a single string.Ú )Úall_special_tokensr-   ÚdecodeÚappendÚstrip)r5   ÚtokensÚcurrent_sub_tokensÚ
out_stringrR   s        r:   Úconvert_tokens_to_stringz)PegasusTokenizer.convert_tokens_to_stringÒ   sˆ   € àÐØˆ
Øò 	1ˆEà˜×/Ñ/Ñ/Ø˜dŸm™m×2Ñ2Ð3EÓFÈÑNÑN�
Ø%'Ñ"à"×)Ñ)¨%Õ0ð	1ð 	�d—m‘m×*Ñ*Ð+=Ó>Ñ>ˆ
Ø×ÑÓ!Ð!r>   c                  ó   — y)zJust EOSr   r"   )r5   Úpairs     r:   Únum_special_tokens_to_addz*PegasusTokenizer.num_special_tokens_to_addà   s   € àr>   c                 ó˜   — t        | j                  «      }|j                  | j                  «       |D �cg c]
  }||v rdnd‘Œ c}S c c}w )Nr   r   )r)   Úall_special_idsÚremoveÚunk_token_id)r5   Úseqri   Úxs       r:   Ú_special_token_maskz$PegasusTokenizer._special_token_maskä   sF   € Ü˜d×2Ñ2Ó3ˆØ×Ñ˜t×0Ñ0Ô1à:=Ö>°Q�Q˜/Ñ)‘¨qÑ0Ò>Ð>ùÒ>s   µAÚtoken_ids_0Útoken_ids_1Úalready_has_special_tokensc                 ó†   — |r| j                  |«      S |€| j                  |«      dgz   S | j                  ||z   «      dgz   S )zCGet list where entries are [1] if a token is [eos] or [pad] else 0.r   )rn   )r5   ro   rp   rq   s       r:   Úget_special_tokens_maskz(PegasusTokenizer.get_special_tokens_maskê   sS   € ñ &Ø×+Ñ+¨KÓ8Ð8ØÐ Ø×+Ñ+¨KÓ8¸A¸3Ñ>Ð>à×+Ñ+¨K¸+Ñ,EÓFÈ!ÈÑLÐLr>   c                 óL   — |€|| j                   gz   S ||z   | j                   gz   S )a@  
        Build model inputs from a sequence or a pair of sequences for sequence classification tasks by concatenating
        and adding special tokens. A PEGASUS sequence has the following format, where `X` represents the sequence:

        - single sequence: `X </s>`
        - pair of sequences: `A B </s>` (not intended use)

        BOS is never used. Pairs of sequences are not the expected use case, but they will be handled without a
        separator.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        )Úeos_token_id)r5   ro   rp   s      r:   Ú build_inputs_with_special_tokensz1PegasusTokenizer.build_inputs_with_special_tokensõ   s7   € ð( ÐØ $×"3Ñ"3Ð!4Ñ4Ð4à˜[Ñ(¨D×,=Ñ,=Ð+>Ñ>Ð>r>   Úsave_directoryÚfilename_prefixc                 óæ  — t         j                  j                  |«      st        j	                  d|› d�«       y t         j                  j                  ||r|dz   ndt        d   z   «      }t         j                  j                  | j                  «      t         j                  j                  |«      k7  rBt         j                  j                  | j                  «      rt        | j                  |«       |fS t         j                  j                  | j                  «      sCt        |d«      5 }| j                  j                  «       }|j                  |«       d d d «       |fS |fS # 1 sw Y   |fS xY w)NzVocabulary path (z) should be a directoryú-r\   r   Úwb)ÚosÚpathÚisdirÚloggerÚerrorÚjoinÚVOCAB_FILES_NAMESÚabspathr   Úisfiler   Úopenr-   Úserialized_model_protoÚwrite)r5   rw   rx   Úout_vocab_fileÚfiÚcontent_spiece_models         r:   Úsave_vocabularyz PegasusTokenizer.save_vocabulary  s%  € Ü�w‰w�}‰}˜^Ô,Ü�L‰LÐ,¨^Ð,<Ð<SÐTÔUØÜŸ™Ÿ™Ø±o˜_¨sÒ2È2ÔQbÐcoÑQpÑpó
ˆô �7‰7�?‰?˜4Ÿ?™?Ó+¬r¯w©w¯©¸~Ó/NÒNÔSU×SZÑSZ×SaÑSaÐbf×bqÑbqÔSrÜ�T—_‘_ nÔ5ð Ð Ð ô —‘—‘ §¡Ô0Ü�n dÓ+ð /¨rØ'+§}¡}×'KÑ'KÓ'MÐ$Ø—‘Ð-Ô.÷/ð Ð Ð �Ð Ð ÷	/ð Ð Ð ús   Ä+,E%Å%E0)z<pad>z</s>z<unk>z<mask_2>z<mask_1>Nég   N)F)NFr<   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r‚   Úvocab_files_namesÚmodel_input_namesr   r   r/   r   r4   ÚpropertyÚintr=   rC   rH   rL   r   rQ   rV   rZ   rd   rg   rn   Úboolrs   rv   r   r‹   Ú__classcell__)r9   s   @r:   r   r   "   sW  ø„ ñ7ðr *ÐØ$Ð&6Ð7Ðð
 ØØØØ"Ø"&ØØ48ñI
ð " $ s¨C x¡.Ñ1ðI
ð 
õI
ðV ð0˜Cò 0ó ð0ð˜4  S ™>ó ò
ò
,ð8˜cð 8 d¨3¡ió 8ð#¨#ð #°#ó #ð
¨#ð °#ó ò"óò?ð inñ	MØð	MØ.6°t©nð	MØaeð	Mà	ˆc‰ó	Mñ?ÐQUÐVYÑQZó ?ñ2!¨cð !ÀHÈSÁMð !Ð]bÐcfÑ]g÷ !r>   r   )r|   Úshutilr   Útypingr   r   r   r   r   Úsentencepiecer+   Útokenization_utilsr
   r   Úutilsr   ÚSPIECE_UNDERLINEr‚   Ú
get_loggerr�   r   r   Ú__all__r"   r>   r:   ú<module>rŸ      s]   ðó 
Ý ß 3Õ 3ã ç AÝ ð Ð à! >Ð2Ð ð 
ˆ×	Ñ	˜HÓ	%€ô{!Ð*ô {!ð| Ð
�r>   