Ë
    S^(hˆ   ã                   óÖ   — d dl Z d dlmZ d dlmZmZ d dlmZ ddlm	Z	 ddl
mZmZ ddlmZ  ed	«        e«       rd
dlmZ ndZ ej"                  e«      ZdddœZ G d„ de	«      ZdgZy)é    N)Úcopyfile)ÚOptionalÚTuple)Ú
processorsé   )ÚPreTrainedTokenizerFast)Úis_sentencepiece_availableÚlogging)Úrequire_versionztokenizers>=0.13.3é   )ÚGemmaTokenizerztokenizer.modelztokenizer.json)Ú
vocab_fileÚtokenizer_filec                   óø   ‡ — e Zd ZdZeZeZdZddgZ		 	 	 	 	 	 	 	 	 dˆ fd„	Z
edefd„«       Zd„ Zed	„ «       Zed
„ «       Zej"                  d„ «       Zej"                  d„ «       Zddedee   dee   fd„Zdd„Zˆ xZS )ÚGemmaTokenizerFastu˜
  
    Construct a Gemma tokenizer fast. Based on byte-level Byte-Pair-Encoding.

    This uses notably ByteFallback and no prefix space. Normalization is applied to replace  `" "` with `"â–�"`

    ```python
    >>> from transformers import GemmaTokenizerFast

    >>> tokenizer = GemmaTokenizerFast.from_pretrained("hf-internal-testing/dummy-gemma")
    >>> tokenizer.encode("Hello this is a test")
    [2, 4521, 736, 603, 476, 2121]
    ```

    If you want to change the `bos_token` or the `eos_token`, make sure to specify them when initializing the model, or
    call `tokenizer.update_post_processor()` to make sure that the post-processing is correctly done (otherwise the
    values of the first token and final token of an encoded sequence will not be correct). For more details, checkout
    [post-processors] (https://huggingface.co/docs/tokenizers/api/post-processors) documentation.


    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`, *optional*):
            [SentencePiece](https://github.com/google/sentencepiece) file (generally has a .model extension) that
            contains the vocabulary necessary to instantiate a tokenizer.
        tokenizer_file (`str`, *optional*):
            [tokenizers](https://github.com/huggingface/tokenizers) file (generally has a .json extension) that
            contains everything needed to load the tokenizer.
        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `False`):
            Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like
            extra spaces.
        unk_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        bos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<bos>"`):
            The beginning of sequence token that was used during pretraining. Can be used a sequence classifier token.
        eos_token (`str` or `tokenizers.AddedToken`, *optional*, defaults to `"<eos>"`):
            The end of sequence token.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The padding token
        add_bos_token (`bool`, *optional*, defaults to `True`):
            Whether or not to add an `bos_token` at the start of sequences.
        add_eos_token (`bool`, *optional*, defaults to `False`):
            Whether or not to add an `eos_token` at the end of sequences.
    ÚleftÚ	input_idsÚattention_maskc
                 ó‚   •— t        ‰| �  d|||||||||	dœ	|
¤Ž || _        |	| _        | j	                  «        || _        y )N)	r   r   Úclean_up_tokenization_spacesÚ	unk_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenÚadd_bos_tokenÚadd_eos_token© )ÚsuperÚ__init__Ú_add_bos_tokenÚ_add_eos_tokenÚupdate_post_processorr   )Úselfr   r   r   r   r   r   r   r   r   ÚkwargsÚ	__class__s              €úo/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/gemma/tokenization_gemma_fast.pyr   zGemmaTokenizerFast.__init__Z   sa   ø€ ô 	‰Ñð 	
Ø!Ø)Ø)EØØØØØ'Ø'ñ	
ð ò	
ð ,ˆÔØ+ˆÔØ×"Ñ"Ô$Ø$ˆ�ó    Úreturnc                 óp   — | j                   r)t        j                  j                  | j                   «      S dS )NF)r   ÚosÚpathÚisfile©r#   s    r&   Úcan_save_slow_tokenizerz*GemmaTokenizerFast.can_save_slow_tokenizerx   s$   € à26·/²/Œr�w‰w�~‰~˜dŸo™oÓ.ÐLÀuÐLr'   c                 óP  — | j                   }| j                  }|€| j                  rt        d«      ‚| j                  }| j
                  }|€| j                  rt        d«      ‚| j                  r|dz   nd› d| j                  rd|z   dz   nd› �}|› | j                  rd|z   d	z   nd› d
| j                  rd|z   d	z   nd› �}g }| j                  r|j                  ||f«       | j                  r|j                  ||f«       t        j                  |||¬«      | j                  _        y)ze
        Updates the underlying post processor with the current `bos_token` and `eos_token`.
        Nz)add_bos_token = True but bos_token = Nonez)add_eos_token = True but eos_token = Nonez:0 Ú z$A:0ú z:0z:1z $B:1)ÚsingleÚpairÚspecial_tokens)r   Úbos_token_idr   Ú
ValueErrorr   Úeos_token_idr   Úappendr   ÚTemplateProcessingÚ
_tokenizerÚpost_processor)r#   Úbosr5   Úeosr7   r2   r3   r4   s           r&   r"   z(GemmaTokenizerFast.update_post_processor}   s<  € ð �n‰nˆØ×(Ñ(ˆØˆ;˜4×-Ò-ÜÐHÓIÐIà�n‰nˆØ×(Ñ(ˆØˆ;˜4×-Ò-ÜÐHÓIÐIà%)×%7Ò%7�S˜5’[¸RÐ@ÀÐ[_×[mÒ[mÀcÈCÁiÐRVÒFVÐsuÐDvÐwˆØ�°×0BÒ0B˜3 ™9 tÒ+ÈÐKÈ5Ðgk×gyÒgyÐRUÐX[ÑR[Ð^bÒRbð  @Bð  QCð  DˆàˆØ×ÒØ×!Ñ! 3¨Ð"5Ô6Ø×ÒØ×!Ñ! 3¨Ð"5Ô6Ü)3×)FÑ)FØ °^ô*
ˆ�‰Õ&r'   c                 ó   — | j                   S ©N)r!   r-   s    r&   r   z GemmaTokenizerFast.add_eos_token—   ó   € à×"Ñ"Ð"r'   c                 ó   — | j                   S r?   )r    r-   s    r&   r   z GemmaTokenizerFast.add_bos_token›   r@   r'   c                 ó2   — || _         | j                  «        y r?   )r!   r"   ©r#   Úvalues     r&   r   z GemmaTokenizerFast.add_eos_tokenŸ   ó   € à#ˆÔØ×"Ñ"Õ$r'   c                 ó2   — || _         | j                  «        y r?   )r    r"   rC   s     r&   r   z GemmaTokenizerFast.add_bos_token¤   rE   r'   Úsave_directoryÚfilename_prefixc                 óÈ  — | j                   st        d«      ‚t        j                  j	                  |«      st
        j                  d|› d�«       y t        j                  j                  ||r|dz   ndt        d   z   «      }t        j                  j                  | j                  «      t        j                  j                  |«      k7  rt        | j                  |«       |fS )NzhYour fast tokenizer does not have the necessary information to save the vocabulary for a slow tokenizer.zVocabulary path (z) should be a directoryú-r0   r   )r.   r6   r*   r+   ÚisdirÚloggerÚerrorÚjoinÚVOCAB_FILES_NAMESÚabspathr   r   )r#   rG   rH   Úout_vocab_files       r&   Úsave_vocabularyz"GemmaTokenizerFast.save_vocabularyª   sµ   € Ø×+Ò+Üðóð ô
 �w‰w�}‰}˜^Ô,Ü�L‰LÐ,¨^Ð,<Ð<SÐTÔUØÜŸ™Ÿ™Ø±o˜_¨sÒ2È2ÔQbÐcoÑQpÑpó
ˆô �7‰7�?‰?˜4Ÿ?™?Ó+¬r¯w©w¯©¸~Ó/NÒNÜ�T—_‘_ nÔ5àÐ Ð r'   c                 óœ   — | j                   r| j                  gng }| j                  r| j                  gng }||z   |z   }|�||z   |z   |z   }|S r?   )r   r5   r   r7   )r#   Útoken_ids_0Útoken_ids_1r5   r7   Úoutputs         r&   Ú build_inputs_with_special_tokensz3GemmaTokenizerFast.build_inputs_with_special_tokens¾   s`   € Ø.2×.@Ò.@˜×)Ñ)Ñ*ÀbˆØ.2×.@Ò.@˜×)Ñ)Ñ*Àbˆà Ñ+¨lÑ:ˆàÐ"Ø˜lÑ*¨[Ñ8¸<ÑGˆFàˆr'   )	NNFz<unk>z<bos>z<eos>z<pad>TFr?   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__rO   Úvocab_files_namesr   Úslow_tokenizer_classÚpadding_sideÚmodel_input_namesr   ÚpropertyÚboolr.   r"   r   r   ÚsetterÚstrr   r   rR   rW   Ú__classcell__)r%   s   @r&   r   r   %   só   ø„ ñ-ð^ *ÐØ)ÐØ€LØ$Ð&6Ð7Ðð ØØ%*ØØØØØØõ%ð< ðM¨ò Mó ðMò
ð4 ñ#ó ð#ð ñ#ó ð#ð ×Ññ%ó ð%ð ×Ññ%ó ð%ñ
!¨cð !ÀHÈSÁMð !Ð]bÐcfÑ]gó !÷(	r'   r   )r*   Úshutilr   Útypingr   r   Ú
tokenizersr   Útokenization_utils_fastr   Úutilsr	   r
   Úutils.versionsr   Útokenization_gemmar   Ú
get_loggerrX   rL   rO   r   Ú__all__r   r'   r&   ú<module>rn      sm   ðó 
Ý ß "å !å >ß 8Ý -ñ Ð$Ô %áÔÞ2à€Nà	ˆ×	Ñ	˜HÓ	%€Ø#4ÐHXÑYÐ ôbÐ0ô bðJ  Ð
 �r'   