Ë
    S^(hp  ã                   ó    — d dl Z d dlmZmZmZmZ d dlZd dlm	Z	 d dl
mZ ddlmZ ddlmZ  G d	„ d
ej                   j"                  «      Zd
gZy)é    N)ÚDictÚListÚOptionalÚUnion)ÚBytePairTokenizer)Úpad_model_inputsé   )Úkerasé   )ÚGPT2Tokenizerc            
       óÊ   ‡ — e Zd ZdZ	 	 ddeeef   dee   dee   dee   fˆ fd„Z	e
defd„«       Ze
d	eeej                  f   fd
„«       Ze
d„ «       Zd„ Zddee   fd„Zˆ xZS )ÚTFGPT2Tokenizera7  
    This is an in-graph tokenizer for GPT2. It should be initialized similarly to other tokenizers, using the
    `from_pretrained()` method. It can also be initialized with the `from_tokenizer()` method, which imports settings
    from an existing standard tokenizer object.

    In-graph tokenizers, unlike other Hugging Face tokenizers, are actually Keras layers and are designed to be run
    when the model is called, rather than during preprocessing. As a result, they have somewhat more limited options
    than standard tokenizer classes. They are most useful when you want to create an end-to-end model that goes
    straight from `tf.string` inputs to outputs.

    Args:
        vocab (Dict[str, int]): Vocabulary dict for Byte Pair Tokenizer
        merges (List[str]): Merges list for Byte Pair Tokenizer
    ÚvocabÚmergesÚ
max_lengthÚpad_token_idc                 ó€   •— t         ‰| �  «        || _        || _        || _        || _        t        |||¬«      | _        y )N)Úsequence_length)ÚsuperÚ__init__r   r   r   r   r   Útf_tokenizer)Úselfr   r   r   r   Ú	__class__s        €úk/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/gpt2/tokenization_gpt2_tf.pyr   zTFGPT2Tokenizer.__init__   s>   ø€ ô 	‰ÑÔØ(ˆÔØ$ˆŒØˆŒ
ØˆŒÜ-¨e°VÈZÔXˆÕó    Ú	tokenizerc                 ó²   — |j                   j                  «       D �cg c]  }dj                  |«      ‘Œ }}|j                  «       } | ||g|¢­i |¤ŽS c c}w )ag  Creates TFGPT2Tokenizer from GPT2Tokenizer

        Args:
            tokenizer (GPT2Tokenizer)

        Examples:

        ```python
        from transformers import AutoTokenizer, TFGPT2Tokenizer

        tokenizer = AutoTokenizer.from_pretrained("openai-community/gpt2")
        tf_tokenizer = TFGPT2Tokenizer.from_tokenizer(tokenizer)
        ```
        ú )Ú	bpe_ranksÚkeysÚjoinÚ	get_vocab)Úclsr   ÚargsÚkwargsÚmr   r   s          r   Úfrom_tokenizerzTFGPT2Tokenizer.from_tokenizer*   sY   € ð  (1×':Ñ':×'?Ñ'?Ó'AÖB !�#—(‘(˜1•+ÐBˆÐBØ×#Ñ#Ó%ˆÙ�5˜&Ð2 4Ò2¨6Ñ2Ð2ùò Cs   �AÚpretrained_model_name_or_pathc                 ó`   — t        j                  |g|¢­i |¤Ž} | j                  |g|¢­i |¤ŽS )a_  Creates TFGPT2Tokenizer from pretrained GPT2Tokenizer

        Args:
            pretrained_model_name_or_path (Union[str, os.PathLike]): Path to pretrained model

        Examples:

        ```python
        from transformers import TFGPT2Tokenizer

        tf_tokenizer = TFGPT2Tokenizer.from_pretrained("openai-community/gpt2")
        ```
        )r   Úfrom_pretrainedr'   )r#   r(   Úinit_inputsr%   r   s        r   r*   zTFGPT2Tokenizer.from_pretrained>   s>   € ô "×1Ñ1Ð2OÐhÐR]ÒhÐagÑhˆ	Ø!ˆs×!Ñ! )ÐD¨kÒD¸VÑDÐDr   c                 ó   —  | di |¤ŽS )z“Creates TFGPT2Tokenizer from configurations

        Args:
            config (Dict): Dictionary with keys such as stated in `get_config`.
        © r-   )r#   Úconfigs     r   Úfrom_configzTFGPT2Tokenizer.from_configP   s   € ñ ‰}�V‰}Ðr   c                 ó`   — | j                   | j                  | j                  | j                  dœS )N©r   r   r   r   r1   )r   s    r   Ú
get_configzTFGPT2Tokenizer.get_configY   s*   € à—Z‘ZØ—k‘kØŸ/™/Ø ×-Ñ-ñ	
ð 	
r   c                 óÊ   — | j                  |«      }t        j                  |«      }| j                  �-|�|n| j                  }|�t        ||| j                  ¬«      \  }}||dœS )N)Úmax_seq_lengthÚ	pad_value)Úattention_maskÚ	input_ids)r   ÚtfÚ	ones_liker   r   r   )r   Úxr   r7   r6   s        r   ÚcallzTFGPT2Tokenizer.calla   sk   € Ø×%Ñ% aÓ(ˆ	ÜŸ™ iÓ0ˆà×ÑÐ(à'1Ð'=™À4Ç?Á?ˆJàÐ%Ü,<Ø¨jÀD×DUÑDUô-Ñ)�	˜>ð #1¸yÑIÐIr   )NN)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚstrÚintr   r   r   Úclassmethodr   r'   r   ÚosÚPathLiker*   r/   r2   r;   Ú__classcell__)r   s   @r   r   r      sÆ   ø„ ñð& %)Ø&*ñYà�C˜�H‰~ðYð �S‘	ðYð ˜S‘Mð	Yð
 ˜s‘mõYð ð3 }ò 3ó ð3ð& ðE¸EÀ#ÀrÇ{Á{ÐBRÑ<Sò Eó ðEð" ñó ðò
ñJ (¨3¡-÷ Jr   r   )rC   Útypingr   r   r   r   Ú
tensorflowr8   Úkeras_nlp.tokenizersr   Útensorflow_textr   Úmodeling_tf_utilsr
   Útokenization_gpt2r   ÚlayersÚLayerr   Ú__all__r-   r   r   ú<module>rO      sB   ðÛ 	ß .Ó .ã Ý 2Ý ,å &Ý ,ôbJ�e—l‘l×(Ñ(ô bJðJ Ð
�r   