Ë
    T^(h 
  ã                   óˆ   — d Z ddlmZmZ ddlmZ ddlmZ ddlm	Z	  ej                  e«      Zdd	d
dœZ G d„ de«      ZdgZy)z)Fast Tokenization classes for OpenAI GPT.é    )ÚOptionalÚTupleé   )ÚPreTrainedTokenizerFast)Úloggingé   )ÚOpenAIGPTTokenizerz
vocab.jsonz
merges.txtztokenizer.json)Ú
vocab_fileÚmerges_fileÚtokenizer_filec                   óf   ‡ — e Zd ZdZeZddgZeZd
ˆ fd„	Z	e
d„ «       Zddedee   dee   fd	„Zˆ xZS )ÚOpenAIGPTTokenizerFasta  
    Construct a "fast" GPT Tokenizer (backed by HuggingFace's *tokenizers* library). Based on Byte-Pair-Encoding with
    the following peculiarities:

    - lower case all inputs
    - uses BERT's BasicTokenizer for pre-BPE tokenization

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
    Ú	input_idsÚattention_maskc                 ó.   •— t        ‰| �  ||f||dœ|¤Ž y )N)r   Ú	unk_token)ÚsuperÚ__init__)Úselfr
   r   r   r   ÚkwargsÚ	__class__s         €úq/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/openai/tokenization_openai_fast.pyr   zOpenAIGPTTokenizerFast.__init__6   s    ø€ Ü‰Ñ˜ [ÐoÀÐ[dÑoÐhnÓoó    c                  ó   — y)NT© )r   s    r   Údo_lower_casez$OpenAIGPTTokenizerFast.do_lower_case9   s   € àr   Úsave_directoryÚfilename_prefixÚreturnc                 óf   — | j                   j                  j                  ||¬«      }t        |«      S )N)Úname)Ú
_tokenizerÚmodelÚsaveÚtuple)r   r   r   Úfiless       r   Úsave_vocabularyz&OpenAIGPTTokenizerFast.save_vocabulary=   s+   € Ø—‘×%Ñ%×*Ñ*¨>ÀÐ*ÓPˆÜ�U‹|Ðr   )NNNz<unk>)N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesÚmodel_input_namesr	   Úslow_tokenizer_classr   Úpropertyr   Ústrr   r   r'   Ú__classcell__)r   s   @r   r   r      sa   ø„ ñð( *ÐØ$Ð&6Ð7ÐØ-Ðõpð ñó ðñ¨cð ÀHÈSÁMð Ð]bÐcfÑ]g÷ r   r   N)r+   Útypingr   r   Útokenization_utils_fastr   Úutilsr   Útokenization_openair	   Ú
get_loggerr(   Úloggerr,   r   Ú__all__r   r   r   ú<module>r:      sP   ðñ 0ç "å >Ý Ý 3ð 
ˆ×	Ñ	˜HÓ	%€à#/ÀÐ`pÑqÐ ô"Ð4ô "ðJ $Ð
$�r   