Ë
    S^(h[  ã                   óH   — d dl mZ d dlmZ d dlmZ d dlmZmZ dede	fd„Z
y)	é    )ÚPath)ÚAny)ÚTikTokenConverter)ÚTIKTOKEN_VOCAB_FILEÚTOKENIZER_FILEÚencodingÚ
output_dirc                 óî  — t        |«      }|j                  d¬«       |dz  t        z  }|t        z  }t	        |j                  «       «      }t	        |j                  «       «      }	 ddlm} ddlm	} t        | t        «      r || «      }  || j                  |«       t        || j                  | j                   ¬«      j#                  «       }|j%                  |«       y	# t        $ r t        d«      ‚w xY w)
a»  
    Converts given `tiktoken` encoding to `PretrainedTokenizerFast` and saves the configuration of converted tokenizer
    on disk.

    Args:
        encoding (`str` or `tiktoken.Encoding`):
            Tokenizer from `tiktoken` library. If `encoding` is `str`, the tokenizer will be loaded with
            `tiktoken.get_encoding(encoding)`.
        output_dir (`str`):
            Save path for converted tokenizer configuration file.
    T)Úexist_okÚtiktokenr   )Úget_encoding)Údump_tiktoken_bpezY`tiktoken` is required to save a `tiktoken` file. Install it with `pip install tiktoken`.)Ú
vocab_fileÚpatternÚadditional_special_tokensN)r   Úmkdirr   r   ÚstrÚabsoluter   r   Útiktoken.loadr   Ú
isinstanceÚ_mergeable_ranksÚImportErrorÚ
ValueErrorr   Ú_pat_strÚ_special_tokensÚ	convertedÚsave)	r   r	   Ú	save_fileÚtokenizer_fileÚsave_file_absoluteÚoutput_file_absoluter   r   Ú	tokenizers	            ú`/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/integrations/tiktoken.pyÚconvert_tiktoken_to_fastr$      sê   € ô �jÓ!€JØ×Ñ˜dÐÔ#à˜ZÑ'Ô*=Ñ=€IØ¤.Ñ0€Nä˜Y×/Ñ/Ó1Ó2ÐÜ˜~×6Ñ6Ó8Ó9Ðð	vÝ)Ý3ä�h¤Ô$Ù# HÓ-ˆHá˜(×3Ñ3Ð5GÔHô "Ø%¨x×/@Ñ/@Ð\d×\tÑ\tôç�iƒkð ð ‡N�NÐ'Õ(øô ò vÜÐtÓuÐuðvús   Á&7C ÃC4N)Úpathlibr   Útypingr   Ú#transformers.convert_slow_tokenizerr   Ú$transformers.tokenization_utils_fastr   r   r   r$   © ó    r#   ú<module>r+      s$   ðÝ Ý å Aß Tð#) sð #)¸ô #)r*   