Ë
    Q^(hš  ã                   óz   — d dl mZmZmZmZmZmZ d dlmZm	Z	m
Z
mZmZ d dlmZ d dlmZ ddlmZ  G d„ de«      Zy	)
é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚNFKCé   )ÚBaseTokenizerc                   ó”  ‡ — e Zd ZdZ	 	 	 	 	 	 	 ddeeeeeef   f      deeeee	eef   e	eef   f   f      deee
f   ded	ed
ee   dee   fˆ fd„Zededefd„«       Zdddgdg dfdeeee   f   dededeeee
f      dedee   defd„Zdddgdg ddfdeee   eee      f   dededeeee
f      dedee   dedee   fd„Zˆ xZS )ÚSentencePieceBPETokenizerzrSentencePiece BPE Tokenizer

    Represents the BPE algorithm, with the pretokenization used by SentencePiece
    Nú<unk>TÚvocabÚmergesÚ	unk_tokenÚreplacementÚadd_prefix_spaceÚdropoutÚfuse_unkc           	      ó¤  •— |�|�t        t        |||||¬«      «      }nt        t        |||¬«      «      }|j                  t        |«      «      �|j	                  t        |«      g«       t        «       |_        |rdnd}	t        j                  ||	¬«      |_	        t        j                  ||	¬«      |_        d||||dœ}
t        ‰| �5  ||
«       y )N)r   r   r   ÚalwaysÚnever)r   Úprepend_schemeÚSentencePieceBPE)Úmodelr   r   r   r   )r
   r   Útoken_to_idÚstrÚadd_special_tokensr   Ú
normalizerr   Ú	MetaspaceÚpre_tokenizerr   ÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   Ú	tokenizerr   Ú
parametersÚ	__class__s              €új/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/tokenizers/implementations/sentencepiece_bpe.pyr*   z"SentencePieceBPETokenizer.__init__   sÏ   ø€ ð Ð Ð!3Ü!¤# e¨V¸WÐPYÐdlÔ"mÓn‰Iä!¤#¨gÀÐU]Ô"^Ó_ˆIà× Ñ ¤ Y£Ó0Ð<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä#›vˆ	ÔÙ%5™¸7ˆÜ"0×":Ñ":À{ÐcqÔ"rˆ	ÔÜ$×.Ñ.¸;ÐWeÔfˆ	Ôð (Ø"Ø&Ø 0Øñ
ˆ
ô 	‰Ñ˜ JÕ/ó    Úvocab_filenameÚmerges_filenamec                 óN   — t        j                  | |«      \  }}t        ||fi |¤ŽS )N)r   Ú	read_filer   )r1   r2   Úkwargsr   r   s        r/   Ú	from_filez#SentencePieceBPETokenizer.from_file1   s(   € äŸ™ n°oÓF‰ˆˆvÜ(¨°ÑA¸&ÑAÐAr0   i0u  é   iè  ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetÚshow_progressc                 óš   — t        j                  ||||||¬«      }t        |t        «      r|g}| j                  j                  ||¬«       y)z%Train the model using the given files©r9   r:   r;   r<   r=   r>   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer#   Ú
_tokenizerÚtrain)	r+   r8   r9   r:   r;   r<   r=   r>   rA   s	            r/   rE   zSentencePieceBPETokenizer.train6   sP   € ô ×%Ñ%Ø!Ø'Ø)Ø)Ø-Ø'ô
ˆô �eœSÔ!Ø�GˆEØ�‰×Ñ˜e¨WÐÕ5r0   ÚiteratorÚlengthc	                 óv   — t        j                  ||||||¬«      }	| j                  j                  ||	|¬«       y)z(Train the model using the given iteratorr@   )rA   rG   N)r   rB   rD   Útrain_from_iterator)
r+   rF   r9   r:   r;   r<   r=   r>   rG   rA   s
             r/   rI   z-SentencePieceBPETokenizer.train_from_iteratorN   sH   € ô ×%Ñ%Ø!Ø'Ø)Ø)Ø-Ø'ô
ˆð 	�‰×+Ñ+ØØØð 	,õ 	
r0   )NNr   u   â–�TNF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r#   r   Úintr   r	   ÚboolÚfloatr*   Ústaticmethodr6   r   rE   r   rI   Ú__classcell__)r.   s   @r/   r   r   
   s÷  ø„ ñð 7;ØOSØ,3Ø Ø!%Ø#'Ø#(ñ0à˜˜c 4¨¨S¨¡>Ð1Ñ2Ñ3ð0ð ˜˜s D¨¨s°C¨x©¸%ÀÀSÀ¹/Ð)IÑ$JÐJÑKÑLð0ð ˜˜j˜Ñ)ð	0ð
 ð0ð ð0ð ˜%‘ð0ð ˜4‘.õ0ðB ðB #ð B¸ò Bó ðBð  ØØ8?°yØ"Ø&(Ø"ñ6à�S˜$˜s™)�^Ñ$ð6ð ð6ð ð	6ð
 ˜U 3¨
 ?Ñ3Ñ4ð6ð ð6ð ˜s™)ð6ð ó6ð6  ØØ8?°yØ"Ø&(Ø"Ø $ñ
à˜ ™ x°¸±Ñ'>Ð>Ñ?ð
ð ð
ð ð	
ð
 ˜U 3¨
 ?Ñ3Ñ4ð
ð ð
ð ˜s™)ð
ð ð
ð ˜‘÷
r0   r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr	   r
   r   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   Úbase_tokenizerr   r   © r0   r/   ú<module>rY      s(   ðß ?× ?ç PÕ PÝ !Ý 'å )ô]
 õ ]
r0   