Ë
    Q^(hÁ  ã                   ó†   — d dl mZmZmZmZmZmZ d dlmZm	Z	m
Z
mZmZmZ d dlmZ d dlmZmZmZ ddlmZ  G d„ de«      Zy	)
é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnion)Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processorsÚtrainers)ÚBPE)Ú	LowercaseÚSequenceÚunicode_normalizer_from_stré   )ÚBaseTokenizerc                   óz  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 ddeeeeeef   f      deeeee	eef   e	eef   f   f      de
de
dee   dee   d	ee   d
ee   de
fˆ fd„Zededefd„«       Zdddg fdeeee   f   dedede
deeeef      f
d„Zdddg dfdeee   eee      f   dedede
deeeef      dee   fd„Zˆ xZS )ÚByteLevelBPETokenizerzjByteLevelBPETokenizer

    Represents a Byte-level BPE as introduced by OpenAI with their GPT-2 model
    NÚvocabÚmergesÚadd_prefix_spaceÚ	lowercaseÚdropoutÚunicode_normalizerÚcontinuing_subword_prefixÚend_of_word_suffixÚtrim_offsetsc
           
      ó   •— |�$|�"t        t        ||||xs d|xs d¬«      «      }
nt        t        «       «      }
g }|r|t        |«      gz  }|r|t        «       gz  }t	        |«      dkD  r)t	        |«      dkD  rt        |«      |
_        n
|d   |
_        t        j                  |¬«      |
_	        t        j                  «       |
_        t        j                  |	¬«      |
_        d|||||||	dœ}t        ‰| �=  |
|«       y )	NÚ )r   r   r   r   r   )r   )r   ÚByteLevelBPE)Úmodelr   r   r   r   r   r   r   )r
   r   r   r   Úlenr   Ú
normalizerr   Ú	ByteLevelÚpre_tokenizerr   Údecoderr   Úpost_processorÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   r   Ú	tokenizerÚnormalizersÚ
parametersÚ	__class__s                €úg/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/tokenizers/implementations/byte_level_bpe.pyr+   zByteLevelBPETokenizer.__init__   s  ø€ ð Ð Ð!3Ü!ÜØØØ#Ø.GÒ.MÈ2Ø'9Ò'?¸Rôó‰Iô "¤#£%Ó(ˆIð ˆáØÔ7Ð8JÓKÐLÑLˆKáØœI›K˜=Ñ(ˆKô ˆ{Ó˜aÒÜ�;Ó !Ò#Ü'/°Ó'<�	Õ$à'2°1¡~�	Ô$ä"0×":Ñ":ÐL\Ô"]ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#7Ñ#7À\Ô#Rˆ	Ô ð $Ø 0Ø"ØØ"4Ø)BØ"4Ø(ñ	
ˆ
ô 	‰Ñ˜ JÕ/ó    Úvocab_filenameÚmerges_filenamec                 óN   — t        j                  | |«      \  }}t        ||fi |¤ŽS )N)r   Ú	read_filer   )r3   r4   Úkwargsr   r   s        r1   Ú	from_filezByteLevelBPETokenizer.from_fileJ   s(   € äŸ™ n°oÓF‰ˆˆvÜ$ U¨FÑ=°fÑ=Ð=r2   i0u  é   TÚfilesÚ
vocab_sizeÚmin_frequencyÚshow_progressÚspecial_tokensc                 óÐ   — t        j                  ||||t        j                  j	                  «       ¬«      }t        |t        «      r|g}| j                  j                  ||¬«       y)z%Train the model using the given files©r;   r<   r=   r>   Úinitial_alphabet)ÚtrainerN)	r   Ú
BpeTrainerr   r&   ÚalphabetÚ
isinstanceÚstrÚ
_tokenizerÚtrain)r,   r:   r;   r<   r=   r>   rB   s          r1   rH   zByteLevelBPETokenizer.trainO   s\   € ô ×%Ñ%Ø!Ø'Ø'Ø)Ü+×5Ñ5×>Ñ>Ó@ô
ˆô �eœSÔ!Ø�GˆEØ�‰×Ñ˜e¨WÐÕ5r2   ÚiteratorÚlengthc                 ó¬   — t        j                  ||||t        j                  j	                  «       ¬«      }| j
                  j                  |||¬«       y)z(Train the model using the given iteratorr@   )rB   rJ   N)r   rC   r   r&   rD   rG   Útrain_from_iterator)r,   rI   r;   r<   r=   r>   rJ   rB   s           r1   rL   z)ByteLevelBPETokenizer.train_from_iteratord   sT   € ô ×%Ñ%Ø!Ø'Ø'Ø)Ü+×5Ñ5×>Ñ>Ó@ô
ˆð 	�‰×+Ñ+ØØØð 	,õ 	
r2   )	NNFFNNNNF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   rF   r   Úintr   ÚboolÚfloatr+   Ústaticmethodr8   r   r	   rH   r   rL   Ú__classcell__)r0   s   @r1   r   r   
   sÌ  ø„ ñð 7;ØOSØ!&ØØ#'Ø,0Ø37Ø,0Ø"ñ80à˜˜c 4¨¨S¨¡>Ð1Ñ2Ñ3ð80ð ˜˜s D¨¨s°C¨x©¸%ÀÀSÀ¹/Ð)IÑ$JÐJÑKÑLð80ð ð	80ð
 ð80ð ˜%‘ð80ð % S™Mð80ð $,¨C¡=ð80ð % S™Mð80ð õ80ðt ð> #ð >¸ò >ó ð>ð  ØØ"Ø79ñ6à�S˜$˜s™)�^Ñ$ð6ð ð6ð ð	6ð
 ð6ð ˜U 3¨
 ?Ñ3Ñ4ó6ð0  ØØ"Ø79Ø $ñ
à˜ ™ x°¸±Ñ'>Ð>Ñ?ð
ð ð
ð ð	
ð
 ð
ð ˜U 3¨
 ?Ñ3Ñ4ð
ð ˜‘÷
r2   r   N)Útypingr   r   r   r   r   r   Ú
tokenizersr	   r
   r   r   r   r   Útokenizers.modelsr   Útokenizers.normalizersr   r   r   Úbase_tokenizerr   r   © r2   r1   ú<module>r\      s+   ðß ?× ?ç \× \Ý !ß SÑ Så )ôp
˜Mõ p
r2   