Ë
    Q^(hZ  ã                   ó†   — d dl mZmZmZmZmZmZ ddlmZm	Z	m
Z
mZmZ ddlmZ ddlmZmZmZmZ ddlmZ  G d„ d	e«      Zy
)é    )ÚDictÚIteratorÚListÚOptionalÚTupleÚUnioné   )Ú
AddedTokenÚ	TokenizerÚdecodersÚpre_tokenizersÚtrainers)ÚBPE)ÚBertNormalizerÚ	LowercaseÚSequenceÚunicode_normalizer_from_stré   )ÚBaseTokenizerc                   ó¸  ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 d!deeeeeef   f      deeeee	eef   e	eef   f   f      deee
f   d	ed
ee   dedee   dedefˆ fd„Zededefd„«       Zdddgdg ddfdeeee   f   dededeeee
f      dedee   d	ee   defd„Zdddgdg dddfdeee   eee      f   dededeeee
f      dedee   d	ee   dedee   fd „Zˆ xZS )"ÚCharBPETokenizeraà  Original BPE Tokenizer

    Represents the BPE algorithm, as introduced by Rico Sennrich
    (https://arxiv.org/abs/1508.07909)

    The defaults settings corresponds to OpenAI GPT BPE tokenizers and differs from the original
    Sennrich subword-nmt implementation by the following options that you can deactivate:
        - adding a normalizer to clean up the text (deactivate with `bert_normalizer=False`) by:
            * removing any control characters and replacing all whitespaces by the classic one.
            * handle chinese chars by putting spaces around them.
            * strip all accents.
        - spitting on punctuation in addition to whitespaces (deactivate it with
          `split_on_whitespace_only=True`)
    Nú<unk>ú</w>TÚvocabÚmergesÚ	unk_tokenÚsuffixÚdropoutÚ	lowercaseÚunicode_normalizerÚbert_normalizerÚsplit_on_whitespace_onlyc
           
      ó¬  •— |�%|�#t        t        |||t        |«      |¬«      «      }
n t        t        t        |«      ||¬«      «      }
|
j                  t        |«      «      �|
j	                  t        |«      g«       g }|r|t        |«      gz  }|r|t        d¬«      gz  }|r|t        «       gz  }t        |«      dkD  r)t        |«      dkD  rt        |«      |
_
        n
|d   |
_
        |	rt        j                  «       |
_        nt        j                  «       |
_        t        j                   |¬«      |
_        d|||||||	d	œ}t$        ‰| �M  |
|«       y )
N)r   r   Úend_of_word_suffix)r   r   r$   F)r   r   r   )r   r   )Úmodelr   r   r   r   r    r!   r"   )r   r   ÚstrÚtoken_to_idÚadd_special_tokensr   r   r   Úlenr   Ú
normalizerr   ÚWhitespaceSplitÚpre_tokenizerÚBertPreTokenizerr   Ú
BPEDecoderÚdecoderÚsuperÚ__init__)Úselfr   r   r   r   r   r   r    r!   r"   Ú	tokenizerÚnormalizersÚ
parametersÚ	__class__s                €úg/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/tokenizers/implementations/char_level_bpe.pyr1   zCharBPETokenizer.__init__   s\  ø€ ð Ð Ð!3Ü!ÜØØØ#Ü! )›nØ'-ôó‰Iô "¤#´°I³ÈÐdjÔ"kÓlˆIà× Ñ ¤ Y£Ó0Ð<Ø×(Ñ(¬#¨i«.Ð)9Ô:ð ˆáØÔ7Ð8JÓKÐLÑLˆKáØœN°UÔ;Ð<Ñ<ˆKáØœI›K˜=Ñ(ˆKô ˆ{Ó˜aÒÜ�;Ó !Ò#Ü'/°Ó'<�	Õ$à'2°1¡~�	Ô$á#Ü&4×&DÑ&DÓ&FˆIÕ#ä&4×&EÑ&EÓ&GˆIÔ#ä$×/Ñ/°vÔ>ˆ	Ôð Ø"ØØØ"Ø"4Ø.Ø(@ñ	
ˆ
ô 	‰Ñ˜ JÕ/ó    Úvocab_filenameÚmerges_filenamec                 óN   — t        j                  | |«      \  }}t        ||fi |¤ŽS )N)r   Ú	read_filer   )r9   r:   Úkwargsr   r   s        r7   Ú	from_filezCharBPETokenizer.from_file\   s(   € äŸ™ n°oÓF‰ˆˆvÜ  vÑ8°Ñ8Ð8r8   i0u  r	   iè  ÚfilesÚ
vocab_sizeÚmin_frequencyÚspecial_tokensÚlimit_alphabetÚinitial_alphabetÚshow_progressc	           	      óœ   — t        j                  |||||||¬«      }	t        |t        «      r|g}| j                  j                  ||	¬«       y)z%Train the model using the given files©r@   rA   rB   rC   rD   r$   rE   )ÚtrainerN)r   Ú
BpeTrainerÚ
isinstancer&   Ú
_tokenizerÚtrain)
r2   r?   r@   rA   rB   rC   rD   r   rE   rH   s
             r7   rL   zCharBPETokenizer.traina   sS   € ô ×%Ñ%Ø!Ø'Ø)Ø)Ø-Ø%Ø'ô
ˆô �eœSÔ!Ø�GˆEØ�‰×Ñ˜e¨WÐÕ5r8   ÚiteratorÚlengthc
           	      óx   — t        j                  |||||||¬«      }
| j                  j                  ||
|	¬«       y)z(Train the model using the given iteratorrG   )rH   rN   N)r   rI   rK   Útrain_from_iterator)r2   rM   r@   rA   rB   rC   rD   r   rE   rN   rH   s              r7   rP   z$CharBPETokenizer.train_from_iterator{   sK   € ô ×%Ñ%Ø!Ø'Ø)Ø)Ø-Ø%Ø'ô
ˆð 	�‰×+Ñ+ØØØð 	,õ 	
r8   )	NNr   r   NFNTF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r   r&   r   Úintr   r
   ÚfloatÚboolr1   Ústaticmethodr>   r   rL   r   rP   Ú__classcell__)r6   s   @r7   r   r   	   s9  ø„ ñð" 7;ØOSØ,3ØØ#'ØØ,0Ø $Ø).ñA0à˜˜c 4¨¨S¨¡>Ð1Ñ2Ñ3ðA0ð ˜˜s D¨¨s°C¨x©¸%ÀÀSÀ¹/Ð)IÑ$JÐJÑKÑLðA0ð ˜˜j˜Ñ)ð	A0ð
 ðA0ð ˜%‘ðA0ð ðA0ð % S™MðA0ð ðA0ð #'õA0ðF ð9 #ð 9¸ò 9ó ð9ð  ØØ8?°yØ"Ø&(Ø &Ø"ñ6à�S˜$˜s™)�^Ñ$ð6ð ð6ð ð	6ð
 ˜U 3¨
 ?Ñ3Ñ4ð6ð ð6ð ˜s™)ð6ð ˜‘ð6ð ó6ð:  ØØ8?°yØ"Ø&(Ø &Ø"Ø $ñ
à˜ ™ x°¸±Ñ'>Ð>Ñ?ð
ð ð
ð ð	
ð
 ˜U 3¨
 ?Ñ3Ñ4ð
ð ð
ð ˜s™)ð
ð ˜‘ð
ð ð
ð ˜‘÷
r8   r   N)Útypingr   r   r   r   r   r   Ú r
   r   r   r   r   Úmodelsr   r4   r   r   r   r   Úbase_tokenizerr   r   © r8   r7   ú<module>r_      s+   ðß ?× ?ç HÕ HÝ ß ZÓ ZÝ )ôM
�}õ M
r8   