Ë
    S^(h&  ã                   óÌ   — d Z ddlZddlZddlmZmZmZ ddlmZm	Z	  e«       rddl
Z
ddlmZ ddlmZ  ej                  e«      Zdd	iZd
„ Z G d„ d«      Z G d„ de«      ZdgZy)z Tokenization classes for CPMAnt.é    N)ÚListÚOptionalÚTuple)Úis_jieba_availableÚrequires_backendsé   )ÚPreTrainedTokenizer)ÚloggingÚ
vocab_filez	vocab.txtc                 óæ   — t        j                  «       }t        | dd¬«      5 }|j                  «       }ddd«       t	        «      D ]  \  }}|j                  d«      }|||<   Œ |S # 1 sw Y   Œ4xY w)z*Loads a vocabulary file into a dictionary.Úrúutf-8©ÚencodingNú
)ÚcollectionsÚOrderedDictÚopenÚ	readlinesÚ	enumerateÚrstrip)r   ÚvocabÚreaderÚtokensÚindexÚtokens         úl/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/cpmant/tokenization_cpmant.pyÚ
load_vocabr   $   sw   € ä×#Ñ#Ó%€EÜ	ˆj˜#¨Ô	0ð $°FØ×!Ñ!Ó#ˆ÷$ä! &Ó)ò ‰ˆˆuØ—‘˜TÓ"ˆØˆˆeŠðð €L÷$ð $ús   £A'Á'A0c                   ó   — e Zd Zdd„Zd„ Zy)ÚWordpieceTokenizerc                 ó.   — || _         || _        || _        y ©N)r   Ú	unk_tokenÚmax_input_chars_per_word)Úselfr   r#   r$   s       r   Ú__init__zWordpieceTokenizer.__init__0   s   € ØˆŒ
Ø"ˆŒØ(@ˆÕ%ó    c                 ó˜  — t        |«      }t        |«      | j                  kD  r| j                  gS d}g }|t        |«      k  r‡t        |«      }d }||k  r0dj	                  ||| «      }|| j
                  v r|}n|dz  }||k  rŒ0|€!|j                  | j                  «       |dz  }n|j                  |«       |}|t        |«      k  rŒ‡|S )Nr   Ú é   )ÚlistÚlenr$   r#   Újoinr   Úappend)r%   r   ÚcharsÚstartÚ
sub_tokensÚendÚ
cur_substrÚsubstrs           r   ÚtokenizezWordpieceTokenizer.tokenize5   sÜ   € Ü�U“ˆÜˆu‹:˜×5Ñ5Ò5Ø—N‘NÐ#Ð#àˆØˆ
Ø”c˜%“jÒ Ü�e“*ˆCØˆJØ˜#’+ØŸ™  u¨SÐ!1Ó2�Ø˜TŸZ™ZÑ'Ø!'�JØØ�q‘�ð ˜#“+ð Ð!Ø×!Ñ! $§.¡.Ô1Ø˜‘
‘à×!Ñ! *Ô-Ø�ð ”c˜%“jÓ ð  Ðr'   N)ú<unk>éÈ   )Ú__name__Ú
__module__Ú__qualname__r&   r5   © r'   r   r    r    /   s   „ óAó
r'   r    c            
       óL  ‡ — e Zd ZdZeZddgZdZ	 	 	 	 	 	 	 	 	 dˆ fd„	Ze	d„ «       Z
e	d„ «       Ze	d„ «       Ze	d	efd
„«       Zd„ Zd„ Zˆ fd„Zd„ Zdee   d	efd„Zd„ Zd„ Zddedee   d	ee   fd„Zddee   dee   d	ee   fd„Z	 ddee   deee      ded	ee   fˆ fd„Zˆ xZS )ÚCpmAntTokenizera²  
    Construct a CPMAnt tokenizer. Based on byte-level Byte-Pair-Encoding.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        bod_token (`str`, *optional*, defaults to `"<d>"`):
            The beginning of document token.
        eod_token (`str`, *optional*, defaults to `"</d>"`):
            The end of document token.
        bos_token (`str`, *optional*, defaults to `"<s>"`):
            The beginning of sequence token.
        eos_token (`str`, *optional*, defaults to `"</s>"`):
            The end of sequence token.
        pad_token (`str`, *optional*, defaults to `"<pad>"`):
            The token used for padding.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token.
        line_token (`str`, *optional*, defaults to `"</n>"`):
            The line token.
        space_token (`str`, *optional*, defaults to `"</_>"`):
            The space token.
    Ú	input_idsÚattention_maskFc                 óX  •— t        | dg«       || _        || _        t        |«      | _        | j                  |	   | j                  d<   | j                  |   | j                  d<   | j                  |	= | j                  |= t        j                  t        | j                  j                  «       d„ ¬«      «      | _        | j                  j                  «       D ��ci c]  \  }}||“Œ
 c}}| _	        t        | j                  |¬«      | _        t        ‰| �4  d||||||||	|
dœ	|¤Ž y c c}}w )	NÚjiebaú r   c                 ó   — | d   S ©Nr*   r;   ©Úxs    r   ú<lambda>z*CpmAntTokenizer.__init__.<locals>.<lambda>„   ó   € ÐZ[Ð\]ÑZ^€ r'   ©Úkey)r   r#   )	Ú	bod_tokenÚ	eod_tokenÚ	bos_tokenÚ	eos_tokenÚ	pad_tokenr#   Ú
line_tokenÚspace_tokenÚpadding_sider;   )r   rK   rL   r   Úencoderr   r   ÚsortedÚitemsÚdecoderr    Úwordpiece_tokenizerÚsuperr&   )r%   r   rK   rL   rM   rN   rO   r#   rP   rQ   rR   ÚkwargsÚkÚvÚ	__class__s                 €r   r&   zCpmAntTokenizer.__init__l   s  ø€ ô 	˜$  	Ô*Ø"ˆŒØ"ˆŒÜ! *Ó-ˆŒØ ŸL™L¨Ñ5ˆ�‰�SÑØ!Ÿ\™\¨*Ñ5ˆ�‰�TÑà�L‰L˜Ð%Ø�L‰L˜Ð$ä"×.Ñ.¬v°d·l±l×6HÑ6HÓ6JÑP^Ô/_Ó`ˆŒØ)-¯©×);Ñ);Ó)=×>¡  A˜˜1™Ó>ˆŒä#5¸D¿L¹LÐT]Ô#^ˆÔ ä‰Ñð 	
ØØØØØØØ!Ø#Ø%ñ	
ð ó	
ùó	 ?s   ÃD&c                 ó4   — | j                   | j                     S r"   )rS   rK   ©r%   s    r   Úbod_token_idzCpmAntTokenizer.bod_token_id–   ó   € à�|‰|˜DŸN™NÑ+Ð+r'   c                 ó4   — | j                   | j                     S r"   )rS   rL   r^   s    r   Úeod_token_idzCpmAntTokenizer.eod_token_idš   r`   r'   c                 ó    — | j                   d   S )Nr   ©rS   r^   s    r   Ú
newline_idzCpmAntTokenizer.newline_idž   s   € à�|‰|˜DÑ!Ð!r'   Úreturnc                 ó,   — t        | j                  «      S r"   )r,   rS   r^   s    r   Ú
vocab_sizezCpmAntTokenizer.vocab_size¢   s   € ä�4—<‘<Ó Ð r'   c                 óB   — t        | j                  fi | j                  ¤ŽS r"   )ÚdictrS   Úadded_tokens_encoderr^   s    r   Ú	get_vocabzCpmAntTokenizer.get_vocab¦   s   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>r'   c                 ó–   — g }t        j                  |d¬«      D ],  }|j                  | j                  j	                  |«      «       Œ. |S )zTokenize a string.F)Úcut_all)rA   ÚcutÚextendrW   r5   )r%   ÚtextÚoutput_tokensrF   s       r   Ú	_tokenizezCpmAntTokenizer._tokenize©   sH   € àˆÜ—‘˜4¨Ô/ò 	GˆAØ× Ñ  ×!9Ñ!9×!BÑ!BÀ1Ó!EÕFð	GàÐr'   c                 óà   •— |D �cg c]
  }|dk\  sŒ	|‘Œ }}|D �cg c]4  }|| j                   k7  sŒ|| j                  k7  sŒ#|| j                  k7  sŒ3|‘Œ6 }}t        ‰| �  |fi |¤ŽS c c}w c c}w )zDecode ids into a string.r   )Úpad_token_idÚeos_token_idÚbos_token_idrX   Ú_decode)r%   Ú	token_idsrY   ÚirF   r\   s        €r   rx   zCpmAntTokenizer._decode°   s   ø€ à )Ö4˜1¨Q°!«V’QÐ4ˆ	Ð4à ö
Ø A¨×):Ñ):Ó$:¸qÀD×DUÑDUÓ?UÐZ[Ð_c×_pÑ_pÓZpŠAð
ˆ	ð 
ô ‰w‰˜yÑ3¨FÑ3Ð3ùò	 5ùò
s    †
A&‘A&›A+°A+Á A+ÁA+c                 ó   — || j                   v S r"   rd   ©r%   r   s     r   ÚcheckzCpmAntTokenizer.check¸   s   € Ø˜Ÿ™Ð$Ð$r'   r   c                 ó$   — dj                  |«      S )Nr)   )r-   )r%   r   s     r   Úconvert_tokens_to_stringz(CpmAntTokenizer.convert_tokens_to_string»   s   € Ø�w‰w�v‹Ðr'   c                 ó€   — | j                   j                  || j                   j                  | j                  «      «      S )z0Converts a token (str) in an id using the vocab.)rS   Úgetr#   r|   s     r   Ú_convert_token_to_idz$CpmAntTokenizer._convert_token_to_id¾   s,   € à�|‰|×Ñ  t§|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHr'   c                 óN   — | j                   j                  || j                  «      S )z=Converts an index (integer) in a token (str) using the vocab.)rV   r�   r#   )r%   r   s     r   Ú_convert_id_to_tokenz$CpmAntTokenizer._convert_id_to_tokenÂ   s   € à�|‰|×Ñ  t§~¡~Ó6Ð6r'   Úsave_directoryÚfilename_prefixc                 ó   — t         j                  j                  |«      r2t         j                  j                  ||r|dz   ndt        d   z   «      }n|r|dz   nd|z   }d}d| j
                  v r)| j
                  d   | j
                  d<   | j
                  d= d| j
                  v r)| j
                  d   | j
                  d<   | j
                  d= t        j                  t        | j
                  j                  «       d	„ ¬
«      «      | _        t        |dd¬«      5 }| j
                  j                  «       D ]>  \  }}||k7  rt        j                  d|› d�«       |}|j                  |dz   «       |dz  }Œ@ 	 d d d «       |fS # 1 sw Y   |fS xY w)Nú-r)   r   r   rB   ú</_>r   ú</n>c                 ó   — | d   S rD   r;   rE   s    r   rG   z1CpmAntTokenizer.save_vocabulary.<locals>.<lambda>Ô   rH   r'   rI   Úwr   r   zSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!r*   )ÚosÚpathÚisdirr-   ÚVOCAB_FILES_NAMESrS   r   r   rT   rU   r   ÚloggerÚwarningÚwrite)r%   r…   r†   r   r   Úwriterr   Útoken_indexs           r   Úsave_vocabularyzCpmAntTokenizer.save_vocabularyÆ   sx  € Ü�7‰7�=‰=˜Ô(ÜŸ™Ÿ™Ø¹/ °3Ò!6ÈrÔUfÐgsÑUtÑ tó‰Jñ 4C˜/¨CÒ/ÈÈnÑ\ˆJØˆØ�$—,‘,ÑØ#'§<¡<°Ñ#4ˆD�L‰L˜Ñ Ø—‘˜SÐ!Ø�4—<‘<ÑØ#'§<¡<°Ñ#5ˆD�L‰L˜Ñ Ø—‘˜TÐ"Ü"×.Ñ.¬v°d·l±l×6HÑ6HÓ6JÑP^Ô/_Ó`ˆŒÜ�*˜c¨GÔ4ð 		¸Ø&*§l¡l×&8Ñ&8Ó&:ò Ñ"��{Ø˜KÒ'Ü—N‘NØ/°
¨|ð <Nð Nôð (�EØ—‘˜U T™\Ô*Ø˜‘
‘ñ÷		ð ˆ}Ð÷		ð ˆ}Ðús   ÄAFÆFÚtoken_ids_0Útoken_ids_1c                 óh   — |€| j                   g|z   S | j                   g|z   | j                   gz   |z   S )a1  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A CPMAnt sequence has the following format:

        - single sequence: `[BOS] Sequence`.

        Args:
            token_ids_0 (`List[int]`): The first tokenized sequence that special tokens will be added.
            token_ids_1 (`List[int]`): The optional second tokenized sequence that special tokens will be added.

        Returns:
            `List[int]`: The model input with special tokens.
        )rw   )r%   r—   r˜   s      r   Ú build_inputs_with_special_tokensz0CpmAntTokenizer.build_inputs_with_special_tokensá   sE   € ð ÐØ×%Ñ%Ð&¨Ñ4Ð4Ø×!Ñ!Ð" [Ñ0°D×4EÑ4EÐ3FÑFÈÑTÐTr'   Úalready_has_special_tokensc                 ó¤   •— |rt         ‰| �  ||d¬«      S |�'dgdgt        |«      z  z   dgz   dgt        |«      z  z   S dgdgt        |«      z  z   S )a¤  
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`): List of IDs.
            token_ids_1 (`List[int]`, *optional*): Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        T)r—   r˜   r›   r*   r   )rX   Úget_special_tokens_maskr,   )r%   r—   r˜   r›   r\   s       €r   r�   z'CpmAntTokenizer.get_special_tokens_maskó   sy   ø€ ñ" &Ü‘7Ñ2Ø'°[Ð]að 3ó ð ð Ð"Ø�3˜1˜#¤ KÓ 0Ñ0Ñ1°Q°CÑ7¸A¸3ÄÀ[ÓAQÑ;QÑRÐRØˆs�q�cœC Ó,Ñ,Ñ-Ð-r'   )	z<d>z</d>z<s>z</s>z<pad>r6   rŠ   r‰   Úleftr"   )NF)r8   r9   r:   Ú__doc__r�   Úvocab_files_namesÚmodel_input_namesÚadd_prefix_spacer&   Úpropertyr_   rb   re   Úintrh   rl   rs   rx   r}   r   Ústrr   r‚   r„   r   r   r–   rš   Úboolr�   Ú__classcell__)r\   s   @r   r=   r=   O   si  ø„ ñð0 *ÐØ$Ð&6Ð7ÐØÐð
 ØØØØØØØØõ(
ðT ñ,ó ð,ð ñ,ó ð,ð ñ"ó ð"ð ð!˜Cò !ó ð!ò?òô4ò%ð¨t°C©yð ¸Só òIò7ñ¨cð ÀHÈSÁMð Ð]bÐcfÑ]gó ñ6U¸DÀ¹Ið UÐTXÐY\ÑT]ð UÐimÐnqÑiró Uð& sxñ.Ø ™9ð.Ø3;¸DÀ¹IÑ3Fð.Økoð.à	ˆc‰÷.ñ .r'   r=   )rŸ   r   r�   Útypingr   r   r   Útransformers.utilsr   r   rA   Útokenization_utilsr	   Úutilsr
   Ú
get_loggerr8   r‘   r�   r   r    r=   Ú__all__r;   r'   r   ú<module>r®      ss   ðñ 'ã Û 	ß (Ñ (ç Dñ ÔÛå 5Ý ð 
ˆ×	Ñ	˜HÓ	%€à! ;Ð/Ð ò÷ñ ô@|.Ð)ô |.ð~ Ð
�r'   