Ë
    S^(hg$  ã                   ó.  — U d Z ddlmZmZmZ ddlmZmZ ddlm	Z	  e	j                  e«      ZdZdZdZdZd	Zd
ZdZedededededediZeeef   ed<   ej1                  «       D � �ci c]  \  } }|| “Œ
 c}} Zeeef   ed<    G d„ de«      ZdgZyc c}} w )z Tokenization classes for CANINE.é    )ÚDictÚListÚOptionalé   )Ú
AddedTokenÚPreTrainedTokenizer)Úloggingi   i à  ià  ià  ià  ià  z[CLS]z[SEP]z[BOS]z[MASK]z[PAD]z
[RESERVED]ÚSPECIAL_CODEPOINTSÚSPECIAL_CODEPOINTS_BY_NAMEc            
       ó„  ‡ — e Zd ZdZ ee«       ee«       ee«       ee«       ee«       ee«      ddfˆ fd„	Z	e
defd„«       Zd„ Zdedee   fd	„Zd
edefd„Zdedefd„Zd„ Z	 ddee   deee      dee   fd„Z	 ddee   deee      dedee   fˆ fd„Z	 ddee   deee      dee   fd„Zddedee   fd„Zˆ xZS )ÚCanineTokenizeraé  
    Construct a CANINE tokenizer (i.e. a character splitter). It turns text into a sequence of characters, and then
    converts each character into its Unicode code point.

    [`CanineTokenizer`] inherits from [`PreTrainedTokenizer`].

    Refer to superclass [`PreTrainedTokenizer`] for usage examples and documentation concerning parameters.

    Args:
        model_max_length (`int`, *optional*, defaults to 2048):
                The maximum sentence length the model accepts.
    Fi   c	                 óÖ  •— t        |t        «      rt        |dd¬«      n|}t        |t        «      rt        |dd¬«      n|}t        |t        «      rt        |dd¬«      n|}t        |t        «      rt        |dd¬«      n|}t        |t        «      rt        |dd¬«      n|}t        |t        «      rt        |dd¬«      n|}i | _        t        j                  «       D ]  \  }
}|
| j                  |<   Œ | j                  j                  «       D ��
ci c]  \  }}
|
|“Œ
 c}
}| _        t        | _        t        | j                  «      | _
        t        ‰| �0  d||||||||dœ|	¤Ž y c c}
}w )NF)ÚlstripÚrstripT)Ú	bos_tokenÚ	eos_tokenÚ	sep_tokenÚ	cls_tokenÚ	pad_tokenÚ
mask_tokenÚadd_prefix_spaceÚmodel_max_length© )Ú
isinstanceÚstrr   Ú_special_codepointsr
   ÚitemsÚ_special_codepoint_stringsÚUNICODE_VOCAB_SIZEÚ_unicode_vocab_sizeÚlenÚ_num_special_tokensÚsuperÚ__init__)Úselfr   r   r   r   r   r   r   r   ÚkwargsÚ	codepointÚnameÚ	__class__s               €úl/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/canine/tokenization_canine.pyr$   zCanineTokenizer.__init__H   sp  ø€ ô JTÐT]Ô_bÔIc”J˜y°¸uÕEÐirˆ	ÜISÐT]Ô_bÔIc”J˜y°¸uÕEÐirˆ	ÜISÐT]Ô_bÔIc”J˜y°¸uÕEÐirˆ	ÜISÐT]Ô_bÔIc”J˜y°¸uÕEÐirˆ	ÜISÐT]Ô_bÔIc”J˜y°¸uÕEÐirˆ	ô KUÐU_ÔadÔJe”Z 
°4ÀÕFÐkuˆ
ð 46ˆÔ Ü1×7Ñ7Ó9ò 	7‰OˆI�tØ-6ˆD×$Ñ$ TÒ*ð	7ð
 48×3KÑ3K×3QÑ3QÓ3S÷;
Ù /  iˆI�t‰Oó;
ˆÔ'ô $6ˆÔ Ü#& t×'?Ñ'?Ó#@ˆÔ ä‰Ñð 
	
ØØØØØØ!Ø-Ø-ñ
	
ð ó
	
ùó;
s   ÄE%Úreturnc                 ó   — | j                   S ©N)r    )r%   s    r*   Ú
vocab_sizezCanineTokenizer.vocab_sizev   s   € à×'Ñ'Ð'ó    c                 óž   — t        | j                  «      D �ci c]  }t        |«      |“Œ }}|j                  | j                  «       |S c c}w r-   )Úranger.   ÚchrÚupdateÚadded_tokens_encoder)r%   ÚiÚvocabs      r*   Ú	get_vocabzCanineTokenizer.get_vocabz   sB   € Ü$)¨$¯/©/Ó$:Ö;˜q”�Q“˜‘Ð;ˆÐ;Ø�‰�T×.Ñ.Ô/Øˆùò <s   ˜A
Útextc                 ó   — t        |«      S )z5Tokenize a string (i.e. perform character splitting).)Úlist)r%   r8   s     r*   Ú	_tokenizezCanineTokenizer._tokenize   s   € ä�D‹zÐr/   Útokenc                 óR   — 	 t        |«      S # t        $ r t        d|› d�«      ‚w xY w)zaConverts a token (i.e. a Unicode character) in an id (i.e. its integer Unicode code point value).zinvalid token: 'ú')ÚordÚ	TypeErrorÚ
ValueError)r%   r<   s     r*   Ú_convert_token_to_idz$CanineTokenizer._convert_token_to_idƒ   s5   € ð	:Ü�u“:ÐøÜò 	:ÜÐ/°¨w°aÐ8Ó9Ð9ð	:ús   ‚
 �&Úindexc                 ór   — 	 |t         v r	t         |   S t        |«      S # t        $ r t        d|› �«      ‚w xY w)z˜
        Converts a Unicode code point (integer) in a token (str). In case it's a special code point, convert to
        human-readable format.
        zinvalid id: )r
   r2   r@   rA   )r%   rC   s     r*   Ú_convert_id_to_tokenz$CanineTokenizer._convert_id_to_tokenŠ   sF   € ð
	5ØÔ*Ñ*Ü)¨%Ñ0Ð0Ü�u“:ÐøÜò 	5Ü˜|¨E¨7Ð3Ó4Ð4ð	5ús   ‚ “
 ž6c                 ó$   — dj                  |«      S )NÚ )Újoin)r%   Útokenss     r*   Úconvert_tokens_to_stringz(CanineTokenizer.convert_tokens_to_string–   s   € Ø�w‰w�v‹Ðr/   Útoken_ids_0Útoken_ids_1c                 ó^   — | j                   g}| j                  g}||z   |z   }|�|||z   z  }|S )a˜  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A CANINE sequence has the following format:

        - single sequence: `[CLS] X [SEP]`
        - pair of sequences: `[CLS] A [SEP] B [SEP]`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        )Úsep_token_idÚcls_token_id©r%   rK   rL   ÚsepÚclsÚresults         r*   Ú build_inputs_with_special_tokensz0CanineTokenizer.build_inputs_with_special_tokens™   sI   € ð& × Ñ Ð!ˆØ× Ñ Ð!ˆà�{Ñ" SÑ(ˆØÐ"Ø�k CÑ'Ñ'ˆFØˆr/   Úalready_has_special_tokensc                 óŽ   •— |rt         ‰| �  ||d¬«      S dgdgt        |«      z  z   dgz   }|�|dgt        |«      z  dgz   z  }|S )aÄ  
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        T)rK   rL   rU   é   r   )r#   Úget_special_tokens_maskr!   )r%   rK   rL   rU   rS   r)   s        €r*   rX   z'CanineTokenizer.get_special_tokens_mask´   sp   ø€ ñ$ &Ü‘7Ñ2Ø'°[Ð]að 3ó ð ð �˜˜œc +Ó.Ñ.Ñ/°1°#Ñ5ˆØÐ"Ø˜�sœS Ó-Ñ-°!°Ñ4Ñ4ˆFØˆr/   c                 ó’   — | j                   g}| j                  g}t        ||z   |z   «      dgz  }|�|t        ||z   «      dgz  z  }|S )aÓ  
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. A CANINE
        sequence pair mask has the following format:

        ```
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |
        ```

        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        r   rW   )rN   rO   r!   rP   s         r*   Ú$create_token_type_ids_from_sequencesz4CanineTokenizer.create_token_type_ids_from_sequencesÐ   sa   € ð. × Ñ Ð!ˆØ× Ñ Ð!ˆä�S˜;Ñ&¨Ñ,Ó-°°Ñ3ˆØÐ"Ø”c˜+¨Ñ+Ó,°¨sÑ2Ñ2ˆFØˆr/   Úsave_directoryÚfilename_prefixc                  ó   — y)Nr   r   )r%   r[   r\   s      r*   Úsave_vocabularyzCanineTokenizer.save_vocabularyð   s   € Ør/   r-   )NF)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r2   ÚCLSÚSEPÚPADÚMASKr$   ÚpropertyÚintr.   r7   r   r   r;   rB   rE   rJ   r   rT   ÚboolrX   rZ   r^   Ú__classcell__)r)   s   @r*   r   r   :   sa  ø„ ññ �c“(Ù�c“(Ù�c“(Ù�c“(Ù�c“(Ù�t“9ØØõ,
ð\ ð(˜Cò (ó ð(òð
˜cð  d¨3¡ió ð:¨#ð :°#ó :ð
5¨#ð 
5°#ó 
5òð JNñØ ™9ðØ3;¸DÀ¹IÑ3Fðà	ˆc‰óð8 sxñØ ™9ðØ3;¸DÀ¹IÑ3FðØkoðà	ˆc‰õð: JNñØ ™9ðØ3;¸DÀ¹IÑ3Fðà	ˆc‰óñ@¨cð ÀHÈSÁM÷ r/   r   N)rb   Útypingr   r   r   Útokenization_utilsr   r   Úutilsr	   Ú
get_loggerr_   Úloggerr   re   rc   rd   ÚBOSrf   ÚRESERVEDr
   rh   r   Ú__annotations__r   r   r   Ú__all__)r'   r(   s   00r*   ú<module>rt      sÝ   ðò 'ç 'Ñ 'ç AÝ ð 
ˆ×	Ñ	˜HÓ	%€ð Ð ð €Ø€Ø€Ø€Ø€Ø€ð ˆØˆØˆØˆ(ØˆØˆlð&Ð �D˜˜c˜‘Nó ð  Vh×UmÑUmÓUo×-pÁ/À)ÈT¨d°I©oÓ-pÐ ˜D  c ™NÓ pôwÐ)ô wðt Ð
�ùó{ .qs   Á'B