Ë
    S^(hÉ.  ã                   ó¬   — d dl Z d dlmZmZmZ d dlZd dlmZ	 d dlm
Z
mZmZmZmZ ddlmZ ddlmZ  G d„ d	ej$                  j&                  «      Zd	gZy)
é    N)ÚListÚOptionalÚUnion)ÚBertTokenizer)ÚFastBertTokenizerÚShrinkLongestTrimmerÚcase_fold_utf8Úcombine_segmentsÚpad_model_inputsé   )Úkerasé   c                   óê   ‡ — e Zd ZdZ	 	 	 	 	 	 	 	 	 	 ddededee   dee   dee   deded	ed
ee   dededefˆ fd„Z	e
dd„«       Ze
deeej                  f   fd„«       Zd„ Z	 	 	 	 	 	 	 dd„Zd„ Zˆ xZS )ÚTFBertTokenizeraà  
    This is an in-graph tokenizer for BERT. It should be initialized similarly to other tokenizers, using the
    `from_pretrained()` method. It can also be initialized with the `from_tokenizer()` method, which imports settings
    from an existing standard tokenizer object.

    In-graph tokenizers, unlike other Hugging Face tokenizers, are actually Keras layers and are designed to be run
    when the model is called, rather than during preprocessing. As a result, they have somewhat more limited options
    than standard tokenizer classes. They are most useful when you want to create an end-to-end model that goes
    straight from `tf.string` inputs to outputs.

    Args:
        vocab_list (`list`):
            List containing the vocabulary.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        cls_token_id (`str`, *optional*, defaults to `"[CLS]"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        sep_token_id (`str`, *optional*, defaults to `"[SEP]"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token_id (`str`, *optional*, defaults to `"[PAD]"`):
            The token used for padding, for example when batching sequences of different lengths.
        padding (`str`, defaults to `"longest"`):
            The type of padding to use. Can be either `"longest"`, to pad only up to the longest sample in the batch,
            or `"max_length", to pad all inputs to the maximum length supported by the tokenizer.
        truncation (`bool`, *optional*, defaults to `True`):
            Whether to truncate the sequence to the maximum length.
        max_length (`int`, *optional*, defaults to `512`):
            The maximum length of the sequence, used for padding (if `padding` is "max_length") and/or truncation (if
            `truncation` is `True`).
        pad_to_multiple_of (`int`, *optional*, defaults to `None`):
            If set, the sequence will be padded to a multiple of this value.
        return_token_type_ids (`bool`, *optional*, defaults to `True`):
            Whether to return token_type_ids.
        return_attention_mask (`bool`, *optional*, defaults to `True`):
            Whether to return the attention_mask.
        use_fast_bert_tokenizer (`bool`, *optional*, defaults to `True`):
            If True, will use the FastBertTokenizer class from Tensorflow Text. If False, will use the BertTokenizer
            class instead. BertTokenizer supports some additional options, but is slower and cannot be exported to
            TFLite.
    Ú
vocab_listÚdo_lower_caseÚcls_token_idÚsep_token_idÚpad_token_idÚpaddingÚ
truncationÚ
max_lengthÚpad_to_multiple_ofÚreturn_token_type_idsÚreturn_attention_maskÚuse_fast_bert_tokenizerc                 ó.  •— t         ‰| �  «        |r#t        |ft        j                  |dœ|¤Ž| _        nÆt        j                  j                  t        j                  j                  |t        j                  t        j                  t        j                  |t        j                  ¬«      t        j                  ¬«      t        j                  ¬«      d¬«      }t        |ft        j                  |dœ|¤Ž| _        || _        || _        |€|j                  d«      n|| _        |€|j                  d	«      n|| _        |€|j                  d
«      n|| _        t'        |dz
  d¬«      | _        || _        || _        || _        |	| _        |
| _        || _        y )N)Útoken_out_typeÚlower_case_nfd_strip_accents)Úout_type)Údtype)ÚkeysÚ	key_dtypeÚvaluesÚvalue_dtyper   )Únum_oov_buckets)r   Ú
lower_casez[CLS]z[SEP]z[PAD]r   ©Úaxis)ÚsuperÚ__init__r   ÚtfÚint64Útf_tokenizerÚlookupÚStaticVocabularyTableÚKeyValueTensorInitializerÚstringÚrangeÚsizeÚBertTokenizerLayerr   r   Úindexr   r   r   r   Úpaired_trimmerr   r   r   r   r   r   )Úselfr   r   r   r   r   r   r   r   r   r   r   r   Útokenizer_kwargsÚlookup_tableÚ	__class__s                  €úk/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/bert/tokenization_bert_tf.pyr+   zTFBertTokenizer.__init__9   sp  ø€ ô  	‰ÑÔÙ"Ü 1Øð!Ü+-¯8©8ÐR_ñ!Øcsñ!ˆDÕô Ÿ9™9×:Ñ:Ü—	‘	×3Ñ3Ø#Ü Ÿi™iÜŸ8™8¤B§G¡G¨JÄÇÁÔ$JÔRT×RZÑRZÔ[Ü "§¡ð	 4ó ð !"ð ;ó ˆLô !3Øð!Ü-/¯X©XÀ-ñ!ØScñ!ˆDÔð %ˆŒØ*ˆÔØ9EÐ9M˜J×,Ñ,¨WÔ5ÐS_ˆÔØ9EÐ9M˜J×,Ñ,¨WÔ5ÐS_ˆÔØ9EÐ9M˜J×,Ñ,¨WÔ5ÐS_ˆÔÜ2°:À±>ÈÔJˆÔØ$ˆŒØˆŒØ$ˆŒØ"4ˆÔØ%:ˆÔ"Ø%:ˆÕ"ó    c           	      ó´  — |j                  dd«      }|€|j                  n|}|j                  dd«      }|€|j                  n|}|j                  dd«      }|€|j                  n|}|j                  dd«      }|€|j                  n|}|j                  «       }t        |j                  «       d„ ¬«      }|D �cg c]  }|d   ‘Œ	 }	} | d
|	||||d	œ|¤ŽS c c}w )aá  
        Initialize a `TFBertTokenizer` from an existing `Tokenizer`.

        Args:
            tokenizer (`PreTrainedTokenizerBase`):
                The tokenizer to use to initialize the `TFBertTokenizer`.

        Examples:

        ```python
        from transformers import AutoTokenizer, TFBertTokenizer

        tokenizer = AutoTokenizer.from_pretrained("google-bert/bert-base-uncased")
        tf_tokenizer = TFBertTokenizer.from_tokenizer(tokenizer)
        ```
        r   Nr   r   r   c                 ó   — | d   S )Nr   © )Úxs    r<   ú<lambda>z0TFBertTokenizer.from_tokenizer.<locals>.<lambda>…   s
   € °A°a±D€ r=   )Úkeyr   ©r   r   r   r   r   r@   )Úpopr   r   r   r   Ú	get_vocabÚsortedÚitems)
ÚclsÚ	tokenizerÚkwargsr   r   r   r   ÚvocabÚentryr   s
             r<   Úfrom_tokenizerzTFBertTokenizer.from_tokenizeri   só   € ð$ Ÿ
™
 ?°DÓ9ˆØ3@Ð3H˜	×/Ò/ÈmˆØ—z‘z .°$Ó7ˆØ1=Ð1E�y×-Ò-È<ˆØ—z‘z .°$Ó7ˆØ1=Ð1E�y×-Ò-È<ˆØ—z‘z .°$Ó7ˆØ1=Ð1E�y×-Ò-È<ˆà×#Ñ#Ó%ˆÜ�u—{‘{“}©.Ô9ˆØ,1Ö2 5�e˜A“hÐ2ˆ
Ð2Ùð 
Ø!Ø'Ø%Ø%Ø%ñ
ð ñ
ð 	
ùò 3s   Â9CÚpretrained_model_name_or_pathc                 ó¢   — 	 t        j                  |g|¢­i |¤Ž} | j                  |fi |¤ŽS #  ddlm}  |j                  |g|¢­i |¤Ž}Y Œ3xY w)a�  
        Instantiate a `TFBertTokenizer` from a pre-trained tokenizer.

        Args:
            pretrained_model_name_or_path (`str` or `os.PathLike`):
                The name or path to the pre-trained tokenizer.

        Examples:

        ```python
        from transformers import TFBertTokenizer

        tf_tokenizer = TFBertTokenizer.from_pretrained("google-bert/bert-base-uncased")
        ```
        r   )ÚBertTokenizerFast)r   Úfrom_pretrainedÚtokenization_bert_fastrQ   rN   )rI   rO   Úinit_inputsrK   rJ   rQ   s         r<   rR   zTFBertTokenizer.from_pretrained�   sj   € ð"	qÜ%×5Ñ5Ð6SÐlÐVaÒlÐekÑlˆIð
 "ˆs×!Ñ! )Ñ6¨vÑ6Ð6øð		qÝAà9Ð)×9Ñ9Ð:WÐpÐZeÒpÐioÑpŠIús	   ‚. ®Ac                 óŠ   — | j                   rt        |«      }| j                  j                  |«      }|j	                  dd«      S )Nr   éÿÿÿÿ)r   r	   r.   ÚtokenizeÚ
merge_dims)r8   ÚtextsÚtokenss      r<   Úunpaired_tokenizez!TFBertTokenizer.unpaired_tokenize©   s>   € Ø×ÒÜ" 5Ó)ˆEØ×"Ñ"×+Ñ+¨EÓ2ˆØ× Ñ   BÓ'Ð'r=   c	                 óî  — |€| j                   }|dvrt        d«      ‚|�|�t        d«      ‚|€| j                  }|€| j                  }|€| j                  }|€| j
                  }|€| j                  }t        |t        j                  «      st        j                  |«      }|�/t        |t        j                  «      st        j                  |«      }|�H|j                  j                  dkD  rt        d«      ‚|j                  j                  dkD  rt        d«      ‚|j                  j                  dk(  r|d d …df   |d d …df   }}| j                  |«      }|€7|r|d d …d |dz
  …f   }t        |f| j                  | j                   ¬	«      \  }	}
nZ| j                  |«      }|r | j"                  j%                  ||g«      \  }}t        ||f| j                  | j                   ¬	«      \  }	}
|d
k(  r:|	j'                  d¬«      }|�(|t        j(                  j+                  | |«       z  }n|}t-        |	|| j.                  ¬«      \  }	}d|	i}|r||d<   |r t-        |
|| j.                  ¬«      \  }
}|
|d<   |S )N)Úlongestr   z1Padding must be either 'longest' or 'max_length'!zJmax_length cannot be overridden at call time when truncating paired texts!r   zJtext argument should not be multidimensional when a text pair is supplied!z)text_pair should not be multidimensional!é   r   )Ústart_of_sequence_idÚend_of_segment_idr]   r(   )Úmax_seq_lengthÚ	pad_valueÚ	input_idsÚattention_maskÚtoken_type_ids)r   Ú
ValueErrorr   r   r   r   r   Ú
isinstancer,   ÚTensorÚconvert_to_tensorÚshapeÚrankr[   r
   r   r   r7   ÚtrimÚbounding_shapeÚmathÚfloordivr   r   )r8   ÚtextÚ	text_pairr   r   r   r   r   r   rc   re   Ú
pad_lengthrd   ÚoutputÚ_s                  r<   ÚcallzTFBertTokenizer.call¯   s›  € ð ˆ?Ø—l‘lˆGØÐ3Ñ3ÜÐPÓQÐQØÐ! iÐ&;äÐiÓjÐjØÐØŸ™ˆJØÐØŸ™ˆJØÐ%Ø!%×!8Ñ!8ÐØ Ð(Ø$(×$>Ñ$>Ð!Ø Ð(Ø$(×$>Ñ$>Ð!Ü˜$¤§	¡	Ô*Ü×'Ñ'¨Ó-ˆDØÐ ¬°I¼r¿y¹yÔ)IÜ×,Ñ,¨YÓ7ˆIØÐ Ø�z‰z�‰ Ò"Ü Ð!mÓnÐnØ�‰×#Ñ# aÒ'Ü Ð!LÓMÐMØ�:‰:�?‰?˜aÒØ"¢1 a 4™j¨$ªq°!¨t©*�)ˆDØ×%Ñ% dÓ+ˆØÐÙØšAÐ/ ¨a¡Ð/Ð/Ñ0�Ü(8Ø�¨d×.?Ñ.?ÐSW×SdÑSdô)Ñ%ˆI‘~ð ×.Ñ.¨yÓ9ˆIÙØ"&×"5Ñ"5×":Ñ":¸DÀ)Ð;LÓ"M‘��iÜ(8Ø�yÐ!¸×8IÑ8IÐ]a×]nÑ]nô)Ñ%ˆI�~ð �iÒØ"×1Ñ1°qÐ1Ó9ˆJØ!Ð-à/´B·G±G×4DÑ4DÀjÀ[ÐRdÓ4eÐ3eÑf‘
à#ˆJä$4°YÈzÐei×evÑevÔ$wÑ!ˆ	�>Ø˜yÐ)ˆÙ Ø'5ˆFÐ#Ñ$Ù Ü 0Ø¨zÀT×EVÑEVô!ÑˆN˜Að (6ˆFÐ#Ñ$Øˆr=   c                 óv   — | j                   | j                  | j                  | j                  | j                  dœS )NrD   rD   )r8   s    r<   Ú
get_configzTFBertTokenizer.get_config÷   s7   € àŸ/™/Ø!×/Ñ/Ø ×-Ñ-Ø ×-Ñ-Ø ×-Ñ-ñ
ð 	
r=   )
NNNr]   Ti   NTTT)rJ   ÚPreTrainedTokenizerBase)NNNNNNN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Úboolr   ÚintÚstrr+   ÚclassmethodrN   r   ÚosÚPathLikerR   r[   ru   rw   Ú__classcell__)r;   s   @r<   r   r      s  ø„ ñ*ð` '+Ø&*Ø&*Ø ØØØ,0Ø&*Ø&*Ø(,ñ.;àð.;ð ð.;ð ˜s‘mð	.;ð
 ˜s‘mð.;ð ˜s‘mð.;ð ð.;ð ð.;ð ð.;ð % S™Mð.;ð  $ð.;ð  $ð.;ð "&õ.;ð` ò$
ó ð$
ðL ð7¸EÀ#ÀrÇ{Á{ÐBRÑ<Sò 7ó ð7ò0(ð ØØØØØ"Ø"óFöP
r=   r   )r�   Útypingr   r   r   Ú
tensorflowr,   Útensorflow_textr   r5   r   r   r	   r
   r   Úmodeling_tf_utilsr   Útokenization_bertÚlayersÚLayerr   Ú__all__r@   r=   r<   ú<module>rŒ      sC   ðÛ 	ß (Ñ (ã Ý ?ß wÕ wå &Ý ,ôr
�e—l‘l×(Ñ(ô r
ðj Ð
�r=   