Ë
    T^(h
•  ã                   óÚ   — d Z ddlZddlmZmZmZmZmZ ddlm	Z	 ddl
mZmZmZmZmZmZmZmZ ddlmZ ddlmZmZ d	d
lmZmZmZ  ej8                  e«      ZdddœZ G d„ de«      Z dgZ!y)z¯
Fast tokenization class for LayoutLMv2. It overwrites 2 methods of the slow tokenizer class, namely _batch_encode_plus
and _encode_plus, in which the Rust tokenizer is used.
é    N)ÚDictÚListÚOptionalÚTupleÚUnion)Únormalizersé   )ÚBatchEncodingÚEncodedInputÚPaddingStrategyÚPreTokenizedInputÚ
TensorTypeÚ	TextInputÚTextInputPairÚTruncationStrategy)ÚPreTrainedTokenizerFast)Úadd_end_docstringsÚloggingé   )Ú"LAYOUTLMV2_ENCODE_KWARGS_DOCSTRINGÚ2LAYOUTLMV2_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRINGÚLayoutLMv2Tokenizerz	vocab.txtztokenizer.json)Ú
vocab_fileÚtokenizer_filec            *       óð  ‡ — e Zd ZdZeZeZddddddddg d	¢g d
¢g d	¢ddddfˆ fd„	Z e	e
e«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d7deeeee   ee   f   deeeee   f      deeee      eeee         f   deeee   eee      f      dedeeeef   deeeef   dee   dedee   dee   deeeef      dee   dee   dededed ed!ed"ef(d#„«       Z e	e
e«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d7d$eee   ee   ee   f   d%ee   deeeee            deeee   eee      f      dedeeeef   deeeef   dee   dedee   dee   deeeef      dee   dee   dededed ed!ed"ef(d&„«       Zd8ded'ee   ded"ee   fd(„Z e	e
e«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d7deeef   dee   deeee         deee      dedeeeef   deeeef   dee   dedee   dee   deeeef      dee   dee   dededed ed!ed"ef(d)„«       Zddddej:                  ej<                  ddddddddddddfd$eee   ee   ee   f   d%ee   deeeee            deeee         ded*ed+edee   dedee   dee   dee   dee   dee   dededed ed!ed"ef(d,„Zddddej:                  ej<                  ddddddddddddfdeeef   dee   deeee         deee      ded*ed+edee   dedee   dee   dee   dee   dee   dededed ed!ed"ef(d-„Z dej:                  dddfd.ee!ee"f   ef   dee   d*edee   dee   dee   d"e#fd/„Z$d9d0„Z%	 d9d1ee   d2eee      d"ee   fd3„Z&d9d4ed5ee   d"e'e   fd6„Z(ˆ xZ)S ):ÚLayoutLMv2TokenizerFasta&  
    Construct a "fast" LayoutLMv2 tokenizer (backed by HuggingFace's *tokenizers* library). Based on WordPiece.

    This tokenizer inherits from [`PreTrainedTokenizerFast`] which contains most of the main methods. Users should
    refer to this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            File containing the vocabulary.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        cls_token_box (`List[int]`, *optional*, defaults to `[0, 0, 0, 0]`):
            The bounding box to use for the special [CLS] token.
        sep_token_box (`List[int]`, *optional*, defaults to `[1000, 1000, 1000, 1000]`):
            The bounding box to use for the special [SEP] token.
        pad_token_box (`List[int]`, *optional*, defaults to `[0, 0, 0, 0]`):
            The bounding box to use for the special [PAD] token.
        pad_token_label (`int`, *optional*, defaults to -100):
            The label to use for padding tokens. Defaults to -100, which is the `ignore_index` of PyTorch's
            CrossEntropyLoss.
        only_label_first_subword (`bool`, *optional*, defaults to `True`):
            Whether or not to only label the first subword, in case word labels are provided.
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see [this
            issue](https://github.com/huggingface/transformers/issues/328)).
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase` (as in the original LayoutLMv2).
    NTz[UNK]z[SEP]z[PAD]z[CLS]z[MASK])r   r   r   r   )éè  r   r   r   iœÿÿÿc                 óÚ  •— t        ‰| �  |f||||||||	|
|||||dœ|¤Ž t        j                  | j                  j
                  j                  «       «      }|j                  d|«      |k7  s|j                  d|«      |k7  r@t        t        |j                  d«      «      }||d<   ||d<    |di |¤Ž| j                  _        || _        |	| _        |
| _        || _        || _        || _        y )N)r   Údo_lower_caseÚ	unk_tokenÚ	sep_tokenÚ	pad_tokenÚ	cls_tokenÚ
mask_tokenÚcls_token_boxÚsep_token_boxÚpad_token_boxÚpad_token_labelÚonly_label_first_subwordÚtokenize_chinese_charsÚstrip_accentsÚ	lowercaser+   Útype© )ÚsuperÚ__init__ÚjsonÚloadsÚbackend_tokenizerÚ
normalizerÚ__getstate__ÚgetÚgetattrr   Úpopr   r%   r&   r'   r(   r)   )Úselfr   r   r   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   ÚkwargsÚpre_tok_stateÚpre_tok_classÚ	__class__s                      €úy/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/layoutlmv2/tokenization_layoutlmv2_fast.pyr0   z LayoutLMv2TokenizerFast.__init__b   s  ø€ ô& 	‰ÑØð	
à)Ø'ØØØØØ!Ø'Ø'Ø'Ø+Ø%=Ø#9Ø'ñ	
ð  ò!	
ô& Ÿ
™
 4×#9Ñ#9×#DÑ#D×#QÑ#QÓ#SÓTˆà×Ñ˜k¨=Ó9¸]ÒJØ× Ñ  °-Ó@ÀMÒQä#¤K°×1BÑ1BÀ6Ó1JÓKˆMØ)6ˆM˜+Ñ&Ø-:ˆM˜/Ñ*Ù0=Ñ0NÀÑ0NˆD×"Ñ"Ô-à*ˆÔð +ˆÔØ*ˆÔØ*ˆÔØ.ˆÔØ(@ˆÕ%ó    Fr   ÚtextÚ	text_pairÚboxesÚword_labelsÚadd_special_tokensÚpaddingÚ
truncationÚ
max_lengthÚstrideÚpad_to_multiple_ofÚpadding_sideÚreturn_tensorsÚreturn_token_type_idsÚreturn_attention_maskÚreturn_overflowing_tokensÚreturn_special_tokens_maskÚreturn_offsets_mappingÚreturn_lengthÚverboseÚreturnc                 óh  — d„ }|�4 ||«      st        d«      ‚t        |t        t        f«      s,t        d«      ‚t        |t        t        f«      st        d«      ‚|�t        |t        t        f«      }n5t        |t        t        f«      xr |xr t        |d   t        t        f«      }|€|n|}|€t        d«      ‚|rYt	        |«      t	        |«      k7  rt        d«      ‚t        ||«      D ]'  \  }}t	        |«      t	        |«      k7  sŒt        d«      ‚ n"t	        |«      t	        |«      k7  rt        d«      ‚|r®|�;t	        |«      t	        |«      k7  r$t        d	t	        |«      › d
t	        |«      › d�«      ‚|�t        t        ||«      «      n|}t        |du«      } | j                  d!i d|“d|“d|“d|“d|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS  | j                  d!i d|“d |“d|“d|“d|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS )"a3  
        Main method to tokenize and prepare for the model one or several sequence(s) or one or several pair(s) of
        sequences with word-level normalized bounding boxes and optional labels.

        Args:
            text (`str`, `List[str]`, `List[List[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence can be a string, a list of strings
                (words of a single example or questions of a batch of examples) or a list of list of strings (batch of
                words).
            text_pair (`List[str]`, `List[List[str]]`):
                The sequence or batch of sequences to be encoded. Each sequence should be a list of strings
                (pretokenized string).
            boxes (`List[List[int]]`, `List[List[List[int]]]`):
                Word-level bounding boxes. Each bounding box should be normalized to be on a 0-1000 scale.
            word_labels (`List[int]`, `List[List[int]]`, *optional*):
                Word-level integer labels (for token classification tasks such as FUNSD, CORD).
        c                 ó  — t        | t        «      ryt        | t        t        f«      rft	        | «      dk(  ryt        | d   t        «      ryt        | d   t        t        f«      r)t	        | d   «      dk(  xs t        | d   d   t        «      S yy)NTr   F)Ú
isinstanceÚstrÚlistÚtupleÚlen)Úts    r>   Ú_is_valid_text_inputz>LayoutLMv2TokenizerFast.__call__.<locals>._is_valid_text_inputÆ   sz   € Ü˜!œSÔ!àÜ˜A¤¤e˜}Ô-ä�q“6˜Q’;àÜ  !¡¤cÔ*àÜ  !¡¤t¬U mÔ4ä˜q ™t›9¨™>ÒE¬Z¸¸!¹¸Q¹ÄÓ-EÐEà àr?   NzStext input must of type `str` (single example) or `List[str]` (batch of examples). zwWords must be of type `List[str]` (single pretokenized example), or `List[List[str]]` (batch of pretokenized examples).r   z-You must provide corresponding bounding boxesz@You must provide words and boxes for an equal amount of examplesz:You must provide as many words as there are bounding boxeszbatch length of `text`: z- does not match batch length of `text_pair`: ú.Úbatch_text_or_text_pairsÚis_pairrB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r@   rA   r.   )	Ú
ValueErrorrV   rX   rY   rZ   ÚzipÚboolÚbatch_encode_plusÚencode_plus)r9   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r:   r\   Ú
is_batchedÚwordsÚwords_exampleÚboxes_exampler^   r_   s                               r>   Ú__call__z LayoutLMv2TokenizerFast.__call__›   sa  € òV	ð( Ð á'¨Ô-Ü Ð!vÓwÐwÜ˜i¬$´¨Ô7Ü ðMóð ô ˜d¤T¬5 MÔ2Ü ðMóð ð
 Ð Ü# D¬4´¨-Ó8‰Jä# D¬4´¨-Ó8Òh¸TÒhÄjÐQUÐVWÑQXÔ[_ÔafÐZgÓFhˆJà!Ð)‘¨yˆØˆ=ÜÐLÓMÐMÙÜ�5‹zœS ›ZÒ'Ü Ð!cÓdÐdÜ03°E¸5Ó0Aò cÑ,�˜}Ü�}Ó%¬¨]Ó);Ó;Ü$Ð%aÓbÐbñcô �5‹zœS ›ZÒ'Ü Ð!]Ó^Ð^áØÐ$¬¨T«´c¸)³nÒ)DÜ Ø.¬s°4«y¨kð :Ü˜I›Ð' qð*óð ð FOÐEZ¤t¬C°°iÓ,@Ô'AÐ`dÐ$Ü˜9¨DÐ0Ó1ˆGØ)�4×)Ñ)ò Ù)Aðáðñ ðñ (ð	ñ
 $6ðñ  ðñ &ðñ &ðñ ðñ $6ðñ *ðñ  .ðñ '<ðñ '<ðñ +Dðñ  ,Fð!ñ" (>ð#ñ$ ,ð%ñ&  Øñ)ð ð. $�4×#Ñ#ò Ùðá#ðñ ðñ (ð	ñ
 $6ðñ  ðñ &ðñ &ðñ ðñ $6ðñ *ðñ  .ðñ '<ðñ '<ðñ +Dðñ  ,Fð!ñ" (>ð#ñ$ ,ð%ñ&  Øñ)ð r?   r^   r_   c           	      óÒ   —  | j                   d||||
|dœ|¤Ž\  }}}} | j                  di d|“d|“d|“d|“d|“d|“d|“d	|“d
|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS )N©rE   rF   rG   rI   rR   r^   r_   rB   rC   rD   Úpadding_strategyÚtruncation_strategyrG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r.   )Ú"_get_padding_truncation_strategiesÚ_batch_encode_plus)r9   r^   r_   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r:   rl   rm   s                          r>   rc   z)LayoutLMv2TokenizerFast.batch_encode_plus3  s  € ð: ElÀD×DkÑDkð E
ØØ!Ø!Ø1ØñE
ð ñE
ÑAÐÐ-¨z¸6ð 'ˆt×&Ñ&ò 
Ù%=ð
áð
ñ ð
ñ $ð	
ñ
  2ð
ñ .ð
ñ !4ð
ñ "ð
ñ ð
ñ  2ð
ñ &ð
ñ *ð
ñ #8ð
ñ #8ð
ñ '@ð
ñ  (Bð!
ñ" $:ð#
ñ$ (ð%
ñ& Øñ)
ð 	
r?   Úpairc                 ót   — |r||fgn|g} | j                   j                  |f|ddœ|¤Ž}|d   j                  S )NF©rD   Úis_pretokenizedr   )Ú
_tokenizerÚencode_batchÚtokens)r9   r@   rp   rD   r:   Úbatched_inputÚ	encodingss          r>   Útokenizez LayoutLMv2TokenizerFast.tokenizep  sQ   € Ù*.˜$ ˜™°T°FˆØ0�D—O‘O×0Ñ0Øð
Ø.@ÐRWñ
Ø[añ
ˆ	ð ˜‰|×"Ñ"Ð"r?   c           	      óÒ   —  | j                   d||||
|dœ|¤Ž\  }}}} | j                  di d|“d|“d|“d|“d|“d|“d|“d	|“d
|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS )aj  
        Tokenize and prepare for the model a sequence or a pair of sequences. .. warning:: This method is deprecated,
        `__call__` should be used instead.

        Args:
            text (`str`, `List[str]`, `List[List[str]]`):
                The first sequence to be encoded. This can be a string, a list of strings or a list of list of strings.
            text_pair (`List[str]` or `List[int]`, *optional*):
                Optional second sequence to be encoded. This can be a list of strings (words of a single example) or a
                list of list of strings (words of a batch of examples).
        rk   r@   rB   rA   rC   rD   rl   rm   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r.   )rn   Ú_encode_plus)r9   r@   rA   rB   rC   rD   rE   rF   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r:   rl   rm   s                          r>   rd   z#LayoutLMv2TokenizerFast.encode_plusx  s  € ðJ ElÀD×DkÑDkð E
ØØ!Ø!Ø1ØñE
ð ñE
ÑAÐÐ-¨z¸6ð !ˆt× Ñ ò 
Ùð
áð
ñ  ð
ñ $ð	
ñ
  2ð
ñ .ð
ñ !4ð
ñ "ð
ñ ð
ñ  2ð
ñ &ð
ñ *ð
ñ #8ð
ñ #8ð
ñ '@ð
ñ  (Bð!
ñ" $:ð#
ñ$ (ð%
ñ& Øñ)
ð 	
r?   rl   rm   c                 ó|  — t        |t        «      st        dt        |«      › d�«      ‚| j	                  ||||	|
|¬«       |r$|D ��cg c]  \  }}|j                  «       |f‘Œ }}}| j                  j                  ||d¬«      }|D �cg c]  }| j                  ||||||�dn|||¬«      ‘Œ! }}i }|d   d   j                  «       D ]'  }|D ���cg c]  \  }}||   D ]  }|‘Œ Œ }}}}|||<   Œ) |D ���cg c]  \  }}|D ]  }|‘Œ Œ }}}}|r2g } t        |«      D ]  \  }!\  }"}| |!gt        |"d   «      z  z  } Œ | |d	<   |d   D ]  }#| j                  |#||«       Œ g }$t        t        |d   «      «      D �]!  }%|r	|d	   |%   }&n|%}&g }'t        |d   |%   ||%   j                  ||%   j                   «      D ]Î  \  }(})}*|*�;|r!|)dk(  r|'j#                  | j$                  «       Œ,|'j#                  ||&   |*   «       ŒD|(| j&                  k(  r|'j#                  | j(                  «       Œo|(| j*                  k(  r|'j#                  | j,                  «       Œš|(| j.                  k(  r|'j#                  | j$                  «       ŒÅt1        d
«      ‚ |$j#                  |'«       �Œ$ |$|d<   |�ôg }+t        t        |d   «      «      D ]Î  }%|r	|d	   |%   }&n|%}&g },t        |d   |%   |d   |%   ||%   j                   «      D ]ƒ  \  }(}-}*|*�`| j2                  r<|-d   dk(  r|,j#                  ||&   |*   «       Œ5|,j#                  | j4                  «       ŒQ|,j#                  ||&   |*   «       Œi|,j#                  | j4                  «       Œ… |+j#                  |,«       ŒÐ |+|d<   |s|d= t7        |||¬«      S c c}}w c c}w c c}}}w c c}}}w )Nz/batch_text_or_text_pairs has to be a list (got ú))rl   rm   rG   rH   rI   rJ   Trr   )ÚencodingrL   rM   rN   rO   rP   rQ   rR   r   Ú	input_idsÚoverflow_to_sample_mappingzId not recognizedÚbboxÚoffset_mappingÚlabels)Útensor_type)rV   rX   Ú	TypeErrorr-   Úset_truncation_and_paddingÚsplitrt   ru   Ú_convert_encodingÚkeysÚ	enumeraterZ   Ú&_eventual_warn_about_too_long_sequenceÚrangera   Úsequence_idsÚword_idsÚappendr'   Úcls_token_idr%   Úsep_token_idr&   Úpad_token_idr`   r)   r(   r
   ).r9   r^   r_   rB   rC   rD   rl   rm   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r@   rA   rx   r~   Útokens_and_encodingsÚsanitized_tokensÚkeyÚitemÚ_ÚeÚstackÚsanitized_encodingsr€   ÚiÚtoksr   Útoken_boxesÚbatch_indexÚoriginal_indexÚtoken_boxes_exampleÚidÚsequence_idÚword_idrƒ   Úlabels_exampleÚoffsets.                                                 r>   ro   z*LayoutLMv2TokenizerFast._batch_encode_plus½  sŸ  € ô4 Ð2´DÔ9ÜÐMÌdÐSkÓNlÐMmÐmnÐoÓpÐpð 	×'Ñ'Ø-Ø 3Ø!ØØ1Ø%ð 	(ô 	
ñ ØYq×'rÁoÀdÈI¨¯©«°yÒ(AÐ'rÐ$Ñ'rà—O‘O×0Ñ0Ø$Ø1Ø ð 1ó 
ˆ	ð2 &ö 
ð ð ×"Ñ"Ø!Ø&;Ø&;Ø*CØ+EàÐ*ñ (,à+Ø+Øð #õ ð 
Ðð  
ð, ÐØ'¨Ñ*¨1Ñ-×2Ñ2Ó4ò 	*ˆCØ&:×NÐN™7˜4 ÀDÈÁIÒN¸q’QÐN�QÐNˆEÒNØ$)Ð˜SÒ!ð	*ð 1E×SÐS¡W Q¨ÈdÒSÈšqÐS˜qÐSÐÒSñ %Ø)+Ð&Ü )Ð*>Ó ?ò K‘�‘9�D˜!Ø*¨q¨c´C¸¸[Ñ8IÓ4JÑ.JÑJÑ*ðKà=WÐÐ9Ñ:à)¨+Ñ6ò 	XˆIØ×7Ñ7¸	À:ÈwÕWð	Xð ˆÜ ¤Ð%5°kÑ%BÓ!CÓDó 	4ˆKÙ(Ø!1Ð2NÑ!OÐP[Ñ!\‘à!,�Ø"$ÐÜ,/Ø  Ñ-¨kÑ:Ø# KÑ0×=Ñ=Ø# KÑ0×9Ñ9ó-ò >Ñ(��K ð
 Ð&Ù ;°!Ò#3Ø+×2Ñ2°4×3EÑ3EÕFà+×2Ñ2°5¸Ñ3HÈÑ3QÕRà˜T×.Ñ.Ò.Ø+×2Ñ2°4×3EÑ3EÕFØ˜t×0Ñ0Ò0Ø+×2Ñ2°4×3EÑ3EÕFØ˜t×0Ñ0Ò0Ø+×2Ñ2°4×3EÑ3EÕFä(Ð)<Ó=Ð=ð%>ð& ×ÑÐ2Ö3ð3	4ð6 $/Ð˜Ñ ð Ð"ØˆFÜ$¤SÐ)9¸+Ñ)FÓ%GÓHò .�Ù,Ø%5Ð6RÑ%SÐT_Ñ%`‘Nà%0�NØ!#�Ü+.Ø$ [Ñ1°+Ñ>Ø$Ð%5Ñ6°{ÑCØ'¨Ñ4×=Ñ=ó,ò DÑ'�B˜ ð
 Ð*Ø×8Ò8Ø% a™y¨Aš~à .× 5Ñ 5°kÀ.Ñ6QÐRYÑ6ZÕ [à .× 5Ñ 5°d×6JÑ6JÕ Kà*×1Ñ1°+¸nÑ2MÈgÑ2VÕWà&×-Ñ-¨d×.BÑ.BÕCðDð  —‘˜nÕ-ð-.ð0 *0Ð˜XÑ&á)Ø$Ð%5Ð6äÐ-Ð/BÐP^Ô_Ð_ùói (sùò 
ùô0 OùäSs   ÁN%Â$N+ÃN0
Ã6N7c                 óÒ  — |r||fgn|g}|g}|�|gnd } | j                   |fi dt        |d u«      “d|“d|“d|“d|“d|“d|“d|	“d	|
“d
|“d|“d|“d|“d|“d|“d|“d|“d|“|¤Ž}|€`|s^t        |j                  «       D ��ci c].  \  }}|t	        |«      dkD  rt        |d   t        «      r|d   n|“Œ0 c}}|j                  «      }| j                  |d   ||«       |S c c}}w )Nr_   rB   rC   rD   rl   rm   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r   r   )	ro   rb   r
   ÚitemsrZ   rV   rX   rx   r‹   )r9   r@   rA   rB   rC   rD   rl   rm   rG   rH   rI   rJ   rK   rL   rM   rN   rO   rP   rQ   rR   r:   rw   Úbatched_boxesÚbatched_word_labelsÚbatched_outputr•   Úvalues                              r>   r{   z$LayoutLMv2TokenizerFast._encode_plus[  s‘  € ñ6 09˜$ 	Ð*Ñ+¸t¸fˆØ˜ˆØ/:Ð/F˜{™mÈDÐØ0˜×0Ñ0Øò
ä˜¨$Ð.Ô/ð
ñ  ð
ñ ,ð	
ñ
  2ð
ñ .ð
ñ !4ð
ñ "ð
ñ ð
ñ  2ð
ñ &ð
ñ *ð
ñ #8ð
ñ #8ð
ñ '@ð
ñ  (Bð!
ñ" $:ð#
ñ$ (ð%
ñ& Øñ)
ˆð2 Ð!Ñ*CÜ*ð '5×&:Ñ&:Ó&<÷á"˜˜Uð ¤S¨£Z°!¢^¼
À5ÈÁ8ÌTÔ8R˜˜qšÐX]Ñ]óð ×(Ñ(óˆNð 	×3Ñ3°NÀ;Ñ4OÐQ[Ð]dÔeàÐùós   Â3C#
Úencoded_inputsc                 ó.  — |€d| j                   v }|| j                   d      }|t        j                  k(  rt        |«      }|�|�||z  dk7  r||z  dz   |z  }|t        j                  k7  xr t        |«      |k7  }|rd|vrdgt        |«      z  |d<   |�r|t        |«      z
  }	|�|n| j
                  }|dk(  r |r|d   dg|	z  z   |d<   d|v r|d   | j                  g|	z  z   |d<   d|v r|d   | j                  g|	z  z   |d<   d|v r|d   | j                  g|	z  z   |d<   d|v r|d   dg|	z  z   |d<   || j                  g|	z  z   || j                   d   <   |S |d	k(  r |rdg|	z  |d   z   |d<   d|v r| j                  g|	z  |d   z   |d<   d|v r| j                  g|	z  |d   z   |d<   d|v r| j                  g|	z  |d   z   |d<   d|v rdg|	z  |d   z   |d<   | j                  g|	z  |z   || j                   d   <   |S t        d
t        |«      z   «      ‚|S )a#  
        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)

        Args:
            encoded_inputs:
                Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
            max_length: maximum length of the returned list and optionally padding length (see below).
                Will truncate by taking into account the special tokens.
            padding_strategy: PaddingStrategy to use for padding.

                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
                - PaddingStrategy.DO_NOT_PAD: Do not pad
                The tokenizer padding sides are defined in self.padding_side:

                    - 'left': pads on the left of the sequences
                    - 'right': pads on the right of the sequences
            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
                `>= 7.5` (Volta).
            padding_side:
                The side on which the model should have padding applied. Should be selected between ['right', 'left'].
                Default value is picked from the class attribute of the same name.
            return_attention_mask:
                (optional) Set to False to avoid returning attention mask (default: set to model specifics)
        Úattention_maskr   r   ÚrightÚtoken_type_idsr�   rƒ   Úspecial_tokens_maskÚleftzInvalid padding strategy:)Úmodel_input_namesr   ÚLONGESTrZ   Ú
DO_NOT_PADrJ   Úpad_token_type_idr'   r(   r’   r`   rW   )
r9   r¬   rG   rl   rI   rJ   rM   Úrequired_inputÚneeds_to_be_paddedÚ
differences
             r>   Ú_padzLayoutLMv2TokenizerFast._padŸ  s  € ðH !Ð(Ø$4¸×8NÑ8NÐ$NÐ!à'¨×(>Ñ(>¸qÑ(AÑBˆàœ×6Ñ6Ò6Ü˜^Ó,ˆJàÐ!Ð&8Ð&DÈ*ÐWiÑJiÐmnÒJnØ%Ð);Ñ;¸qÑ@ÐDVÑVˆJà-´×1KÑ1KÑKÒqÔPSÐTbÓPcÐgqÑPqÐñ !Ð%5¸^Ñ%KØ01¨s´S¸Ó5HÑ/HˆNÐ+Ñ,âØ#¤c¨.Ó&9Ñ9ˆJØ+7Ð+C™<È×IZÑIZˆLØ˜wÒ&Ù(Ø7EÐFVÑ7WÐ[\ÐZ]Ð`jÑZjÑ7j�NÐ#3Ñ4Ø# ~Ñ5à&Ð'7Ñ8¸D×<RÑ<RÐ;SÐV`Ñ;`Ñ`ð #Ð#3Ñ4ð ˜^Ñ+Ø-;¸FÑ-CÀt×GYÑGYÐFZÐ]gÑFgÑ-g�N 6Ñ*Ø˜~Ñ-Ø/=¸hÑ/GÈ4×K_ÑK_ÐJ`ÐcmÑJmÑ/m�N 8Ñ,Ø(¨NÑ:Ø<JÐK`Ñ<aÐefÐdgÐjtÑdtÑ<t�NÐ#8Ñ9Ø<JÈd×N_ÑN_ÐM`ÐcmÑMmÑ<m�˜t×5Ñ5°aÑ8Ñ9ð$ Ðð#  Ò'Ù(Ø89°s¸ZÑ7GÈ.ÐYiÑJjÑ7j�NÐ#3Ñ4Ø# ~Ñ5Ø8<×8NÑ8NÐ7OÐR\Ñ7\Ð_mØ(ñ`ñ 8�NÐ#3Ñ4ð ˜^Ñ+Ø.2×.@Ñ.@Ð-AÀJÑ-NÐQ_Ð`fÑQgÑ-g�N 6Ñ*Ø˜~Ñ-Ø04×0DÑ0DÐ/EÈ
Ñ/RÐUcÐdlÑUmÑ/m�N 8Ñ,Ø(¨NÑ:Ø=>¸CÀ*Ñ<LÈ~Ð^sÑOtÑ<t�NÐ#8Ñ9Ø=A×=NÑ=NÐ<OÐR\Ñ<\Ð_mÑ<m�˜t×5Ñ5°aÑ8Ñ9ð Ðô !Ð!<¼sÀ<Ó?PÑ!PÓQÐQàÐr?   c                 ól   — | j                   g|z   | j                  gz   }|r||| j                  gz   z  }|S )a–  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. A BERT sequence has the following format:

        - single sequence: `[CLS] X [SEP]`
        - pair of sequences: `[CLS] A [SEP] B [SEP]`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
        )r�   r‘   )r9   Útoken_ids_0Útoken_ids_1Úoutputs       r>   Ú build_inputs_with_special_tokensz8LayoutLMv2TokenizerFast.build_inputs_with_special_tokensø  sE   € ð" ×#Ñ#Ð$ {Ñ2°d×6GÑ6GÐ5HÑHˆáØ�k T×%6Ñ%6Ð$7Ñ7Ñ7ˆFàˆr?   r¼   r½   c                 ó´   — | j                   g}| j                  g}|€t        ||z   |z   «      dgz  S t        ||z   |z   «      dgz  t        ||z   «      dgz  z   S )a§  
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. A BERT sequence
        pair mask has the following format: :: 0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1 | first sequence | second
        sequence | If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        r   r   )r‘   r�   rZ   )r9   r¼   r½   ÚsepÚclss        r>   Ú$create_token_type_ids_from_sequencesz<LayoutLMv2TokenizerFast.create_token_type_ids_from_sequences  st   € ð" × Ñ Ð!ˆØ× Ñ Ð!ˆØÐÜ�s˜[Ñ(¨3Ñ.Ó/°1°#Ñ5Ð5Ü�3˜Ñ$ sÑ*Ó+¨q¨cÑ1´C¸ÀcÑ8IÓ4JÈaÈSÑ4PÑPÐPr?   Úsave_directoryÚfilename_prefixc                 óf   — | j                   j                  j                  ||¬«      }t        |«      S )N)Úname)rt   ÚmodelÚsaverY   )r9   rÄ   rÅ   Úfiless       r>   Úsave_vocabularyz'LayoutLMv2TokenizerFast.save_vocabulary'  s+   € Ø—‘×%Ñ%×*Ñ*¨>ÀÐ*ÓPˆÜ�U‹|Ðr?   )NNNTFNNr   NNNNNFFFFT)NF)N)*Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚVOCAB_FILES_NAMESÚvocab_files_namesr   Úslow_tokenizer_classr0   r   r   r   r   r   r   r   r   Úintrb   rW   r   r   r   r
   ri   r   rc   ry   rd   rµ   ÚDO_NOT_TRUNCATEro   r{   r   r   Údictrº   r¿   rÃ   r   rË   Ú__classcell__)r=   s   @r>   r   r   1   s~	  ø„ ñ+ðZ *ÐØ.Ðð ØØØØØØØÚ"Ú.Ú"ØØ!%Ø#Øõ!7Añr Ð:Ð<nÓoð RVØ?CØCGØ#'Ø5:Ø;?Ø$(ØØ,0Ø&*Ø;?Ø04Ø04Ø*/Ø+0Ø',Ø#Øñ)Uà�IÐ0°$°y±/À4ÐHYÑCZÐZÑ[ðUð ˜EÐ"3°TÐ:KÑ5LÐ"LÑMÑNðUð �T˜$˜s™)‘_ d¨4°°S±	©?Ñ&;Ð;Ñ<ð	Uð
 ˜e D¨¡I¨t°D¸±I©Ð$>Ñ?Ñ@ðUð !ðUð �t˜S /Ð1Ñ2ðUð ˜$ Ð%7Ð7Ñ8ðUð ˜S‘MðUð ðUð % S™MðUð ˜s‘mðUð !  s¨J Ñ!7Ñ8ðUð  (¨™~ðUð  (¨™~ðUð  $(ð!Uð" %)ð#Uð$ !%ð%Uð& ð'Uð( ð)Uð, 
ò-Uó pðUñn Ð:Ð<nÓoð #'Ø15ØCGØ#'Ø5:Ø;?Ø$(ØØ,0Ø&*Ø;?Ø04Ø04Ø*/Ø+0Ø',Ø#Øñ1:
à"'Ø�‰OØ�ÑØÐ"Ñ#ð%ñ#
ð:
ð ˜$‘ð:
ð ˜˜T $ s¡)™_Ñ-Ñ.ð:
ð ˜e D¨¡I¨t°D¸±I©Ð$>Ñ?Ñ@ð:
ð !ð:
ð �t˜S /Ð1Ñ2ð:
ð ˜$ Ð%7Ð7Ñ8ð:
ð ˜S‘Mð:
ð ð:
ð % S™Mð:
ð  ˜s‘mð!:
ð" !  s¨J Ñ!7Ñ8ð#:
ð$  (¨™~ð%:
ð&  (¨™~ð':
ð( $(ð):
ð* %)ð+:
ð, !%ð-:
ð. ð/:
ð0 ð1:
ð4 
ò5:
ó pð:
ñx#˜Sð #¨°©ð #ÐRVð #ÐmqÐruÑmvó #ñ Ð:Ð<nÓoð 26Ø+/Ø+/Ø#'Ø5:Ø;?Ø$(ØØ,0Ø&*Ø;?Ø04Ø04Ø*/Ø+0Ø',Ø#Øñ)B
à�IÐ0Ð0Ñ1ðB
ð Ð-Ñ.ðB
ð ˜˜T #™Y™Ñ(ð	B
ð
 ˜d 3™iÑ(ðB
ð !ðB
ð �t˜S /Ð1Ñ2ðB
ð ˜$ Ð%7Ð7Ñ8ðB
ð ˜S‘MðB
ð ðB
ð % S™MðB
ð ˜s‘mðB
ð !  s¨J Ñ!7Ñ8ðB
ð  (¨™~ðB
ð  (¨™~ðB
ð  $(ð!B
ð" %)ð#B
ð$ !%ð%B
ð& ð'B
ð( ð)B
ð, 
ò-B
ó pðB
ðV #'Ø15Ø15Ø#'Ø,;×,FÑ,FØ2D×2TÑ2TØ$(ØØ,0Ø&*Ø(,Ø04Ø04Ø*/Ø+0Ø',Ø#Øñ1\`à"'Ø�‰OØ�ÑØÐ"Ñ#ð%ñ#
ð\`ð ˜$‘ð\`ð ˜˜T $ s¡)™_Ñ-Ñ.ð\`ð ˜d 4¨¡9™oÑ.ð\`ð !ð\`ð *ð\`ð 0ð\`ð ˜S‘Mð\`ð ð\`ð % S™Mð\`ð  ˜s‘mð!\`ð" ! ™ð#\`ð$  (¨™~ð%\`ð&  (¨™~ð'\`ð( $(ð)\`ð* %)ð+\`ð, !%ð-\`ð. ð/\`ð0 ð1\`ð2 
ó3\`ðB 26Ø+/Ø+/Ø#'Ø,;×,FÑ,FØ2D×2TÑ2TØ$(ØØ,0Ø&*Ø)-Ø04Ø04Ø*/Ø+0Ø',Ø#Øñ)Bà�IÐ0Ð0Ñ1ðBð Ð-Ñ.ðBð ˜˜T #™Y™Ñ(ð	Bð
 ˜d 3™iÑ(ðBð !ðBð *ðBð 0ðBð ˜S‘MðBð ðBð % S™MðBð ˜s‘mðBð ! ™ðBð  (¨™~ðBð  (¨™~ðBð  $(ð!Bð" %)ð#Bð$ !%ð%Bð& ð'Bð( ð)Bð, 
ó-BðN %)Ø,;×,FÑ,FØ,0Ø&*Ø04ñWà˜d 3¨Ð#4Ñ5°}ÐDÑEðWð ˜S‘MðWð *ð	Wð
 % S™MðWð ˜s‘mðWð  (¨™~ðWð 
óWórð2 JNñQØ ™9ðQØ3;¸DÀ¹IÑ3FðQà	ˆc‰óQñ.¨cð ÀHÈSÁMð Ð]bÐcfÑ]g÷ r?   r   )"rÏ   r1   Útypingr   r   r   r   r   Ú
tokenizersr   Útokenization_utils_baser
   r   r   r   r   r   r   r   Útokenization_utils_fastr   Úutilsr   r   Útokenization_layoutlmv2r   r   r   Ú
get_loggerrÌ   ÚloggerrÐ   r   Ú__all__r.   r?   r>   ú<module>rà      ss   ðñó
 ß 5Õ 5å "÷	÷ 	ó 	õ ?ß 0÷ñ ð 
ˆ×	Ñ	˜HÓ	%€à#.ÐBRÑSÐ ôxÐ5ô xðv %Ð
%�r?   