Ë
    T^(h¸Î ã                   óL  — d Z ddlZddlZddlZddlZddlZddlZddlZddlZddl	m
Z
 ddlmZmZmZmZmZmZmZ ddlZddlmZmZmZmZ ddlmZmZmZmZmZm Z  ddl!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'  e&«       rddl(Z) e'jT                  e+«      Z,d	d
iZ- G d„ de"«      Z. ej^                  dg d¢«      Z0 e
d¬«       G d„ d«      «       Z1e
 G d„ d«      «       Z2 e
d¬«       G d„ d«      «       Z3de4fd„Z5d„ Z6d„ Z7dZ8 G d„ de«      Z9 G d„ d«      Z: G d „ d!«      Z; G d"„ d#ejx                  «      Z=e
 G d$„ d%«      «       Z>e
 G d&„ d'«      «       Z?e
 G d(„ d)«      «       Z@e
 G d*„ d+«      «       ZAe
 G d,„ d-«      «       ZB ej^                  d.g d/¢«      ZC eCdd0d0«      ZD eCddd0«      ZE eCddd«      ZF eCd0dd0«      ZG eCd0dd«      ZHd1eGfd2eDfd3eDfd4eEfd5eEfd6eHfd7eHfd8eHfd9eHfd:eFfd;eFfd<eFfd=eFfd>eEfd5eEfd?eFfd=eFfd@eFfdAeFfdBeHfdCeHffZIdDZJdE„ ZKdF„ ZL eL«       ZMdGZNg dH¢ZOg dI¢ZPg dJ¢ZQ ej¤                  dK«      ZSdLZTdMZU eVdN«      ZWdO„ ZXdP„ ZYdQ„ ZZdR„ Z[dS„ Z\dT„ Z]dU„ Z^dV„ Z_dW„ Z`eeVeeeV   eeV   eeV   f   f   Zaee?geeVebf   f   ZcdZddXZedYZfdZZgd[„ Zhd\„ Zid]„ Zjd^„ Zkd_„ Zld`„ Zmda„ Zndb„ Zodc„ Zpdd„ Zqde„ Zrdgdf„ZsdgZty)hz#Tokenization class for TAPAS model.é    N)Ú	dataclass)ÚCallableÚDictÚ	GeneratorÚListÚOptionalÚTupleÚUnioné   )ÚPreTrainedTokenizerÚ_is_controlÚ_is_punctuationÚ_is_whitespace)ÚENCODE_KWARGS_DOCSTRINGÚVERY_LARGE_INTEGERÚBatchEncodingÚEncodedInputÚPreTokenizedInputÚ	TextInput)ÚExplicitEnumÚPaddingStrategyÚ
TensorTypeÚadd_end_docstringsÚis_pandas_availableÚloggingÚ
vocab_filez	vocab.txtc                   ó   — e Zd ZdZdZdZy)ÚTapasTruncationStrategyz}
    Possible values for the `truncation` argument in [`~TapasTokenizer.__call__`]. Useful for tab-completion in an IDE.
    Údrop_rows_to_fitÚdo_not_truncateN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚDROP_ROWS_TO_FITÚDO_NOT_TRUNCATE© ó    új/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/tapas/tokenization_tapas.pyr   r   3   s   „ ñð *ÐØ'�Or(   r   Ú
TokenValue)ÚtokenÚ	column_idÚrow_idT)Úfrozenc                   ó,   — e Zd ZU eed<   eed<   eed<   y)ÚTokenCoordinatesÚcolumn_indexÚ	row_indexÚtoken_indexN)r!   r"   r#   ÚintÚ__annotations__r'   r(   r)   r0   r0   ?   s   … àÓØƒNØÔr(   r0   c                   ó:   — e Zd ZU eeee         ed<   ee   ed<   y)ÚTokenizedTableÚrowsÚselected_tokensN)r!   r"   r#   r   Ústrr5   r0   r'   r(   r)   r7   r7   F   s!   … à
ˆt�D˜‘I‰Ñ
ÓØÐ*Ñ+Ô+r(   r7   c                   óN   — e Zd ZU ee   ed<   ee   ed<   ee   ed<   ee   ed<   y)ÚSerializedExampleÚtokensÚ
column_idsÚrow_idsÚsegment_idsN)r!   r"   r#   r   r:   r5   r4   r'   r(   r)   r<   r<   L   s*   … à�‰IÓØ�S‘	ÓØ�#‰YÓØ�c‘Ôr(   r<   r+   c                 ó$   — | j                  d«      S )Nú##)Ú
startswith)r+   s    r)   Ú_is_inner_wordpiecerD   T   s   € Ø×Ñ˜DÓ!Ð!r(   c                 óæ   — t        j                  «       }t        | dd¬«      5 }|j                  «       }ddd«       t	        «      D ]  \  }}|j                  d«      }|||<   Œ |S # 1 sw Y   Œ4xY w)z*Loads a vocabulary file into a dictionary.Úrúutf-8©ÚencodingNú
)ÚcollectionsÚOrderedDictÚopenÚ	readlinesÚ	enumerateÚrstrip)r   ÚvocabÚreaderr=   Úindexr+   s         r)   Ú
load_vocabrT   X   sw   € ä×#Ñ#Ó%€EÜ	ˆj˜#¨Ô	0ð $°FØ×!Ñ!Ó#ˆ÷$ä! &Ó)ò ‰ˆˆuØ—‘˜TÓ"ˆØˆˆeŠðð €L÷$ð $ús   £A'Á'A0c                 óN   — | j                  «       } | sg S | j                  «       }|S )z@Runs basic whitespace cleaning and splitting on a piece of text.)ÚstripÚsplit)Útextr=   s     r)   Úwhitespace_tokenizerY   c   s%   € à�:‰:‹<€DÙØˆ	Ø�Z‰Z‹\€FØ€Mr(   aô  
            add_special_tokens (`bool`, *optional*, defaults to `True`):
                Whether or not to encode the sequences with the special tokens relative to their model.
            padding (`bool`, `str` or [`~utils.PaddingStrategy`], *optional*, defaults to `False`):
                Activates and controls padding. Accepts the following values:

                - `True` or `'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
                  sequence if provided).
                - `'max_length'`: Pad to a maximum length specified with the argument `max_length` or to the maximum
                  acceptable input length for the model if that argument is not provided.
                - `False` or `'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of different
                  lengths).
            truncation (`bool`, `str` or [`TapasTruncationStrategy`], *optional*, defaults to `False`):
                Activates and controls truncation. Accepts the following values:

                - `True` or `'drop_rows_to_fit'`: Truncate to a maximum length specified with the argument `max_length`
                  or to the maximum acceptable input length for the model if that argument is not provided. This will
                  truncate row by row, removing rows from the table.
                - `False` or `'do_not_truncate'` (default): No truncation (i.e., can output batch with sequence lengths
                  greater than the model maximum admissible input size).
            max_length (`int`, *optional*):
                Controls the maximum length to use by one of the truncation/padding parameters.

                If left unset or set to `None`, this will use the predefined model maximum length if a maximum length
                is required by one of the truncation/padding parameters. If the model has no specific maximum input
                length (like XLNet) truncation/padding to a maximum length will be deactivated.
            is_split_into_words (`bool`, *optional*, defaults to `False`):
                Whether or not the input is already pre-tokenized (e.g., split into words). If set to `True`, the
                tokenizer assumes the input is already split into words (for instance, by splitting it on whitespace)
                which it will tokenize. This is useful for NER or token classification.
            pad_to_multiple_of (`int`, *optional*):
                If set will pad the sequence to a multiple of the provided value. This is especially useful to enable
                the use of Tensor Cores on NVIDIA hardware with compute capability `>= 7.5` (Volta).
            return_tensors (`str` or [`~utils.TensorType`], *optional*):
                If set, will return tensors instead of list of python integers. Acceptable values are:

                - `'tf'`: Return TensorFlow `tf.constant` objects.
                - `'pt'`: Return PyTorch `torch.Tensor` objects.
                - `'np'`: Return Numpy `np.ndarray` objects.
c            ,       óR  ‡ — e Zd ZdZeZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 didedee   dee   dededed	ee	e
      fˆ fd
„Zed„ «       Zed„ «       Zd„ Zd„ Zd„ Zd„ Zd„ Zdjde
dee
   dee
   fd„Zde	e   de	e   de	e   fd„Zde	e   de	e   de	e   fd„Zde	e   de	e   de	e   fd„Zde	e   de	e   de	e   fd„Z	 djde	e   dee	e      de	e   fd„Z	 dkde	e   dee	e      dede	e   fˆ fd „Z ee«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dld!d"d#eee e!e"e	e    e	e!   e	e"   f      d$eee	e   e	e	e      f      d%eee	e    e	e	e       f      d&ed'eee
e#f   d(eee
e$f   d)ee   d*ee   d+ee
   d,eee
e%f      d-ee   d.ee   d/ed0ed1ed2ed3ede&f&d4„«       Z' ee(e«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dld!d"d#eee	e    e	e!   e	e"   f      d$ee	e	e         d%ee	e	e          d&ed'eee
e#f   d(eee
e$f   d)ee   d*ee   d+ee
   d,eee
e%f      d-ee   d.ee   d/ed0ed1ed2ed3ede&f&d5„«       Z)d6„ Z*	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dmd#ee	e    e	e!   e	e"   f   d$ee	e	e         d%ee	e	e          d&ed'eee
e#f   d(eee
e$f   d)ee   d*ee   d+ee
   d,eee
e%f      d-ee   d.ee   d/ed0ed1ed2ed3ede&f$d7„Z+	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dnd8d"d9ee	e    e	e!   e	e"   f   d:ee,   d;ee	e	e
         d$ee	e	e         d%ee	e	e          d&ed'eee
e#f   d(eee
e$f   d)ee   d*ee   d+ee
   d,eee
e%f      d-ee   d.ee   d0ed1ed2ed3ed<ede&f*d=„Z- ee(«      	 	 	 	 	 	 dod!d"d>eee e!e"f      d&ed'eee
e#f   d(eee
e$f   d)ee   d,eee
e%f      de	e   fd?„«       Z. ee(e«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dpd!d"d>eee e!e"f      d$ee	e      d%ee	e       d&ed'eee
e#f   d(eee
e$f   d)ee   d*ee   d+ee
   d,eee
e%f      d-ee   d.ee   d0ed1ed2ed3ede&f$d@„«       Z/	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dqd!d"d>ee e!e"f   d$ee	e      d%ee	e       d&ed'eee
e#f   d(eee
e$f   d)ee   d*ee   d+ee
   d,eee
e%f      d-ee   d.ee   d0ed1ed2ed3ef"dA„Z0 ee(e«      	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dnd8d"dBee e!e"f   d:ee,   dCee,   d$ee	e      d%ee	e       d&ed'eee
e#f   d(eee
e$f   d)ee   d*ee   d+ee
   d,eee
e%f      d-ee   d.ee   d0ed1ed2ed3ed<ede&f*dD„«       Z1dCe	e
   d:e,dEedFed)edGee
e$f   deeef   fdH„Z2	 djdI„Z3dJ„ Z4djdK„Z5de6eddf   fdL„Z7dM„ Z8dN„ Z9dO„ Z:dP„ Z;dQ„ Z<dR„ Z=dS„ Z>dT„ Z?dU„ Z@dV„ ZAdW„ ZBdX„ ZCdY„ ZDdZ„ ZEd[„ ZFd\„ ZGd]„ ZHd^„ ZId_„ ZJd`„ ZKda„ ZLdb„ ZMde#jœ                  dddfdceeOe
e"f   e&f   d)ee   dde#d*ee   d+ee
   d.ee   dePfde„ZQdf„ ZRdg„ ZSdrdh„ZTˆ xZUS )sÚTapasTokenizeraÊ  
    Construct a TAPAS tokenizer. Based on WordPiece. Flattens a table and one or more related sentences to be used by
    TAPAS models.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods. [`TapasTokenizer`] creates several token type ids to
    encode tabular structure. To be more precise, it adds 7 token type ids, in the following order: `segment_ids`,
    `column_ids`, `row_ids`, `prev_labels`, `column_ranks`, `inv_column_ranks` and `numeric_relations`:

    - segment_ids: indicate whether a token belongs to the question (0) or the table (1). 0 for special tokens and
      padding.
    - column_ids: indicate to which column of the table a token belongs (starting from 1). Is 0 for all question
      tokens, special tokens and padding.
    - row_ids: indicate to which row of the table a token belongs (starting from 1). Is 0 for all question tokens,
      special tokens and padding. Tokens of column headers are also 0.
    - prev_labels: indicate whether a token was (part of) an answer to the previous question (1) or not (0). Useful in
      a conversational setup (such as SQA).
    - column_ranks: indicate the rank of a table token relative to a column, if applicable. For example, if you have a
      column "number of movies" with values 87, 53 and 69, then the column ranks of these tokens are 3, 1 and 2
      respectively. 0 for all question tokens, special tokens and padding.
    - inv_column_ranks: indicate the inverse rank of a table token relative to a column, if applicable. For example, if
      you have a column "number of movies" with values 87, 53 and 69, then the inverse column ranks of these tokens are
      1, 3 and 2 respectively. 0 for all question tokens, special tokens and padding.
    - numeric_relations: indicate numeric relations between the question and the tokens of the table. 0 for all
      question tokens, special tokens and padding.

    [`TapasTokenizer`] runs end-to-end tokenization on a table and associated sentences: punctuation splitting and
    wordpiece.

    Args:
        vocab_file (`str`):
            File containing the vocabulary.
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        do_basic_tokenize (`bool`, *optional*, defaults to `True`):
            Whether or not to do basic tokenization before WordPiece.
        never_split (`Iterable`, *optional*):
            Collection of tokens which will never be split during tokenization. Only has an effect when
            `do_basic_tokenize=True`
        unk_token (`str`, *optional*, defaults to `"[UNK]"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
        sep_token (`str`, *optional*, defaults to `"[SEP]"`):
            The separator token, which is used when building a sequence from multiple sequences, e.g. two sequences for
            sequence classification or for a text and a question for question answering. It is also used as the last
            token of a sequence built with special tokens.
        pad_token (`str`, *optional*, defaults to `"[PAD]"`):
            The token used for padding, for example when batching sequences of different lengths.
        cls_token (`str`, *optional*, defaults to `"[CLS]"`):
            The classifier token which is used when doing sequence classification (classification of the whole sequence
            instead of per-token classification). It is the first token of the sequence when built with special tokens.
        mask_token (`str`, *optional*, defaults to `"[MASK]"`):
            The token used for masking values. This is the token used when training this model with masked language
            modeling. This is the token which the model will try to predict.
        empty_token (`str`, *optional*, defaults to `"[EMPTY]"`):
            The token used for empty cell values in a table. Empty cell values include "", "n/a", "nan" and "?".
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters. This should likely be deactivated for Japanese (see this
            [issue](https://github.com/huggingface/transformers/issues/328)).
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase` (as in the original BERT).
        cell_trim_length (`int`, *optional*, defaults to -1):
            If > 0: Trim cells so that the length is <= this value. Also disables further cell trimming, should thus be
            used with `truncation` set to `True`.
        max_column_id (`int`, *optional*):
            Max column id to extract.
        max_row_id (`int`, *optional*):
            Max row id to extract.
        strip_column_names (`bool`, *optional*, defaults to `False`):
            Whether to add empty strings instead of column names.
        update_answer_coordinates (`bool`, *optional*, defaults to `False`):
            Whether to recompute the answer coordinates from the answer text.
        min_question_length (`int`, *optional*):
            Minimum length of each question in terms of tokens (will be skipped otherwise).
        max_question_length (`int`, *optional*):
            Maximum length of each question in terms of tokens (will be skipped otherwise).
        clean_up_tokenization_spaces (`bool`, *optional*, defaults to `True`):
            Whether or not to cleanup spaces after decoding, cleanup consists in removing potential artifacts like
            extra spaces.
    NÚcell_trim_lengthÚmax_column_idÚ
max_row_idÚstrip_column_namesÚupdate_answer_coordinatesÚmodel_max_lengthÚadditional_special_tokensc                 ó.  •— t        «       st        d«      ‚|�|
|vr|j                  |
«       n|
g}t        j                  j                  |«      st        d|› d�«      ‚t        |«      | _        t        j                  | j                  j                  «       D ��cg c]	  \  }}||f‘Œ c}}«      | _        || _        |rt        ||||¬«      | _        t!        | j                  t#        |«      ¬«      | _        || _        |�|n	|�|nt(        | _        |�|n	|�|nt(        | _        || _        || _        || _        || _        t7        ‰| �p  di d|“d|“d|“d	|“d
|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“d|“|¤Ž y c c}}w )Nz+Pandas is required for the TAPAS tokenizer.z&Can't find a vocabulary file at path 'z'. To load the vocabulary from a Google pretrained model use `tokenizer = BertTokenizer.from_pretrained(PRETRAINED_MODEL_NAME)`)Údo_lower_caseÚnever_splitÚtokenize_chinese_charsÚstrip_accents)rQ   Ú	unk_tokenrd   Údo_basic_tokenizere   rh   Ú	sep_tokenÚ	pad_tokenÚ	cls_tokenÚ
mask_tokenÚempty_tokenrf   rg   r\   r]   r^   r_   r`   Úmin_question_lengthÚmax_question_lengthra   rb   Úclean_up_tokenization_spacesr'   )r   ÚImportErrorÚappendÚosÚpathÚisfileÚ
ValueErrorrT   rQ   rK   rL   ÚitemsÚids_to_tokensri   ÚBasicTokenizerÚbasic_tokenizerÚWordpieceTokenizerr:   Úwordpiece_tokenizerr\   r   r]   r^   r_   r`   ro   rp   ÚsuperÚ__init__)Úselfr   rd   ri   re   rh   rj   rk   rl   rm   rn   rf   rg   r\   r]   r^   r_   r`   ro   rp   ra   rb   rq   ÚkwargsÚtokÚidsÚ	__class__s                             €r)   r   zTapasTokenizer.__init__ë   sN  ø€ ô4 #Ô$ÜÐKÓLÐLà$Ð0ØÐ";Ñ;Ø)×0Ñ0°Õ=à)4¨Ð%ä�w‰w�~‰~˜jÔ)ÜØ8¸¸ð E`ð `óð ô   
Ó+ˆŒ
Ü(×4Ñ4ÐQU×Q[ÑQ[×QaÑQaÓQc×5dÁXÀSÈ#°s¸C²jÓ5dÓeˆÔØ!2ˆÔÙÜ#1Ø+Ø'Ø'=Ø+ô	$ˆDÔ ô $6¸D¿J¹JÔRUÐV_ÓR`Ô#aˆÔ ð !1ˆÔð Ð(ñ ð  Ð+ñ "ä#ð 	Ôð Ð%ñ ð  Ð+ñ "ä#ð 	Œð #5ˆÔØ)BˆÔ&Ø#6ˆÔ Ø#6ˆÔ ä‰Ñò 	
Ù'ð	
á/ð	
ñ $ð	
ñ  ð		
ñ
  ð	
ñ  ð	
ñ  ð	
ñ "ð	
ñ $ð	
ñ $:ð	
ñ (ð	
ñ .ð	
ñ (ð	
ñ "ð	
ñ  2ð	
ñ  '@ð!	
ñ" !4ð#	
ñ$ !4ð%	
ñ& .ð'	
ñ( '@ð)	
ñ* *FØó-	
ùóA 6es   ÂF
c                 ó.   — | j                   j                  S ©N)r{   rd   ©r€   s    r)   rd   zTapasTokenizer.do_lower_caseM  s   € à×#Ñ#×1Ñ1Ð1r(   c                 ó,   — t        | j                  «      S r†   )ÚlenrQ   r‡   s    r)   Ú
vocab_sizezTapasTokenizer.vocab_sizeQ  s   € ä�4—:‘:‹Ðr(   c                 óB   — t        | j                  fi | j                  ¤ŽS r†   )ÚdictrQ   Úadded_tokens_encoderr‡   s    r)   Ú	get_vocabzTapasTokenizer.get_vocabU  s   € Ü�D—J‘JÑ< $×";Ñ";Ñ<Ð<r(   c                 óˆ  — t        |«      t        k(  r| j                  d   gS g }| j                  rv| j                  j                  || j                  ¬«      D ]J  }|| j                  j                  v r|j                  |«       Œ-|| j                  j                  |«      z  }ŒL |S | j                  j                  |«      }|S )Nr   )re   )
Úformat_textÚ
EMPTY_TEXTrb   ri   r{   ÚtokenizeÚall_special_tokensre   rs   r}   )r€   rX   Úsplit_tokensr+   s       r)   Ú	_tokenizezTapasTokenizer._tokenizeX  sÁ   € Ü�tÓ¤
Ò*Ø×2Ñ2°1Ñ5Ð6Ð6ØˆØ×!Ò!Ø×-Ñ-×6Ñ6°tÈ×I`ÑI`Ð6Óaò M�à˜D×0Ñ0×<Ñ<Ñ<Ø ×'Ñ'¨Õ.à  D×$<Ñ$<×$EÑ$EÀeÓ$LÑL‘LðMð Ðð  ×3Ñ3×<Ñ<¸TÓBˆLØÐr(   c                 ó€   — | j                   j                  || j                   j                  | j                  «      «      S )z0Converts a token (str) in an id using the vocab.)rQ   Úgetrh   )r€   r+   s     r)   Ú_convert_token_to_idz#TapasTokenizer._convert_token_to_idg  s(   € à�z‰z�~‰~˜e T§Z¡Z§^¡^°D·N±NÓ%CÓDÐDr(   c                 óN   — | j                   j                  || j                  «      S )z=Converts an index (integer) in a token (str) using the vocab.)ry   r—   rh   )r€   rS   s     r)   Ú_convert_id_to_tokenz#TapasTokenizer._convert_id_to_tokenk  s   € à×!Ñ!×%Ñ% e¨T¯^©^Ó<Ð<r(   c                 ód   — dj                  |«      j                  dd«      j                  «       }|S )z:Converts a sequence of tokens (string) in a single string.ú z ##Ú )ÚjoinÚreplacerV   )r€   r=   Ú
out_strings      r)   Úconvert_tokens_to_stringz'TapasTokenizer.convert_tokens_to_stringo  s,   € à—X‘X˜fÓ%×-Ñ-¨e°RÓ8×>Ñ>Ó@ˆ
ØÐr(   Úsave_directoryÚfilename_prefixÚreturnc                 óà  — d}t         j                  j                  |«      r2t         j                  j                  ||r|dz   ndt        d   z   «      }n|r|dz   nd|z   }t        |dd¬«      5 }t        | j                  j                  «       d„ ¬	«      D ]>  \  }}||k7  rt        j                  d
|› d�«       |}|j                  |dz   «       |dz  }Œ@ 	 d d d «       |fS # 1 sw Y   |fS xY w)Nr   ú-r�   r   ÚwrG   rH   c                 ó   — | d   S ©Né   r'   )Úkvs    r)   ú<lambda>z0TapasTokenizer.save_vocabulary.<locals>.<lambda>}  s   € ÐPRÐSTÑPU€ r(   ©ÚkeyzSaving vocabulary to z\: vocabulary indices are not consecutive. Please check that the vocabulary is not corrupted!rJ   rª   )rt   ru   Úisdirrž   ÚVOCAB_FILES_NAMESrM   ÚsortedrQ   rx   ÚloggerÚwarningÚwrite)r€   r¢   r£   rS   r   Úwriterr+   r3   s           r)   Úsave_vocabularyzTapasTokenizer.save_vocabularyt  sû   € ØˆÜ�7‰7�=‰=˜Ô(ÜŸ™Ÿ™Ø¹/ °3Ò!6ÈrÔUfÐgsÑUtÑ tó‰Jñ 4C˜/¨CÒ/ÈÈnÑ\ˆJÜ�*˜c¨GÔ4ð 		¸Ü&,¨T¯Z©Z×-=Ñ-=Ó-?ÑEUÔ&Vò Ñ"��{Ø˜KÒ'Ü—N‘NØ/°
¨|ð <Nð Nôð (�EØ—‘˜U T™\Ô*Ø˜‘
‘ñ÷		ð ˆ}Ð÷		ð ˆ}Ðús   Á.A(C"Ã"C-Ú	query_idsÚtable_valuesc                 óD   — dgdt        |«      z   dz   t        |«      z   z  S )aÚ  
        Creates the attention mask according to the query token IDs and a list of table values.

        Args:
            query_ids (`List[int]`): list of token IDs corresponding to the ID.
            table_values (`List[TableValue]`): lift of table values, which are named tuples containing the
                token value, the column ID and the row ID of said token.

        Returns:
            `List[int]`: List of ints containing the attention mask values.
        rª   ©r‰   )r€   r·   r¸   s      r)   Ú$create_attention_mask_from_sequencesz3TapasTokenizer.create_attention_mask_from_sequencesˆ  s)   € ð ˆs�aœ#˜i›.Ñ(¨1Ñ,¬s°<Ó/@Ñ@ÑAÐAr(   c                 ó|   — |rt        t        |Ž «      d   ng }dgdt        |«      z   dz   z  dgt        |«      z  z   S )aê  
        Creates the segment token type IDs according to the query token IDs and a list of table values.

        Args:
            query_ids (`List[int]`): list of token IDs corresponding to the ID.
            table_values (`List[TableValue]`): lift of table values, which are named tuples containing the
                token value, the column ID and the row ID of said token.

        Returns:
            `List[int]`: List of ints containing the segment token type IDs values.
        r   rª   ©ÚlistÚzipr‰   )r€   r·   r¸   Ú	table_idss       r)   Ú,create_segment_token_type_ids_from_sequencesz;TapasTokenizer.create_segment_token_type_ids_from_sequences–  sH   € ñ 4@”Dœ˜lÐ+Ó,¨QÒ/ÀRˆ	Øˆs�aœ#˜i›.Ñ(¨1Ñ,Ñ-°°´c¸)³nÑ0DÑDÐDr(   c                 ót   — |rt        t        |Ž «      d   ng }dgdt        |«      z   dz   z  t        |«      z   S )aè  
        Creates the column token type IDs according to the query token IDs and a list of table values.

        Args:
            query_ids (`List[int]`): list of token IDs corresponding to the ID.
            table_values (`List[TableValue]`): lift of table values, which are named tuples containing the
                token value, the column ID and the row ID of said token.

        Returns:
            `List[int]`: List of ints containing the column token type IDs values.
        rª   r   r½   )r€   r·   r¸   Útable_column_idss       r)   Ú+create_column_token_type_ids_from_sequencesz:TapasTokenizer.create_column_token_type_ids_from_sequences§  sD   € ñ ;Gœ4¤ \Ð 2Ó3°AÒ6ÈBÐØˆs�aœ#˜i›.Ñ(¨1Ñ,Ñ-´Ð5EÓ0FÑFÐFr(   c                 ót   — |rt        t        |Ž «      d   ng }dgdt        |«      z   dz   z  t        |«      z   S )aâ  
        Creates the row token type IDs according to the query token IDs and a list of table values.

        Args:
            query_ids (`List[int]`): list of token IDs corresponding to the ID.
            table_values (`List[TableValue]`): lift of table values, which are named tuples containing the
                token value, the column ID and the row ID of said token.

        Returns:
            `List[int]`: List of ints containing the row token type IDs values.
        é   r   rª   r½   )r€   r·   r¸   Útable_row_idss       r)   Ú(create_row_token_type_ids_from_sequencesz7TapasTokenizer.create_row_token_type_ids_from_sequences¸  sB   € ñ 8DœœS ,Ð/Ó0°Ò3ÈˆØˆs�aœ#˜i›.Ñ(¨1Ñ,Ñ-´°]Ó0CÑCÐCr(   Útoken_ids_0Útoken_ids_1c                 ó^   — |€t        d«      ‚| j                  g|z   | j                  gz   |z   S )a¦  
        Build model inputs from a question and flattened table for question answering or sequence classification tasks
        by concatenating and adding special tokens.

        Args:
            token_ids_0 (`List[int]`): The ids of the question.
            token_ids_1 (`List[int]`, *optional*): The ids of the flattened table.

        Returns:
            `List[int]`: The model input with special tokens.
        z=With TAPAS, you must provide both question IDs and table IDs.)rw   Úcls_token_idÚsep_token_id)r€   rÉ   rÊ   s      r)   Ú build_inputs_with_special_tokensz/TapasTokenizer.build_inputs_with_special_tokensÉ  s=   € ð ÐÜÐ\Ó]Ð]à×!Ñ!Ð" [Ñ0°D×4EÑ4EÐ3FÑFÈÑTÐTr(   Úalready_has_special_tokensc                 ó¬   •— |rt         ‰| �  ||d¬«      S |�'dgdgt        |«      z  z   dgz   dgt        |«      z  z   S dgdgt        |«      z  z   dgz   S )aº  
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of question IDs.
            token_ids_1 (`List[int]`, *optional*):
                List of flattened table IDs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        T)rÉ   rÊ   rÏ   rª   r   )r~   Úget_special_tokens_maskr‰   )r€   rÉ   rÊ   rÏ   r„   s       €r)   rÑ   z&TapasTokenizer.get_special_tokens_maskÜ  s€   ø€ ñ& &Ü‘7Ñ2Ø'°[Ð]að 3ó ð ð Ð"Ø�3˜1˜#¤ KÓ 0Ñ0Ñ1°Q°CÑ7¸A¸3ÄÀ[ÓAQÑ;QÑRÐRØˆs�q�cœC Ó,Ñ,Ñ-°°Ñ3Ð3r(   Útablezpd.DataFrameÚqueriesÚanswer_coordinatesÚanswer_textÚadd_special_tokensÚpaddingÚ
truncationÚ
max_lengthÚpad_to_multiple_ofÚpadding_sideÚreturn_tensorsÚreturn_token_type_idsÚreturn_attention_maskÚreturn_overflowing_tokensÚreturn_special_tokens_maskÚreturn_offsets_mappingÚreturn_lengthÚverbosec                 óN  — t        |t        j                  «      sJ d«       ‚d}|�t        |t        «      rd}n9t        |t        t
        f«      r#t        |«      dk(  st        |d   t        «      rd}|st        d«      ‚t        |t        t
        f«      }|rH | j                  di d|“d|“d|“d	|“d
|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS  | j                  di d|“d|“d|“d	|“d
|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS )a  
        Main method to tokenize and prepare for the model one or several sequence(s) related to a table.

        Args:
            table (`pd.DataFrame`):
                Table containing tabular data. Note that all cell values must be text. Use *.astype(str)* on a Pandas
                dataframe to convert it to string.
            queries (`str` or `List[str]`):
                Question or batch of questions related to a table to be encoded. Note that in case of a batch, all
                questions must refer to the **same** table.
            answer_coordinates (`List[Tuple]` or `List[List[Tuple]]`, *optional*):
                Answer coordinates of each table-question pair in the batch. In case only a single table-question pair
                is provided, then the answer_coordinates must be a single list of one or more tuples. Each tuple must
                be a (row_index, column_index) pair. The first data row (not the column header row) has index 0. The
                first column has index 0. In case a batch of table-question pairs is provided, then the
                answer_coordinates must be a list of lists of tuples (each list corresponding to a single
                table-question pair).
            answer_text (`List[str]` or `List[List[str]]`, *optional*):
                Answer text of each table-question pair in the batch. In case only a single table-question pair is
                provided, then the answer_text must be a single list of one or more strings. Each string must be the
                answer text of a corresponding answer coordinate. In case a batch of table-question pairs is provided,
                then the answer_coordinates must be a list of lists of strings (each list corresponding to a single
                table-question pair).
        z"Table must be of type pd.DataFrameFTr   zgqueries input must of type `str` (single example), `List[str]` (batch or single pretokenized example). rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   Úqueryr'   )
Ú
isinstanceÚpdÚ	DataFramer:   r¾   Útupler‰   rw   Úbatch_encode_plusÚencode_plus)r€   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   r�   Úvalid_queryÚ
is_batcheds                         r)   Ú__call__zTapasTokenizer.__call__ø  s'  € ôp ˜%¤§¡Ô.ÐTÐ0TÓTÐ.ð ˆð ˆ?œj¨´#Ô6Ø‰KÜ˜¤$¬ Ô/Ü�7‹|˜qÒ ¤J¨w°q©z¼3Ô$?Ø"�áÜðóð ô   ¬$´¨Ó7ˆ
áØ)�4×)Ñ)ò Ùðáðñ $6ðñ (ð	ñ
 $6ðñ  ðñ &ðñ &ðñ $6ðñ *ðñ  .ðñ '<ðñ '<ðñ +Dðñ ,Fðñ  (>ð!ñ" ,ð#ñ$  Øñ'ð ð, $�4×#Ñ#ò Ùðáðñ $6ðñ (ð	ñ
 $6ðñ  ðñ &ðñ &ðñ $6ðñ *ðñ  .ðñ '<ðñ '<ðñ +Dðñ ,Fðñ  (>ð!ñ" ,ð#ñ$  Øñ'ð r(   c                 ó8  — |�|st        d«      ‚|r|r|s|rt        d«      ‚|€|€dgt        |«      z  x}}d|v rt        d«      ‚|rt        d«      ‚ | j                  di d|“d|“d	|“d
|“d|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“d|“|¤ŽS )aÃ  
        Prepare a table and a list of strings for the model.

        <Tip warning={true}>

        This method is deprecated, `__call__` should be used instead.

        </Tip>

        Args:
            table (`pd.DataFrame`):
                Table containing tabular data. Note that all cell values must be text. Use *.astype(str)* on a Pandas
                dataframe to convert it to string.
            queries (`List[str]`):
                Batch of questions related to a table to be encoded. Note that all questions must refer to the **same**
                table.
            answer_coordinates (`List[Tuple]` or `List[List[Tuple]]`, *optional*):
                Answer coordinates of each table-question pair in the batch. Each tuple must be a (row_index,
                column_index) pair. The first data row (not the column header row) has index 0. The first column has
                index 0. The answer_coordinates must be a list of lists of tuples (each list corresponding to a single
                table-question pair).
            answer_text (`List[str]` or `List[List[str]]`, *optional*):
                Answer text of each table-question pair in the batch. In case a batch of table-question pairs is
                provided, then the answer_coordinates must be a list of lists of strings (each list corresponding to a
                single table-question pair). Each string must be the answer text of a corresponding answer coordinate.
        Nú·Asking to return token_type_ids while setting add_special_tokens to False results in an undefined behavior. Please set add_special_tokens to True or set return_token_type_ids to None.úWIn case you provide answers, both answer_coordinates and answer_text should be providedÚis_split_into_wordsú<Currently TapasTokenizer only supports questions as strings.úªreturn_offset_mapping is not available when using Python tokenizers. To use this feature, change your tokenizer to one deriving from transformers.PreTrainedTokenizerFast.rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   r'   )rw   r‰   ÚNotImplementedErrorÚ_batch_encode_plus)r€   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   r�   s                       r)   rê   z TapasTokenizer.batch_encode_plusp  sO  € ðn !Ð,Ñ5GÜð5óð ñ ¡{Ñ<NÑS^ÜÐvÓwÐwØÐ'¨KÐ,?Ø04¨v¼¸G»Ñ/DÐDÐ à  FÑ*Ü%Ð&dÓeÐeá!Ü%ð8óð ð 'ˆt×&Ñ&ò 
Ùð
áð
ñ  2ð
ñ $ð	
ñ
  2ð
ñ ð
ñ "ð
ñ "ð
ñ  2ð
ñ &ð
ñ *ð
ñ #8ð
ñ #8ð
ñ '@ð
ñ (Bð
ñ  $:ð!
ñ" (ð#
ñ$ Øñ'
ð 	
r(   c                 ó   — | j                  |«      }| j                  �1t        |«      | j                  kD  rt        j	                  d«       dg fS | j
                  �1t        |«      | j
                  k  rt        j	                  d«       dg fS ||fS )zITokenizes the query, taking into account the max and min question length.zDSkipping query as its tokens are longer than the max question lengthr�   zESkipping query as its tokens are shorter than the min question length)r’   rp   r‰   r²   r³   ro   )r€   rå   Úquery_tokenss      r)   Ú_get_question_tokensz#TapasTokenizer._get_question_tokensÓ  s…   € ð —}‘} UÓ+ˆØ×#Ñ#Ð/´C¸Ó4EÈ×H`ÑH`Ò4`Ü�N‰NÐaÔbØ�r�6ˆMØ×#Ñ#Ð/´C¸Ó4EÈ×H`ÑH`Ò4`Ü�N‰NÐbÔcØ�r�6ˆMà�lÐ"Ð"r(   c                 ó  — | j                  |«      }g }t        |«      D ]/  \  }}| j                  |«      \  }}|||<   |j                  |«       Œ1 | j	                  |||||||||||	|
|d||||||¬«      }t        |«      S )NT)Útokenized_tableÚqueries_tokensrÔ   r×   rØ   rÕ   rÖ   rÙ   rÚ   rÛ   rÜ   Úprepend_batch_axisrÞ   rÝ   rß   rà   râ   rã   )Ú_tokenize_tablerO   rù   rs   Ú_batch_prepare_for_modelr   )r€   rÒ   rÓ   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rß   rà   rá   râ   rã   r�   Útable_tokensrü   Úidxrå   rø   Úbatch_outputss                             r)   rö   z!TapasTokenizer._batch_encode_plusà  s½   € ð4 ×+Ñ+¨EÓ2ˆàˆÜ# GÓ,ò 	0‰JˆC�Ø"&×";Ñ";¸EÓ"BÑˆE�<Ø ˆG�C‰LØ×!Ñ! ,Õ/ð	0ð
 ×5Ñ5ØØØ(Ø)Ø1ØØ!Ø#Ø1Ø!Ø1Ø%Ø)Ø#Ø"7Ø"7Ø&?Ø'AØ'Øð) 6ó 
ˆô. ˜]Ó+Ð+r(   Ú	raw_tableÚraw_queriesrû   rü   rý   c                 ó¸  — i }t        t        ||||«      «      D ]™  \  }}|\  }}}}| j                  |||||||t        j                  j
                  |	|
d d d|||d d||dk7  r||dz
     nd |dk7  r||dz
     nd ¬«      }|j                  «       D ]"  \  }}||vrg ||<   ||   j                  |«       Œ$ Œ› | j                  |||
|||¬«      }t        ||¬«      }|S )NFr   rª   )rû   rø   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÞ   rÝ   rà   râ   rÜ   rý   rã   Úprev_answer_coordinatesÚprev_answer_text)r×   rÙ   rÚ   rÛ   rÞ   )Útensor_type)
rO   r¿   Úprepare_for_modelr   Ú
DO_NOT_PADÚvaluerx   rs   Úpadr   ) r€   r  r  rû   rü   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rà   rá   râ   rã   rý   r�   r  rS   ÚexampleÚ	raw_queryrø   Úanswer_coordsÚ
answer_txtÚoutputsr®   r  s                                    r)   rÿ   z'TapasTokenizer._batch_prepare_for_model  s<  € ð8 ˆä'¬¨K¸ÐI[Ð]hÓ(iÓjò 	1‰NˆE�7ØAHÑ>ˆI�| ]°JØ×,Ñ,ØØØ /Ø)Ø#0Ø&Ø#5Ü'×2Ñ2×8Ñ8Ø%Ø%Ø#'Ø!Ø&+Ø&;Ø+EØ+Ø#Ø#(ØØINÐRSÊÐ(:¸5À1¹9Ò(EÐY]Ø;@ÀAº: ¨U°Q©YÒ!7È4ð+ -ó ˆGð0 &Ÿm™m›oò 1‘
��UØ˜mÑ+Ø)+�M #Ñ&Ø˜cÑ"×)Ñ)¨%Õ0ñ1ð5	1ð> Ÿ™ØØØ!Ø1Ø%Ø"7ð !ó 
ˆô & mÀÔPˆàÐr(   rå   c           
      ó@   —  | j                   |f||||||dœ|¤Ž}	|	d   S )af  
        Prepare a table and a string for the model. This method does not return token type IDs, attention masks, etc.
        which are necessary for the model to work correctly. Use that method if you want to build your processing on
        your own, otherwise refer to `__call__`.

        Args:
            table (`pd.DataFrame`):
                Table containing tabular data. Note that all cell values must be text. Use *.astype(str)* on a Pandas
                dataframe to convert it to string.
            query (`str` or `List[str]`):
                Question related to a table to be encoded.
        )rå   rÖ   r×   rØ   rÙ   rÜ   Ú	input_ids)rë   )
r€   rÒ   rå   rÖ   r×   rØ   rÙ   rÜ   r�   Úencoded_inputss
             r)   ÚencodezTapasTokenizer.encodee  sG   € ð< *˜×)Ñ)Øð	
àØ1ØØ!Ø!Ø)ñ	
ð ñ	
ˆð ˜kÑ*Ð*r(   c                 ó  — |�|st        d«      ‚|r|r|s|rt        d«      ‚d|v rt        d«      ‚|rt        d«      ‚ | j                  di d|“d|“d|“d	|“d
|“d|“d|“d|“d|	“d|
“d|“d|“d|“d|“d|“d|“d|“|¤ŽS )a7  
        Prepare a table and a string for the model.

        Args:
            table (`pd.DataFrame`):
                Table containing tabular data. Note that all cell values must be text. Use *.astype(str)* on a Pandas
                dataframe to convert it to string.
            query (`str` or `List[str]`):
                Question related to a table to be encoded.
            answer_coordinates (`List[Tuple]` or `List[List[Tuple]]`, *optional*):
                Answer coordinates of each table-question pair in the batch. The answer_coordinates must be a single
                list of one or more tuples. Each tuple must be a (row_index, column_index) pair. The first data row
                (not the column header row) has index 0. The first column has index 0.
            answer_text (`List[str]` or `List[List[str]]`, *optional*):
                Answer text of each table-question pair in the batch. The answer_text must be a single list of one or
                more strings. Each string must be the answer text of a corresponding answer coordinate.
        rð   rñ   rò   ró   rô   rÒ   rå   rÔ   rÕ   rÖ   rØ   r×   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rà   rá   râ   rã   r'   )rw   rõ   Ú_encode_plus)r€   rÒ   rå   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rà   rá   râ   rã   r�   s                      r)   rë   zTapasTokenizer.encode_plus�  s#  € ðZ !Ð,Ñ5GÜð5óð ñ ¡{Ñ<NÑS^ÜÐvÓwÐwà  FÑ*Ü%Ð&dÓeÐeá!Ü%ð8óð ð !ˆt× Ñ ò 
Ùð
áð
ñ  2ð
ñ $ð	
ñ
  2ð
ñ "ð
ñ ð
ñ "ð
ñ  2ð
ñ &ð
ñ *ð
ñ #8ð
ñ #8ð
ñ (Bð
ñ $:ð
ñ  (ð!
ñ" Øñ%
ð 	
r(   c                 óÆ   — |€d}t         j                  d«       | j                  |«      }| j                  |«      \  }}| j	                  |||||||||||	|
|d|||||¬«      S )Nr�   z‡TAPAS is a question answering model but you have not passed a query. Please be aware that the model will probably not behave correctly.T)rû   rø   rÔ   rÕ   rÖ   rØ   r×   rÙ   rÚ   rÛ   rÜ   rý   rÞ   rÝ   rà   râ   rã   )r²   r³   rþ   rù   r	  )r€   rÒ   rå   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rà   rá   râ   rã   r�   r   rø   s                        r)   r  zTapasTokenizer._encode_plusæ  s—   € ð2 ˆ=ØˆEÜ�N‰Nð<ôð
 ×+Ñ+¨EÓ2ˆØ"×7Ñ7¸Ó>Ñˆˆ|à×%Ñ%ØØØ(Ø%Ø1Ø#Ø1Ø!ØØ!Ø1Ø%Ø)Ø#Ø"7Ø"7Ø'AØ'Øð' &ó 
ð 	
r(   r  rø   c                 óæ  — t        |t        «      r(|r|
€|�t        j                  }n,t        j                  }nt        |t        «      st        |«      }t        |	t        «      r$|	rt
        j                  }	n,t
        j                  }	nt        |	t
        «      st        |	«      }	i }d}d\  }}d|v rd|v rd}|d   }|d   }| j                  ||	t
        j                  k7  «      }| j                  |«      }| j                  |«      \  }}}|	t
        j                  k7  r| j                  |||||
|	¬«      \  }}t        | j                  ||||«      «      }| j                  |«      }t        |«      dkD  rt        t!        |Ž «      d   nt        t!        |Ž «      } | j                  t        | «      «      } d|v r|d   rt#        d	«      ‚|r| j%                  || «      }!n|| z   }!|
�(t        |!«      |
kD  rt#        d
t        |!«      › d|
› �«      ‚|!|d<   | j'                  ||«      }"| j)                  ||«      }#| j+                  ||«      }$|r|€|€dgt        |$«      z  }%n| j-                  |#|$|||«      }%t/        |«      }t1        |«      }| j3                  |#|$|«      \  }&}'| j5                  ||#|$|«      }(|€d| j6                  v }|€d| j6                  v }|r| j9                  ||«      })|)|d<   |�L|�J| j-                  |#|$|||«      }*| j;                  ||#|$«      }+| j=                  ||#|$«      },|*|d<   |+|d<   |,|d<   |r5|"|#|$|%|&|'|(g}-t        t!        |-Ž «      D �.cg c]  }.t        |.«      ‘Œ }-}.|-|d<   |r*|r| j?                  || «      |d<   ndgt        |!«      z  |d<   |
€zt        |d   «      | j@                  kD  r_|r]| jB                  jE                  dd«      s2tF        jI                  dt        |d   «      › d| j@                  › d�«       d| jB                  d<   |t        j                  k7  s|r!| jK                  ||
|jL                  |||¬«      }|rt        |d   «      |d<   tO        |||¬«      }/|/S c c}.w )a�  
        Prepares a sequence of input id so that it can be used by the model. It adds special tokens, truncates
        sequences if overflowing while taking into account the special tokens.

        Args:
            raw_table (`pd.DataFrame`):
                The original table before any transformation (like tokenization) was applied to it.
            raw_query (`TextInput` or `PreTokenizedInput` or `EncodedInput`):
                The original query before any transformation (like tokenization) was applied to it.
            tokenized_table (`TokenizedTable`):
                The table after tokenization.
            query_tokens (`List[str]`):
                The query after tokenization.
            answer_coordinates (`List[Tuple]` or `List[List[Tuple]]`, *optional*):
                Answer coordinates of each table-question pair in the batch. The answer_coordinates must be a single
                list of one or more tuples. Each tuple must be a (row_index, column_index) pair. The first data row
                (not the column header row) has index 0. The first column has index 0.
            answer_text (`List[str]` or `List[List[str]]`, *optional*):
                Answer text of each table-question pair in the batch. The answer_text must be a single list of one or
                more strings. Each string must be the answer text of a corresponding answer coordinate.
        F)NNr  r  T)Útruncation_strategyr   rß   z?TAPAS does not return overflowing tokens as it works on tables.z�Could not encode the query and table header given the maximum length. Encoding the query and table header results in a length of z( which is higher than the max_length of r  Útoken_type_idsÚattention_maskÚlabelsÚnumeric_valuesÚnumeric_values_scaleÚspecial_tokens_maskz4sequence-length-is-longer-than-the-specified-maximumzcToken indices sequence length is longer than the specified maximum sequence length for this model (z > zJ). Running this sequence through the model will result in indexing errors.)rÙ   r×   rÚ   rÛ   rÞ   Úlength)r  rý   )(ræ   Úboolr   Ú
MAX_LENGTHr
  r   r%   r&   Ú_get_num_rowsÚ_get_num_columnsÚ_get_table_boundariesÚ_get_truncated_table_rowsr¾   Ú_get_table_valuesÚconvert_tokens_to_idsr‰   r¿   rw   rÎ   rÁ   rÄ   rÈ   Úget_answer_idsÚadd_numeric_table_valuesÚadd_numeric_values_to_questionÚ_get_numeric_column_ranksÚ_get_numeric_relationsÚmodel_input_namesr»   Ú_get_numeric_valuesÚ_get_numeric_values_scalerÑ   ra   Údeprecation_warningsr—   r²   r³   r  r  r   )0r€   r  r  rû   rø   rÔ   rÕ   rÖ   r×   rØ   rÙ   rÚ   rÛ   rÜ   rÝ   rÞ   rà   rá   râ   rã   rý   r�   r  Úis_part_of_batchr  r  Únum_rowsÚnum_columnsÚ_Ú
num_tokensÚ
table_datar·   rÀ   r  r@   r>   r?   Úprev_labelsÚcolumn_ranksÚinv_column_ranksÚnumeric_relationsr  r  r  r  r  rƒ   r  s0                                                   r)   r	  z TapasTokenizer.prepare_for_model  s4  € ôd �gœtÔ$Ù˜JÐ2Ð6HÐ6TÜ)×4Ñ4‘ä)×4Ñ4‘Ü˜G¤_Ô5Ü% gÓ.ˆGä�j¤$Ô'ÙÜ4×EÑE‘
ä4×DÑD‘
Ü˜JÔ(?Ô@Ü0°Ó<ˆJàˆà ÐØ4>Ñ1ÐÐ!1Ø$¨Ñ.Ð3EÈÑ3OØ#ÐØ&,Ð-FÑ&GÐ#Ø%Ð&8Ñ9Ðà×%Ñ% i°Ô?V×?fÑ?fÑ1fÓgˆØ×+Ñ+¨IÓ6ˆØ×5Ñ5°oÓFÑˆˆ1ˆjàÔ0×@Ñ@Ò@Ø#'×#AÑ#AØ˜o¨x¸ÀjÐfpð $Bó $Ñ ˆH�jô ˜$×0Ñ0°À+ÈxÐYcÓdÓeˆ
à×.Ñ.¨|Ó<ˆ	Ü14°Z³À1Ò1D”Dœ˜jÐ)Ó*¨1Ò-Ì$ÌsÐT^ÐO_ÓJ`ˆ	Ø×.Ñ.¬t°I«Ó?ˆ	à&¨&Ñ0°VÐ<WÒ5XÜÐ^Ó_Ð_áØ×=Ñ=¸iÈÓS‰Ià! IÑ-ˆIàÐ!¤c¨)£n°zÒ&AÜð1Ü14°Y³Ð0@Ð@hÐisÐhtðvóð ð
 '0ˆ�{Ñ#à×GÑGÈ	ÐS]Ó^ˆØ×EÑEÀiÐQ[Ó\ˆ
Ø×?Ñ?À	È:ÓVˆÙÐ$;Ð$CÐHXÐH`à˜#¤ G£Ñ,‰Kà×-Ñ-Ø˜G ZÐ1AÐCZóˆKô -¨YÓ7ˆ	Ü2°9Ó=ˆ	ð *.×)GÑ)GÈ
ÐT[Ð]fÓ)gÑ&ˆÐ&Ø ×7Ñ7¸	À:ÈwÐXaÓbÐð !Ð(Ø$4¸×8NÑ8NÐ$NÐ!Ø Ð(Ø$4¸×8NÑ8NÐ$NÐ!á Ø!×FÑFÀyÐR\Ó]ˆNØ/=ˆNÐ+Ñ,àÐ)¨kÐ.EØ×(Ñ(¨°W¸jÈ+ÐWiÓjˆFØ!×5Ñ5°iÀÈWÓUˆNØ#'×#AÑ#AÀ)ÈZÐY`Ó#aÐ à'-ˆN˜8Ñ$Ø/=ˆNÐ+Ñ,Ø5IˆNÐ1Ñ2á àØØØØØ Ø!ðˆNô 48¼¸^Ð8LÓ3MÖN¨Cœd 3�iÐNˆNÐNØ/=ˆNÐ+Ñ,á%Ù!Ø8<×8TÑ8TÐU^Ð`iÓ8j�Ð4Ò5à9:¸¼cÀ)»nÑ8L�Ð4Ñ5ð Ð¤# n°[Ñ&AÓ"BÀT×EZÑEZÒ"ZÑ_fØ×,Ñ,×0Ñ0Ð1gÐinÔoÜ—‘ð'Ü'*¨>¸+Ñ+FÓ'GÐ&HÈÈD×LaÑLaÐKbð cQðQôð
 aeˆD×%Ñ%Ð&\Ñ]ð ”o×0Ñ0Ò0Ñ4IØ!ŸX™XØØ%ØŸ™Ø#5Ø)Ø&;ð &ó ˆNñ Ü'*¨>¸+Ñ+FÓ'GˆN˜8Ñ$ä%Ø¨ÐK]ô
ˆð ÐùòK Os   ÍQ.r4  r5  r  c                 ó  — t        |t        «      st        |«      }|€| j                  }|t        j                  k(  r&	 | j	                  |||||¬«      }|�n.|dz  }|dk  rn#Œ%|t        j
                  k7  rt        d|› d�«      ‚|xs dfS )a¨  
        Truncates a sequence pair in-place following the strategy.

        Args:
            query_tokens (`List[str]`):
                List of strings corresponding to the tokenized query.
            tokenized_table (`TokenizedTable`):
                Tokenized table
            num_rows (`int`):
                Total number of table rows
            num_columns (`int`):
                Total number of table columns
            max_length (`int`):
                Total maximum length.
            truncation_strategy (`str` or [`TapasTruncationStrategy]`):
                Truncation strategy to use. Seeing as this method should only be called when truncating, the only
                available strategy is the `"drop_rows_to_fit"` strategy.

        Returns:
            `Tuple(int, int)`: tuple containing the number of rows after truncation, and the number of tokens available
            for each table element.
        )r4  r5  rÙ   rª   zUnknown truncation strategy ú.)ræ   r   ra   r%   Ú_get_max_num_tokensr&   rw   )r€   rø   rû   r4  r5  rÙ   r  r7  s           r)   r'  z(TapasTokenizer._get_truncated_table_rowsâ  s¾   € ô> Ð-Ô/FÔGÜ"9Ð:MÓ"NÐàÐØ×.Ñ.ˆJàÔ"9×"JÑ"JÒJØØ!×5Ñ5Ø  /¸HÐR]Ðjtð 6ó �
ð Ð)àð ˜A‘�à˜a’<Øð ð !Ô$;×$KÑ$KÒKÜÐ;Ð<OÐ;PÐPQÐRÓSÐSà˜š qÐ(Ð(r(   c           
      óB  — g }g }|D ]O  }| j                   r!|j                  | j                  d«      «       Œ0|j                  | j                  |«      «       ŒQ |j                  |«       |j                  «       D ]?  \  }}g }|D ]"  }|j                  | j                  |«      «       Œ$ |j                  |«       ŒA g }t	        |«      D ]H  \  }	}t	        |«      D ]5  \  }
}t	        |«      D ]"  \  }}|j                  t        |	|
|¬«      «       Œ$ Œ7 ŒJ t        ||¬«      S )zÁ
        Tokenizes column headers and cell texts of a table.

        Args:
            table (`pd.Dataframe`):
                Table. Returns: `TokenizedTable`: TokenizedTable object.
        r�   ©r2   r1   r3   )r8   r9   )r_   rs   r’   ÚiterrowsrO   r0   r7   )r€   rÒ   Útokenized_rowsÚtokenized_rowÚcolumnr  ÚrowÚcellÚtoken_coordinatesr2   r1   r3   r6  s                r)   rþ   zTapasTokenizer._tokenize_table  s<  € ð ˆØˆàò 	<ˆFØ×&Ò&Ø×$Ñ$ T§]¡]°2Ó%6Õ7à×$Ñ$ T§]¡]°6Ó%:Õ;ð		<ð
 	×Ñ˜mÔ,ð Ÿ™Ó(ò 	1‰HˆC�ØˆMØò :�Ø×$Ñ$ T§]¡]°4Ó%8Õ9ð:à×!Ñ! -Õ0ð		1ð ÐÜ'¨Ó7ò 		‰NˆI�sÜ&/°£nò Ñ"�˜dÜ&/°£oò ‘N�K Ø%×,Ñ,Ü(Ø&/Ø)5Ø(3ôõññð		ô ØØ-ô
ð 	
r(   c                 ó   — t        |«      dz   S )NrÆ   rº   )r€   Úquestion_tokenss     r)   Ú_question_encoding_costz&TapasTokenizer._question_encoding_costH  s   € ä�?Ó# aÑ'Ð'r(   c                 óF   — |�|n| j                   | j                  |«      z
  S )al  
        Computes the number of tokens left for the table after tokenizing a question, taking into account the max
        sequence length of the model.

        Args:
            question_tokens (`List[String]`):
                List of question tokens. Returns: `int`: the number of tokens left for the table, given the model max
                length.
        )ra   rK  )r€   rJ  rÙ   s      r)   Ú_get_token_budgetz TapasTokenizer._get_token_budgetL  s0   € ð )Ð4‘
¸$×:OÑ:OÐSW×SoÑSoØóT
ñ 
ð 	
r(   c              #   ó¦  K  — |j                   D ]¾  }|j                  |dz   k\  rŒ|j                  |k\  rŒ&|j                  |j                     |j                     }||j                     }|j                  }|dk\  r't        ||   «      r|dz  }|dk\  rt        ||   «      rŒ||k\  rŒ™t        ||j                  dz   |j                  «      –— ŒÀ y­w)zFIterates over partial table and returns token, column and row indexes.rª   r   N)r9   r2   r1   r8   r3   rD   Ú
TableValue)	r€   rÒ   r5  r4  r7  ÚtcrG  r+   Úword_begin_indexs	            r)   r(  z TapasTokenizer._get_table_valuesZ  sÔ   è ø€ à×'Ñ'ò 	GˆBà�|‰|˜x¨!™|Ò+ØØ�‰ +Ò-ØØ—:‘:˜bŸl™lÑ+¨B¯O©OÑ<ˆDØ˜Ÿ™Ñ(ˆEØ!Ÿ~™~Ðð # aÒ'Ô,?ÀÐEUÑ@VÔ,WØ  AÑ%Ð ð # aÒ'Ô,?ÀÐEUÑ@VÕ,Wà :Ò-ØÜ˜U B§O¡O°aÑ$7¸¿¹ÓFÓFñ	Gùs   ‚BCÂ"/Cc                 ó(  — d}d}d}|j                   D ]y  }t        ||j                  dz   «      }t        ||j                  dz   «      }t        ||j                  dz   «      }t        | j                  |«      }t        | j                  |«      }Œ{ |||fS )z2Return maximal number of rows, columns and tokens.r   rª   )r9   Úmaxr1   r2   r3   Úminr]   r^   )r€   rÒ   Úmax_num_tokensÚmax_num_columnsÚmax_num_rowsrP  s         r)   r&  z$TapasTokenizer._get_table_boundariesm  s–   € àˆØˆØˆØ×'Ñ'ò 	>ˆBÜ! /°2·?±?ÀQÑ3FÓGˆOÜ˜|¨R¯\©\¸AÑ-=Ó>ˆLÜ  °·±À!Ñ1CÓDˆNÜ! $×"4Ñ"4°oÓFˆOÜ˜tŸ™°Ó=‰Lð	>ð ˜_¨nÐ<Ð<r(   c           	      óJ   — t        d„ | j                  ||||«      D «       «      S )Nc              3   ó    K  — | ]  }d –— Œ y­w)rª   Nr'   )Ú.0r6  s     r)   ú	<genexpr>z1TapasTokenizer._get_table_cost.<locals>.<genexpr>{  s   è ø€ Ò_˜”1Ñ_ùs   ‚)Úsumr(  )r€   rÒ   r5  r4  r7  s        r)   Ú_get_table_costzTapasTokenizer._get_table_costz  s%   € ÜÑ_˜d×4Ñ4°U¸KÈÐS]Ó^Ô_Ó_Ð_r(   c                 óD  — | j                  ||«      }| j                  |«      \  }}}| j                  dk\  r|| j                  kD  r| j                  }d}	t        |dz   «      D ]   }	| j	                  ||||	dz   «      }
|
|kD  sŒ  n |	|k  r| j                  dk\  ry|	dk(  ry|	S )zCComputes max number of tokens that can be squeezed into the budget.r   rª   N)rM  r&  r\   Úranger]  )r€   rJ  rû   r5  r4  rÙ   Útoken_budgetr6  rU  r7  Úcosts              r)   r?  z"TapasTokenizer._get_max_num_tokens}  sÂ   € à×-Ñ-¨o¸zÓJˆØ#×9Ñ9¸/ÓJÑˆˆ1ˆnØ× Ñ  AÒ%¨.¸4×;PÑ;PÒ*PØ!×2Ñ2ˆNØˆ
Ü °Ñ 2Ó3ò 	ˆJØ×'Ñ'¨¸ÀhÐPZÐ]^ÑP^Ó_ˆDØ�lÓ"Ùð	ð ˜Ò&Ø×$Ñ$¨Ò)àØ˜QŠØØÐr(   c                 óX   — |j                   d   }|| j                  k\  rt        d«      ‚|S )Nrª   zToo many columns)Úshaper]   rw   )r€   rÒ   r5  s      r)   r%  zTapasTokenizer._get_num_columns�  s/   € Ø—k‘k !‘nˆØ˜$×,Ñ,Ò,ÜÐ/Ó0Ð0ØÐr(   c                 ó~   — |j                   d   }|| j                  k\  r|r| j                  dz
  }|S t        d«      ‚|S )Nr   rª   zToo many rows)rc  r^   rw   )r€   rÒ   r   r4  s       r)   r$  zTapasTokenizer._get_num_rows–  sE   € Ø—;‘;˜q‘>ˆØ�t—‘Ò&ÙØŸ?™?¨QÑ.�ð ˆô ! Ó1Ð1Øˆr(   c                 óP  — g }g }g }g }|j                  | j                  «       |j                  d«       |j                  d«       |j                  d«       |D ]F  }|j                  |«       |j                  d«       |j                  d«       |j                  d«       ŒH ||||fS )z!Serializes texts in index arrays.r   )rs   rl   )r€   rJ  r=   r@   r>   r?   r+   s          r)   Ú_serialize_textzTapasTokenizer._serialize_textŸ  s¤   € àˆØˆØˆ
Øˆð 	�‰�d—n‘nÔ%Ø×Ñ˜1ÔØ×Ñ˜!ÔØ�‰�qÔà$ò 	ˆEØ�M‰M˜%Ô Ø×Ñ˜qÔ!Ø×Ñ˜aÔ Ø�N‰N˜1Õð		ð �{ J°Ð7Ð7r(   c                 óª  — | j                  |«      \  }}}}	|j                  | j                  «       |j                  d«       |j                  d«       |	j                  d«       | j                  ||||«      D ]J  \  }
}}|j                  |
«       |j                  d«       |j                  |«       |	j                  |«       ŒL t	        ||||	¬«      S )zSerializes table and text.r   rª   )r=   r@   r>   r?   )rf  rs   rj   r(  r<   )r€   rJ  rÒ   r5  r4  r7  r=   r@   r>   r?   r+   r,   r-   s                r)   Ú
_serializezTapasTokenizer._serialize´  sÏ   € ð 48×3GÑ3GÈÓ3XÑ0ˆ�˜Z¨ð 	�‰�d—n‘nÔ%Ø×Ñ˜1ÔØ×Ñ˜!ÔØ�‰�qÔà(,×(>Ñ(>¸uÀkÐS[Ð]gÓ(hò 	#Ñ$ˆE�9˜fØ�M‰M˜%Ô Ø×Ñ˜qÔ!Ø×Ñ˜iÔ(Ø�N‰N˜6Õ"ð		#ô !ØØ#Ø!Øô	
ð 	
r(   c                 ó|   — i }|j                  «       D ]&  \  }}||   }|j                  €Œ|j                  ||<   Œ( |S r†   )rB  Únumeric_value)r€   rÒ   Ú	col_indexÚtable_numeric_valuesr2   rF  rG  s          r)   Ú_get_column_valuesz!TapasTokenizer._get_column_valuesÒ  sR   € Ø!ÐØ#Ÿn™nÓ.ò 	E‰NˆI�sØ�y‘>ˆDØ×!Ñ!Ñ-Ø26×2DÑ2DÐ$ YÒ/ð	Eð $Ð#r(   c              #   óv   K  — t        t        |«      «      D ]  }||   dz
  |k(  sŒ||   dz
  |k(  sŒ|–— Œ  y ­wr©   )r_  r‰   )r€   r>   r?   r,   r-   rS   s         r)   Ú_get_cell_token_indexesz&TapasTokenizer._get_cell_token_indexesÚ  sE   è ø€ Üœ3˜z›?Ó+ò 	ˆEØ˜%Ñ  1Ñ$¨	Ó1°g¸e±nÀqÑ6HÈFÓ6RØ“ñ	ùs   ‚#9¦9²9c           	      óÌ  — dgt        |«      z  }dgt        |«      z  }|��*t        t        |j                  «      «      D �]  }| j                  ||«      }|sŒ	 t	        |j                  «       «      }|j                  «       D �	�
ci c]  \  }	}
|	 ||
«      “Œ }}	}
t        j                  t        «      }|j                  «       D ]  \  }	}
||
   j                  |	«       Œ t        |j                  «       «      }t        |«      D ]A  \  }}
||
   D ]4  }	| j                  ||||	«      D ]  }|dz   ||<   t        |«      |z
  ||<   Œ Œ6 ŒC �Œ ||fS # t        $ r Y �Œw xY wc c}
}	w )z-Returns column ranks for all numeric columns.r   rª   )r‰   r_  Úcolumnsrm  Úget_numeric_sort_key_fnÚvaluesrw   rx   rK   Údefaultdictr¾   rs   r±   ÚkeysrO   ro  )r€   r>   r?   rÒ   ÚranksÚ	inv_ranksrk  rl  Úkey_fnr2   r  Útable_numeric_values_invÚunique_valuesÚrankrS   s                  r)   r-  z(TapasTokenizer._get_numeric_column_ranksß  s¡  € ð �”c˜*“oÑ%ˆØ�Cœ#˜j›/Ñ)ˆ	ð ÑÜ"¤3 u§}¡}Ó#5Ó6ó I�	Ø'+×'>Ñ'>¸uÀiÓ'PÐ$á+ØðÜ4Ð5I×5PÑ5PÓ5RÓS�Fð Zn×YsÑYsÓYu×'vÑEUÀYÐPU¨	±6¸%³=Ñ(@Ð'vÐ$Ñ'vä+6×+BÑ+BÄ4Ó+HÐ(Ø(<×(BÑ(BÓ(Dò FÑ$�I˜uØ,¨UÑ3×:Ñ:¸9ÕEðFô !'Ð'?×'DÑ'DÓ'FÓ G�ä#,¨]Ó#;ò I‘K�D˜%Ø%=¸eÑ%Dò I˜	Ø%)×%AÑ%AÀ*ÈgÐW`ÐbkÓ%lò I˜EØ+/°!©8˜E %™LÜ/2°=Ó/AÀDÑ/H˜I eÒ,ñIñIòIð'Ið2 �iÐÐøô# "ò Úðüó (ws   ÁEÂE Å	EÅEc                 ó’   — |syt        |j                  «       «      }|j                  |«       	 t        |«      S # t        $ r Y yw xY w)aÇ  
        Returns the sort key function for comparing value to table values. The function returned will be a suitable
        input for the key param of the sort(). See number_annotation_utils._get_numeric_sort_key_fn for details

        Args:
            table_numeric_values: Numeric values of a column
            value: Numeric value in the question

        Returns:
            A function key function to compare column and question values.
        N)r¾   rs  rs   rr  rw   )r€   rl  r  Ú
all_valuess       r)   Ú_get_numeric_sort_key_fnz'TapasTokenizer._get_numeric_sort_key_fn  sN   € ñ $ØÜÐ.×5Ñ5Ó7Ó8ˆ
Ø×Ñ˜%Ô ð	Ü*¨:Ó6Ð6øÜò 	Ùð	ús   ¯
: º	AÁAc           	      óâ  — dgt        |«      z  }t        j                  t        «      }|�¬|�ª|j                  D ]›  }|j
                  D ]Š  }t        t        |j                  «      «      D ]g  }	| j                  ||	«      }
| j                  |
|«      }|€Œ*|
j                  «       D ]+  \  }}t        |||«      }|€Œ||	|f   j                  |«       Œ- Œi ŒŒ Œ� |j                  «       D ]…  \  \  }	}}d}|D ]X  }|j                  t        j                  j                  k\  sJ ‚|d|j                  t        j                  j                  z
  z  z  }ŒZ | j!                  |||	|«      D ]  }|||<   Œ	 Œ‡ |S )a%  
        Returns numeric relations embeddings

        Args:
            question: Question object.
            column_ids: Maps word piece position to column id.
            row_ids: Maps word piece position to row id.
            table: The table containing the numeric cell values.
        r   rÆ   )r‰   rK   rt  ÚsetÚnumeric_spansrs  r_  rq  rm  r~  rx   Úget_numeric_relationÚaddr  ÚRelationÚEQro  )r€   Úquestionr>   r?   rÒ   r<  Úcell_indices_to_relationsÚnumeric_value_spanr  r1   rl  Úsort_key_fnr2   Ú
cell_valueÚrelationÚ	relationsÚrelation_set_indexÚcell_token_indexs                     r)   r.  z%TapasTokenizer._get_numeric_relations  sµ  € ð ˜C¤# j£/Ñ1Ðô
 %0×$;Ñ$;¼CÓ$@Ð!ØÐ EÐ$5Ø&.×&<Ñ&<ò 
aÐ"Ø/×6Ñ6ò 	a�EÜ(-¬c°%·-±-Ó.@Ó(Aò a˜Ø/3×/FÑ/FÀuÈlÓ/[Ð,Ø&*×&CÑ&CÐDXÐZ_Ó&`˜Ø&Ð.Ø$Ø5I×5OÑ5OÓ5Qò aÑ1˜I zÜ';¸EÀ:È{Ó'[˜HØ'Ñ3Ø 9¸,È	Ð:QÑ R× VÑ VÐW_Õ `ñañañ	að
að 5N×4SÑ4SÓ4Uò 	IÑ0Ñ%ˆ\˜9 yØ!"ÐØ%ò P�Ø—~‘~¬¯©×):Ñ):Ò:Ð:Ð:Ø" a¨H¯N©N¼X¿[¹[×=NÑ=NÑ,NÑ&OÑOÑ"ðPð %)×$@Ñ$@ÀÈWÐVbÐdmÓ$nò IÐ Ø6HÐ!Ð"2Ò3ñIð	Ið !Ð r(   c                 ó~  — t        d«      gt        |«      z  }|�¢|j                  d   }|j                  d   }t        |«      D ]v  }t        |«      D ]f  }|j                  ||f   j
                  }	|	€Œ!|	j                  €Œ.|	j                  }
|
t        d«      k(  rŒI| j                  ||||«      D ]  }|
||<   Œ	 Œh Œx |S )z6Returns numeric values for computation of answer loss.Únanr   rª   Úinf)Úfloatr‰   rc  r_  Úilocrj  Úfloat_valuero  )r€   rÒ   r>   r?   r  r4  r5  rk  r2   rj  r”  rS   s               r)   r0  z"TapasTokenizer._get_numeric_values@  sá   € ô   ›,˜¬#¨j«/Ñ9ˆàÐØ—{‘{ 1‘~ˆHØŸ+™+ a™.ˆKä" ;Ó/ò 
@�	Ü!& x£ò 	@�IØ$)§J¡J¨y¸)Ð/CÑ$D×$RÑ$R�MØ$Ñ0Ø(×4Ñ4Ð<Ø$Ø&3×&?Ñ&?˜Ø&¬%°«,Ò6Ø$Ø%)×%AÑ%AÀ*ÈgÐW`ÐbkÓ%lò @˜EØ4?˜N¨5Ò1ñ@ñ	@ð
@ð Ðr(   c           
      ó.  — dgt        |«      z  }|€|S |j                  d   }|j                  d   }t        |«      D ]U  }t        |«      D ]E  }t        | j	                  ||||«      «      }	t        |	«      }
|
dkD  sŒ1|	D ]  }t        |
«      ||<   Œ ŒG ŒW |S )zDReturns a scale to each token to down weigh the value of long words.ç      ð?r   rª   )r‰   rc  r_  r¾   ro  r’  )r€   rÒ   r>   r?   r  r4  r5  rk  r2   ÚindicesÚnum_indicesrS   s               r)   r1  z(TapasTokenizer._get_numeric_values_scaleW  s»   € ð !$˜u¤s¨:£Ñ6Ðàˆ=Ø'Ð'à—;‘;˜q‘>ˆØ—k‘k !‘nˆä˜{Ó+ò 	IˆIÜ" 8›_ò I�	Ü˜t×;Ñ;¸JÈÐQZÐ\eÓfÓg�Ü! '›l�Ø “?Ø!(ò I˜Ü6;¸KÓ6HÐ,¨UÒ3ñIñ	Ið	Ið $Ð#r(   c                 ó  — t        |«      | j                  kD  r)|j                  «        t        |«      | j                  kD  rŒ)t        |«      | j                  k  r+|j                  d«       t        |«      | j                  k  rŒ*y y ©Nr   )r‰   ra   Úpoprs   )r€   Úinputss     r)   Ú_pad_to_seq_lengthz!TapasTokenizer._pad_to_seq_lengthl  sc   € Ü�&‹k˜D×1Ñ1Ò1Ø�J‰JŒLô �&‹k˜D×1Ñ1Ó1ä�&‹k˜D×1Ñ1Ò1Ø�M‰M˜!Ôô �&‹k˜D×1Ñ1Õ1r(   c                 ó  — dgt        |«      z  }t        «       }t        «       }|D ]K  }|\  }}	|j                  ||	f«       | j                  ||||	«      D ]  }
|j                  ||	f«       d||
<   Œ ŒM t        |«      t        |«      z
  }||fS )z2Maps lists of answer coordinates to token indexes.r   rª   )r‰   r€  rƒ  ro  )r€   r>   r?   Úanswers_listÚ
answer_idsÚfound_answersÚall_answersÚanswersr1   r2   rS   Úmissing_counts               r)   Ú$_get_all_answer_ids_from_coordinatesz3TapasTokenizer._get_all_answer_ids_from_coordinatesr  s«   € ð �Sœ3˜z›?Ñ*ˆ
Ü›ˆÜ“eˆØ#ò 	&ˆGØ&-Ñ#ˆL˜)Ø�O‰O˜\¨9Ð5Ô6Ø×5Ñ5°jÀ'È<ÐYbÓcò &�Ø×!Ñ! <°Ð";Ô<Ø$%�
˜5Ò!ñ&ð	&ô ˜KÓ(¬3¨}Ó+=Ñ=ˆØ˜=Ð(Ð(r(   c                 ó<   — d„ }| j                  || ||«      ¬«      S )a  
        Maps answer coordinates of a question to token indexes.

        In the SQA format (TSV), the coordinates are given as (row, column) tuples. Here, we first swap them to
        (column, row) format before calling _get_all_answer_ids_from_coordinates.
        c                 ó<   — | D �cg c]  }|d   |d   f‘Œ c}S c c}w ©Nrª   r   r'   )Úanswer_coordinates_questionÚcoordss     r)   Ú_to_coordinatesz;TapasTokenizer._get_all_answer_ids.<locals>._to_coordinatesŽ  s#   € Ø9TÖU¨v�V˜A‘Y  q¡	Ò*ÒUÐUùÒUs   …)rŸ  )r¥  )r€   r>   r?   rÔ   r«  s        r)   Ú_get_all_answer_idsz"TapasTokenizer._get_all_answer_ids†  s/   € ò	Vð ×8Ñ8Ø˜©Ð?QÓ/Rð 9ó 
ð 	
r(   c                 óø   — t        j                  d|› d|› �«       t        dt        |«      z   t        |«      z
  «      D ]9  }t	        |«      D ]&  \  }}|||z      j
                  |j
                  k7  sŒ& Œ6 |c S  y)z.Return start index of segment in text or None.ztext: rœ   rª   N)r   Úinfor_  r‰   rO   Úpiece)r€   rX   ÚsegmentrS   Ú	seg_indexÚ	seg_tokens         r)   Ú_find_tokenszTapasTokenizer._find_tokens•  s„   € ä�‰�v˜d˜V 1 W IÐ.Ô/Ü˜1œs 4›y™=¬3¨w«<Ñ7Ó8ò 	ˆEÜ(1°'Ó(:ò Ñ$�	˜9Ø˜ 	Ñ)Ñ*×0Ñ0°I·O±OÓCÙðð ’ð	ð r(   c              #   óò   K  — t        j                  d|› �«       t        |j                  «      D ]C  \  }}|dk(  rŒt        |«      D ]*  \  }}| j	                  ||«      }|€Œt        |||¬«      –— Œ, ŒE y­w)z4Returns all occurrences of answer_text in the table.zanswer text: r   NrA  )r   r®  rO   r8   r³  r0   )r€   rû   rÕ   r2   rF  rk  rG  r3   s           r)   Ú)_find_answer_coordinates_from_answer_textz8TapasTokenizer._find_answer_coordinates_from_answer_text   sˆ   è ø€ ô 	�‰�} [ MÐ2Ô3Ü'¨×(<Ñ(<Ó=ò 	‰NˆI�sØ˜AŠ~àÜ#,¨S£>ò ‘�	˜4Ø"×/Ñ/°°kÓB�ØÑ*Ü*Ø"+Ø%.Ø$/ôó ññ		ùs   ‚AA7Á"A7c                 óÄ  — dgt        |«      z  }|D ]Ë  }| j                  ||«      D ]´  }t        | j                  |||j                  |j
                  dz
  ¬«      «      }|j                  «        g }	|rD|j                  |d   z   }
|
t        |«      z   }|D ]  }||
k\  sŒ	||k  sŒ|	j                  |«       Œ! t        |	«      t        |«      k(  sŒ¨|	D ]  }d||<   Œ	  ŒË ŒÍ |S )zDMaps question with answer texts to the first matching token indexes.r   rª   )r,   r-   )	r‰   rµ  r¾   ro  r1   r2   Úsortr3   rs   )r€   r>   r?   rû   Úanswer_textsr   rÕ   ÚcoordinatesÚindexesÚcoordinate_answer_idsÚbegin_indexÚ	end_indexrS   s                r)   Ú"_find_answer_ids_from_answer_textsz1TapasTokenizer._find_answer_ids_from_answer_texts´  s  € ð �Sœ3˜z›?Ñ*ˆ
Ø'ò 	ˆKØ#×MÑMØØó ò �ô Ø×0Ñ0Ø"ØØ"-×":Ñ":Ø*×4Ñ4°qÑ8ð	 1ó ó�ð —‘”Ø(*Ð%ÙØ"-×"9Ñ"9¸GÀA¹JÑ"F�KØ +¬c°+Ó.>Ñ >�IØ!(ò @˜Ø  KÓ/°E¸IÓ4EØ1×8Ñ8¸Õ?ð@ô Ð,Ó-´°[Ó1AÓAØ!6ò .˜Ø,-˜
 5Ò)ð.áñ3ð	ð6 Ðr(   c                 óL   — | j                  |||«      \  }}|rt        d«      ‚|S )z7Maps answer coordinates of a question to token indexes.zCouldn't find all answers)r¬  rw   )r€   r>   r?   rÔ   r   r¤  s         r)   Ú_get_answer_idszTapasTokenizer._get_answer_idsÚ  s1   € à$(×$<Ñ$<¸ZÈÐRdÓ$eÑ!ˆ
�MáÜÐ8Ó9Ð9ØÐr(   c                 ó®   — | j                   r2| j                  ||||D �cg c]  }| j                  |«      ‘Œ c}¬«      S | j                  |||«      S c c}w )N)r¸  )r`   r¾  r’   rÀ  )r€   r>   r?   rû   Úanswer_texts_questionr©  Úats          r)   r*  zTapasTokenizer.get_answer_idsâ  sa   € Ø×)Ò)Ø×:Ñ:ØØØØ:OÖP°B˜dŸm™m¨BÕ/ÒPð	 ;ó ð ð ×#Ñ# J°Ð9TÓUÐUùò Qs   ŸAr  Úpadding_strategyc                 ó*  — |€d| j                   v }|t        j                  k(  rt        |d   «      }|�|�||z  dk7  r||z  dz   |z  }|t        j                  k7  xr t        |d   «      |k7  }|rd|vrdgt        |d   «      z  |d<   |�r†|t        |d   «      z
  }|�|n| j
                  }|dk(  r¢|r|d   dg|z  z   |d<   d|v r|d   | j                  gdz  g|z  z   |d<   d|v r|d   dg|z  z   |d<   d	|v r|d	   t        d
«      g|z  z   |d	<   d|v r|d   dg|z  z   |d<   d|v r|d   dg|z  z   |d<   |d   | j                  g|z  z   |d<   |S |dk(  r¢|rdg|z  |d   z   |d<   d|v r| j                  gdz  g|z  |d   z   |d<   d|v rdg|z  |d   z   |d<   d	|v rt        d
«      g|z  |d	   z   |d	<   d|v rdg|z  |d   z   |d<   d|v rdg|z  |d   z   |d<   | j                  g|z  |d   z   |d<   |S t        dt        |«      z   «      ‚|S )a#  
        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)

        Args:
            encoded_inputs:
                Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
            max_length: maximum length of the returned list and optionally padding length (see below).
                Will truncate by taking into account the special tokens.
            padding_strategy: PaddingStrategy to use for padding.

                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
                - PaddingStrategy.DO_NOT_PAD: Do not pad
                The tokenizer padding sides are defined in self.padding_side:

                    - 'left': pads on the left of the sequences
                    - 'right': pads on the right of the sequences
            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
                `>= 7.5` (Volta).
            padding_side:
                The side on which the model should have padding applied. Should be selected between ['right', 'left'].
                Default value is picked from the class attribute of the same name.
            return_attention_mask:
                (optional) Set to False to avoid returning attention mask (default: set to model specifics)
        r  r  r   rª   Úrightr  é   r  r  r�  r  r–  r   ÚleftzInvalid padding strategy:)r/  r   ÚLONGESTr‰   r
  rÛ   Úpad_token_type_idr’  Úpad_token_idrw   r:   )	r€   r  rÙ   rÄ  rÚ   rÛ   rÞ   Úneeds_to_be_paddedÚ
differences	            r)   Ú_padzTapasTokenizer._padì  sm  € ðH !Ð(Ø$4¸×8NÑ8NÐ$NÐ!àœ×6Ñ6Ò6Ü˜^¨KÑ8Ó9ˆJàÐ!Ð&8Ð&DÈ*ÐWiÑJiÐmnÒJnØ%Ð);Ñ;¸qÑ@ÐDVÑVˆJð ¤× :Ñ :Ñ:Òm¼sÀ>ÐR]ÑC^Ó?_ÐcmÑ?mð 	ñ
 !Ð%5¸^Ñ%KØ01¨s´S¸ÈÑ9TÓ5UÑ/UˆNÐ+Ñ,âØ#¤c¨.¸Ñ*EÓ&FÑFˆJØ+7Ð+C™<È×IZÑIZˆLØ˜wÒ&Ù(Ø7EÐFVÑ7WÐ[\ÐZ]Ð`jÑZjÑ7j�NÐ#3Ñ4Ø# ~Ñ5à&Ð'7Ñ8¸T×=SÑ=SÐ<TÐWXÑ<XÐ;YÐ\fÑ;fÑfð #Ð#3Ñ4ð ˜~Ñ-Ø/=¸hÑ/GÈ1È#ÐPZÑJZÑ/Z�N 8Ñ,Ø# ~Ñ5Ø7EÐFVÑ7WÔ[`ÐafÓ[gÐZhÐkuÑZuÑ7u�NÐ#3Ñ4Ø)¨^Ñ;à&Ð'=Ñ>À#ÀÈÑASÑSð #Ð#9Ñ:ð )¨NÑ:Ø<JÐK`Ñ<aÐefÐdgÐjtÑdtÑ<t�NÐ#8Ñ9Ø.<¸[Ñ.IÈT×M^ÑM^ÐL_ÐblÑLlÑ.l�˜{Ñ+ð, Ðð+  Ò'Ù(Ø89°s¸ZÑ7GÈ.ÐYiÑJjÑ7j�NÐ#3Ñ4Ø# ~Ñ5Ø9=×9OÑ9OÐ8PÐSTÑ8TÐ7UÐXbÑ7bÐesØ(ñfñ 8�NÐ#3Ñ4ð ˜~Ñ-Ø01¨s°ZÑ/?À.ÐQYÑBZÑ/Z�N 8Ñ,Ø# ~Ñ5Ü8=¸e»°~È
Ñ7RÐUcÐdtÑUuÑ7u�NÐ#3Ñ4Ø)¨^Ñ;Ø>A¸UÀZÑ=OÐR`Ø.ñSñ >�NÐ#9Ñ:ð )¨NÑ:Ø=>¸CÀ*Ñ<LÈ~Ð^sÑOtÑ<t�NÐ#8Ñ9Ø/3×/@Ñ/@Ð.AÀJÑ.NÐQ_Ð`kÑQlÑ.l�˜{Ñ+ð Ðô !Ð!<¼sÀ<Ó?PÑ!PÓQÐQàÐr(   c              #   óŒ   K  — t        |«      D ]2  \  }}||   }||   dz
  }||   dz
  }	|dk\  sŒ!|	dk\  sŒ'|dk(  sŒ-||f–— Œ4 y ­wr¨  )rO   )
r€   Úprobabilitiesr@   r?   r>   ÚiÚpÚ
segment_idÚcolrF  s
             r)   Ú_get_cell_token_probsz$TapasTokenizer._get_cell_token_probsO  s^   è ø€ Ü˜mÓ,ò 	‰DˆAˆqØ$ Q™ˆJØ˜Q‘- !Ñ#ˆCØ˜!‘*˜q‘.ˆCØ�a‹x˜C 1›H¨°q«Ø˜�d“
ñ	ùs   ‚,A¯AµA»	Ac                 óN  — t        j                  t        «      }| j                  ||||«      D ]+  \  }}||   dz
  }||   dz
  }	|||	f   j	                  |«       Œ- |j                  «       D �
�ci c])  \  }
}|
t        j                  |«      j                  «       “Œ+ c}}
S c c}}
w )z?Computes average probability per cell, aggregating over tokens.rª   )	rK   rt  r¾   rÕ  rs   rx   ÚnpÚarrayÚmean)r€   rÐ  r@   r?   r>   Úcoords_to_probsrÑ  ÚprobrÔ  rF  rª  Ú
cell_probss               r)   Ú_get_mean_cell_probsz#TapasTokenizer._get_mean_cell_probsW  s¦   € ä%×1Ñ1´$Ó7ˆØ×1Ñ1°-ÀÈgÐWaÓbò 	5‰GˆAˆtØ˜Q‘- !Ñ#ˆCØ˜!‘*˜q‘.ˆCØ˜S #˜JÑ'×.Ñ.¨tÕ4ð	5ð O^×NcÑNcÓNe×fÑ8J¸À
�œŸ™ Ó,×1Ñ1Ó3Ñ3ÓfÐfùÓfs   Á/.B!c                 ód  — |j                  «       }|�|j                  «       }|j                  «       D ��ci c]  \  }}|dk7  sŒ||j                  «       “Œ }}}d||dk  <   ddt        j                  | «      z   z  |d   z  }g d¢}|d   }	|d   dd…dd…|j	                  d	«      f   }
|d   dd…dd…|j	                  d
«      f   }|d   dd…dd…|j	                  d«      f   }|	j
                  d   }g }t        |«      D ]û  }||   j                  «       }|
|   }||   }||   }|j                  «       }|j                  «       }|dk(  r|dk(  rŒP| j                  ||j                  «       |j                  «       |j                  «       «      }g }t        |«      D ]B  }t        |«      D ]2  }|j                  ||fd«      }|€Œ||kD  sŒ |j                  ||f«       Œ4 ŒD t        |«      }|j                  |«       Œý |f}|�$|j                  d¬«      }||j                  «       f}|S c c}}w )aé  
        Converts logits of [`TapasForQuestionAnswering`] to actual predicted answer coordinates and optional
        aggregation indices.

        The original implementation, on which this function is based, can be found
        [here](https://github.com/google-research/tapas/blob/4908213eb4df7aa988573350278b44c4dbe3f71b/tapas/experiments/prediction_utils.py#L288).

        Args:
            data (`dict`):
                Dictionary mapping features to actual values. Should be created using [`TapasTokenizer`].
            logits (`torch.Tensor` or `tf.Tensor` of shape `(batch_size, sequence_length)`):
                Tensor containing the logits at the token level.
            logits_agg (`torch.Tensor` or `tf.Tensor` of shape `(batch_size, num_aggregation_labels)`, *optional*):
                Tensor containing the aggregation logits.
            cell_classification_threshold (`float`, *optional*, defaults to 0.5):
                Threshold to be used for cell selection. All table cells for which their probability is larger than
                this threshold will be selected.

        Returns:
            `tuple` comprising various elements depending on the inputs:

            - predicted_answer_coordinates (`List[List[[tuple]]` of length `batch_size`): Predicted answer coordinates
              as a list of lists of tuples. Each element in the list contains the predicted answer coordinates of a
              single example in the batch, as a list of tuples. Each tuple is a cell, i.e. (row index, column index).
            - predicted_aggregation_indices (`List[int]`of length `batch_size`, *optional*, returned when
              `logits_aggregation` is provided): Predicted aggregation operator indices of the aggregation head.
        NÚtraininggÍÌÌÌÌ,VÀrª   r  )r@   r>   r?   r9  r:  r;  r<  r  r  r@   r?   r>   r   éÿÿÿÿ)Úaxis)Únumpyrx   r×  ÚexprS   rc  r_  ÚtolistrS  rÝ  r—   rs   r±   Úargmax)r€   ÚdataÚlogitsÚ
logits_aggÚcell_classification_thresholdr®   r  rÐ  Útoken_typesr  r@   r?   r>   Ú	num_batchÚpredicted_answer_coordinatesrÑ  Úprobabilities_exampleÚsegment_ids_exampleÚrow_ids_exampleÚcolumn_ids_exampleÚ	max_widthÚ
max_heightÚcell_coords_to_probrÔ   rÔ  rF  Ú	cell_probÚoutputÚpredicted_aggregation_indicess                                r)   Úconvert_logits_to_predictionsz,TapasTokenizer.convert_logits_to_predictions`  sŽ  € ð: —‘“ˆØÐ!Ø#×)Ñ)Ó+ˆJØ59·Z±Z³\×W¡z s¨EÀSÈJÓEV��U—[‘[“]Ñ"ÐWˆÑWð
 "'ˆˆv˜‰~Ñð ˜Q¤§¡¨¨£Ñ0Ñ1°DÐ9IÑ4JÑJˆò
ˆð ˜Ñ%ˆ	ØÐ+Ñ,ªQ²°;×3DÑ3DÀ]Ó3SÐ-SÑTˆØÐ'Ñ(ªªA¨{×/@Ñ/@ÀÓ/KÐ)KÑLˆØÐ*Ñ+ªAªq°+×2CÑ2CÀLÓ2QÐ,QÑRˆ
ð —O‘O AÑ&ˆ	Ø')Ð$Ü�yÓ!ò 	DˆAØ$1°!Ñ$4×$;Ñ$;Ó$=Ð!Ø"-¨a¡.ÐØ% a™jˆOØ!+¨A¡Ðà*×.Ñ.Ó0ˆIØ(×,Ñ,Ó.ˆJà˜AŠ~ *°¢/Øà"&×";Ñ";Ø%Ø#×*Ñ*Ó,Ø×&Ñ&Ó(Ø"×)Ñ)Ó+ó	#Ðð "$ÐÜ˜YÓ'ò B�Ü  Ó,ò B�CØ 3× 7Ñ 7¸¸c¸
ÀDÓ I�IØ Ñ,Ø$Ð'DÓDØ.×5Ñ5°s¸C°jÕAñ	BðBô "(Ð(:Ó!;ÐØ(×/Ñ/Ð0BÕCð9	Dð< /Ð0ˆàÐ!Ø,6×,=Ñ,=À2Ð,=Ó,FÐ)Ø2Ð4Q×4XÑ4XÓ4ZÐ[ˆFàˆùóA Xs   ¶H,ÁH,)TTNz[UNK]z[SEP]z[PAD]z[CLS]z[MASK]z[EMPTY]TNrà  NNFFNNi   NTr†   )NF)NNNTFFNNNNNNFFFFT)NNTFFNNNNTNFFFFT)NNNNTFFNNNNTTFFFTF)NTFFNN)NNNTFFNNNNNNFFFT)NNTFFNNNNTTFFFT)Ng      à?)Vr!   r"   r#   r$   r°   Úvocab_files_namesr4   r   r"  r   r:   r   Úpropertyrd   rŠ   rŽ   r•   r˜   rš   r¡   r	   r¶   rO  r»   rÁ   rÄ   rÈ   rÎ   rÑ   r   Ú-TAPAS_ENCODE_PLUS_ADDITIONAL_KWARGS_DOCSTRINGr
   r   r   r   r   r   r   r   rî   r   rê   rù   rö   r7   rÿ   r  rë   r  r	  r'  rþ   rK  rM  r   r(  r&  r]  r?  r%  r$  rf  rh  rm  ro  r-  r~  r.  r0  r1  r�  r¥  r¬  r³  rµ  r¾  rÀ  r*  r
  r   rŒ   rÎ  rÕ  rÝ  r÷  Ú__classcell__)r„   s   @r)   r[   r[   –   s¥  ø„ ñPðd *Ðð
 ØØØØØØØØØ#ØØ "Ø'+Ø$(Ø#(Ø*/Ø Ø Ø #Ø9=Ø%)ñ/`
ð ð`
ð   ‘}ð`
ð  ˜S‘Mð!`
ð" !ð#`
ð$ $(ð%`
ð* ð+`
ð, $,¨D°©IÑ#6õ-`
ðD ñ2ó ð2ð ñó ðò=òòEò=òñ
¨cð ÀHÈSÁMð Ð]bÐcfÑ]gó ð(B¸dÀ3¹ið BÐW[Ð\fÑWgð BÐlpÐqtÑluó BðEØ˜c™ðEØ26°zÑ2BðEà	ˆc‰óEð"GØ˜c™ðGØ26°zÑ2BðGà	ˆc‰óGð"DØ˜c™ðDØ26°zÑ2BðDà	ˆc‰óDð$ JNñUØ ™9ðUØ3;¸DÀ¹IÑ3FðUà	ˆc‰óUð( sxñ4Ø ™9ð4Ø3;¸DÀ¹IÑ3Fð4Økoð4à	ˆc‰õ4ñ8 ÐEÓFð ØNRØOSØ#'Ø5:Ø@EØ$(Ø,0Ø&*Ø;?Ø04Ø04Ø*/Ø+0Ø',Ø#Øñ9uàðuð ØØØ!ØØ�Y‘ØÐ&Ñ'Ø�\Ñ"ð$ññ	
ðuð % U¨4°©;¸¸TÀ%¹[Ñ8IÐ+IÑ%JÑKðuð ˜e D¨¡O°T¸$¸y¹/Ñ5JÐ$JÑKÑLðuð !ðuð  �t˜S /Ð1Ñ2ð!uð" ˜$ Ð%<Ð<Ñ=ð#uð$ ˜S‘Mð%uð& % S™Mð'uð( ˜s‘mð)uð* !  s¨J Ñ!7Ñ8ð+uð,  (¨™~ð-uð.  (¨™~ð/uð0 $(ð1uð2 %)ð3uð4 !%ð5uð6 ð7uð8 ð9uð< 
ò=uó Gðuñn Ð/Ð1^Ó_ð Ø:>Ø7;Ø#'Ø5:Ø@EØ$(Ø,0Ø&*Ø;?Ø04Ø04Ø*/Ø+0Ø',Ø#Øñ3`
àð`
ð ØØ�Y‘ØÐ&Ñ'Ø�\Ñ"ð$ññ
ð`
ð % T¨$¨u©+Ñ%6Ñ7ð`
ð ˜d 4¨	¡?Ñ3Ñ4ð`
ð !ð`
ð �t˜S /Ð1Ñ2ð`
ð ˜$ Ð%<Ð<Ñ=ð`
ð ˜S‘Mð`
ð  % S™Mð!`
ð" ˜s‘mð#`
ð$ !  s¨J Ñ!7Ñ8ð%`
ð&  (¨™~ð'`
ð(  (¨™~ð)`
ð* $(ð+`
ð, %)ð-`
ð. !%ð/`
ð0 ð1`
ð2 ð3`
ð6 
ò7`
ó `ð`
òD#ð* ;?Ø7;Ø#'Ø5:Ø@EØ$(Ø,0Ø&*Ø;?Ø04Ø04Ø*/Ø+0Ø',Ø#Øñ/9,ð Ø�‰OØÐ"Ñ#Ø�Ñð ñ
ð9,ð % T¨$¨u©+Ñ%6Ñ7ð9,ð ˜d 4¨	¡?Ñ3Ñ4ð9,ð !ð9,ð �t˜S /Ð1Ñ2ð9,ð ˜$ Ð%<Ð<Ñ=ð9,ð ˜S‘Mð9,ð % S™Mð9,ð ˜s‘mð9,ð  !  s¨J Ñ!7Ñ8ð!9,ð"  (¨™~ð#9,ð$  (¨™~ð%9,ð& $(ð'9,ð( %)ð)9,ð* !%ð+9,ð, ð-9,ð. ð/9,ð2 
ó39,ðF 59Ø48Ø:>Ø7;Ø#'Ø5:Ø@EØ$(Ø,0Ø&*Ø;?Ø04Ø04Ø+0Ø',Ø#ØØ#(ñ3Hà!ðHð Ø�‰OØÐ"Ñ#Ø�Ñð ñ
ðHð " .Ñ1ðHð !  d¨3¡i¡Ñ1ðHð % T¨$¨u©+Ñ%6Ñ7ðHð ˜d 4¨	¡?Ñ3Ñ4ðHð !ðHð �t˜S /Ð1Ñ2ðHð ˜$ Ð%<Ð<Ñ=ðHð ˜S‘MðHð  % S™Mð!Hð" ˜s‘mð#Hð$ !  s¨J Ñ!7Ñ8ð%Hð&  (¨™~ð'Hð(  (¨™~ð)Hð* %)ð+Hð, !%ð-Hð. ð/Hð0 ð1Hð2 !ð3Hð6 
ó7HñT Ð/Ó0ð Ø#'Ø5:Ø@EØ$(Ø;?ñ(+àð(+ð ØØØ!Øðññ
ð(+ð !ð(+ð �t˜S /Ð1Ñ2ð(+ð ˜$ Ð%<Ð<Ñ=ð(+ð ˜S‘Mð(+ð !  s¨J Ñ!7Ñ8ð(+ð  
ˆc‰ò!(+ó 1ð(+ñT Ð/Ð1^Ó_ð Ø48Ø15Ø#'Ø5:Ø@EØ$(Ø,0Ø&*Ø;?Ø04Ø04Ø+0Ø',Ø#Øñ1S
àðS
ð ØØØ!Øðññ
ðS
ð % T¨%¡[Ñ1ðS
ð ˜d 9™oÑ.ðS
ð !ðS
ð �t˜S /Ð1Ñ2ðS
ð ˜$ Ð%<Ð<Ñ=ðS
ð ˜S‘MðS
ð  % S™Mð!S
ð" ˜s‘mð#S
ð$ !  s¨J Ñ!7Ñ8ð%S
ð&  (¨™~ð'S
ð(  (¨™~ð)S
ð* %)ð+S
ð, !%ð-S
ð. ð/S
ð0 ð1S
ð4 
ò5S
ó `ðS
ðz 59Ø15Ø#'Ø5:Ø@EØ$(Ø,0Ø&*Ø;?Ø04Ø04Ø+0Ø',Ø#Øñ-7
àð7
ð ØØØðñ
ð7
ð % T¨%¡[Ñ1ð7
ð ˜d 9™oÑ.ð7
ð !ð7
ð �t˜S /Ð1Ñ2ð7
ð ˜$ Ð%<Ð<Ñ=ð7
ð ˜S‘Mð7
ð % S™Mð7
ð ˜s‘mð7
ð  !  s¨J Ñ!7Ñ8ð!7
ð"  (¨™~ð#7
ð$  (¨™~ð%7
ð& %)ð'7
ð( !%ð)7
ð* ð+7
ð, ó-7
ñr Ð/Ð1^Ó_ð 59Ø15Ø48Ø15Ø#'Ø5:Ø@EØ$(Ø,0Ø&*Ø;?Ø04Ø04Ø+0Ø',Ø#ØØ#(ñ3@à!ð@ð ØØØðñ
ð@ð " .Ñ1ð@ð ˜~Ñ.ð@ð % T¨%¡[Ñ1ð@ð ˜d 9™oÑ.ð@ð !ð@ð �t˜S /Ð1Ñ2ð@ð ˜$ Ð%<Ð<Ñ=ð@ð ˜S‘Mð@ð  % S™Mð!@ð" ˜s‘mð#@ð$ !  s¨J Ñ!7Ñ8ð%@ð&  (¨™~ð'@ð(  (¨™~ð)@ð* %)ð+@ð, !%ð-@ð. ð/@ð0 ð1@ð2 !ð3@ð6 
ò7@ó `ð@ðD7)à˜3‘ið7)ð (ð7)ð ð	7)ð
 ð7)ð ð7)ð # 3Ð(?Ð#?Ñ@ð7)ð 
ˆs�Cˆx‰ó7)ðv ó+
òZ(ó
ðGÈYÐWaÐcgÐimÐWmÑMnó Gò&=ò`òò&òò8ò*
ò<$òò
! òFò*'!òRò.$ò*ò)ò(
ò	òò($òLòVð %)Ø,;×,FÑ,FØ,0Ø&*Ø04ñ_à˜d 3¨Ð#4Ñ5°}ÐDÑEð_ð ˜S‘Mð_ð *ð	_ð
 % S™Mð_ð ˜s‘mð_ð  (¨™~ð_ð 
ó_òFòg÷`r(   r[   c                   óJ   — e Zd ZdZ	 	 	 	 	 d
d„Zdd„Zd„ Zdd„Zd„ Zd„ Z	d	„ Z
y)rz   aª  
    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).

    Args:
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        never_split (`Iterable`, *optional*):
            Collection of tokens which will never be split during tokenization. Only has an effect when
            `do_basic_tokenize=True`
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters.

            This should likely be deactivated for Japanese (see this
            [issue](https://github.com/huggingface/transformers/issues/328)).
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase` (as in the original BERT).
        do_split_on_punc (`bool`, *optional*, defaults to `True`):
            In some instances we want to skip the basic punctuation splitting so that later tokenization can capture
            the full context of the words, such as contractions.
    Nc                 ód   — |€g }|| _         t        |«      | _        || _        || _        || _        y r†   )rd   r€  re   rf   rg   Údo_split_on_punc)r€   rd   re   rf   rg   rþ  s         r)   r   zBasicTokenizer.__init__Ý  s<   € ð ÐØˆKØ*ˆÔÜ˜{Ó+ˆÔØ&<ˆÔ#Ø*ˆÔØ 0ˆÕr(   c                 óJ  — |r$| j                   j                  t        |«      «      n| j                   }| j                  |«      }| j                  r| j                  |«      }t        j                  d|«      }t        |«      }g }|D ]€  }||vrY| j                  r0|j                  «       }| j                  dur/| j                  |«      }n| j                  r| j                  |«      }|j                  | j                  ||«      «       Œ‚ t        dj                  |«      «      }|S )aj  
        Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer.

        Args:
            never_split (`List[str]`, *optional*)
                Kept for backward compatibility purposes. Now implemented directly at the base class level (see
                [`PreTrainedTokenizer.tokenize`]) List of token not to split.
        ÚNFCFrœ   )re   Úunionr€  Ú_clean_textrf   Ú_tokenize_chinese_charsÚunicodedataÚ	normalizerY   rd   Úlowerrg   Ú_run_strip_accentsÚextendÚ_run_split_on_puncrž   )r€   rX   re   Úunicode_normalized_textÚorig_tokensr”   r+   Úoutput_tokenss           r)   r’   zBasicTokenizer.tokenizeí  s  € ñ CN�d×&Ñ&×,Ñ,¬S°Ó-=Ô>ÐSW×ScÑScˆØ×Ñ Ó%ˆð ×&Ò&Ø×/Ñ/°Ó5ˆDä"-×"7Ñ"7¸¸tÓ"DÐÜ)Ð*AÓBˆØˆØ ò 	MˆEØ˜KÑ'Ø×%Ò%Ø!ŸK™K›M�EØ×)Ñ)°Ñ6Ø $× 7Ñ 7¸Ó >™Ø×'Ò'Ø ×3Ñ3°EÓ:�EØ×Ñ × 7Ñ 7¸¸{Ó KÕLð	Mô ,¨C¯H©H°\Ó,BÓCˆØÐr(   c                 óº   — t        j                  d|«      }g }|D ].  }t        j                  |«      }|dk(  rŒ|j                  |«       Œ0 dj	                  |«      S )z$Strips accents from a piece of text.ÚNFDÚMnr�   )r  r  Úcategoryrs   rž   )r€   rX   rõ  ÚcharÚcats        r)   r  z!BasicTokenizer._run_strip_accents  s^   € ä×$Ñ$ U¨DÓ1ˆØˆØò 	 ˆDÜ×&Ñ& tÓ,ˆCØ�dŠ{ØØ�M‰M˜$Õð		 ð
 �w‰w�v‹Ðr(   c                 óv  — | j                   r|�||v r|gS t        |«      }d}d}g }|t        |«      k  rb||   }t        |«      r|j	                  |g«       d}n)|r|j	                  g «       d}|d   j	                  |«       |dz  }|t        |«      k  rŒb|D �cg c]  }dj                  |«      ‘Œ c}S c c}w )z&Splits punctuation on a piece of text.r   TFrà  rª   r�   )rþ  r¾   r‰   r   rs   rž   )	r€   rX   re   ÚcharsrÑ  Ústart_new_wordrõ  r  Úxs	            r)   r	  z!BasicTokenizer._run_split_on_punc  s¿   € à×$Ò$¨Ð)@ÀTÈ[ÑEXØ�6ˆMÜ�T“
ˆØˆØˆØˆØ”#�e“*ŠnØ˜‘8ˆDÜ˜tÔ$Ø—‘˜t˜fÔ%Ø!%‘á!Ø—M‘M "Ô%Ø!&�Ø�r‘
×!Ñ! $Ô'Ø�‰FˆAð ”#�e“*‹nð %+Ö+˜q�—‘˜•
Ò+Ð+ùÒ+s   ÂB6c                 óø   — g }|D ]c  }t        |«      }| j                  |«      r4|j                  d«       |j                  |«       |j                  d«       ŒS|j                  |«       Œe dj                  |«      S )z)Adds whitespace around any CJK character.rœ   r�   )ÚordÚ_is_chinese_charrs   rž   ©r€   rX   rõ  r  Úcps        r)   r  z&BasicTokenizer._tokenize_chinese_chars4  sm   € àˆØò 	$ˆDÜ�T“ˆBØ×$Ñ$ RÔ(Ø—‘˜cÔ"Ø—‘˜dÔ#Ø—‘˜cÕ"à—‘˜dÕ#ð	$ð �w‰w�v‹Ðr(   c                 ó¦   — |dk\  r|dk  sF|dk\  r|dk  s<|dk\  r|dk  s2|dk\  r|dk  s(|d	k\  r|d
k  s|dk\  r|dk  s|dk\  r|dk  s
|dk\  r|dk  ryy)z6Checks whether CP is the codepoint of a CJK character.i N  iÿŸ  i 4  i¿M  i   iß¦ i § i?· i@· i¸ i ¸ i¯Î i ù  iÿú  i ø iú TFr'   )r€   r  s     r)   r  zBasicTokenizer._is_chinese_charA  sr   € ð �6Š\˜b FšlØ�f’  v¢Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�f’  v¢Ø�g’ "¨¢-ààr(   c                 óÔ   — g }|D ]Q  }t        |«      }|dk(  s|dk(  st        |«      rŒ$t        |«      r|j                  d«       ŒA|j                  |«       ŒS dj	                  |«      S )zBPerforms invalid character removal and whitespace cleanup on text.r   iýÿ  rœ   r�   )r  r   r   rs   rž   r  s        r)   r  zBasicTokenizer._clean_textY  sf   € àˆØò 	$ˆDÜ�T“ˆBØ�QŠw˜" š,¬+°dÔ*;ØÜ˜dÔ#Ø—‘˜cÕ"à—‘˜dÕ#ð	$ð �w‰w�v‹Ðr(   )TNTNTr†   )r!   r"   r#   r$   r   r’   r  r	  r  r  r  r'   r(   r)   rz   rz   Æ  s<   „ ñð0 ØØ#ØØó1ó $òL	ó,ò,òó0r(   rz   c                   ó   — e Zd ZdZdd„Zd„ Zy)r|   zRuns WordPiece tokenization.c                 ó.   — || _         || _        || _        y r†   )rQ   rh   Úmax_input_chars_per_word)r€   rQ   rh   r   s       r)   r   zWordpieceTokenizer.__init__k  s   € ØˆŒ
Ø"ˆŒØ(@ˆÕ%r(   c                 ó  — g }t        |«      D ]÷  }t        |«      }t        |«      | j                  kD  r|j	                  | j
                  «       ŒBd}d}g }|t        |«      k  rst        |«      }d}	||k  r:dj                  ||| «      }
|dkD  rd|
z   }
|
| j                  v r|
}	n|dz  }||k  rŒ:|	€d}n"|j	                  |	«       |}|t        |«      k  rŒs|r|j	                  | j
                  «       Œç|j                  |«       Œù |S )aå  
        Tokenizes a piece of text into its word pieces. This uses a greedy longest-match-first algorithm to perform
        tokenization using the given vocabulary.

        For example, `input = "unaffable"` wil return as output `["un", "##aff", "##able"]`.

        Args:
            text: A single token or whitespace separated tokens. This should have
                already been passed through *BasicTokenizer*.

        Returns:
            A list of wordpiece tokens.
        Fr   Nr�   rB   rª   T)	rY   r¾   r‰   r   rs   rh   rž   rQ   r  )r€   rX   r  r+   r  Úis_badÚstartÚ
sub_tokensÚendÚ
cur_substrÚsubstrs              r)   r’   zWordpieceTokenizer.tokenizep  s&  € ð ˆÜ(¨Ó.ò 	1ˆEÜ˜“KˆEÜ�5‹z˜D×9Ñ9Ò9Ø×$Ñ$ T§^¡^Ô4ØàˆFØˆEØˆJØœ#˜e›*Ò$Ü˜%“j�Ø!�
Ø˜c’kØŸW™W U¨5°Ð%5Ó6�FØ˜q’yØ!%¨¡˜Ø §¡Ñ+Ø%+˜
ØØ˜1‘H�Cð ˜c“kð Ð%Ø!�FØØ×!Ñ! *Ô-Ø�ð œ#˜e›*Ó$ñ" Ø×$Ñ$ T§^¡^Õ4à×$Ñ$ ZÕ0ð;	1ð< Ðr(   N)éd   )r!   r"   r#   r$   r   r’   r'   r(   r)   r|   r|   h  s   „ Ù&óAó
.r(   r|   c                   ó0   — e Zd ZdZdZdZdZdZdZdZ	dZ
d	Zy
)r„  rª   rÆ   r   é   é   é   rÇ  é   é	   N)r!   r"   r#   ÚHEADER_TO_CELLÚCELL_TO_HEADERÚQUERY_TO_HEADERÚQUERY_TO_CELLÚROW_TO_CELLÚCELL_TO_ROWr…  ÚLTÚGTr'   r(   r)   r„  r„  ª  s/   „ Ø€NØ€NØ€OØ€MØ€KØ€KØ	
€BØ	
€BØ	
�Br(   r„  c                   óJ   — e Zd ZU dZee   ed<   dZee   ed<   dZee   ed<   y)ÚDateNÚyearÚmonthÚday)	r!   r"   r#   r9  r   r4   r5   r:  r;  r'   r(   r)   r8  r8  ¶  s,   … à€Dˆ(�3‰-ÓØ€Eˆ8�C‰=ÓØ€Cˆ�#‰Ôr(   r8  c                   ó6   — e Zd ZU dZee   ed<   dZee   ed<   y)ÚNumericValueNr”  Údate)	r!   r"   r#   r”  r   r’  r5   r>  r8  r'   r(   r)   r=  r=  ½  s   … à#'€K�˜%‘Ó'Ø€Dˆ(�4‰.Ôr(   r=  c                   óJ   — e Zd ZU dZee   ed<   dZee   ed<   dZe	e
   ed<   y)ÚNumericValueSpanNr¼  r½  rs  )r!   r"   r#   r¼  r   r4   r5   r½  rs  r   r=  r'   r(   r)   r@  r@  Ã  s-   … à!%€K�˜#‘Ó%Ø#€Iˆx˜‰}Ó#Ø!%€FˆD�ÑÔ%r(   r@  c                   ó,   — e Zd ZU eed<   dZee   ed<   y)ÚCellrX   Nrj  )r!   r"   r#   r:   r5   rj  r   r=  r'   r(   r)   rB  rB  Ê  s   … à
ƒIØ,0€M�8˜LÑ)Ô0r(   rB  c                   ó<   — e Zd ZU eed<   eed<   dZeee      ed<   y)ÚQuestionÚoriginal_textrX   Nr�  )	r!   r"   r#   r:   r5   r�  r   r   r@  r'   r(   r)   rD  rD  Ð  s"   … àÓØ
ƒIØ6:€M�8˜DÐ!1Ñ2Ñ3Ô:r(   rD  Ú	_DateMask)r9  r:  r;  Fú%Bú%Yz%Ysz%b %Yz%B %Yz%B %dz%b %dz%d %bz%d %Bz	%B %d, %Yz%d %B %Yz%m-%d-%Yz%Y-%m-%dz%Y-%mz%d %b %Yz	%b %d, %Yz%d.%m.%Yz	%A, %b %dz	%A, %B %d))z%Aú\w+)rG  rI  )rH  z\d{4})z%bz\w{3})z%dú\d{1,2})z%mrJ  c                 óZ  — | \  }}|}|j                  dt        j                  d«      «      }|j                  dt        j                  d«      «      }|j                  dd«      }t        D ]  \  }}|j                  ||«      }Œ d|vsJ |«       ‚||t        j                  d|z   dz   «      fS )z<Compute a regex for each date pattern to use as a prefilter.r>  r¦   rœ   z\s+ú%ú^ú$)rŸ   ÚreÚescapeÚ_FIELD_TO_REGEXÚcompile)ÚdpÚpatternÚmaskÚregexÚfieldÚfield_regexs         r)   Ú_process_date_patternrY  	  s¨   € à�M€GˆTØ€EØ�M‰M˜#œrŸy™y¨›~Ó.€EØ�M‰M˜#œrŸy™y¨›~Ó.€EØ�M‰M˜#˜vÓ&€EÜ-ò 2Ñˆˆ{Ø—‘˜e [Ó1‰ð2ð �eÑÐ"˜UÓ"ÐØ�Dœ"Ÿ*™* S¨5¡[°3Ñ%6Ó7Ð7Ð7r(   c                  ó.   — t        d„ t        D «       «      S )Nc              3   ó2   K  — | ]  }t        |«      –— Œ y ­wr†   )rY  )rZ  rS  s     r)   r[  z)_process_date_patterns.<locals>.<genexpr>	  s   è ø€ ÒD¨rÔ& r×*ÑDùs   ‚)ré   Ú_DATE_PATTERNSr'   r(   r)   Ú_process_date_patternsr]  	  s   € ÜÑD´^ÔDÓDÐDr(   r+  )ÚzeroÚoneÚtwoÚthreeÚfourÚfiveÚsixÚsevenÚeightÚnineÚtenÚelevenÚtwelve)ÚzerothÚfirstÚsecondÚthirdÚfourthÚfithÚsixthÚseventhÚeighthÚninthÚtenthÚeleventhÚtwelfth)ÚstÚndÚrdÚthz.((^|\s)[+-])?((\.\d+)|(\d+(,\d\d\d)*(\.\d*)?))i¤  ià  ÚINFc                 óX  — | j                   t        k  s| j                   t        kD  rt        d| j                   › �«      ‚t	        «       }|j                   r| j                   |_         |j
                  r| j
                  |_        |j                  r| j                  |_        t        |¬«      S )zYConverts date (datetime Python object) to a NumericValue object with a Date object value.zInvalid year: )r>  )r9  Ú	_MIN_YEARÚ	_MAX_YEARrw   r8  r:  r;  r=  )r>  rU  Únew_dates      r)   Ú_get_numeric_value_from_dater�  P	  sw   € à‡y�y”9Ò §	¡	¬IÒ 5Ü˜>¨$¯)©)¨Ð5Ó6Ð6ä‹v€HØ‡y‚yØŸ	™	ˆŒØ‡z‚zØŸ™ˆŒØ‡x‚xØ—x‘xˆŒÜ˜XÔ&Ð&r(   c                 ó$   — | d   | d   z
  | d    fS )zHSorts span by decreasing length first and increasing first index second.rª   r   r'   )Úspans    r)   Ú_get_span_length_keyr„  _	  s    € à�‰7�T˜!‘WÑ˜t A™w˜hÐ&Ð&r(   c                 ó   — t        | ¬«      S )zDConverts float (Python) to a NumericValue object with a float value.)r”  )r=  )r  s    r)   Ú_get_numeric_value_from_floatr†  d	  s   € ä EÔ*Ð*r(   c                 ó*  — t        j                  dd| «      } t        D ]U  \  }}}|j                  | «      sŒ	 t        j                  j                  | |«      j                  «       }	 t        ||«      c S  y# t        $ r Y Œdw xY w# t        $ r Y Œsw xY w)zAAttempts to format a text as a standard date string (yyyy-mm-dd).zSept\bÚSepN)	rO  ÚsubÚ_PROCESSED_DATE_PATTERNSÚmatchÚdatetimeÚstrptimer>  rw   r�  )rX   Ú
in_patternrU  rV  r>  s        r)   Ú_parse_dater�  j	  s�   € ä�6‰6�)˜U DÓ)€DÜ#;ò 
Ñˆ
�D˜%Ø�{‰{˜4Ô Øð	Ü×$Ñ$×-Ñ-¨d°JÓ?×DÑDÓFˆDð	Ü/°°dÓ;Ò;ð
ð øô ò 	Ùð	ûô ò 	Ùð	ús#   ¸.A7Á'BÁ7	BÂBÂ	BÂBc                 óú   — t         D ]$  }| j                  |«      sŒ| dt        |«        }  n | j                  dd«      } 	 t	        | «      }t        j                  |«      ry|t        k(  ry|S # t
        $ r Y yw xY w)z,Parses simple cardinal and ordinals numbers.Nú,r�   )	Ú_ORDINAL_SUFFIXESÚendswithr‰   rŸ   r’  rw   ÚmathÚisnanÚ_INF)rX   Úsuffixr  s      r)   Ú_parse_numberr˜  {	  s†   € ä#ò ˆØ�=‰=˜Õ Ø˜œ3˜v›;˜,Ð'ˆDÙðð �<‰<˜˜RÓ €DðÜ�d“ˆô ‡z�z�%ÔØØ”‚}ØØ€Løô ò Ùðús   ÁA. Á.	A:Á9A:c              #   ó(  K  — g }t        | «      D ]~  \  }}|j                  «       sŒ|dk(  s| |dz
     j                  «       s|j                  |«       |dz   t        | «      k(  s| |dz      j                  «       rŒk|| d D ]  }||dz   f–— Œ Œ€ y­w)zÿ
    Split a text into all possible ngrams up to 'max_ngram_length'. Split points are white space and punctuation.

    Args:
      text: Text to split.
      max_ngram_length: maximal ngram length.
    Yields:
      Spans, tuples of begin-end index.
    r   rª   N)rO   Úisalnumrs   r‰   )rX   Úmax_ngram_lengthÚstart_indexesrS   r  Ústart_indexs         r)   Úget_all_spansrž  �	  s¥   è ø€ ð €MÜ  “ò -‰ˆˆtØ�|‰|Œ~ØØ�AŠ:˜T %¨!¡)™_×4Ñ4Ô6Ø× Ñ  Ô'Ø�1‰9œ˜D›	Ò!¨¨e°a©i©×)@Ñ)@Õ)BØ,Ð.>Ð->Ð-?Ð@ò -�Ø! 5¨1¡9Ð,Ó,ñ-ñ-ùs   ‚A8BÁ;Bc                 ó\   — dj                  | j                  «       j                  «       «      S )Nrœ   )rž   r  rW   ©rX   s    r)   Únormalize_for_matchr¡  ¢	  s    € Ø�8‰8�D—J‘J“L×&Ñ&Ó(Ó)Ð)r(   c                 ó(  — | j                  «       j                  «       } | dk(  s
| dk(  s| dk(  rt        } t        j                  dd| «      j                  dd«      } dj                  | j                  «       «      } | j                  «       } | r| S t        S )z"Lowercases and strips punctuation.zn/aú?r�  z[^\w\d]+rœ   r6  )r  rV   r‘   rO  r‰  rŸ   rž   rW   r   s    r)   r�   r�   ¦	  sz   € à�:‰:‹<×ÑÓ€DØˆu‚}˜ š t¨u¢}Üˆä�6‰6�+˜s DÓ)×1Ñ1°#°sÓ;€DØ�8‰8�D—J‘J“LÓ!€DØ�:‰:‹<€DÙØˆÜÐr(   c           	      ó˜  — t        j                  t        «      }t        j	                  | «      D ]\  }| |j                  «       |j                  «        }t        |«      }|€Œ2||j                  «          j                  t        |«      «       Œ^ t        | d¬«      D ]É  \  }}||f|v rŒ| || }t        |«      }|�|||f   j                  t        |«      «       t        t        «      D ]4  \  }}||k(  sŒ|||f   j                  t        t        |«      «      «        n t        t        «      D ]4  \  }}||k(  sŒ|||f   j                  t        t        |«      «      «        ŒÉ ŒË t        | t         ¬«      D ].  \  }}| || }t#        |«      }|€Œ|||f   j                  |«       Œ0 t%        |j'                  «       d„ d¬«      }	g }
|	D ];  \  }}|
D ]  \  }}|d   |d   k  sŒ|d   |d   k  sŒ Œ( |
j                  ||f«       Œ= |
j)                  d„ ¬«       g }|
D ](  \  }}|j                  t+        |d   |d   |¬	«      «       Œ* |S )
z’
    Extracts longest number and date spans.

    Args:
      text: text to annotate

    Returns:
      List of longest numeric value spans.
    rª   )r›  c                 ó   — t        | d   «      S rš  )r„  ©Ú
span_values    r)   r¬   zparse_text.<locals>.<lambda>Ü	  s   € Ô=QÐR\Ð]^ÑR_Ó=`€ r(   T)r®   Úreverser   c                 ó   — | d   d   S rš  r'   r¦  s    r)   r¬   zparse_text.<locals>.<lambda>å	  s   € ¨z¸!©}¸QÑ/?€ r(   r­   )r¼  r½  rs  )rK   rt  r¾   Ú_NUMBER_PATTERNÚfinditerr#  r%  r˜  rƒ  rs   r†  rž  rO   Ú_NUMBER_WORDSr’  Ú_ORDINAL_WORDSÚ_MAX_DATE_NGRAM_SIZEr�  r±   rx   r·  r@  )rX   Ú	span_dictr‹  Ú	span_textÚnumberr¼  r½  Úwordr>  ÚspansÚselected_spansrƒ  r  Úselected_spanr6  Únumeric_value_spansrs  s                    r)   Ú
parse_textr·  ´	  s™  € ô ×'Ñ'¬Ó-€IÜ ×)Ñ)¨$Ó/ò RˆØ˜Ÿ™›¨¯©«Ð5ˆ	Ü˜yÓ)ˆØÑØ�e—j‘j“lÑ#×*Ñ*Ô+HÈÓ+PÕQð	Rô #0°ÀqÔ"Iò Ñˆ�YØ˜Ð# yÑ0ØØ˜ YÐ/ˆ	ä˜yÓ)ˆØÐØ�k 9Ð,Ñ-×4Ñ4Ô5RÐSYÓ5ZÔ[Ü%¤mÓ4ò 	‰LˆF�DØ˜DÓ Ø˜+ yÐ0Ñ1×8Ñ8Ô9VÔW\Ð]cÓWdÓ9eÔfÙð	ô &¤nÓ5ò 	‰LˆF�DØ˜DÓ Ø˜+ yÐ0Ñ1×8Ñ8Ô9VÔW\Ð]cÓWdÓ9eÔfÙñ	ðô" #0°ÔG[Ô"\ò ;Ñˆ�YØ˜ YÐ/ˆ	Ü˜9Ó%ˆØÑØ�k 9Ð,Ñ-×4Ñ4°TÕ:ð	;ô �9—?‘?Ó$Ñ*`ÐjnÔo€EØ€NØò 1‰ˆˆeØ .ò 	1ÑˆM˜1Ø˜QÑ 4¨¡7Ó*¨t°A©w¸-ÈÑ:JÓ/JÙð	1ð ×!Ñ! 4¨ -Õ0ð1ð ×ÑÑ?ÐÔ@àÐØ&ò l‰ˆˆfØ×"Ñ"Ô#3ÀÀQÁÐSWÐXYÑSZÐciÔ#jÕkðlàÐr(   ÚEMPTYr±  r>  c                 óf   — | j                   �t        S | j                  �t        S t	        d| › �«      ‚)NúUnknown type: )r”  ÚNUMBER_TYPEr>  Ú	DATE_TYPErw   )rj  s    r)   Ú_get_value_typer½  þ	  s8   € Ø× Ñ Ð,ÜÐØ	×	Ñ	Ð	'ÜÐÜ
�~ m _Ð5Ó
6Ð6r(   c                 ót  — | j                   �| j                   S | j                  �‡| j                  }g d¢}|j                  �t        |j                  «      |d<   |j                  �t        |j                  «      |d<   |j
                  �t        |j
                  «      |d<   t        |«      S t        d| › �«      ‚)z7Maps a NumericValue proto to a float or tuple of float.)NNNr   rª   rÆ   rº  )r”  r>  r9  r’  r:  r;  ré   rw   )rj  r>  Úvalue_tuples      r)   Ú_get_value_as_primitive_valuerÀ  
  s¦   € à× Ñ Ð,Ø×(Ñ(Ð(Ø×ÑÐ%Ø×!Ñ!ˆÚ(ˆà�9‰9Ð Ü" 4§9¡9Ó-ˆK˜‰NØ�:‰:Ð!Ü" 4§:¡:Ó.ˆK˜‰NØ�8‰8ÐÜ" 4§8¡8›_ˆK˜‰NÜ�[Ó!Ð!Ü
�~ m _Ð5Ó
6Ð6r(   c                 ó>   — | D �ch c]  }t        |«      ’Œ c}S c c}w r†   )r½  )r  r  s     r)   Ú_get_all_typesrÂ  
  s   € Ø0>Ö? uŒO˜EÕ"Ò?Ð?ùÒ?s   …c                 óŒ  ‡— t        | «      }t        |«      dk7  rt        d| › �«      ‚t        t	        |«      «      }|t
        k(  rt        S t        t        t        «      «      Š| D ]F  }t        |«      }t        |t        «      sJ ‚t        |«      D ]  \  }}|�Œ	‰j                  |«       Œ ŒH ‰st        d| › �«      ‚ˆfd„}|S )a$  
    Creates a function that can be used as a sort key or to compare the values. Maps to primitive types and finds the
    biggest common subset. Consider the values "05/05/2010" and "August 2007". With the corresponding primitive values
    (2010.,5.,5.) and (2007.,8., None). These values can be compared by year and date so we map to the sequence (2010.,
    5.), (2007., 8.). If we added a third value "2006" with primitive value (2006., None, None), we could only compare
    by the year so we would map to (2010.,), (2007.,) and (2006.,).

    Args:
     numeric_values: Values to compare

    Returns:
     A function that can be used as a sort key function (mapping numeric values to a comparable tuple)

    Raises:
      ValueError if values don't have a common type or are not comparable.
    rª   zNo common value type in zNo common value in c                 óD   •‡— t        | «      Št        ˆfd„‰D «       «      S )Nc              3   ó(   •K  — | ]	  }‰|   –— Œ y ­wr†   r'   )rZ  rS   r  s     €r)   r[  z@get_numeric_sort_key_fn.<locals>._sort_key_fn.<locals>.<genexpr>F
  s   øè ø€ Ò= e�U˜5•\Ñ=ùs   ƒ)rÀ  ré   )rj  r  Úvalid_indexess    @€r)   Ú_sort_key_fnz-get_numeric_sort_key_fn.<locals>._sort_key_fnD
  s   ù€ Ü-¨mÓ<ˆÜÓ=¨}Ô=Ó=Ð=r(   )rÂ  r‰   rw   ÚnextÚiterr»  rÀ  r€  r_  Ú_DATE_TUPLE_SIZEræ   ré   rO   Údiscard)	r  Úvalue_typesÚ
value_typerj  r  Útuple_indexÚinner_valuerÇ  rÆ  s	           @r)   rr  rr  
  sÛ   ø€ ô" ! Ó0€KÜ
ˆ;Ó˜1ÒÜÐ3°NÐ3CÐDÓEÐEä”d˜;Ó'Ó(€JØ”[Ò ä,Ð,ô œÔ.Ó/Ó0€Mà'ò 3ˆÜ-¨mÓ<ˆÜ˜%¤Ô'Ð'Ð'Ü(1°%Ó(8ò 	3Ñ$ˆK˜ØÑ"Ø×%Ñ% kÕ2ñ	3ð3ñ ÜÐ.¨~Ð.>Ð?Ó@Ð@ô>ð Ðr(   c                 ó0  — t        j                  «       }| j                  «       D ]  }|j                  t	        |«      «       Œ |si S t        |j                  «       «      }|t        | «      |z  k  ri S t        «       }|j                  «       D ]  \  }}||k(  sŒ|j                  |«       Œ t        |«      dkD  rt        |v sJ ‚t        }	nt        t        |«      «      }	i }
| j                  «       D ]!  \  }}|D ]  }t        |«      |	k(  sŒ||
|<    Œ! Œ# |
S )a(  
    Finds the most common numeric values in a column and returns them

    Args:
        row_index_to_values:
            For each row index all the values in that cell.
        min_consolidation_fraction:
            Fraction of cells that need to have consolidated value.
        debug_info:
            Additional information only used for logging

    Returns:
        For each row index the first value that matches the most common value. Rows that don't have a matching value
        are dropped. Empty list if values can't be consolidated.
    rª   )rK   ÚCounterrs  ÚupdaterÂ  rS  r‰   r€  rx   rƒ  r¼  rÈ  rÉ  r½  )Úrow_index_to_valuesÚmin_consolidation_fractionÚ
debug_infoÚtype_countsr  Ú	max_countÚvalid_typesrÍ  ÚcountÚmax_typeÚnew_row_index_to_valuerS   rs  r  s                 r)   Ú_consolidate_numeric_valuesrÜ  K
  s.  € ô  ×%Ñ%Ó'€KØ-×4Ñ4Ó6ò ;ˆØ×Ñœ>¨.Ó9Õ:ð;áØˆ	Ü�K×&Ñ&Ó(Ó)€IØ”3Ð*Ó+Ð.HÑHÒHàˆ	ä“%€KØ(×.Ñ.Ó0ò (Ñˆ
�EØ�IÓØ�O‰O˜JÕ'ð(ô ˆ;Ó˜!ÒÜ˜KÑ'Ð'Ð'Ü‰äœ˜[Ó)Ó*ˆàÐØ,×2Ñ2Ó4ò ‰ˆˆvàò 	ˆEÜ˜uÓ%¨Ó1Ø05Ð& uÑ-Ùñ	ðð "Ð!r(   c                 óJ   — t        | «      }t        j                  d„ |D «       Ž S )z'Parses text and returns numeric values.c              3   ó4   K  — | ]  }|j                   –— Œ y ­wr†   )rs  )rZ  rƒ  s     r)   r[  z&_get_numeric_values.<locals>.<genexpr>}
  s   è ø€ ÒC¨T˜TŸ[�[ÑCùs   ‚)r·  Ú	itertoolsÚchain)rX   r�  s     r)   r0  r0  z
  s!   € ä˜tÓ$€MÜ�?‰?ÑC°]ÔCÐDÐDr(   c                 ó˜   — i }| j                  «       D ]4  \  }}t        ||   j                  «      }t        t	        |«      «      ||<   Œ6 |S )a=  
    Parses text in column and returns a dict mapping row_index to values. This is the _get_column_values function from
    number_annotation_utils.py of the original implementation

    Args:
      table: Pandas dataframe
      col_index: integer, indicating the index of the column to get the numeric values of
    )rB  r¡  rX   r¾   r0  )rÒ   rk  Úindex_to_valuesr2   rF  rX   s         r)   rm  rm  €
  sV   € ð €OØŸ.™.Ó*ò E‰ˆ	�3Ü" 3 y¡>×#6Ñ#6Ó7ˆÜ%)Ô*=¸dÓ*CÓ%Dˆ˜	Ò"ðEð Ðr(   c                 ó¢   —  || «      }  ||«      }| |k(  rt         j                  S | |k  rt         j                  S | |kD  rt         j                  S y)z7Compares two values and returns their relation or None.N)r„  r…  r5  r6  )r  Úother_valuer‰  s      r)   r‚  r‚  �
  sO   € á˜Ó€EÙ˜kÓ*€KØ�ÒÜ�{‰{ÐØˆ{ÒÜ�{‰{ÐØˆ{ÒÜ�{‰{ÐØr(   c                 óN   — | }t        | «      } t        | «      }t        || |¬«      S )z'Adds numeric value spans to a question.)rE  rX   r�  )r¡  r·  rD  )r†  rE  r�  s      r)   r,  r,  �
  s*   € à€MÜ" 8Ó,€HÜ˜xÓ(€MÜ -°hÈmÔ\Ð\r(   c                 ó.   — t        | t        «      rdS | dfS )z@Return an empty string and True if 'text' is in invalid unicode.)r�   TF)ræ   Úbytesr   s    r)   Úfilter_invalid_unicoderè  ¥
  s   € ä# D¬%Ô0ˆ:ÐC°t¸U°mÐCr(   c           
      ó˜  — t        | d«      sd| _        | j                  «       D ]Q  \  }}t        |«      D ]>  \  }}t	        |«      \  }}|sŒt        j                  d| j                  › d|› d|› �«       Œ@ ŒS t        | j                  «      D ];  \  }}t	        |«      \  }}|sŒt        j                  d| j                  › d|› �«       Œ= y)zü
    Removes invalid unicode from table. Checks whether a table cell text contains an invalid unicode encoding. If yes,
    reset the table cell text to an empty str and log a warning for each invalid cell

    Args:
        table: table to clean.
    Útable_idr   z(Scrub an invalid table body @ table_id: z, row_index: z, col_index: z*Scrub an invalid table header @ table_id: N)Úhasattrrê  rB  rO   rè  r   r³   rq  )rÒ   r2   rF  rk  rG  Ú
is_invalidrE  s          r)   Ú!filter_invalid_unicode_from_tablerí  ª
  sÝ   € ô �5˜*Ô%ØˆŒàŸ.™.Ó*ò ‰ˆ	�3Ü(¨›~ò 	‰OˆI�tÜ5°dÓ;ÑˆD�*ÚÜ—‘Ø>¸u¿~¹~Ð>NÈmÐ\eÐ[fð g"Ø"+ ð.õñ	ðô ' u§}¡}Ó5ò sÑˆ	�6Ü3°FÓ;Ñˆ�
ÚÜ�O‰OÐHÈÏÉÐHXÐXeÐfoÐepÐqÕrñsr(   c                 ó’  — | j                  «       } t        | «       | j                  «       D ]3  \  }}t        |«      D ]   \  }}t	        |¬«      | j
                  ||f<   Œ" Œ5 t        | j                  «      D ]M  \  }}t        t        | |«      |||f¬«      }|j                  «       D ]  \  }}	|	| j
                  ||f   _
        Œ ŒO | S )a’  
    Parses text in table column-wise and adds the consolidated values. Consolidation refers to finding values with a
    common types (date or number)

    Args:
        table:
            Table to annotate.
        min_consolidation_fraction:
            Fraction of cells in a column that need to have consolidated value.
        debug_info:
            Additional information used for logging.
    r   )rÔ  rÕ  )Úcopyrí  rB  rO   rB  r“  rq  rÜ  rm  rx   rj  )
rÒ   rÔ  rÕ  r2   rF  rk  rG  rE  Úcolumn_valuesrj  s
             r)   r+  r+  Ä
  sß   € ð �J‰J‹L€Eä% eÔ,ð  Ÿ.™.Ó*ò ?‰ˆ	�3Ü(¨›~ò 	?‰OˆI�tÜ/3¸¬ˆE�J‰J�y )Ð+Ò,ñ	?ð?ô
 ' u§}¡}Ó5ò KÑˆ	�6Ü3Ü˜u iÓ0Ø'AØ" FÐ+ô
ˆð )6×(;Ñ(;Ó(=ò 	KÑ$ˆI�}Ø=JˆE�J‰J�y )Ð+Ñ,Õ:ñ	KðKð €Lr(   )gffffffæ?N)ur$   rK   rŒ  Úenumrß  r”  rt   rO  r  Údataclassesr   Útypingr   r   r   r   r   r	   r
   râ  r×  Útokenization_utilsr   r   r   r   Útokenization_utils_baser   r   r   r   r   r   Úutilsr   r   r   r   r   r   Úpandasrç   Ú
get_loggerr!   r²   r°   r   Ú
namedtuplerO  r0   r7   r<   r:   rD   rT   rY   rú  r[   rz   r|   ÚEnumr„  r8  r=  r@  rB  rD  rF  Ú_YEARÚ_YEAR_MONTHÚ_YEAR_MONTH_DAYÚ_MONTHÚ
_MONTH_DAYr\  rQ  rY  r]  rŠ  r®  r¬  r­  r’  rR  rª  r~  r  r’  r–  r�  r„  r†  r�  r˜  rž  r¡  r�   r·  Ú_PrimitiveNumericValueÚEllipsisÚ
_SortKeyFnrÊ  r‘   r»  r¼  r½  rÀ  rÂ  rr  rÜ  r0  rm  r‚  r,  rè  rí  r+  Ú__all__r'   r(   r)   ú<module>r     s   ðñ *ã Û Û Û Û Û 	Û 	Û Ý !ß J× JÑ Jã ç cÓ c÷÷ ÷ q× pñ ÔÛà	ˆ×	Ñ	˜HÓ	%€ð " ;Ð/Ð ô(˜lô (ð $ˆ[×#Ñ# LÒ2RÓS€
ñ �$Ô÷ð ó ðð ÷,ð ,ó ð,ñ
 �$Ô÷ð ó ðð"˜só "òòð'1Ð -ôTjÐ(ô j÷`9^ñ ^÷D6ñ 6ôD	ˆt�y‰yô 	ð ÷ð ó ðð ÷ ð  ó ð ð
 ÷&ð &ó ð&ð ÷1ð 1ó ð1ð
 ÷;ð ;ó ð;ð #ˆK×"Ñ" ;Ò0HÓI€	á�$˜˜uÓ%€Ù˜˜d EÓ*€Ù˜D $¨Ó-€Ù	�5˜$ Ó	&€Ù�u˜d DÓ)€
ð
 
ˆ6€NØ	ˆ5€MØ
ˆE€NØˆkÐØˆkÐØˆjÐØˆjÐØˆjÐØˆjÐØ�/Ð"Ø�Ð!Ø�Ð!Ø�Ð!ØˆkÐØˆkÐØ�Ð!Ø�Ð!Ø�/Ð"Ø�Ð!Ø�*ÐØ�*Ðð+€ð2€ò8òEñ 2Ó3Ð àÐ ò€ò €ò  -Ð à�"—*‘*ÐNÓO€ð €	Ø€	áˆUƒ|€ò'ò'ò
+òò"ò$-ò**òò6ð~ ˜u e¨H°U©O¸XÀe¹_ÈhÐW\ÉoÐ,]Ñ&^Ð^Ñ_Ð Ø�|�n e¨E°8¨OÑ&<Ð<Ñ=€
àÐ à€
à€Ø€	ò7ò7ò$@ò,ò^,"ò^Eòò 
ò]òDò
só4!ðH Ð
�r(   