Ë
    S^(hëa  ã                   óÒ   — d dl Z d dlZd dlZd dlZd dlmZmZmZ ddlm	Z	m
Z
mZmZ ddlmZ  ej                  e«      ZdddœZd	„ Zd
„ Zd„ Zd„ Z G d„ d«      Z G d„ de	«      ZdgZy)é    N)ÚListÚOptionalÚTupleé   )ÚPreTrainedTokenizerÚ_is_controlÚ_is_punctuationÚ_is_whitespace)Úloggingz
vocab.jsonz
merges.txt)Ú
vocab_fileÚmerges_filec                 ób   — t        «       }| d   }| dd D ]  }|j                  ||f«       |}Œ |S )zƒ
    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
    strings)
    r   é   N)ÚsetÚadd)ÚwordÚpairsÚ	prev_charÚchars       ún/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/herbert/tokenization_herbert.pyÚ	get_pairsr   "   sF   € ô
 ‹E€EØ�Q‘€IØ�Q�R�ò ˆØ�	‰	�9˜dÐ#Ô$Ø‰	ðð €Ló    c                 ó*  — | j                  dd«      } t        j                  dd| «      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  d	d
«      } | j                  dd
«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  d d!«      } | j                  d"d#«      } | j                  d$d%«      } | j                  d&d'«      } | j                  d(d)«      } | j                  d*d+«      } | j                  d,d-«      } t        j                  d.d| «      } | j                  d/d0«      } | j                  d1d2«      } | j                  d3d4«      } | j                  d5d6«      } | j                  d7d8«      } | j                  d9d:«      } | j                  d;d<«      } | j                  d=d>«      } | j                  d?d@«      } | S )Azz
    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/replace-unicode-punctuation.perl
    u   ï¼Œú,u   ã€‚\s*z. u   ã€�u   â€�ú"u   â€œu   âˆ¶ú:u   ï¼šu   ï¼Ÿú?u   ã€Šu   ã€‹u   ï¼‰ú)u   ï¼�ú!u   ï¼ˆú(u   ï¼›ú;u   ï¼‘Ú1u   ã€�u   ã€Œu   ï¼�Ú0u   ï¼“Ú3u   ï¼’Ú2u   ï¼•Ú5u   ï¼–Ú6u   ï¼™Ú9u   ï¼—Ú7u   ï¼˜Ú8u   ï¼”Ú4u   ï¼Ž\s*u   ï½žú~u   â€™ú'u   â€¦z...u   â”�ú-u   ã€ˆú<u   ã€‰ú>u   ã€�ú[u   ã€‘ú]u   ï¼…ú%)ÚreplaceÚreÚsub)Útexts    r   Úreplace_unicode_punctr8   0   sM  € ð �<‰<˜˜sÓ#€DÜ�6‰6�)˜T 4Ó(€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DÜ�6‰6�)˜T 4Ó(€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜uÓ%€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ€Kr   c                 ó¦   — g }| D ]:  }t        j                  |«      }|j                  d«      rŒ*|j                  |«       Œ< dj	                  |«      S )zw
    Port of https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/remove-non-printing-char.perl
    ÚCÚ )ÚunicodedataÚcategoryÚ
startswithÚappendÚjoin)r7   Úoutputr   Úcats       r   Úremove_non_printing_charrC   \   sS   € ð €FØò ˆÜ×"Ñ" 4Ó(ˆØ�>‰>˜#ÔØØ�‰�dÕð	ð
 �7‰7�6‹?Ðr   c                 óN   — | j                  «       } | sg S | j                  «       }|S )z@Runs basic whitespace cleaning and splitting on a piece of text.)ÚstripÚsplit)r7   Útokenss     r   Úwhitespace_tokenizerH   j   s%   € à�:‰:‹<€DÙØˆ	Ø�Z‰Z‹\€FØ€Mr   c                   óJ   — e Zd ZdZ	 	 	 	 	 d
d„Zdd„Zd„ Zdd„Zd„ Zd„ Z	d	„ Z
y)ÚBasicTokenizeraª  
    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).

    Args:
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        never_split (`Iterable`, *optional*):
            Collection of tokens which will never be split during tokenization. Only has an effect when
            `do_basic_tokenize=True`
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters.

            This should likely be deactivated for Japanese (see this
            [issue](https://github.com/huggingface/transformers/issues/328)).
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase` (as in the original BERT).
        do_split_on_punc (`bool`, *optional*, defaults to `True`):
            In some instances we want to skip the basic punctuation splitting so that later tokenization can capture
            the full context of the words, such as contractions.
    Nc                 ód   — |€g }|| _         t        |«      | _        || _        || _        || _        y ©N)Údo_lower_caser   Únever_splitÚtokenize_chinese_charsÚstrip_accentsÚdo_split_on_punc)ÚselfrM   rN   rO   rP   rQ   s         r   Ú__init__zBasicTokenizer.__init__‹   s<   € ð ÐØˆKØ*ˆÔÜ˜{Ó+ˆÔØ&<ˆÔ#Ø*ˆÔØ 0ˆÕr   c                 óJ  — |r$| j                   j                  t        |«      «      n| j                   }| j                  |«      }| j                  r| j                  |«      }t        j                  d|«      }t        |«      }g }|D ]€  }||vrY| j                  r0|j                  «       }| j                  dur/| j                  |«      }n| j                  r| j                  |«      }|j                  | j                  ||«      «       Œ‚ t        dj                  |«      «      }|S )aj  
        Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer.

        Args:
            never_split (`List[str]`, *optional*)
                Kept for backward compatibility purposes. Now implemented directly at the base class level (see
                [`PreTrainedTokenizer.tokenize`]) List of token not to split.
        ÚNFCFú )rN   Úunionr   Ú_clean_textrO   Ú_tokenize_chinese_charsr<   Ú	normalizerH   rM   ÚlowerrP   Ú_run_strip_accentsÚextendÚ_run_split_on_puncr@   )rR   r7   rN   Úunicode_normalized_textÚorig_tokensÚsplit_tokensÚtokenÚoutput_tokenss           r   ÚtokenizezBasicTokenizer.tokenize›   s  € ñ CN�d×&Ñ&×,Ñ,¬S°Ó-=Ô>ÐSW×ScÑScˆØ×Ñ Ó%ˆð ×&Ò&Ø×/Ñ/°Ó5ˆDä"-×"7Ñ"7¸¸tÓ"DÐÜ)Ð*AÓBˆØˆØ ò 	MˆEØ˜KÑ'Ø×%Ò%Ø!ŸK™K›M�EØ×)Ñ)°Ñ6Ø $× 7Ñ 7¸Ó >™Ø×'Ò'Ø ×3Ñ3°EÓ:�EØ×Ñ × 7Ñ 7¸¸{Ó KÕLð	Mô ,¨C¯H©H°\Ó,BÓCˆØÐr   c                 óº   — t        j                  d|«      }g }|D ].  }t        j                  |«      }|dk(  rŒ|j                  |«       Œ0 dj	                  |«      S )z$Strips accents from a piece of text.ÚNFDÚMnr;   )r<   rZ   r=   r?   r@   )rR   r7   rA   r   rB   s        r   r\   z!BasicTokenizer._run_strip_accentsÁ   s^   € ä×$Ñ$ U¨DÓ1ˆØˆØò 	 ˆDÜ×&Ñ& tÓ,ˆCØ�dŠ{ØØ�M‰M˜$Õð		 ð
 �w‰w�v‹Ðr   c                 óv  — | j                   r|�||v r|gS t        |«      }d}d}g }|t        |«      k  rb||   }t        |«      r|j	                  |g«       d}n)|r|j	                  g «       d}|d   j	                  |«       |dz  }|t        |«      k  rŒb|D �cg c]  }dj                  |«      ‘Œ c}S c c}w )z&Splits punctuation on a piece of text.r   TFéÿÿÿÿr   r;   )rQ   ÚlistÚlenr	   r?   r@   )	rR   r7   rN   ÚcharsÚiÚstart_new_wordrA   r   Úxs	            r   r^   z!BasicTokenizer._run_split_on_puncÌ   s¿   € à×$Ò$¨Ð)@ÀTÈ[ÑEXØ�6ˆMÜ�T“
ˆØˆØˆØˆØ”#�e“*ŠnØ˜‘8ˆDÜ˜tÔ$Ø—‘˜t˜fÔ%Ø!%‘á!Ø—M‘M "Ô%Ø!&�Ø�r‘
×!Ñ! $Ô'Ø�‰FˆAð ”#�e“*‹nð %+Ö+˜q�—‘˜•
Ò+Ð+ùÒ+s   ÂB6c                 óø   — g }|D ]c  }t        |«      }| j                  |«      r4|j                  d«       |j                  |«       |j                  d«       ŒS|j                  |«       Œe dj                  |«      S )z)Adds whitespace around any CJK character.rV   r;   )ÚordÚ_is_chinese_charr?   r@   ©rR   r7   rA   r   Úcps        r   rY   z&BasicTokenizer._tokenize_chinese_charsâ   sm   € àˆØò 	$ˆDÜ�T“ˆBØ×$Ñ$ RÔ(Ø—‘˜cÔ"Ø—‘˜dÔ#Ø—‘˜cÕ"à—‘˜dÕ#ð	$ð �w‰w�v‹Ðr   c                 ó¦   — |dk\  r|dk  sF|dk\  r|dk  s<|dk\  r|dk  s2|dk\  r|dk  s(|d	k\  r|d
k  s|dk\  r|dk  s|dk\  r|dk  s
|dk\  r|dk  ryy)z6Checks whether CP is the codepoint of a CJK character.i N  iÿŸ  i 4  i¿M  i   iß¦ i § i?· i@· i¸ i ¸ i¯Î i ù  iÿú  i ø iú TF© )rR   rt   s     r   rr   zBasicTokenizer._is_chinese_charï   sr   € ð �6Š\˜b FšlØ�f’  v¢Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�f’  v¢Ø�g’ "¨¢-ààr   c                 óÔ   — g }|D ]Q  }t        |«      }|dk(  s|dk(  st        |«      rŒ$t        |«      r|j                  d«       ŒA|j                  |«       ŒS dj	                  |«      S )zBPerforms invalid character removal and whitespace cleanup on text.r   iýÿ  rV   r;   )rq   r   r
   r?   r@   rs   s        r   rX   zBasicTokenizer._clean_text  sf   € àˆØò 	$ˆDÜ�T“ˆBØ�QŠw˜" š,¬+°dÔ*;ØÜ˜dÔ#Ø—‘˜cÕ"à—‘˜dÕ#ð	$ð �w‰w�v‹Ðr   )TNTNTrL   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__rS   rd   r\   r^   rY   rr   rX   rv   r   r   rJ   rJ   t   s<   „ ñð0 ØØ#ØØó1ó $òL	ó,ò,òó0r   rJ   c                   ód  ‡ — e Zd ZdZeZddddddddg d	¢ddfˆ fd
„	Zed„ «       Zd„ Z	d„ Z
d„ Zd„ Zed„ «       Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	 d#dee   deee      dee   fd„Z	 d$dee   deee      dedee   fˆ fd„Z	 d#dee   deee      dee   fd„Zd#dedee   dee   fd „Zd!„ Zd"„ Zˆ xZ S )%ÚHerbertTokenizeraÎ  
    Construct a BPE tokenizer for HerBERT.

    Peculiarities:

    - uses BERT's pre-tokenizer: BaseTokenizer splits tokens on spaces, and also on punctuation. Each occurrence of a
      punctuation character will be treated separately.

    - Such pretokenized input is BPE subtokenized

    This tokenizer inherits from [`XLMTokenizer`] which contains most of the methods. Users should refer to the
    superclass for more information regarding methods.
    Nz<s>z<unk>z<pad>z<mask>z</s>F)
z
<special0>z
<special1>z
<special2>z
<special3>z
<special4>z
<special5>z
<special6>z
<special7>z
<special8>z
<special9>c                 ó¤  •— 	 dd l }|| _        i | _        i | _        h d£| _        |
| _        || _        || _        |�|�t        |«      t        |«      k(  sJ ‚d | _
        d | _        t        |d¬«      5 }t        j                  |«      | _        d d d «       | j                  j!                  «       D ��ci c]  \  }}||“Œ
 c}}| _        t        |d¬«      5 }|j%                  «       j'                  d«      d d }d d d «       D �cg c]  }t)        |j'                  «       d d «      ‘Œ  }}t+        t-        |t/        t        |«      «      «      «      | _        i | _        t5        ‰| �l  d||	||||||||
d d	œ|¤Ž t9        d
| j:                  d
d
¬«      | _        y # t        $ r t        d«      ‚w xY w# 1 sw Y   �Œ#xY wc c}}w # 1 sw Y   ŒÊxY wc c}w )Nr   zrYou need to install sacremoses to use HerbertTokenizer. See https://pypi.org/project/sacremoses/ for installation.>   ÚjaÚthÚzhúutf-8©Úencodingú
ri   é   )Ú	unk_tokenÚ	bos_tokenÚ	sep_tokenÚ	pad_tokenÚ	cls_tokenÚ
mask_tokenÚadditional_special_tokensÚlang2idÚid2langÚdo_lowercase_and_remove_accentÚtokenizer_fileF)rM   rN   rO   rP   rv   )Ú
sacremosesÚImportErrorÚsmÚcache_moses_punct_normalizerÚcache_moses_tokenizerÚlang_with_custom_tokenizerr�   rŽ   r�   rk   Úja_word_tokenizerÚzh_word_tokenizerÚopenÚjsonÚloadÚencoderÚitemsÚdecoderÚreadrF   ÚtupleÚdictÚzipÚrangeÚ	bpe_ranksÚcacheÚsuperrS   rJ   Úall_special_tokensÚbert_pre_tokenizer)rR   r   r   r‘   r‹   r‡   rŠ   rŒ   r‰   rˆ   r�   r�   rŽ   r�   Úkwargsr’   Úvocab_handleÚkÚvÚmerges_handleÚmergesÚmergeÚ	__class__s                         €r   rS   zHerbertTokenizer.__init__&  sÝ  ø€ ð8	Ûð ˆŒð -/ˆÔ)à%'ˆÔ"Ú*<ˆÔ'à.LˆÔ+ØˆŒØˆŒØÐ 7Ð#6Ü�w“<¤3 w£<Ò/Ð/Ð/à!%ˆÔØ!%ˆÔä�* wÔ/ð 	3°<ÜŸ9™9 \Ó2ˆDŒL÷	3à)-¯©×);Ñ);Ó)=×>¡  A˜˜1™Ó>ˆŒÜ�+¨Ô0ð 	;°MØ"×'Ñ'Ó)×/Ñ/°Ó5°c°rÐ:ˆF÷	;à8>Ö?¨u”%˜Ÿ™› b qÐ)Õ*Ð?ˆÐ?Üœc &¬%´°F³Ó*<Ó=Ó>ˆŒØˆŒ
ä‰Ñð 	
ØØØØØØ!Ø&?ØØØ+IØñ	
ð ò	
ô #1ØØ×/Ñ/Ø#(Øô	#
ˆÕøô_ ò 	ÜðMóð ð	ú÷.	3ñ 	3üã>÷	;ð 	;üâ?s/   ƒF Á2F.Â2F;Ã#GÄ#GÆF+Æ.F8ÇG
c                 ó   — | j                   S rL   )r�   ©rR   s    r   rM   zHerbertTokenizer.do_lower_casez  s   € ð ×2Ñ2Ð2r   c                 ó¶   — || j                   vr,| j                  j                  |¬«      }|| j                   |<   n| j                   |   }|j                  |«      S )N©Úlang)r•   r”   ÚMosesPunctNormalizerrZ   )rR   r7   r¶   Úpunct_normalizers       r   Úmoses_punct_normz!HerbertTokenizer.moses_punct_norm€  sZ   € Ø�t×8Ñ8Ñ8Ø#Ÿw™w×;Ñ;ÀÐ;ÓFÐØ6FˆD×-Ñ-¨dÒ3à#×@Ñ@ÀÑFÐØ×)Ñ)¨$Ó/Ð/r   c                 ó¼   — || j                   vr,| j                  j                  |¬«      }|| j                   |<   n| j                   |   }|j                  |dd¬«      S )Nrµ   F)Ú
return_strÚescape)r–   r”   ÚMosesTokenizerrd   )rR   r7   r¶   Úmoses_tokenizers       r   Úmoses_tokenizezHerbertTokenizer.moses_tokenize‰  s_   € Ø�t×1Ñ1Ñ1Ø"Ÿg™g×4Ñ4¸$Ð4Ó?ˆOØ/>ˆD×&Ñ& tÒ,à"×8Ñ8¸Ñ>ˆOØ×'Ñ'¨¸ÀuÐ'ÓMÐMr   c                 óV   — t        |«      }| j                  ||«      }t        |«      }|S rL   )r8   r¹   rC   )rR   r7   r¶   s      r   Úmoses_pipelinezHerbertTokenizer.moses_pipeline’  s-   € Ü$ TÓ*ˆØ×$Ñ$ T¨4Ó0ˆÜ'¨Ó-ˆØˆr   c                 óþ  — | j                   €<	 dd l}|j                  dt        j                  j	                  d«      › d�«      | _         t        | j                   j                  |«      «      S # t
        t        f$ r€ t        j                  d«       t        j                  d«       t        j                  d«       t        j                  d«       t        j                  d	«       t        j                  d
«       ‚ w xY w)Nr   z-model r,   z/local/share/kytea/model.binz™Make sure you install KyTea (https://github.com/neubig/kytea) and it's python wrapper (https://github.com/chezou/Mykytea-python) with the following stepsz81. git clone git@github.com:neubig/kytea.git && cd kyteaz2. autoreconf -iz#3. ./configure --prefix=$HOME/localz4. make && make installz5. pip install kytea)r˜   ÚMykyteaÚosÚpathÚ
expanduserÚAttributeErrorr“   ÚloggerÚerrorrj   ÚgetWS)rR   r7   rÃ   s      r   Úja_tokenizezHerbertTokenizer.ja_tokenize™  sÐ   € Ø×!Ñ!Ð)ðÛà)0¯©ØœbŸg™g×0Ñ0°Ó5Ð6Ð6RÐSó*�Ô&ô �D×*Ñ*×0Ñ0°Ó6Ó7Ð7øô #¤KÐ0ò 
Ü—‘ð[ôô —‘ÐWÔXÜ—‘Ð/Ô0Ü—‘ÐBÔCÜ—‘Ð6Ô7Ü—‘Ð3Ô4Øð
ús   Ž;A- Á-BC<c                 ó,   — t        | j                  «      S rL   )rk   r�   r³   s    r   Ú
vocab_sizezHerbertTokenizer.vocab_size®  s   € ô �4—<‘<Ó Ð r   c                 óB   — t        | j                  fi | j                  ¤ŽS rL   )r¢   r�   Úadded_tokens_encoderr³   s    r   Ú	get_vocabzHerbertTokenizer.get_vocab´  s   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>r   c                 ó  ‡ — t        |d d «      |d   dz   fz   }|‰ j                  v r‰ j                  |   S t        |«      }|s|dz   S 	 t        |ˆ fd„¬«      }|‰ j                  vrnÎ|\  }}g }d}|t        |«      k  r�	 |j                  ||«      }	|j                  |||	 «       |	}||   |k(  r6|t        |«      dz
  k  r%||dz      |k(  r|j                  ||z   «       |dz  }n|j                  ||   «       |dz  }|t        |«      k  rŒ�t        |«      }|}t        |«      dk(  rnt        |«      }Œídj                  |«      }|d	k(  rd
}|‰ j                  |<   |S # t        $ r |j                  ||d  «       Y Œpw xY w)Nri   ú</w>c                 óN   •— ‰j                   j                  | t        d«      «      S )NÚinf)r¥   ÚgetÚfloat)ÚpairrR   s    €r   ú<lambda>z&HerbertTokenizer.bpe.<locals>.<lambda>Â  s   ø€ °·±×1CÑ1CÀDÌ%ÐPUË,Ó1W€ r   ©Úkeyr   r   r†   rV   z
  </w>z
</w>)r¡   r¦   r   Úminr¥   rk   Úindexr]   Ú
ValueErrorr?   r@   )
rR   rb   r   r   ÚbigramÚfirstÚsecondÚnew_wordrm   Újs
   `         r   ÚbpezHerbertTokenizer.bpe¸  sª  ø€ Ü�U˜3˜B�ZÓ  E¨"¡I°Ñ$6Ð#8Ñ8ˆØ�D—J‘JÑØ—:‘:˜eÑ$Ð$Ü˜$“ˆáØ˜6‘>Ð!àÜ˜Ó$WÔXˆFØ˜TŸ^™^Ñ+ØØ"‰MˆE�6ØˆHØˆAØ”c˜$“i’-ðØŸ
™
 5¨!Ó,�Að
 —O‘O D¨¨1 IÔ.Ø�Aà˜‘7˜eÒ#¨¬C°«I¸©MÒ(9¸dÀ1ÀqÁ5¹kÈVÒ>SØ—O‘O E¨F¡NÔ3Ø˜‘F‘Aà—O‘O D¨¡GÔ,Ø˜‘F�Að ”c˜$“i“-ô  ˜X“ˆHØˆDÜ�4‹y˜AŠ~Øä! $›�ð9 ð: �x‰x˜‹~ˆØ�:ÒØˆDØ ˆ�
‰
�5ÑØˆøô/ "ò Ø—O‘O D¨¨ HÔ-Ùðús   ÂE  Å F Å?F c                 óÄ   — | j                   j                  |«      }g }|D ]=  }|sŒ|j                  t        | j	                  |«      j                  d«      «      «       Œ? |S )NrV   )r©   rd   r]   rj   rã   rF   )rR   r7   Ú
pre_tokensra   rb   s        r   Ú	_tokenizezHerbertTokenizer._tokenizeä  s_   € Ø×,Ñ,×5Ñ5°dÓ;ˆ
àˆØò 	FˆEÚØ×#Ñ#¤D¨¯©°%«×)>Ñ)>¸sÓ)CÓ$DÕEð	Fð Ðr   c                 ó€   — | j                   j                  || j                   j                  | j                  «      «      S )z0Converts a token (str) in an id using the vocab.)r�   rÕ   r‡   )rR   rb   s     r   Ú_convert_token_to_idz%HerbertTokenizer._convert_token_to_idï  s,   € à�|‰|×Ñ  t§|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHr   c                 óN   — | j                   j                  || j                  «      S )z=Converts an index (integer) in a token (str) using the vocab.)rŸ   rÕ   r‡   )rR   rÜ   s     r   Ú_convert_id_to_tokenz%HerbertTokenizer._convert_id_to_tokenô  s   € à�|‰|×Ñ  t§~¡~Ó6Ð6r   c                 ód   — dj                  |«      j                  dd«      j                  «       }|S )z:Converts a sequence of tokens (string) in a single string.r;   rÒ   rV   )r@   r4   rE   )rR   rG   Ú
out_strings      r   Úconvert_tokens_to_stringz)HerbertTokenizer.convert_tokens_to_stringù  s+   € à—W‘W˜V“_×,Ñ,¨V°SÓ9×?Ñ?ÓAˆ
ØÐr   Útoken_ids_0Útoken_ids_1Úreturnc                 óf   — | j                   g}| j                  g}|€||z   |z   S ||z   |z   |z   |z   S )a�  
        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
        adding special tokens. An XLM sequence has the following format:

        - single sequence: `<s> X </s>`
        - pair of sequences: `<s> A </s> B </s>`

        Args:
            token_ids_0 (`List[int]`):
                List of IDs to which the special tokens will be added.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.

        )Úbos_token_idÚsep_token_id)rR   rî   rï   ÚbosÚseps        r   Ú build_inputs_with_special_tokensz1HerbertTokenizer.build_inputs_with_special_tokensÿ  sP   € ð( × Ñ Ð!ˆØ× Ñ Ð!ˆàÐØ˜Ñ$ sÑ*Ð*Ø�[Ñ  3Ñ&¨Ñ4°sÑ:Ð:r   Úalready_has_special_tokensc                 ó´   •— |rt         ‰| �  ||d¬«      S |�+dgdgt        |«      z  z   dgz   dgt        |«      z  z   dgz   S dgdgt        |«      z  z   dgz   S )aÄ  
        Retrieve sequence ids from a token list that has no special tokens added. This method is called when adding
        special tokens using the tokenizer `prepare_for_model` method.

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.
            already_has_special_tokens (`bool`, *optional*, defaults to `False`):
                Whether or not the token list is already formatted with special tokens for the model.

        Returns:
            `List[int]`: A list of integers in the range [0, 1]: 1 for a special token, 0 for a sequence token.
        T)rî   rï   r÷   r   r   )r§   Úget_special_tokens_maskrk   )rR   rî   rï   r÷   r±   s       €r   rù   z(HerbertTokenizer.get_special_tokens_mask  s‰   ø€ ñ& &Ü‘7Ñ2Ø'°[Ð]að 3ó ð ð Ð"Ø�3˜1˜#¤ KÓ 0Ñ0Ñ1°Q°CÑ7¸A¸3ÄÀ[ÓAQÑ;QÑRÐVWÐUXÑXÐXØˆs�q�cœC Ó,Ñ,Ñ-°°Ñ3Ð3r   c                 ó´   — | j                   g}| j                  g}|€t        ||z   |z   «      dgz  S t        ||z   |z   «      dgz  t        ||z   «      dgz  z   S )aÑ  
        Create a mask from the two sequences passed to be used in a sequence-pair classification task. An XLM sequence
        pair mask has the following format:

        ```
        0 0 0 0 0 0 0 0 0 0 0 1 1 1 1 1 1 1 1 1
        | first sequence    | second sequence |
        ```

        If `token_ids_1` is `None`, this method only returns the first portion of the mask (0s).

        Args:
            token_ids_0 (`List[int]`):
                List of IDs.
            token_ids_1 (`List[int]`, *optional*):
                Optional second list of IDs for sequence pairs.

        Returns:
            `List[int]`: List of [token type IDs](../glossary#token-type-ids) according to the given sequence(s).
        r   r   )ró   Úcls_token_idrk   )rR   rî   rï   rõ   Úclss        r   Ú$create_token_type_ids_from_sequencesz5HerbertTokenizer.create_token_type_ids_from_sequences8  st   € ð. × Ñ Ð!ˆØ× Ñ Ð!ˆØÐÜ�s˜[Ñ(¨3Ñ.Ó/°1°#Ñ5Ð5Ü�3˜Ñ$ sÑ*Ó+¨q¨cÑ1´C¸ÀcÑ8IÓ4JÈaÈSÑ4PÑPÐPr   Úsave_directoryÚfilename_prefixc           	      ó.  — t         j                  j                  |«      st        j	                  d|› d�«       y t         j                  j                  ||r|dz   ndt        d   z   «      }t         j                  j                  ||r|dz   ndt        d   z   «      }t        |dd¬	«      5 }|j                  t        j                  | j                  d
dd¬«      dz   «       d d d «       d}t        |dd¬	«      5 }t        | j                  j                  «       d„ ¬«      D ]M  \  }}	||	k7  rt        j                  d|› d�«       |	}|j                  dj                  |«      dz   «       |dz  }ŒO 	 d d d «       ||fS # 1 sw Y   Œ�xY w# 1 sw Y   ||fS xY w)NzVocabulary path (z) should be a directoryr.   r;   r   r   Úwr‚   rƒ   r†   TF)ÚindentÚ	sort_keysÚensure_asciir…   r   c                 ó   — | d   S )Nr   rv   )Úkvs    r   rØ   z2HerbertTokenizer.save_vocabulary.<locals>.<lambda>f  s   € ÐY[Ð\]ÑY^€ r   rÙ   zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!rV   r   )rÄ   rÅ   ÚisdirrÈ   rÉ   r@   ÚVOCAB_FILES_NAMESrš   Úwriter›   Údumpsr�   Úsortedr¥   rž   Úwarning)
rR   rþ   rÿ   r   Ú
merge_fileÚfrÜ   ÚwriterÚ
bpe_tokensÚtoken_indexs
             r   Úsave_vocabularyz HerbertTokenizer.save_vocabularyV  s�  € Ü�w‰w�}‰}˜^Ô,Ü�L‰LÐ,¨^Ð,<Ð<SÐTÔUØÜ—W‘W—\‘\Ø±o˜_¨sÒ2È2ÔQbÐcoÑQpÑpó
ˆ
ô —W‘W—\‘\Ø±o˜_¨sÒ2È2ÔQbÐcpÑQqÑqó
ˆ
ô �*˜c¨GÔ4ð 	c¸Ø�G‰G”D—J‘J˜tŸ|™|°AÀÐTYÔZÐ]aÑaÔb÷	cð ˆÜ�*˜c¨GÔ4ð 		¸Ü+1°$·.±.×2FÑ2FÓ2HÑN^Ô+_ò Ñ'�
˜KØ˜KÒ'Ü—N‘NØ/°
¨|ð <Mð Môð (�EØ—‘˜SŸX™X jÓ1°DÑ8Ô9Ø˜‘
‘ñ÷		ð ˜:Ð%Ð%÷	cð 	cú÷		ð ˜:Ð%Ð%ús   Â*6E<Ã8A7FÅ<FÆFc                 óD   — | j                   j                  «       }d |d<   |S )Nr”   )Ú__dict__Úcopy)rR   Ústates     r   Ú__getstate__zHerbertTokenizer.__getstate__s  s"   € Ø—‘×"Ñ"Ó$ˆØˆˆd‰Øˆr   c                 óZ   — || _         	 dd l}|| _        y # t        $ r t        d«      ‚w xY w)Nr   znYou need to install sacremoses to use XLMTokenizer. See https://pypi.org/project/sacremoses/ for installation.)r  r’   r“   r”   )rR   Údr’   s      r   Ú__setstate__zHerbertTokenizer.__setstate__y  s>   € ØˆŒð	Ûð ˆ�øô ò 	ÜðMóð ð	ús   ‰ •*rL   )NF)!rx   ry   rz   r{   r  Úvocab_files_namesrS   ÚpropertyrM   r¹   r¿   rÁ   rË   rÍ   rÐ   rã   ræ   rè   rê   rí   r   Úintr   rö   Úboolrù   rý   Ústrr   r  r  r  Ú__classcell__)r±   s   @r   r}   r}     su  ø„ ñð *Ðð ØØØØØØØ',ò#
ð Øõ3R
ðh ñ3ó ð3ò0òNòò8ð* ñ!ó ð!ò?ò*òXòIò
7ò
ð JNñ;Ø ™9ð;Ø3;¸DÀ¹IÑ3Fð;à	ˆc‰ó;ð: sxñ4Ø ™9ð4Ø3;¸DÀ¹IÑ3Fð4Økoð4à	ˆc‰õ4ð< JNñQØ ™9ðQØ3;¸DÀ¹IÑ3FðQà	ˆc‰óQñ<&¨cð &ÀHÈSÁMð &Ð]bÐcfÑ]gó &ò:ör   r}   )r›   rÄ   r5   r<   Útypingr   r   r   Útokenization_utilsr   r   r	   r
   Úutilsr   Ú
get_loggerrx   rÈ   r  r   r8   rC   rH   rJ   r}   Ú__all__rv   r   r   ú<module>r&     s‚   ðó Û 	Û 	Û ß (Ñ (ç cÓ cÝ ð 
ˆ×	Ñ	˜HÓ	%€ð ØñÐ ò
ò(òX
ò÷^ñ ^ôBoÐ*ô oðd Ð
�r   