Ë
    S^(h.ø  ã                   óÚ  — d Z ddlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZ ddlmZmZmZ ddlmZmZmZmZ ddlmZ  ej.                  e«      Zdœd	„Zd
edefd„Zd„ Z G d„ d«      Z G d„ de«      Z dedefd„Z! G d„ d«      Z" G d„ de"«      Z# G d„ de"«      Z$ G d„ de"«      Z% G d„ de"«      Z& G d„ de"«      Z' G d „ d!e"«      Z( G d"„ d#e"«      Z) G d$„ d%e"«      Z* G d&„ d'e"«      Z+ G d(„ d)e"«      Z, G d*„ d+e"«      Z- G d,„ d-e"«      Z. G d.„ d/e.«      Z/ G d0„ d1e.«      Z0 G d2„ d3e.«      Z1 G d4„ d5e.«      Z2 G d6„ d7e.«      Z3 G d8„ d9e.«      Z4 G d:„ d;e.«      Z5 G d<„ d=e.«      Z6 G d>„ d?e.«      Z7 G d@„ dAe.«      Z8 G dB„ dCe.«      Z9 G dD„ dEe.«      Z: G dF„ dGe.«      Z; G dH„ dIe.«      Z< G dJ„ dKe.«      Z= G dL„ dMe.«      Z> G dN„ dOe"«      Z? G dP„ dQe.«      Z@ G dR„ dSe"«      ZA G dT„ dUe"«      ZB G dV„ dWe"«      ZC G dX„ dYe.«      ZD G dZ„ d[e.«      ZE G d\„ d]e.«      ZF G d^„ d_e"«      ZG G d`„ dae.«      ZH G db„ dce.«      ZIdd„ ZJ G de„ df«      ZKi dge/“dhe+“die0“dje#“dke@“dleC“dme1“dneA“doe(“dpe#“dqe-“dre2“dse#“dte#“due#“dve#“dwe#“i dxe/“dye%“dze(“d{e)“d|e#“d}e#“d~e+“de7“d€e+“d�e+“d‚e#“dƒeG“d„e3“d…e4“d†e&“d‡e#“dˆe+“¥i d‰e5“dŠe'“d‹e<“dŒe*“d�e#“dŽe9“d�e:“d�e#“d‘e+“d’e,“d“e6“d”e#“d•e=“d–e>“d—e?“d˜e7“d™e8“¥e$eDeFeFeEeFdšœ¥ZLd�defd›„ZMy)žzù
Utilities to convert slow tokenizers in their fast tokenizers counterparts.

All the conversions are grouped here to gather SentencePiece dependencies outside of the fast tokenizers files and
allow to make our dependency on SentencePiece optional.
é    N)Úversion)Ú
AddedTokenÚRegexÚ	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramÚ	WordPieceé   )Úis_protobuf_availableÚis_sentencepiece_availableÚloggingÚrequires_backends)ÚPROTOBUF_IMPORT_ERRORc                 ó  — t        «       rddlm} |S t        «       rSdd l}t        j                  |j                  j                  «      t        j                  d«      k  rddl	m} |S ddl	m
} |S t        t        j                  | «      «      ‚)Nr   )Úsentencepiece_model_pb2z4.0.0)Úsentencepiece_model_pb2_new)r   Úsentencepiecer   r   Úgoogle.protobufr   ÚparseÚprotobufÚ__version__Útransformers.utilsr   ÚImportErrorr   Úformat)Úerror_messager   Úgoogles      úa/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/convert_slow_tokenizer.pyÚimport_protobufr"   "   sl   € Ü!Ô#Ý9à&Ð&ÜÔÛä�=‰=˜Ÿ™×4Ñ4Ó5¼¿¹ÀgÓ8NÒNÝBð 'Ð&õ bØ&Ð&äÔ/×6Ñ6°}ÓEÓFÐFó    Úadd_prefix_spaceÚreturnc                 ó4   — | rd}t        |dd«      sd}|S d}|S )NÚalwaysÚlegacyTÚfirstÚnever)Úgetattr)r$   Úoriginal_tokenizerÚprepend_schemes      r!   Ú_get_prepend_schemer.   3   s1   € ÙØ!ˆÜÐ)¨8°TÔ:Ø$ˆNð Ðð !ˆØÐr#   c                 ó‚  ‡ — |d u}|rt        |«      n‰ }g }|j                  «       D ]j  \  }}g }t        dt        |«      «      D ]*  }|d | ||d  }	}|‰ v sŒ|	‰ v sŒ|j	                  ||	|f«       Œ, t        |ˆ fd„¬«      }|j                  |«       Œl t        |d„ |¬«      }|D �
cg c]  }
|
d   |
d   f‘Œ }}
|S c c}
w )Nr   c                 ó$   •— ‰| d      ‰| d      fS ©Nr   r   © )ÚxÚvocabs    €r!   ú<lambda>z!generate_merges.<locals>.<lambda>H   s   ø€ ¨U°1°Q±4©[¸%ÀÀ!Á¹+Ð,F€ r#   ©Úkeyc                 óB   — | d   t        | d   «      t        | d   «      fS )Né   r   r   )Úlen©Úvals    r!   r5   z!generate_merges.<locals>.<lambda>K   s!   € ¨S°©V´S¸¸Q¹³[Ä#ÀcÈ!ÁfÃ+Ð,N€ r#   ©r7   Úreverser   )ÚdictÚitemsÚranger:   ÚappendÚsortedÚextend)r4   Úvocab_scoresr>   ÚmergesÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rr<   s   `          r!   Úgenerate_mergesrM   =   sê   ø€ Ø $Ð&€GÙ)0”4˜Ô%°e€Là€FØ*×0Ñ0Ó2ò Ñˆˆ{ØˆÜ˜1œc %›jÓ)ò 	>ˆEØ$ V e˜}¨e°E°F¨m�WˆGØ˜%Ò G¨uÒ$4Ø—‘˜g w°Ð<Õ=ð	>ô �uÓ"FÔGˆØ�‰�eÕðô �FÑ NÐX_Ô`€FØ*0Ö1 3ˆs�1‰v�s˜1‘vÒÐ1€FÐ1Ø€Mùò 2s   Â'B<c                   óD   — e Zd ZdZdefd„Zddeeeef   e	e   f   fd„Z
y)ÚSentencePieceExtractorzl
    Extractor implementation for SentencePiece trained models. https://github.com/google/sentencepiece
    Úmodelc                 óv   — t        | d«       ddlm}  |«       | _        | j                  j	                  |«       y )Nr   r   )ÚSentencePieceProcessor)r   r   rR   ÚspÚLoad)ÚselfrP   rR   s      r!   Ú__init__zSentencePieceExtractor.__init__U   s)   € Ü˜$ Ô0Ý8á(Ó*ˆŒØ�‰�‰�UÕr#   Nr%   c                 ó°   — | j                   }t        |j                  «       «      D �ci c]  }|j                  |«      |“Œ }}t	        ||«      }||fS c c}w )úÂ
        By default will return vocab and merges with respect to their order, by sending `vocab_scores` we're going to
        order the merges with respect to the piece scores instead.
        )rS   rA   ÚGetPieceSizeÚid_to_piecerM   ©rU   rE   rS   rJ   r4   rF   s         r!   ÚextractzSentencePieceExtractor.extract\   sV   € ð
 �W‰WˆÜ;@ÀÇÁÓARÓ;SÖT°%�—‘ Ó&¨Ñ-ÐTˆÐTä  ¨Ó5ˆà�fˆ}Ðùò	 Us   ¨A©N)Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrrV   Útupler?   ÚintÚlistr\   r2   r#   r!   rO   rO   P   s5   „ ñð˜có ñ
¨E°$°s¸C°x±.À$ÀuÁ+Ð2MÑ,Nô 
r#   rO   c                   ó4   — e Zd Zddeeeef   ee   f   fd„Zy)ÚGemmaSentencePieceExtractorNr%   c                 óà   — | j                   }t        |j                  «       «      D �ci c]  }|j                  |«      |“Œ }}d|vr|j	                  d«      |d<   t        ||«      }||fS c c}w )rX   ú	ú<0x09>)rS   rA   rY   rZ   ÚgetrM   r[   s         r!   r\   z#GemmaSentencePieceExtractor.extractj   sr   € ð
 �W‰WˆÜ;@ÀÇÁÓARÓ;SÖT°%�—‘ Ó&¨Ñ-ÐTˆÐTð �uÑØŸ)™) HÓ-ˆE�$‰KÜ  ¨Ó5ˆØ�fˆ}Ðùò Us   ¨A+r]   )	r^   r_   r`   rc   r?   rb   rd   re   r\   r2   r#   r!   rg   rg   i   s$   „ ñ¨E°$°s¸C°x±.À$ÀuÁ+Ð2MÑ,Nô r#   rg   Úpiecec                 ó^   — t        | «      dk  xs | d   dk7  xs | d   j                  «        S )Nr9   éÿÿÿÿú,éþÿÿÿ)r:   Úisdigit)rl   s    r!   Úcheck_number_commarr   z   s3   € Üˆu‹:˜‰>ÒH˜U 2™Y¨#Ñ-ÒH°U¸2±Y×5FÑ5FÓ5HÐ1HÐHr#   c                   ó   — e Zd Zd„ Zdefd„Zy)Ú	Converterc                 ó   — || _         y r]   )r,   )rU   r,   s     r!   rV   zConverter.__init__   s
   € Ø"4ˆÕr#   r%   c                 ó   — t        «       ‚r]   )ÚNotImplementedError©rU   s    r!   Ú	convertedzConverter.converted‚   s   € Ü!Ó#Ð#r#   N)r^   r_   r`   rV   r   ry   r2   r#   r!   rt   rt   ~   s   „ ò5ð$˜9ô $r#   rt   c                   ó   — e Zd Zdefd„Zy)ÚBertConverterr%   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )N©Ú	unk_tokenFÚbasic_tokenizerT©Ú
clean_textÚhandle_chinese_charsÚstrip_accentsÚ	lowercaseú:0 $A:0 ú:0ú:0 $B:1 ú:1©ÚsingleÚpairÚspecial_tokensú##©Úprefix©r,   r4   r   r   rb   r~   Úhasattrr   Útokenize_chinese_charsrƒ   Údo_lower_caser   ÚBertNormalizerÚ
normalizerr	   ÚBertPreTokenizerÚpre_tokenizerÚ	cls_tokenÚ	sep_tokenÚcls_token_idÚsep_token_idr
   ÚTemplateProcessingÚpost_processorr   Údecoder©
rU   r4   Ú	tokenizerr’   rƒ   r“   ÚclsÚseprš   r›   s
             r!   ry   zBertConverter.converted‡   ó…  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr#   N©r^   r_   r`   r   ry   r2   r#   r!   r{   r{   †   ó   „ ð#˜9ô #r#   r{   c                   ó   — e Zd Zdefd„Zy)ÚSplinterConverterr%   c           
      ó‚  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }t	        | j                   j&                  «      }d}	| j                   j(                  }
| j                   j*                  }| j                   j,                  }| j                   j/                  d«      }| j                   j0                  dk(  r|› d|› d	|	› d	|› d
|› d�
}n|› d|› d
|› d	|	› d	|› d�
}t3        j4                  |› d|› d�|||
f||f||f|	|fg¬«      |_        t9        j                  d¬«      |_        |S )Nr}   Fr   Tr€   ú.Úrightr…   ú r‡   rˆ   r†   r‰   r�   rŽ   )r,   r4   r   r   rb   r~   r‘   r   r’   rƒ   r“   r   r”   r•   r	   r–   r—   r˜   r™   Úquestion_tokenrš   r›   Úquestion_token_idÚconvert_tokens_to_idsÚpadding_sider
   rœ   r�   r   rž   )rU   r4   r    r’   rƒ   r“   r¡   r¢   ÚquestionÚdotrš   r›   r­   Údot_token_idr‹   s                  r!   ry   zSplinterConverter.converted®   s"  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆÜ�t×.Ñ.×=Ñ=Ó>ˆØˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆØ ×3Ñ3×EÑEÐØ×.Ñ.×DÑDÀSÓIˆà×"Ñ"×/Ñ/°7Ò:Ø�U˜( 8 *¨A¨c¨U°!°C°5¸ÀÀÀRÐH‰Dà�U˜( 3 % x°¨z¸¸3¸%¸qÀÀÀRÐHˆDä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Øà�lÐ#Ø�lÐ#ØÐ,Ð-Ø�lÐ#ð	ô	$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr#   Nr¤   r2   r#   r!   r§   r§   ­   s   „ ð.˜9ô .r#   r§   c                   ó   — e Zd Zdefd„Zy)ÚFunnelConverterr%   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )Nr}   Fr   Tr€   z:2 $A:0 r†   r‡   rˆ   r‰   r�   rŽ   r�   rŸ   s
             r!   ry   zFunnelConverter.convertedà   r£   r#   Nr¤   r2   r#   r!   r´   r´   ß   r¥   r#   r´   c                   ó   — e Zd Zdefd„Zy)ÚMPNetConverterr%   c                 ór  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	|› d
�||f||	fg¬«      |_        t1        j                  d¬«      |_        |S )Nr}   Fr   Tr€   r…   r†   z:0 r‡   rˆ   r‰   r�   rŽ   r�   rŸ   s
             r!   ry   zMPNetConverter.converted  s‹  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   S¨¨¨X°c°U¸"Ð=à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr#   Nr¤   r2   r#   r!   r·   r·     r¥   r#   r·   c                   ó   — e Zd Zdefd„Zy)ÚOpenAIGPTConverterr%   c           
      ó  — | j                   j                  }t        | j                   j                  j	                  «       «      }| j                   j
                  }t        t        ||d t        |«      dd¬«      «      }|j                  t        |«      «      �|j                  t        |«      g«       t        j                  d¬«      |_        t        j                  «       |_        t#        j$                  d¬«      |_        |S )Nú</w>F)r4   rF   Údropoutr~   Úend_of_word_suffixÚfuse_unkT)r„   ©Úsuffix)r,   Úencoderre   Ú	bpe_ranksÚkeysr~   r   r   rb   Útoken_to_idÚadd_special_tokensr   r”   r•   r	   r–   r—   r   Ú
BPEDecoderrž   ©rU   r4   rF   r~   r    s        r!   ry   zOpenAIGPTConverter.converted.  sÔ   € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØÜ˜i›.Ø#)Øôó	
ˆ	ð × Ñ ¤ Y£Ó0Ð<Ø×(Ñ(¬#¨i«.Ð)9Ô:ä*×9Ñ9ÀDÔIˆ	ÔÜ"0×"AÑ"AÓ"Cˆ	ÔÜ$×/Ñ/°vÔ>ˆ	ÔàÐr#   Nr¤   r2   r#   r!   rº   rº   -  s   „ ð˜9ô r#   rº   c                   ó<   — e Zd Zddeeef   deeeef      defd„Z	y)ÚGPT2ConverterNr4   rF   r%   c           
      óN  — |s| j                   j                  }|st        | j                   j                  «      }t	        t        ||d ddd¬«      «      }t        | j                   dd«      }t        j                  |¬«      |_	        t        j                  «       |_        t        | j                   dd«      rT| j                   j                  }| j                   j                  }t        j                  |› d�|› d�||fg¬	«      |_        |S t        j                  d¬
«      |_        |S )NÚ F©r4   rF   r½   Úcontinuing_subword_prefixr¾   r¿   r$   ©r$   Úadd_bos_tokenz:0 $A:0z:0 $A:0 $B:1r‰   ©Útrim_offsets)r,   rÂ   re   rÃ   r   r   r+   r	   Ú	ByteLevelr—   r   rž   Ú	bos_tokenÚbos_token_idr
   rœ   r�   )rU   r4   rF   r    r$   ÚbosrÕ   s          r!   ry   zGPT2Converter.convertedI  s  € ÙØ×+Ñ+×3Ñ3ˆEÙÜ˜$×1Ñ1×;Ñ;Ó<ˆFäÜØØØØ*,Ø#%Øôó	
ˆ	ô # 4×#:Ñ#:Ð<NÐPUÓVÐÜ"0×":Ñ":ÐL\Ô"]ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ�4×*Ñ*¨O¸UÔCØ×)Ñ)×3Ñ3ˆCØ×2Ñ2×?Ñ?ˆLÜ'1×'DÑ'DØ˜˜g�Ø�u˜LÐ)à˜,Ð'ð ô(ˆIÔ$ð Ðô (2×';Ñ';ÈÔ'OˆIÔ$ØÐr#   ©NN©
r^   r_   r`   r?   rb   rd   re   rc   r   ry   r2   r#   r!   rÊ   rÊ   H  s2   „ ñ"˜t C¨ H™~ð "¸dÀ5ÈÈcÈÁ?Ñ>Sð "Ð_hô "r#   rÊ   c                   ó   — e Zd Zdefd„Zy)ÚHerbertConverterr%   c           	      óŒ  — d}d}| j                   j                  }t        | j                   j                  j	                  «       «      }||d   d   v r|dd  }t        t        ||d | j                   j                  |¬«      «      }t        j                  dd¬«      |_
        t        j                  «       |_        t        j                  |¬«      |_        t#        j$                  | j                   j&                  | j                   j(                  f| j                   j*                  | j                   j,                  f¬	«      |_        |S )
Nz	#version:r¼   r   r   )r½   r~   r¾   F)r„   rƒ   rÀ   )r¢   r¡   )r,   rÂ   re   rÃ   rÄ   r   r   r~   r   r”   r•   r	   r–   r—   r   rÇ   rž   r
   ÚBertProcessingr™   r›   r˜   rš   r�   )rU   Útokenizer_info_strÚtoken_suffixr4   rF   r    s         r!   ry   zHerbertConverter.convertedo  s  € Ø(ÐØˆà×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ ¨¡¨1¡Ñ-Ø˜A˜B�ZˆFäÜØØØØ×1Ñ1×;Ñ;Ø#/ôó
ˆ	ô  +×9Ñ9ÀEÐY^Ô_ˆ	ÔÜ"0×"AÑ"AÓ"Cˆ	ÔÜ$×/Ñ/°|ÔDˆ	ÔÜ#-×#<Ñ#<Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYô$
ˆ	Ô ð
 Ðr#   Nr¤   r2   r#   r!   rÚ   rÚ   n  ó   „ ð˜9ô r#   rÚ   c                   ó<   — e Zd Zddeeef   deeeef      defd„Z	y)ÚQwen2ConverterNr4   rF   r%   c                 ó0  — |s| j                   j                  }|s-t        | j                   j                  j	                  «       «      }t        t        ||d d dddd¬«      «      }t        j                  «       |_	        t        j                  t        j                  t        d«      dd¬«      t        j                  t        | j                   dd«      d¬«      g«      |_        t#        j                  «       |_        t'        j                  d¬	«      |_        |S )
NrÌ   F)r4   rF   r½   r~   rÎ   r¾   r¿   Úbyte_fallbackzn(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+Úisolated©ÚbehaviorÚinvertr$   ©r$   Ú	use_regexrÑ   )r,   rÂ   re   rÃ   rÄ   r   r   r   ÚNFCr•   r	   ÚSequenceÚSplitr   rÓ   r+   r—   r   rž   r
   r�   )rU   r4   rF   r    s       r!   ry   zQwen2Converter.convertedŽ  s  € ÙØ×+Ñ+×3Ñ3ˆEÙÜ˜$×1Ñ1×;Ñ;×@Ñ@ÓBÓCˆFäÜØØØØØ*,Ø#%ØØ#ô	ó
ˆ	ô  +Ÿ™Ó0ˆ	Ôä"0×"9Ñ"9ä×$Ñ$Üð Nóð (Ø ôô ×(Ñ(Ü%,¨T×-DÑ-DÐFXÐZ_Ó%`Ø#ôðó#
ˆ	Ôô  %×.Ñ.Ó0ˆ	ÔÜ#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐr#   r×   rØ   r2   r#   r!   rá   rá   �  s2   „ ñ(˜t C¨ H™~ð (¸dÀ5ÈÈcÈÁ?Ñ>Sð (Ð_hô (r#   rá   c                   ó   — e Zd Zdefd„Zy)ÚRobertaConverterr%   c           
      óÔ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  |j                  |j                   f|j"                  |j$                  f|j                  d¬«      |_        |S )NrÌ   FrÍ   rÏ   T©r¢   r¡   r$   rÒ   )r,   rÂ   re   rÃ   rÄ   r   r   r	   rÓ   r$   r—   r   rž   r
   ÚRobertaProcessingr™   r›   r˜   rš   r�   ©rU   Úotr4   rF   r    s        r!   ry   zRobertaConverter.convertedº  sÄ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#?Ñ#?Ø—‘˜rŸ™Ð/Ø—‘˜rŸ™Ð/Ø×0Ñ0Øô	$
ˆ	Ô ð Ðr#   Nr¤   r2   r#   r!   rî   rî   ¹  ó   „ ð˜9ô r#   rî   c                   ó   — e Zd Zdefd„Zy)ÚRoFormerConverterr%   c           	      óV  — ddl m} | j                  j                  }t	        t        |t        | j                  j                  «      ¬«      «      }d}d}t        | j                  d«      r@| j                  j                  j                  }| j                  j                  j                  }t        j                  dd||¬«      |_        t        j                   j#                   ||«      «      |_        t        | j                  j&                  «      }t        | j                  j(                  «      }| j                  j*                  }| j                  j,                  }	t/        j0                  |› d|› d	�|› d|› d
|› d�||f||	fg¬«      |_        t5        j
                  d¬«      |_        |S )Nr   )ÚJiebaPreTokenizerr}   Fr   Tr€   r…   r†   r‡   rˆ   r‰   r�   rŽ   )Ú"models.roformer.tokenization_utilsrø   r,   r4   r   r   rb   r~   r‘   r   rƒ   r“   r   r”   r•   r	   ÚPreTokenizerÚcustomr—   r˜   r™   rš   r›   r
   rœ   r�   r   rž   )
rU   rø   r4   r    rƒ   r“   r¡   r¢   rš   r›   s
             r!   ry   zRoFormerConverter.converted×  sy  € ÝIà×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	àˆØˆÜ�4×*Ñ*Ð,=Ô>Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!&Ø'Ø#ô	 
ˆ	Ôô #1×"=Ñ"=×"DÑ"DÑEVÐW\ÓE]Ó"^ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr#   Nr¤   r2   r#   r!   rö   rö   Ö  r¥   r#   rö   c                   ó   — e Zd Zdefd„Zy)ÚDebertaConverterr%   c           
      óÖ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  ddd| j                   j                  d«      fd| j                   j                  d«      fg¬	«      |_        |S )
NrÌ   FrÍ   rÏ   ú[CLS]:0 $A:0 [SEP]:0ú![CLS]:0 $A:0 [SEP]:0 $B:1 [SEP]:1ú[CLS]ú[SEP]r‰   )r,   rÂ   re   rÃ   rÄ   r   r   r	   rÓ   r$   r—   r   rž   r
   rœ   r®   r�   rò   s        r!   ry   zDebertaConverter.convertedþ  sÕ   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#@Ñ#@Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô$
ˆ	Ô ð Ðr#   Nr¤   r2   r#   r!   rý   rý   ý  rß   r#   rý   c                   ó`   ‡ — e Zd ZdZeZi Zˆ fd„Zd„ Zd„ Z	d„ Z
d„ Zd„ Zd„ Zd	„ Zd
efd„Zˆ xZS )ÚSpmConverterFc                 ó¦  •— t        | d«       t        ‰| �  |Ž  t        «       }|j	                  «       }t        | j                  j                  d«      5 }|j                  |j                  «       «       d d d «       || _
        | j                  j                  j                  r#| j                  st        j                  d«       y y y # 1 sw Y   ŒTxY w)Nr   Úrba  The sentencepiece tokenizer that you are converting to a fast tokenizer uses the byte fallback option which is not implemented in the fast tokenizers. In practice this means that the fast version of the tokenizer can produce unknown tokens whereas the sentencepiece version would have converted these unknown tokens into a sequence of byte tokens matching the original piece of text.)r   ÚsuperrV   r"   Ú
ModelProtoÚopenr,   Ú
vocab_fileÚParseFromStringÚreadÚprotoÚtrainer_specrã   Úhandle_byte_fallbackÚwarningsÚwarn)rU   ÚargsÚ	model_pb2ÚmÚfÚ	__class__s        €r!   rV   zSpmConverter.__init__!  s°   ø€ Ü˜$ 
Ô+ä‰Ñ˜$Ñô $Ó%ˆ	à× Ñ Ó"ˆÜ�$×)Ñ)×4Ñ4°dÓ;ð 	(¸qØ×Ñ˜aŸf™f›hÔ'÷	(àˆŒ
à�:‰:×"Ñ"×0Ò0¸×9RÒ9RÜ�M‰Mðeõð :SÐ0÷		(ð 	(ús   Á CÃCc                 ól   — |j                   D �cg c]  }|j                  |j                  f‘Œ c}S c c}w r]   ©Úpiecesrl   Úscore©rU   r  rl   s      r!   r4   zSpmConverter.vocab6  s'   € Ø8=¿¹ÖE¨u�—‘˜eŸk™kÒ*ÒEÐEùÒEs   �1c                 ó.   — |j                   j                  S r]   )r  Úunk_id©rU   r  s     r!   r  zSpmConverter.unk_id9  s   € Ø×!Ñ!×(Ñ(Ð(r#   c                 ó~  — |j                   j                  }| j                  |«      }|dk(  r1t        t	        || j                  |«      | j                  ¬«      «      }n£|dk(  r“| j                  | j                  j                  «      j                  |«      \  }}t        |«      D ���	ci c]  \  }\  }}	||“Œ }
}}}	t        t        |
||j                   j                  d| j                  d ¬«      «      }nt        d«      ‚t        |j                  «      D ��cg c]I  \  }}|j                   dv r6||j"                  |j                   dk(  xs |j"                  | j$                  v f‘ŒK }}}|j'                  t)        |d	„ ¬
«      D ���cg c]  \  }}}t+        |d|¬«      ‘Œ c}}}«       |S c c}	}}w c c}}w c c}}}w )Nr   ©r  rã   r9   T)r~   r¿   rã   r½   z]You're trying to run a `Unigram` model but you're file was trained with a different algorithm©é   é   r"  c                 ó   — | d   S ©Nr   r2   ©r3   s    r!   r5   z(SpmConverter.tokenizer.<locals>.<lambda>h  ó   € ÐQRÐSTÑQU€ r#   r6   F©Ú
normalizedÚspecial)r  Ú
model_typer4   r   r   r  r  ÚSpmExtractorr,   r
  r\   Ú	enumerater   Ú	unk_pieceÚ	Exceptionr  Útyperl   rŒ   Ú
add_tokensrC   r   )rU   r  r+  rE   r    Ú_rF   ÚiÚwordr  Ú	bpe_vocabÚidÚpÚspm_added_tokensÚtokenr*  s                   r!   r    zSpmConverter.tokenizer<  s´  € Ø×'Ñ'×2Ñ2ˆ
Ø—z‘z %Ó(ˆà˜Š?Ü!ÜØ ØŸ;™; uÓ-Ø"&×";Ñ";ôó‰Ið ˜1Š_Ø×)Ñ)¨$×*AÑ*A×*LÑ*LÓM×UÑUÐVbÓc‰IˆAˆvÜ9BÀ<Ó9P×QÐQÑ%5 Q©¨¨u˜˜q™ÐQˆIÒQÜ!ÜØØØ#×0Ñ0×:Ñ:Ø!Ø"&×";Ñ";Ø ôó	‰Iô Øoóð ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÖDôô	
ð ÐùôC Rùó*
ùôs   Â)F+ÄAF2ÆF8c                 ó   — |j                   j                  }t        j                  dd¬«      t        j                  t        d«      d«      g}|st        j                  |«      S t        j                  t        j                  |«      g|z   «      S )NFT)Úleftrª   ú {2,}õ   â–�)Únormalizer_specÚprecompiled_charsmapr   ÚStripÚReplacer   rë   ÚPrecompiled©rU   r  r?  Ú_normalizerss       r!   r•   zSpmConverter.normalizern  s{   € Ø$×4Ñ4×IÑIÐä×Ñ 5°Ô5Ü×Ñ¤ g£°Ó6ð
ˆñ $Ü×'Ñ'¨Ó5Ð5ä×'Ñ'¬×)@Ñ)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgr#   c                 ó\   — t        || j                  «      }t        j                  ||¬«      S ©N©Úreplacementr-   )r.   r,   r	   Ú	Metaspace©rU   rH  r$   r-   s       r!   r—   zSpmConverter.pre_tokenizery  s)   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×'Ñ'°KÐP^Ô_Ð_r#   c                  ó   — y r]   r2   rx   s    r!   r�   zSpmConverter.post_processor}  s   € Ør#   c                 ó\   — t        || j                  «      }t        j                  ||¬«      S rF  )r.   r,   r   rI  rJ  s       r!   rž   zSpmConverter.decoder€  s(   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×!Ñ!¨kÈ.ÔYÐYr#   r%   c                 óz  — | j                  | j                  «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j
                  }| j                  ||«      }|�||_        | j                  ||«      |_        | j                  «       }|r||_        |S )Nr=  Tr$   )	r    r  r•   r‘   r,   r$   r—   rž   r�   )rU   r    r•   rH  r$   r—   r�   s          r!   ry   zSpmConverter.converted„  s¸   € Ø—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$àÐr#   )r^   r_   r`   r  rO   r,  rŒ   rV   r4   r  r    r•   r—   r�   rž   r   ry   Ú__classcell__©r  s   @r!   r  r    sL   ø„ Ø ÐØ)€LØ€Nôò*Fò)ò0òd	hò`òòZð˜9÷ r#   r  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚAlbertConverterc                 óÌ   — |j                   D �cg c]J  }t        |j                  «      r|j                  |j                  fn|j                  |j                  dz
  f‘ŒL c}S c c}w ©Néd   ©r  rr   rl   r  r  s      r!   r4   zAlbertConverter.vocabž  ó^   € ð Ÿ™ö
àô +=¸U¿[¹[Ô*IˆU�[‰[˜%Ÿ+™+Ñ&ÐPU×P[ÑP[Ð]b×]hÑ]hÐknÑ]nÐOoÑoò
ð 	
ùò 
ó   �AA!c                 ó‚  — t        j                  dd«      t        j                  dd«      g}| j                  j                  sF|j	                  t        j
                  «       «       |j	                  t        j                  «       «       | j                  j                  r#|j	                  t        j                  «       «       |j                  j                  }|r$|j	                  t        j                  |«      «       |j	                  t        j                  t        d«      d«      «       t        j                  |«      S ©Nz``ú"z''r<  r«   ©r   rA  r,   Úkeep_accentsrB   ÚNFKDÚStripAccentsr“   Ú	Lowercaser>  r?  rB  r   rë   ©rU   r  Úlist_normalizersr?  s       r!   r•   zAlbertConverter.normalizer¤  óï   € ä×Ñ  cÓ*Ü×Ñ  cÓ*ð
Ðð ×&Ñ&×3Ò3Ø×#Ñ#¤K×$4Ñ$4Ó$6Ô7Ø×#Ñ#¤K×$<Ñ$<Ó$>Ô?Ø×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<à$×4Ñ4×IÑIÐáØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRà×Ñ¤× 3Ñ 3´E¸'³NÀCÓ HÔIÜ×#Ñ#Ð$4Ó5Ð5r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S ©Nrÿ   r   r  r  r‰   ©r
   rœ   r,   r®   rx   s    r!   r�   zAlbertConverter.post_processor·  óR   € Ü×,Ñ,Ø)Ø4à˜$×1Ñ1×GÑGÈÓPÐQØ˜$×1Ñ1×GÑGÈÓPÐQðô
ð 	
r#   N©r^   r_   r`   r4   r•   r�   r2   r#   r!   rQ  rQ  �  ó   „ ò
ò6ó&
r#   rQ  c                   ó   — e Zd Zd„ Zd„ Zy)ÚBarthezConverterc                 ó
   — d}|S ©Nr"  r2   ©rU   r  r  s      r!   r  zBarthezConverter.unk_idÃ  ó   € ØˆØˆr#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S ©Nz<s> $A </s>z<s> $A </s> </s> $B </s>ú<s>ú</s>r‰   re  rx   s    r!   r�   zBarthezConverter.post_processorÇ  óR   € Ü×,Ñ,Ø Ø+à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r#   N)r^   r_   r`   r  r�   r2   r#   r!   rj  rj  Â  s   „ òó
r#   rj  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚCamembertConverterc                 ó�   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|dgz  }|S c c}w )N))z
<s>NOTUSEDç        ©ú<pad>rw  )z</s>NOTUSEDrw  ©z<unk>rw  )z<unk>NOTUSEDiœÿÿÿr   ©z<mask>rw  r  ©rU   r  r4   rl   s       r!   r4   zCamembertConverter.vocabÓ  sP   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØ�/Ð"Ñ"ˆØˆùò Ló   —Ac                  ó   — yrl  r2   r  s     r!   r  zCamembertConverter.unk_idà  s   € àr#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rp  re  rx   s    r!   r�   z!CamembertConverter.post_processorä  rs  r#   N©r^   r_   r`   r4   r  r�   r2   r#   r!   ru  ru  Ò  s   „ òòó
r#   ru  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚDebertaV2Converterc                 ó  — g }| j                   j                  r%|j                  t        j                  d¬«      «       t        || j                   «      }|j                  t        j                  ||¬«      «       t        j                  |«      S )Nrä   )ræ   rG  )r,   Úsplit_by_punctrB   r	   ÚPunctuationr.   rI  rë   )rU   rH  r$   Úlist_pretokenizersr-   s        r!   r—   z DebertaV2Converter.pre_tokenizerð  sq   € ØÐØ×"Ñ"×1Ò1Ø×%Ñ%¤n×&@Ñ&@È*Ô&UÔVÜ,Ð-=¸t×?VÑ?VÓWˆØ×!Ñ!¤.×":Ñ":À{ÐcqÔ"rÔsÜ×&Ñ&Ð'9Ó:Ð:r#   c                 ó¼  — g }| j                   j                  r#|j                  t        j                  «       «       |j                  t        j
                  «       «       |j                  j                  }|r$|j                  t        j                  |«      «       |j                  t        j                  t        d«      d«      «       t        j                  |«      S )Nr<  r«   )r,   r“   rB   r   r_  r@  r>  r?  rB  rA  r   rë   r`  s       r!   r•   zDebertaV2Converter.normalizerø  s¥   € ØÐØ×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<Ø×Ñ¤× 1Ñ 1Ó 3Ô4à$×4Ñ4×IÑIÐÙØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRØ×Ñ¤× 3Ñ 3´E¸'³NÀCÓ HÔIä×#Ñ#Ð$4Ó5Ð5r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rd  re  rx   s    r!   r�   z!DebertaV2Converter.post_processor  rf  r#   N)r^   r_   r`   r—   r•   r�   r2   r#   r!   r‚  r‚  ï  s   „ ò;ò6ó
r#   r‚  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚMBartConverterc                 óž   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|dgz  }|S c c}w )N©©rq  rw  rx  ©rr  rw  rz  r"  )©Úar_ARrw  ©Úcs_CZrw  ©Úde_DErw  ©Úen_XXrw  ©Úes_XXrw  ©Úet_EErw  ©Úfi_FIrw  ©Úfr_XXrw  ©Úgu_INrw  ©Úhi_INrw  ©Úit_ITrw  ©Úja_XXrw  ©Úkk_KZrw  ©Úko_KRrw  ©Últ_LTrw  ©Úlv_LVrw  ©Úmy_MMrw  ©Úne_NPrw  ©Únl_XXrw  ©Úro_ROrw  ©Úru_RUrw  ©Úsi_LKrw  ©Útr_TRrw  ©Úvi_VNrw  ©Úzh_CNrw  r{  r  r|  s       r!   r4   zMBartConverter.vocab  sa   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò 
ñ 	
ˆð6 	�/Ð"Ñ"ˆØˆùò; Ló   —A
c                  ó   — yrl  r2   r  s     r!   r  zMBartConverter.unk_id7  ó   € Ør#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz$A </s> en_XXz$A $B </s> en_XXr–  rr  r‰   re  rx   s    r!   r�   zMBartConverter.post_processor:  óR   € Ü×,Ñ,Ø"Ø#à˜$×1Ñ1×GÑGÈÓPÐQØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r#   Nr€  r2   r#   r!   rŠ  rŠ    s   „ ò$òLó
r#   rŠ  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚMBart50Converterc                 óž   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|dgz  }|S c c}w )NrŒ  r"  )4r�  r‘  r“  r•  r—  r™  r›  r�  rŸ  r¡  r£  r¥  r§  r©  r«  r­  r¯  r±  r³  rµ  r·  r¹  r»  r½  r¿  )Úaf_ZArw  )Úaz_AZrw  )Úbn_INrw  )Úfa_IRrw  )Úhe_ILrw  )Úhr_HRrw  )Úid_IDrw  )Úka_GErw  )Úkm_KHrw  )Úmk_MKrw  )Úml_INrw  )Úmn_MNrw  )Úmr_INrw  )Úpl_PLrw  )Úps_AFrw  )Úpt_XXrw  )Úsv_SErw  )Úsw_KErw  )Úta_INrw  )Úte_INrw  )Úth_THrw  )Útl_XXrw  )Úuk_UArw  )Úur_PKrw  )Úxh_ZArw  )Úgl_ESrw  )Úsl_SIrw  r{  r  r|  s       r!   r4   zMBart50Converter.vocabF  sa   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò  Rñ  	RˆØ�/Ð"Ñ"ˆØˆùò LrÁ  c                  ó   — yrl  r2   r  s     r!   r  zMBart50Converter.unk_idR  rÃ  r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nzen_XX $A </s>zen_XX $A $B </s>r–  rr  r‰   re  rx   s    r!   r�   zMBart50Converter.post_processorU  rÅ  r#   Nr€  r2   r#   r!   rÇ  rÇ  E  s   „ ò
òó
r#   rÇ  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚNllbConverterc                 ó„   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )NrŒ  r"  r  r|  s       r!   r4   zNllbConverter.vocaba  óC   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò Ló   —=c                  ó   — yrl  r2   r  s     r!   r  zNllbConverter.unk_idk  rÃ  r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nzeng_Latn $A </s>zeng_Latn $A $B </s>Úeng_Latnrr  r‰   re  rx   s    r!   r�   zNllbConverter.post_processorn  sR   € Ü×,Ñ,Ø%Ø&à˜T×4Ñ4×JÑJÈ:ÓVÐWØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r#   Nr€  r2   r#   r!   rç  rç  `  s   „ òòó
r#   rç  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚSeamlessM4TConverterc                 ó„   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )N)rx  rz  r�  rŽ  r"  r  r|  s       r!   r4   zSeamlessM4TConverter.vocabz  ré  rê  c                 ó.   — | j                   j                  S r]   )r,   Úunk_token_idr  s     r!   r  zSeamlessM4TConverter.unk_id„  s   € Ø×&Ñ&×3Ñ3Ð3r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz__eng__ $A </s>z__eng__ $A $B </s>Ú__eng__rr  r‰   re  rx   s    r!   r�   z#SeamlessM4TConverter.post_processor‡  sR   € Ü×,Ñ,Ø$Ø%à˜D×3Ñ3×IÑIÈ)ÓTÐUØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r#   Nr€  r2   r#   r!   rï  rï  y  s   „ òò4ó
r#   rï  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXLMRobertaConverterc                 ó�   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|dgz  }|S c c}w )NrŒ  r"  r{  r  r|  s       r!   r4   zXLMRobertaConverter.vocab“  sP   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØ�/Ð"Ñ"ˆØˆùò Lr}  c                 ó
   — d}|S rl  r2   rm  s      r!   r  zXLMRobertaConverter.unk_idž  rn  r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rp  re  rx   s    r!   r�   z"XLMRobertaConverter.post_processor¢  rs  r#   Nr€  r2   r#   r!   rö  rö  ’  ó   „ ò	òó
r#   rö  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXLNetConverterc                 óÌ   — |j                   D �cg c]J  }t        |j                  «      r|j                  |j                  fn|j                  |j                  dz
  f‘ŒL c}S c c}w rS  rU  r  s      r!   r4   zXLNetConverter.vocab®  rV  rW  c                 ó‚  — t        j                  dd«      t        j                  dd«      g}| j                  j                  sF|j	                  t        j
                  «       «       |j	                  t        j                  «       «       | j                  j                  r#|j	                  t        j                  «       «       |j                  j                  }|r$|j	                  t        j                  |«      «       |j	                  t        j                  t        d«      d«      «       t        j                  |«      S rY  r[  r`  s       r!   r•   zXLNetConverter.normalizer´  rb  r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz$A:0 <sep>:0 <cls>:2z!$A:0 <sep>:0 $B:1 <sep>:1 <cls>:2z<sep>z<cls>r‰   re  rx   s    r!   r�   zXLNetConverter.post_processorÇ  rf  r#   Nrg  r2   r#   r!   rü  rü  ­  rh  r#   rü  c                   ó   — e Zd Zy)ÚReformerConverterN©r^   r_   r`   r2   r#   r!   r  r  Ò  ó   „ Ør#   r  c                   ó   — e Zd Zd„ Zd„ Zy)ÚRemBertConverterc                 ób  — t        j                  dd«      t        j                  dd«      t        j                  t        d«      d«      g}| j                  j                  sF|j                  t        j                  «       «       |j                  t        j                  «       «       | j                  j                  r#|j                  t        j                  «       «       |j                  j                  }|r$|j                  t        j                  |«      «       t        j                  |«      S rY  )r   rA  r   r,   r\  rB   r]  r^  r“   r_  r>  r?  rB  rë   r`  s       r!   r•   zRemBertConverter.normalizerØ  sä   € ä×Ñ  cÓ*Ü×Ñ  cÓ*Ü×Ñ¤ g£°Ó4ð
Ðð
 ×&Ñ&×3Ò3Ø×#Ñ#¤K×$4Ñ$4Ó$6Ô7Ø×#Ñ#¤K×$<Ñ$<Ó$>Ô?Ø×"Ñ"×0Ò0Ø×#Ñ#¤K×$9Ñ$9Ó$;Ô<à$×4Ñ4×IÑIÐáØ×#Ñ#¤K×$;Ñ$;Ð<PÓ$QÔRä×#Ñ#Ð$4Ó5Ð5r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rd  re  rx   s    r!   r�   zRemBertConverter.post_processorë  rf  r#   N)r^   r_   r`   r•   r�   r2   r#   r!   r  r  Ö  s   „ ò6ó&
r#   r  c                   ó   — e Zd Zy)ÚBertGenerationConverterNr  r2   r#   r!   r	  r	  ö  r  r#   r	  c                   ó$   — e Zd Zd„ Zd„ Zd„ Zd„ Zy)ÚPegasusConverterc                 óv  — | j                   j                  df| j                   j                  dfg}| j                   j                  �|| j                   j                  dfgz  }| j                   j                  �I| j                   j
                  | j                   j                  k  r|| j                   j                  dfgz  }|t        d| j                   j                  «      D �cg c]
  }d|› d�df‘Œ c}z  }||j                  dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w c c}w )Nrw  r9   z<unk_ú>g      YÀ)r,   Ú	pad_tokenÚ	eos_tokenÚmask_token_sentÚ
mask_tokenÚmask_token_idÚoffsetrA   r  rl   r  )rU   r  r4   r3  rl   s        r!   r4   zPegasusConverter.vocabû  s%  € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 ×"Ñ"×2Ñ2Ð>Ø�t×.Ñ.×>Ñ>ÀÐDÐEÑEˆEð ×#Ñ#×.Ñ.Ð:Ø×'Ñ'×5Ñ5¸×8OÑ8O×8VÑ8VÒVà�t×.Ñ.×9Ñ9¸3Ð?Ð@Ñ@ˆEà´%¸¸4×;RÑ;R×;YÑ;YÓ2ZÖ[¨Q�U˜1˜#˜Q�< Ò(Ò[Ñ[ˆØ¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò \ùÚKs   Ã%D1ÄD6c                 ó\   — |j                   j                  | j                  j                  z   S r]   )r  r  r,   r  r  s     r!   r  zPegasusConverter.unk_id  s%   € Ø×!Ñ!×(Ñ(¨4×+BÑ+B×+IÑ+IÑIÐIr#   c                 óª   — t        || j                  «      }t        j                  t        j                  «       t        j
                  ||¬«      g«      S rF  )r.   r,   r	   rë   ÚWhitespaceSplitrI  rJ  s       r!   r—   zPegasusConverter.pre_tokenizer  sJ   € Ü,Ð-=¸t×?VÑ?VÓWˆÜ×&Ñ&ä×.Ñ.Ó0Ü×(Ñ(°[ÐQ_Ô`ðó
ð 	
r#   c                 óš   — | j                   j                  }|| j                   j                  fg}t        j                  d|gdd|g|¬«      S )Nú$Aú$Br‰   )r,   r  Úeos_token_idr
   rœ   )rU   ÚeosrŒ   s      r!   r�   zPegasusConverter.post_processor  sR   € Ø×%Ñ%×/Ñ/ˆà�$×)Ñ)×6Ñ6Ð7ð
ˆô ×,Ñ,°T¸3°KÀtÈTÐSVÐFWÐhvÔwÐwr#   N)r^   r_   r`   r4   r  r—   r�   r2   r#   r!   r  r  ú  s   „ òò&Jò
óxr#   r  c                   ó   — e Zd Zd„ Zd„ Zy)ÚT5Converterc                 óò   — | j                   j                  }|j                  D �cg c]  }|j                  |j                  f‘Œ }}|t        |dz
  dd«      D �cg c]
  }d|› d�df‘Œ c}z  }|S c c}w c c}w )Nr   rn   z
<extra_id_r  rw  )r,   Ú
_extra_idsr  rl   r  rA   )rU   r  Únum_extra_idsrl   r4   r3  s         r!   r4   zT5Converter.vocab#  sw   € Ø×/Ñ/×:Ñ:ˆØ9>¿¹ÖF°�%—+‘+˜uŸ{™{Ò+ÐFˆÐFØ´E¸-È!Ñ:KÈRÐQSÓ4TÖU¨q�Z ˜s !Ð$ cÒ*ÒUÑUˆØˆùò GùÚUs   ¥A/ÁA4c                 ór   — t        j                  ddgg d¢d| j                  j                  d«      fg¬«      S ©Nr  rr  )r  rr  r  rr  r‰   re  rx   s    r!   r�   zT5Converter.post_processor)  ó=   € Ü×,Ñ,Ø˜&�>Ú-à˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r#   N)r^   r_   r`   r4   r�   r2   r#   r!   r  r  "  s   „ òó
r#   r  c                   ó   — e Zd Zd„ Zy)ÚUdopConverterc                 ór   — t        j                  ddgg d¢d| j                  j                  d«      fg¬«      S r"  re  rx   s    r!   r�   zUdopConverter.post_processor4  r#  r#   N©r^   r_   r`   r�   r2   r#   r!   r%  r%  3  s   „ ó
r#   r%  c                   ó   — e Zd Zdefd„Zy)ÚWhisperConverterr%   c           
      óÈ  — | j                   j                  }t        | j                   j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  | j                   j                  ¬«      |_
        t        j                  «       |_        | j                   j                  }| j                   j                  |«      }| j                   j                  }| j                   j                   }dj#                  |D �cg c]  }|› d�‘Œ	 c}«      }	t%        j&                  |	› d|› d�|	› d|› d	�||fgt)        ||«      ¢¬
«      |_        |S c c}w )NrÌ   FrÍ   rÏ   r«   r†   z $A:0 z $A:0 $B:1 rˆ   r‰   )r,   rÂ   re   rÃ   rÄ   r   r   r	   rÓ   r$   r—   r   rž   Úprefix_tokensÚconvert_ids_to_tokensr  r  Újoinr
   rœ   Úzipr�   )
rU   r4   rF   r    Úprefix_token_idsÚprefixesr  r  r9  Úprefix_templates
             r!   ry   zWhisperConverter.converted?  sR  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈD×LcÑLc×LtÑLtÔ"uˆ	ÔÜ$×.Ñ.Ó0ˆ	Ôà×2Ñ2×@Ñ@ÐØ×*Ñ*×@Ñ@ÐAQÓRˆØ×%Ñ%×/Ñ/ˆØ×.Ñ.×;Ñ;ˆØŸ(™(¸hÖ#G°U u g¨R¢LÒ#GÓHˆÜ#-×#@Ñ#@Ø%Ð& f¨S¨E°Ð4Ø#Ð$ K°¨u°BÐ7à�lÐ#ðä�XÐ/Ó0ðô$
ˆ	Ô ð Ðùò $Hs   ÄENr¤   r2   r#   r!   r)  r)  >  s   „ ð ˜9ô  r#   r)  c                   ó   — e Zd Zd„ Zy)ÚBigBirdConverterc           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S rd  re  rx   s    r!   r�   zBigBirdConverter.post_processorc  rf  r#   Nr'  r2   r#   r!   r3  r3  b  s   „ ó
r#   r3  c                   ó   — e Zd Zdefd„Zy)ÚCLIPConverterr%   c                 óp  — | j                   j                  }t        | j                   j                  j	                  «       «      }| j                   j
                  }t        t        ||d dddt        |«      ¬«      «      }t        j                  t        j                  «       t        j                  t        d«      d«      t        j                  «       g«      |_        t!        j                  t!        j"                  t        d«      dd	¬
«      t!        j$                  d¬«      g«      |_        t)        j$                  «       |_        t-        j.                  | j                   j0                  | j                   j2                  f| j                   j4                  | j                   j6                  fdd¬«      |_        |S )NrÌ   r¼   F©r4   rF   r½   rÎ   r¾   r¿   r~   z\s+r«   z9's|'t|'re|'ve|'m|'ll|'d|[\p{L}]+|[\p{N}]|[^\s\p{L}\p{N}]+ÚremovedTrå   rÏ   rð   )r,   rÂ   re   rÃ   rÄ   r~   r   r   rb   r   rë   rê   rA  r   r_  r•   r	   rì   rÓ   r—   r   rž   r
   rñ   r  r  rÔ   rÕ   r�   rÈ   s        r!   ry   zCLIPConverter.convertedo  sk  € Ø×'Ñ'×/Ñ/ˆÜ�d×-Ñ-×7Ñ7×<Ñ<Ó>Ó?ˆØ×+Ñ+×5Ñ5ˆ	äÜØØØØ*,Ø#)ØÜ˜i›.ôó

ˆ	ô  +×3Ñ3Ü�_‰_Ó¤× 3Ñ 3´E¸&³MÀ3Ó GÌ×I^ÑI^ÓI`Ðaó 
ˆ	Ôô #1×"9Ñ"9ä×$Ñ$ÜÐZÓ[Ø&Øôô
 ×(Ñ(¸%Ô@ðó	#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôô $.×#?Ñ#?Ø×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ×(Ñ(×2Ñ2°D×4KÑ4K×4XÑ4XÐYØ"Øô	$
ˆ	Ô ð Ðr#   Nr¤   r2   r#   r!   r6  r6  n  s   „ ð'˜9ô 'r#   r6  c                   ó   — e Zd Zdefd„Zy)ÚLayoutLMv2Converterr%   c           	      ól  — | j                   j                  }t        t        |t	        | j                   j
                  «      ¬«      «      }d}d}d}t        | j                   d«      r`| j                   j                  j                  }| j                   j                  j                  }| j                   j                  j                  }t        j                  d|||¬«      |_        t        j                  «       |_        t	        | j                   j"                  «      }t	        | j                   j$                  «      }| j                   j&                  }| j                   j(                  }	t+        j,                  |› d|› d�|› d|› d|› d	�||f||	fg¬
«      |_        t1        j                  d¬«      |_        |S )Nr}   FTr   r€   r…   r†   r‡   rˆ   r‰   r�   rŽ   r�   rŸ   s
             r!   ry   zLayoutLMv2Converter.convertedš  s…  € Ø×'Ñ'×-Ñ-ˆÜœi¨¼¸T×=TÑ=T×=^Ñ=^Ó9_Ô`Óaˆ	à!&ÐØˆØˆÜ�4×*Ñ*Ð,=Ô>Ø%)×%<Ñ%<×%LÑ%L×%cÑ%cÐ"Ø ×3Ñ3×CÑC×QÑQˆMØ ×3Ñ3×CÑC×QÑQˆMä*×9Ñ9ØØ!7Ø'Ø#ô	 
ˆ	Ôô #1×"AÑ"AÓ"Cˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜( 3 % rÐ*Ø�5˜   X¨c¨U°"Ð5à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ô %×.Ñ.°dÔ;ˆ	ÔàÐr#   Nr¤   r2   r#   r!   r;  r;  ™  r¥   r#   r;  c                   ó   — e Zd Zdefd„Zy)ÚBlenderbotConverterr%   c           
      ó®  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd¬«      «      }t        j                  |j                  ¬«      |_
        t        j                  «       |_        t        j                  d|j                  › d�|j                  |j                   fg¬«      |_        |S )NrÌ   FrÍ   rÏ   z$A:0 r†   )rŠ   rŒ   )r,   rÂ   re   rÃ   rÄ   r   r   r	   rÓ   r$   r—   r   rž   r
   rœ   r  r  r�   rò   s        r!   ry   zBlenderbotConverter.convertedÁ  s½   € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%Øôó	
ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	ÔÜ#-×#@Ñ#@Ø˜2Ÿ<™<˜.¨Ð+à—‘˜rŸ™Ð/ðô$
ˆ	Ô ð Ðr#   Nr¤   r2   r#   r!   r>  r>  À  rô   r#   r>  c                   ó   — e Zd Zd„ Zd„ Zd„ Zy)ÚXGLMConverterc                 ó’   — g d¢}||j                   dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|g d¢z  }|S c c}w )NrŒ  r"  ))z<madeupword0>rw  )z<madeupword1>rw  )z<madeupword2>rw  )z<madeupword3>rw  )z<madeupword4>rw  )z<madeupword5>rw  )z<madeupword6>rw  r  r|  s       r!   r4   zXGLMConverter.vocabÞ  sT   € ò
ˆð 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØò  zñ  	zˆØˆùò Ls   —Ac                 ó
   — d}|S rl  r2   rm  s      r!   r  zXGLMConverter.unk_idé  rn  r#   c           	      ó¢   — t        j                  ddd| j                  j                  d«      fd| j                  j                  d«      fg¬«      S )Nz</s> $Az</s> $A </s> </s> $Brq  rr  r‰   re  rx   s    r!   r�   zXGLMConverter.post_processorí  sR   € Ü×,Ñ,ØØ'à˜×/Ñ/×EÑEÀeÓLÐMØ˜×0Ñ0×FÑFÀvÓNÐOðô
ð 	
r#   Nr€  r2   r#   r!   rA  rA  Ý  rú  r#   rA  c                   ó<   — e Zd ZdZeZddhZ	 d„ Zd„ Zd„ Z	d„ Z
d„ Zy	)
ÚGemmaConverterTz<start_of_turn>z<end_of_turn>c                 ó.   — t        j                  dd«      S ©Nr«   r=  )r   rA  r  s     r!   r•   zGemmaConverter.normalizer  s   € Ü×"Ñ" 3¨Ó.Ð.r#   c                 ót  — | j                   j                  df| j                   j                  df| j                   j                  dfg}||j                  dd  D �cg c]  }|j
                  |j                  f‘Œ c}z  }t        d„ |D «       «      s#t        d„ t        |«      D «       d «      }|�d||<   |S c c}w )Nrw  r"  c              3   ó,   K  — | ]  }|d    dk(  –— Œ y­w)r   ri   Nr2   )Ú.0r3   s     r!   ú	<genexpr>z'GemmaConverter.vocab.<locals>.<genexpr>  s   è ø€ Ò/ A�1�Q‘4˜4•<Ñ/ùs   ‚c              3   ó8   K  — | ]  \  }}|d    dk(  sŒ|–— Œ y­w)r   rj   Nr2   )rK  r3  r3   s      r!   rL  z'GemmaConverter.vocab.<locals>.<genexpr>  s   è ø€ Ò"V©¨¨AÀQÀqÁTÈXÓEU¤1Ñ"Vùs   ‚“)ri   rw  )
r,   r  r  rÔ   r  rl   r  ÚanyÚnextr-  )rU   r  r4   rl   Úoverride_indexs        r!   r4   zGemmaConverter.vocab  s·   € à×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4Ø×$Ñ$×.Ñ.°Ð4ð
ˆð
 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆô Ñ/¨Ô/Ô/Ü!Ñ"V´¸5Ó1AÔ"VÐX\Ó]ˆNØÐ)Ø(3��nÑ%àˆùò Ls   ÁB5c                 ó.   — t        j                  dd«      S )Nr«   Úmerged_with_previous)r	   rì   ©rU   rH  r$   s      r!   r—   zGemmaConverter.pre_tokenizer  s   € Ü×#Ñ# CÐ)?Ó@Ð@r#   c                 ó
   — d}|S rl  r2   rm  s      r!   r  zGemmaConverter.unk_id  rn  r#   c                 ó¢   — t        j                  t        j                  dd«      t        j                  «       t        j                  «       g«      S )Nr=  r«   )r   rë   rA  ÚByteFallbackÚFuserS  s      r!   rž   zGemmaConverter.decoder"  s?   € Ü× Ñ ä× Ñ  ¨Ó,Ü×%Ñ%Ó'Ü—‘“ðó
ð 	
r#   N)r^   r_   r`   r  rg   r,  rŒ   r•   r4   r—   r  rž   r2   r#   r!   rF  rF  ø  s6   „ ØÐØ.€Là'¨Ð9€Nðò/òò Aòó
r#   rF  c                   ó4   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	y)	ÚLlamaConverterTc                 ó(  — | j                   j                  d«      df| j                   j                  d«      df| j                   j                  d«      dfg}||j                  dd  D �cg c]  }|j                  |j                  f‘Œ c}z  }|S c c}w )Nr   rw  r   r9   r"  )r,   r,  r  rl   r  r|  s       r!   r4   zLlamaConverter.vocab/  sŽ   € à×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCØ×$Ñ$×:Ñ:¸1Ó=¸sÐCð
ˆð
 	¸%¿,¹,ÀqÀrÐ:JÖK°�5—;‘; §¡Ò,ÒKÑKˆØˆùò Ls   Á)Bc                 ó
   — d}|S r%  r2   rm  s      r!   r  zLlamaConverter.unk_id8  rn  r#   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S ©Nr=  r«   r   )Úcontentr;  ©r   rA  rV  rW  r@  rë   ©rU   rH  r$   Úsequences       r!   rž   zLlamaConverter.decoder<  ó\   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆñ
 ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r#   c                 óø   — t        | j                  dd«      rcg }t        | j                  dd«      r|t        j                  d¬«      gz  }|t        j                  dd¬«      gz  }t        j
                  |«      S y )Nr(   Tr$   r=  )Úprependr«   )Úpatternr^  )r+   r,   r   ÚPrependrA  rë   )rU   r  ra  s      r!   r•   zLlamaConverter.normalizerF  sr   € Ü�4×*Ñ*¨H°dÔ;ØˆHÜ�t×.Ñ.Ð0BÀDÔIØœ[×0Ñ0¸Ô?Ð@Ñ@�Øœ×,Ñ,°SÀ%ÔHÐIÑIˆHÜ×'Ñ'¨Ó1Ð1Ør#   c                 óŽ   — t        | j                  dd«      s.t        || j                  «      }t        j                  ||d¬«      S y )Nr(   TF©rH  r-   Úsplit)r+   r,   r.   r	   rI  rJ  s       r!   r—   zLlamaConverter.pre_tokenizerO  sA   € Ü�t×.Ñ.°¸$Ô?Ü0Ð1AÀ4×CZÑCZÓ[ˆNÜ!×+Ñ+¸ÐTbÐjoÔpÐpØr#   c                  ó   — y r]   r2   rx   s    r!   r�   zLlamaConverter.post_processorU  s   € àr#   N)
r^   r_   r`   r  r4   r  rž   r•   r—   r�   r2   r#   r!   rY  rY  ,  s&   „ ØÐòòò+òòór#   rY  c                   ó   — e Zd Zdefd„Zy)ÚMarkupLMConverterr%   c                 óŠ  — | j                   }|j                  }t        |j                  j	                  «       «      }t        t        ||d ddd| j                   j                  ¬«      «      }t        j                  |j                  ¬«      |_        t        j                  «       |_        t        | j                   j                  «      }t        | j                   j                   «      }| j                   j"                  }| j                   j$                  }t'        j(                  |› d|› �|› d|› d|› �||f||fg¬«      |_        |S )NrÌ   Fr8  rÏ   z $A z $B r‰   )r,   rÂ   re   rÃ   rÄ   r   r   r~   r	   rÓ   r$   r—   r   rž   rb   r˜   r™   rš   r›   r
   rœ   r�   )	rU   ró   r4   rF   r    r¡   r¢   rš   r›   s	            r!   ry   zMarkupLMConverter.converted[  s,  € Ø×$Ñ$ˆØ—
‘
ˆÜ�b—l‘l×'Ñ'Ó)Ó*ˆäÜØØØØ*,Ø#%ØØ×1Ñ1×;Ñ;ôó

ˆ	ô #1×":Ñ":ÈB×L_ÑL_Ô"`ˆ	ÔÜ$×.Ñ.Ó0ˆ	Ôä�$×)Ñ)×3Ñ3Ó4ˆÜ�$×)Ñ)×3Ñ3Ó4ˆØ×.Ñ.×;Ñ;ˆØ×.Ñ.×;Ñ;ˆä#-×#@Ñ#@Ø�U˜$˜s˜eÐ$Ø�5˜˜S˜E  c UÐ+à�lÐ#Ø�lÐ#ðô$
ˆ	Ô ð Ðr#   Nr¤   r2   r#   r!   rl  rl  Z  s   „ ð"˜9ô "r#   rl  c                   ó*   — e Zd ZdZdd„Zd„ Zd„ Zd„ Zy)ÚMoshiConverterTNc                 ó  — t        | d«       t        j                  | |«       t        «       }|j	                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY w©Nr   r  ©	r   rt   rV   r"   r  r	  r  r  r  )rU   r
  Úmodel_max_lengthÚkwargsr  r  r  s          r!   rV   zMoshiConverter.__init__ƒ  sr   € Ü˜$ 
Ô+ä×Ñ˜4 Ô,ô $Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
úó   Á	 A9Á9B	c                 óâ   — |j                   j                  }t        j                  dd«      g}|st        j                  |«      S t        j                  t        j
                  |«      g|z   «      S rH  )r>  r?  r   rA  rë   rB  rC  s       r!   r•   zMoshiConverter.normalizer�  sg   € Ø$×4Ñ4×IÑIÐä×Ñ  UÓ+ð
ˆñ $Ü×'Ñ'¨Ó5Ð5ä×'Ñ'¬×)@Ñ)@ÐAUÓ)VÐ(WÐZfÑ(fÓgÐgr#   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S r]  r_  r`  s       r!   rž   zMoshiConverter.decoderš  rb  r#   c                 ó6   — d}t        j                  ||d¬«      S )Nr)   Frh  )r	   rI  rJ  s       r!   r—   zMoshiConverter.pre_tokenizer¤  s   € Ø ˆÜ×'Ñ'°KÐP^ÐfkÔlÐlr#   r]   )r^   r_   r`   r  rV   r•   rž   r—   r2   r#   r!   ro  ro  €  s   „ ØÐóòhò+ómr#   ro  c                   óB   — e Zd ZdZdd„Zd„ Zd„ Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zy)ÚHeliumConverterTNc                 ó  — t        | d«       t        j                  | |«       t        «       }|j	                  «       }t        |d«      5 }|j                  |j                  «       «       d d d «       || _        y # 1 sw Y   || _        y xY wrq  rr  )rU   r
  r  r  r  r  s         r!   rV   zHeliumConverter.__init__¬  sp   € Ü˜$ 
Ô+ä×Ñ˜4 Ô,ä#Ó%ˆ	à× Ñ Ó"ˆÜ�*˜dÓ#ð 	( qØ×Ñ˜aŸf™f›hÔ'÷	(àˆ�
÷	(àˆ�
úru  c                 óV  — | j                  |«      }t        t        || j                  |«      | j                  ¬«      «      }t        |j                  «      D ��cg c]I  \  }}|j                  dv r6||j                  |j                  dk(  xs |j                  | j                  v f‘ŒK }}}|j                  t        |d„ ¬«      D ���cg c]  \  }}}t        |d|d¬«      ‘Œ c}}}«       |j                  t        d	dd¬
«      g«       |j                  dd¬«       |S c c}}w c c}}}w )Nr   r!  r"  c                 ó   — | d   S r%  r2   r&  s    r!   r5   z+HeliumConverter.tokenizer.<locals>.<lambda>Í  r'  r#   r6   FT)r)  r*  Úsingle_wordú
r(  ry  )r  Úpad_id)r4   r   r   r  r  r-  r  r0  rl   rŒ   r1  rC   r   Úenable_padding)	rU   r  rE   r    r6  r7  r8  r9  r*  s	            r!   r    zHeliumConverter.tokenizer¸  s   € Ø—z‘z %Ó(ˆÜÜØØ—{‘{ 5Ó)Ø"×7Ñ7ôó
ˆ	ô # 5§<¡<Ó0÷
á��AØ�v‰v˜Ñð �—‘˜!Ÿ&™& A™+ÒG¨¯©°D×4GÑ4GÐ)GÒHð
Ðñ 
ð
 	×Ñô +1Ð1AÁ~Ô*V÷ð á&�B˜˜wô ˜5¨U¸GÐQUÖVôô	
ð 	×Ñœj¨¸%ÈÔOÐPÔQØ× Ñ ¨7¸1Ð Ô=ØÐùó
ùôs   ÁADÃD$c                 ó¨   — g }|j                   D ]@  }|j                  dk(  r|d|j                  fgz  }Œ%||j                  |j                  fgz  }ŒB |S )Nz<0x0A>r  r  r|  s       r!   r4   zHeliumConverter.vocabÔ  s]   € ØˆØ—\‘\ò 	6ˆEØ�{‰{˜hÒ&Ø˜4 §¡Ð-Ð.Ñ.‘à˜5Ÿ;™;¨¯©Ð4Ð5Ñ5‘ð		6ð
 ˆr#   c                 ó
   — d}|S r%  r2   rm  s      r!   r  zHeliumConverter.unk_idÝ  rn  r#   c                 óÜ   — t        j                  dd«      t        j                  «       t        j                  «       g}|t        j                  dd¬«      gz  }t        j
                  |«      S r]  r_  r`  s       r!   rž   zHeliumConverter.decoderá  sY   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆð
 	”X—^‘^¨C°aÔ8Ð9Ñ9ˆÜ× Ñ  Ó*Ð*r#   c                 ó~   — t        j                  t        j                  d«      t        j                  dd«      g«      S rH  )r   rë   rf  rA  r  s     r!   r•   zHeliumConverter.normalizerê  s2   € Ü×#Ñ#¤[×%8Ñ%8¸Ó%=¼{×?RÑ?RÐSWÐY^Ó?_Ð$`ÓaÐar#   c                 óV   — t        j                  t        j                  dd«      g«      S )Nr  Ú
contiguous)r	   rë   rì   rS  s      r!   r—   zHeliumConverter.pre_tokenizerí  s#   € Ü×&Ñ&¬×(<Ñ(<¸TÀ<Ó(PÐ'QÓRÐRr#   c                 ó<   — t        j                  ddgg d¢dg¬«      S )Nrq  r  )rq  r  rq  r  )rq  r   r‰   )r
   rœ   rx   s    r!   r�   zHeliumConverter.post_processorð  s/   € Ü×,Ñ,àØðòð ðô
ð 	
r#   r]   )r^   r_   r`   r  rV   r    r4   r  rž   r•   r—   r�   r2   r#   r!   rz  rz  ©  s2   „ ØÐó
òò8òò+òbòSó
r#   rz  c            	      óò  — t        t        t        d«      t        d«      dz   «      «      t        t        t        d«      t        d«      dz   «      «      z   t        t        t        d«      t        d«      dz   «      «      z   } | dd }d	}t        d
«      D ]1  }|| vsŒ| j                  |«       |j                  d
|z   «       |dz  }Œ3 |D �cg c]  }t	        |«      ‘Œ }}t        t        | |«      «      S c c}w )a8  
    Returns list of utf-8 byte and a mapping to unicode strings. We specifically avoids mapping to whitespace/control
    characters the bpe code barfs on.

    The reversible bpe codes work on unicode strings. This means you need a large # of unicode characters in your vocab
    if you want to avoid UNKs. When you're at something like a 10B token dataset you end up needing around 5K for
    decent coverage. This is a significant percentage of your normal, say, 32K bpe vocab. To avoid that, we want lookup
    tables between utf-8 bytes and unicode strings.
    ú!ú~r   õ   Â¡õ   Â¬õ   Â®õ   Ã¿Nr   é   )re   rA   ÚordrB   Úchrr?   r.  )ÚbsÚcsÚnÚbs       r!   Úbytes_to_unicoder—    sæ   € ô 	ŒU”3�s“8œS ›X¨™\Ó*Ó+¬d´5¼¸T»ÄCÈÃIÐPQÁMÓ3RÓ.SÑSÔVZÔ[`ÔadÐeiÓajÔloÐptÓluÐxyÑlyÓ[zÓV{Ñ{ð ð 
‰Aˆ€BØ	€AÜ�4‹[ò ˆØ�BŠ;Ø�I‰I�aŒLØ�I‰I�d˜Q‘hÔØ�‰F‰Að	ð
 Ö	�QŒ#ˆa�&Ð	€BÐ	Ü”�B˜“ÓÐùò 
s   ÃC4c                   óJ   ‡ — e Zd ZdZ	 	 	 	 dˆ fd„	Zdefd„Zd„ Zdefd„Z	ˆ xZ
S )	ÚTikTokenConverterz'
    A general tiktoken converter.
    c                 ó¤   •— t        ‰| �  |Ž  || _        || _        || _        t        |«      t        u r|j                  «       | _        y || _        y r]   )	r  rV   r
  re  r$   r0  r?   rÄ   Úadditional_special_tokens)rU   r
  re  r$   r›  r  rt  r  s          €r!   rV   zTikTokenConverter.__init__   sV   ø€ ô 	‰Ñ˜$ÑØ$ˆŒØˆŒØ 0ˆÔä04Ð5NÓ0OÔSWÑ0WÐ%×*Ñ*Ó,ð 	Õ&Ø]vð 	Õ&r#   Útiktoken_urlc                 ó0  ‡‡— 	 ddl m}  ||«      Št	        «       Šˆfd„}g }i }‰j                  «       D ]�  \  }}|| ||«      <   t        |«      dk(  rŒ g }t        dt        |«      «      D ]2  }	|d |	 ||	d  }}
|
‰v sŒ|‰v sŒ|
|z   ‰v sŒ|j                  |
||f«       Œ4 t        |ˆfd„d¬«      }|j                  |«       Œ� t        |d	„ d¬«      }|D �cg c]  } ||d   «       ||d   «      f‘Œ }}||fS # t        $ r t        d«      ‚w xY wc c}w )
Nr   )Úload_tiktoken_bpezY`tiktoken` is required to read a `tiktoken` file. Install it with `pip install tiktoken`.c           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w )NrÌ   zlatin-1)r-  Údecoder‘  )r–  ÚcharÚbyte_encoders     €r!   Útoken_bytes_to_stringzPTikTokenConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string<  s2   ø€ Ø—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTs    <r   c                 ó$   •— ‰| d      ‰| d      fS r1   r2   )r3   rÃ   s    €r!   r5   zCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>J  s   ø€ °¸1¸Q¹4±À)ÈAÈaÉDÁ/Ð0R€ r#   Fr=   c                 ó   — | d   S )Nr9   r2   r;   s    r!   r5   zCTikTokenConverter.extract_vocab_merges_from_model.<locals>.<lambda>L  s
   € °°A±€ r#   )Útiktoken.loadrž  r/  Ú
ValueErrorr—  r@   r:   rA   rB   rC   rD   )rU   rœ  rž  r£  rF   r4   r9  ÚrankrI   rJ   rK   rL   r<   rÃ   r¢  s                @@r!   Úextract_vocab_merges_from_modelz1TikTokenConverter.extract_vocab_merges_from_model1  sY  ù€ ð	Ý7ñ & lÓ3ˆ	Ü'Ó)ˆô	Vð ˆØˆØ$Ÿ?™?Ó,ò 
	!‰KˆE�4Ø26ˆEÑ'¨Ó.Ñ/Ü�5‹z˜QŠØØˆEÜ˜q¤# e£*Ó-ò ;�Ø#(¨¨% =°%¸¸°-˜�Ø˜iÒ'¨G°yÒ,@ÀgÐPWÑFWÐ\eÒEeØ—L‘L '¨7°DÐ!9Õ:ð;ô ˜5Ó&RÐ\aÔbˆEØ�M‰M˜%Õ ð
	!ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ðøô5 ò 	ÜØkóð ð	üò2 ds   „C; ÃDÃ;Dc                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S )NF)r¿   Úignore_mergesT)r©  r
  r   r   r‘   rP   r«  )rU   rE   rF   r    s       r!   r    zTikTokenConverter.tokenizerP  sN   € Ø#×CÑCÀDÇOÁOÓTÑˆ�fÜœc ,°ÀÔGÓHˆ	Ü�9—?‘? OÔ4Ø,0ˆI�O‰OÔ)ØÐr#   r%   c           
      óÊ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        |j                  | j                  D �cg c]  }t        |dd¬«      ‘Œ c}«       t        j                  d¬«      |_        |S c c}w )Nrä   Frå   rè   Tr(  rÑ   )r    r	   rë   rì   r   re  rÓ   r$   r—   r   rž   rÆ   r›  r   r
   r�   )rU   r    r9  s      r!   ry   zTikTokenConverter.convertedW  s¸   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	Ôà×$Ñ$ØLP×LjÑLjÖkÀ5ŒZ˜¨%¸Ö>Òkô	
ô $.×#7Ñ#7ÀUÔ#Kˆ	Ô àÐùò ls   Â'C )Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)r^   r_   r`   ra   rV   rb   r©  r    r   ry   rN  rO  s   @r!   r™  r™    s;   ø„ ñð ð KØØ"&õ
ð"¸Có ò>ð˜9÷ r#   r™  ÚAlbertTokenizerÚBartTokenizerÚBarthezTokenizerÚBertTokenizerÚBigBirdTokenizerÚBlenderbotTokenizerÚCamembertTokenizerÚCLIPTokenizerÚCodeGenTokenizerÚConvBertTokenizerÚDebertaTokenizerÚDebertaV2TokenizerÚDistilBertTokenizerÚDPRReaderTokenizerÚDPRQuestionEncoderTokenizerÚDPRContextEncoderTokenizerÚElectraTokenizerÚFNetTokenizerÚFunnelTokenizerÚGPT2TokenizerÚHerbertTokenizerÚLayoutLMTokenizerÚLayoutLMv2TokenizerÚLayoutLMv3TokenizerÚLayoutXLMTokenizerÚLongformerTokenizerÚLEDTokenizerÚLxmertTokenizerÚMarkupLMTokenizerÚMBartTokenizerÚMBart50TokenizerÚMPNetTokenizerÚMobileBertTokenizerÚMvpTokenizerÚNllbTokenizerÚOpenAIGPTTokenizerÚPegasusTokenizerÚQwen2TokenizerÚRealmTokenizerÚReformerTokenizerÚRemBertTokenizerÚRetriBertTokenizerÚRobertaTokenizerÚRoFormerTokenizerÚSeamlessM4TTokenizerÚSqueezeBertTokenizerÚT5TokenizerÚUdopTokenizerÚWhisperTokenizerÚXLMRobertaTokenizerÚXLNetTokenizer)ÚSplinterTokenizerÚXGLMTokenizerÚLlamaTokenizerÚCodeLlamaTokenizerÚGemmaTokenizerÚPhi3Tokenizerc                 óv  — | j                   j                  }|t        v r!|st        |   } || «      j                  «       S 	 t        j                  d«       t        | j                  | j                  ¬«      j                  «       S # t        $ r* t        dt        t        j                  «       «      › �«      ‚w xY w)aˆ  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].
       from_tiktoken (bool, optional): Whether to use the `tiktoken` library to convert the tokenizer instead of sentencepiece.
            Defaults to False.

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    zConverting from Tiktoken)r
  r›  zÌConverting from SentencePiece and Tiktoken failed, if a converter for SentencePiece is available, provide a model path with a SentencePiece tokenizer.model file.Currently available slow->fast converters: )r  r^   ÚSLOW_TO_FAST_CONVERTERSry   ÚloggerÚinfor™  r
  r›  r/  r§  re   rÄ   )Útransformer_tokenizerÚfrom_tiktokenÚtokenizer_class_nameÚconverter_classs       r!   Úconvert_slow_tokenizerrî  §  s¹   € ð  1×:Ñ:×CÑCÐØÔ6Ñ6¹}Ü1Ð2FÑGˆÙÐ4Ó5×?Ñ?ÓAÐAð	Ü�K‰KÐ2Ô3Ü$Ø0×;Ñ;Ø*?×*YÑ*Yô÷ ‰i‹kðøô ò 	Üð>ä>BÔCZ×C_ÑC_ÓCaÓ>bÐ=cðeóð ð	ús   ÁAB Â3B8)rÌ   )F)Nra   r  Ú	packagingr   Ú
tokenizersr   r   r   r   r   r	   r
   Útokenizers.modelsr   r   r   Úutilsr   r   r   r   Úutils.import_utilsr   Ú
get_loggerr^   rè  r"   Úboolrb   r.   rM   rO   rg   rr   rt   r{   r§   r´   r·   rº   rÊ   rÚ   rá   rî   rö   rý   r  rQ  rj  ru  r‚  rŠ  rÇ  rç  rï  rö  rü  r  r  r	  r  r  r%  r)  r3  r6  r;  r>  rA  rF  rY  rl  ro  rz  r—  r™  rç  rî  r2   r#   r!   ú<module>rö     sg  ðñó å ß f× fÑ fß 5Ñ 5ç `Ó `Ý 5ð 
ˆ×	Ñ	˜HÓ	%€óGð"¨$ð Àsó ò÷&ñ ô2Ð"8ô ð"I˜cð I dó I÷$ñ $ô$�Iô $ôN/˜	ô /ôd$�iô $ôN$�Yô $ôN˜ô ô6#�Iô #ôL�yô ô>)�Yô )ôX�yô ô:$˜	ô $ôN�yô ô>~�9ô ~ôB"
�lô "
ôJ
�|ô 
ô 
˜ô 
ô:
˜ô 
ôB2
�\ô 2
ôj
�|ô 
ô6
�Lô 
ô2
˜<ô 
ô2
˜,ô 
ô6"
�\ô "
ôJ	˜ô 	ô
�|ô 
ô@	˜lô 	ô%x�|ô %xôP
�,ô 
ô"
�Lô 
ô!�yô !ôH	
�|ô 	
ô(�Iô (ôV$˜)ô $ôN˜)ô ô:
�Lô 
ô61
�\ô 1
ôh+�\ô +ô\#˜	ô #ôL&m�\ô &môRV
�lô V
òt÷0Lñ Lð^:Ø�ð:àÐ%ð:ð Ð(ð:ð �]ð	:ð
 Ð(ð:ð Ð.ð:ð Ð,ð:ð �]ð:ð ˜ð:ð ˜ð:ð Ð(ð:ð Ð,ð:ð ˜=ð:ð ˜-ð:ð " =ð:ð  ! -ð!:ð" ˜ñ#:ð$ �_ð%:ð& �ð':ð( �]ð):ð* Ð(ð+:ð, ˜ð-:ð. ˜=ð/:ð0 Ð+ð1:ð2 Ð-ð3:ð4 Ð+ð5:ð6 Ð$ð7:ð8 �}ð9:ð: Ð*ð;:ð< �nð=:ð> Ð(ð?:ð@ �nðA:ðB ˜=ðC:ðD Ð$òE:ðF �]ðG:ðH Ð,ðI:ðJ Ð(ðK:ðL �nðM:ðN �mðO:ðP Ð*ðQ:ðR Ð(ðS:ðT ˜-ðU:ðV Ð(ðW:ðX Ð*ðY:ðZ Ð0ð[:ð\ ˜Mð]:ð^ �;ð_:ð` �]ða:ðb Ð(ðc:ðd Ð.ðe:ðf �nñg:ðh +Ø"Ø$Ø(Ø$Ø#òs:Ð ñz!È)ô !r#   