Ë
    S^(h±  ã                   ód   — d dl mZmZmZmZmZ d dlmZ d dlm	Z	 d dl
mZ  G d„ d«      Zdefd„Zy	)
é    )ÚRegexÚ	TokenizerÚdecodersÚpre_tokenizersÚ
processors)ÚBPE)ÚLlamaTokenizerFast)Úbytes_to_unicodec                   óJ   ‡ — e Zd ZdZ	 	 	 	 dˆ fd„	Zdefd„Zd„ Zdefd„Z	ˆ xZ
S )	ÚMistralConverterz'
    A general tiktoken converter.
    c                 óV   •— t        ‰| �  |Ž  || _        || _        || _        || _        y )N)ÚsuperÚ__init__ÚvocabÚpatternÚadd_prefix_spaceÚadditional_special_tokens)Úselfr   r   r   r   ÚargsÚkwargsÚ	__class__s          €ú_/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/integrations/mistral.pyr   zMistralConverter.__init__   s0   ø€ ô 	‰Ñ˜$ÑØˆŒ
ØˆŒØ 0ˆÔØ)BˆÕ&ó    r   c                 ó&  ‡‡— |Št        «       Šˆfd„}g }i }t        ‰j                  «       «      D ]¤  \  }\  }}|| j                  vr‰|| ||«      <   t	        |«      dk(  rŒ1g }t        dt	        |«      «      D ]2  }|d | ||d  }
}	|	‰v sŒ|
‰v sŒ|	|
z   ‰v sŒ|j                  |	|
|f«       Œ4 t        |ˆfd„d¬«      }|j                  |«       Œ |||<   Œ¦ t        |d„ d¬«      }|D �cg c]  } ||d   «       ||d   «      f‘Œ }}||fS c c}w )Nc           	      ó‚   •— dj                  | j                  d«      D �cg c]  }‰t        |«         ‘Œ c}«      S c c}w )NÚ zlatin-1)ÚjoinÚdecodeÚord)ÚbÚcharÚbyte_encoders     €r   Útoken_bytes_to_stringzOMistralConverter.extract_vocab_merges_from_model.<locals>.token_bytes_to_string    s2   ø€ Ø—7‘7ÀÇÁÈÓ@SÖT¸˜L¬¨T«Ó3ÒTÓUÐUùÒTs    <é   c                 ó$   •— ‰| d      ‰| d      fS )Nr   r$   © )ÚxÚ	bpe_rankss    €r   ú<lambda>zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>/   s   ø€ °Y¸qÀ¹t±_ÀiÐPQÐRSÑPTÁoÐ4V€ r   F)ÚkeyÚreversec                 ó   — | d   S )Né   r&   )Úvals    r   r)   zBMistralConverter.extract_vocab_merges_from_model.<locals>.<lambda>3   s
   € °°A±€ r   r   )	r
   Ú	enumerateÚitemsr   ÚlenÚrangeÚappendÚsortedÚextend)r   r   r#   ÚmergesÚidxÚtokenÚrankÚlocalÚindexÚpiece_lÚpiece_rr.   r(   r"   s               @@r   Úextract_vocab_merges_from_modelz0MistralConverter.extract_vocab_merges_from_model   sK  ù€ Øˆ	Ü'Ó)ˆô	Vð ˆØˆÜ"+¨I¯O©OÓ,=Ó">ò 	#ÑˆC‘�%˜Ø˜D×:Ñ:Ñ:Ø69�Ñ+¨EÓ2Ñ3Ü�u“: ’?ØØ�Ü" 1¤c¨%£jÓ1ò ?�EØ',¨V¨e }°e¸E¸F°m˜W�GØ )Ò+°¸9Ò0DÈ'ÐT[ÑJ[Ð`iÒIiØŸ™ g¨w¸Ð%=Õ>ð?ô ˜uÓ*VÐ`eÔf�Ø—‘˜eÕ$à"��e’ð	#ô ˜Ñ$6ÀÔFˆØ\bÖcÐUXÑ(¨¨Q©Ó0Ñ2GÈÈAÉÓ2OÒPÐcˆÐcØ�fˆ}Ðùò ds   Ã+Dc                 ó¾   — | j                  | j                  «      \  }}t        t        ||d¬«      «      }t	        |j
                  d«      rd|j
                  _        |S )NF)Úfuse_unkÚignore_mergesT)r>   r   r   r   ÚhasattrÚmodelrA   )r   Úvocab_scoresr6   Ú	tokenizers       r   rE   zMistralConverter.tokenizer7   sN   € Ø#×CÑCÀDÇJÁJÓOÑˆ�fÜœc ,°ÀÔGÓHˆ	Ü�9—?‘? OÔ4Ø,0ˆI�O‰OÔ)ØÐr   Úreturnc                 óŒ  — | j                  «       }t        j                  t        j                  t	        | j
                  «      dd¬«      t        j                  | j                  d¬«      g«      |_        t        j                  «       |_
        |j                  | j                  «       t        j                  d¬«      |_        |S )NÚisolatedF)ÚbehaviorÚinvert)r   Ú	use_regex)Útrim_offsets)rE   r   ÚSequenceÚSplitr   r   Ú	ByteLevelr   Úpre_tokenizerr   ÚdecoderÚadd_special_tokensr   r   Úpost_processor)r   rE   s     r   Ú	convertedzMistralConverter.converted>   s™   € Ø—N‘NÓ$ˆ	Ü"0×"9Ñ"9ä×$Ñ$¤U¨4¯<©<Ó%8À:ÐV[Ô\Ü×(Ñ(¸$×:OÑ:OÐ[`Ôaðó#
ˆ	Ôô %×.Ñ.Ó0ˆ	ÔØ×$Ñ$ T×%CÑ%CÔDä#-×#7Ñ#7ÀUÔ#Kˆ	Ô àÐr   )Nzs(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+FN)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   Ústrr>   rE   r   rT   Ú__classcell__)r   s   @r   r   r      s<   ø„ ñð ð KØØ"&õCð°Só ò6ð˜9÷ r   r   Útokenizer_filec                 óæ  — ddl m} |j                  | «      }|j                  j                  j
                  }|j                  j                  j                  D �cg c]  }t        |d«      r|j                  n|‘Œ }}|D �ci c]  }||j                  |«      “Œ }}|j                  |«       |}t        t        ||¬«      j                  «       ¬«      }|j                  d|i«       |S c c}w c c}w )z1Convert a "tekken" tokenizer to a fast Tokenizer.r   )ÚMistralTokenizerÚvalue)r   r   )Útokenizer_objectr   )Ú(mistral_common.tokens.tokenizers.mistralr]   Ú	from_fileÚinstruct_tokenizerrE   Ú_tekken_token2id_nospecialÚ_all_special_tokensrB   r^   r;   Úupdater	   r   rT   rR   )r[   r]   Úmistral_tokenizerr   r8   Úall_specialÚspecials_tokensrE   s           r   Úconvert_tekken_tokenizerri   N   sø   € õ Jð )×2Ñ2°>ÓBÐð ×0Ñ0×:Ñ:×UÑU€Eð '×9Ñ9×CÑC×WÑWöàô ˜u gÔ.ˆ�Š°EÑ9ð€Kð ð EPÖP¸5�u˜k×/Ñ/°Ó6Ñ6ÐP€OÐPØ×Ñ˜5Ô!Ø€Eô #Ü)°ÐQ\Ô]×gÑgÓiô€Ið
 × Ñ Ð"=¸{Ð!KÔLàÐùò!ùò Qs   Á!C)ÂC.N)Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr   Útransformersr	   Ú#transformers.convert_slow_tokenizerr
   r   rY   ri   r&   r   r   ú<module>rn      s-   ðß MÕ MÝ !å +Ý @÷Cñ CðL¨Sô r   