Ë
    S^(h¯g  ã                   ó8  — d Z ddlmZ ddlZddlmZmZmZmZm	Z	 ddl
mZmZ ddlmZ ddlmZmZmZmZmZ dd	lmZ dd
lmZ  ej2                  e«      Zi ddddœ“ddddddddddddœ
“ddddddddddddœ
“ddddddddddddœ
“dddddddddddddd œ“d!dddddddddddœ
“d"d#d$d%d&d'œ“d(dddddddddddœ
“d)d*dd+dd,d-œ“d.d/d0d1d2d3d4dd,d5d6dd7œ“d8dddddddd9dd:œ	“d;d*d<d=d>d+d,d?œ“d@dddddddAdBœ“dCdddd,ddDdEdFddGœ	“dHdddddddddIddœ
“dJddddddddddddKœ“ZdLdMdNdOdPd#d$d%d&dQdRœ
dSdd#d$d%d&dTœdUœZdV„ Z G dW„ dX«      Z G dY„ dZe«      Z  G d[„ d\e«      Z! G d]„ d^e«      Z" G d_„ d`e«      Z# G da„ dbe«      Z$ G dc„ dde«      Z%e e!e!e"e#e#e#e#e#e$e#e#e%deœZ&dfefdg„Z'y)hz
Integration with GGML / The file is copied and adapted from https://github.com/99991/pygguf
with extra methods beings exposed
é    )ÚarrayN)Ú	TokenizerÚdecodersÚnormalizersÚpre_tokenizersÚ
processors)ÚBPEÚUnigramé   )Ú
AddedToken)ÚGemmaConverterÚGPT2ConverterÚLlamaConverterÚQwen2ConverterÚT5Converter)Úlogging)ÚtqdmÚgeneralÚ
model_typeÚ_model_name_or_path)ÚarchitectureÚnameÚllamaÚmax_position_embeddingsÚnum_hidden_layersÚintermediate_sizeÚhidden_sizeÚhead_dimÚ
rope_thetaÚnum_attention_headsÚnum_key_value_headsÚrms_norm_epsÚ
vocab_size)
Úcontext_lengthÚblock_countÚfeed_forward_lengthÚembedding_lengthúrope.dimension_countúrope.freq_baseúattention.head_countúattention.head_count_kvú attention.layer_norm_rms_epsilonr#   ÚmistralÚqwen2Úqwen2moeÚnum_expertsÚnum_experts_per_tok)r$   r%   r&   r'   r(   r)   r*   r+   r,   r#   Úexpert_countÚexpert_used_countÚfalconÚ	tokenizerÚbos_token_idÚeos_token_idÚunk_token_idÚpad_token_id)úggml.bos_token_idúggml.eos_token_idúggml.unknown_token_idúggml.padding_token_idÚphi3ÚbloomÚn_layerÚn_headÚlayer_norm_epsilon)r%   r'   r*   r#   úattention.layer_norm_epsilonÚt5Ún_positionsÚ
num_layersÚd_ffÚd_modelÚd_kvÚ	num_headsÚrelative_attention_num_bucketsÚdecoder_start_token_id)r$   r%   r&   r'   úattention.key_lengthr*   r+   rC   z attention.relative_buckets_countrL   r#   ÚstablelmÚlayer_norm_eps)	r$   r%   r&   r'   r(   r*   r+   rC   r#   Úgpt2Ún_ctxÚn_embdr&   )r%   r$   r'   r&   r*   rC   Ú
starcoder2Únorm_epsilon)r%   r$   r'   r&   r*   r+   rC   ÚmambaÚconv_kernelÚ
state_sizeÚtime_step_rank)	r#   r$   r'   r,   r%   zssm.conv_kernelzssm.state_sizezssm.time_step_rankzssm.inner_sizeÚnemotronÚnorm_epsÚgemma2)r$   r%   r&   r'   r(   r)   rM   r*   r+   r,   r#   Útokenizer_typeÚtokensÚscoresÚ
token_typeÚmergesÚadd_prefix_space)
ú
ggml.modelzggml.tokenszggml.scoreszggml.token_typezggml.mergesr:   r;   r<   r=   zggml.add_space_prefixÚchat_template)rc   rb   r:   r;   r<   r=   )r5   Útokenizer_configc                 óŽ  — t        |t        «      s|g}t        |«      dk(  r|d   }d }n|d   dk7  rt        d«      ‚|\  }}|dv rt	        | d   «      } | S |dv rt        | d   «      } | S |dv rt        | d   «      } | S |dv r3t        d	t        | «      «      j                  «       j                  «       } | S |d
v rt        | |«      } | S )Né   r   é	   zPReceived multiple types, therefore expected the first type to indicate an array.)r   rf   r   é   é   é   é
   é   )é   é   )é   )é   ÚB)rg   )Ú
isinstanceÚlistÚlenÚ
ValueErrorÚintÚfloatÚboolr   ÚtobytesÚdecodeÚ_gguf_parse_value)Ú_valueÚ	data_typeÚarray_data_types      ú\/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/integrations/ggml.pyr{   r{   ë   s÷   € Ü�i¤Ô&Ø�Kˆ	Ü
ˆ9ƒ~˜ÒØ˜a‘Lˆ	Ø‰à�Q‰<˜1ÒÜÐoÓpÐpØ%.Ñ"ˆ	�?àÐ.Ñ.Ü�V˜A‘Y“ˆð €Mð 
�gÑ	Ü�v˜a‘yÓ!ˆð €Mð 
�cÑ	Ü�f˜Q‘i“ˆð
 €Mð	 
�cÑ	Ü�sœD ›LÓ)×1Ñ1Ó3×:Ñ:Ó<ˆð €Mð 
�cÑ	Ü" 6¨?Ó;ˆØ€Mó    c                   ó   — e Zd Zd„ Zy)ÚGGUFTokenizerSkeletonc                 óp  ‡— |j                  «       D ]  \  }}t        | ||«       Œ t        | d«      �s)t        | d«      rt        | d«      st        d«      ‚| j                  }| j
                  }t        |«      D ��ci c]  \  }}|||   “Œ c}}Št        j                  d«       g }t        ‰j                  «       «      D ]k  \  }	}
g }t        dt        |	«      «      D ]*  }|	d | |	|d  }}||v sŒ||v sŒ|j                  |||
f«       Œ, t        |ˆfd„d¬	«      }|j                  |«       Œm t        |d
„ d¬	«      }|D �cg c]  }|d   |d   f‘Œ }}|| _        nt| j                  D �	cg c]  }	t!        |	j#                  d«      «      ‘Œ c}	| _        t        | d«      s1t        t        | j                  «      «      D �cg c]  }d ‘Œ c}| _        t        | d«      sg | _        t        | d«      sd | _        t        | d«      r| j&                  €| j(                  | _        y y y c c}}w c c}w c c}	w c c}w )Nr`   r]   r^   z\tokens and scores need to be passed for a LLaMa tokenizer without merges to be instantiated.z:Merges were not in checkpoint, building merges on the fly.rf   c                 ó$   •— ‰| d      ‰| d      fS )Nr   rf   © )ÚxÚvocabs    €r   ú<lambda>z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s   ø€ °U¸1¸Q¹4±[À%ÈÈ!ÉÁ+Ð4N€ r€   T)ÚkeyÚreversec                 ó   — | d   S )Nr   r…   )Úvals    r   rˆ   z0GGUFTokenizerSkeleton.__init__.<locals>.<lambda>  s
   € °C¸±F€ r€   r   ú Úadded_tokensr8   Úunknown_token_id)ÚitemsÚsetattrÚhasattrru   r]   r^   Ú	enumerateÚloggerÚwarningr   Úrangert   ÚappendÚsortedÚextendr`   ÚtupleÚsplitrŽ   r8   r�   )ÚselfÚdict_ÚkÚvr]   r^   ÚiÚtr`   ÚmergeÚpiece_scoreÚlocalÚindexÚpiece_lÚpiece_rrŒ   Ú_r‡   s                    @r   Ú__init__zGGUFTokenizerSkeleton.__init__  s   ø€ Ø—K‘K“Mò 	 ‰DˆAˆqÜ�D˜!˜QÕð	 ô �t˜XÕ&Ü˜4 Ô*´'¸$ÀÔ2IÜ Øróð ð —[‘[ˆFØ—[‘[ˆFÜ.7¸Ó.?×@¡d a¨�Q˜˜q™	‘\Ó@ˆEä�N‰NÐWÔXØˆFÜ&*¨5¯;©;«=Ó&9ò %Ñ"��{Ø�Ü" 1¤c¨%£jÓ1ò F�EØ',¨V¨e }°e¸E¸F°m˜W�GØ &Ò(¨W¸Ò->ØŸ™ g¨w¸Ð%DÕEðFô ˜uÓ*NÐX\Ô]�Ø—‘˜eÕ$ð%ô ˜FÑ(:ÀDÔIˆFØ28Ö9¨3�s˜1‘v˜s 1™vÒ&Ð9ˆFÐ9Ø ˆD�Kà@DÇÁÖL°uœ5 §¡¨SÓ!1Õ2ÒLˆDŒKÜ˜4 Ô*Ü-2´3°t·{±{Ó3CÓ-DÖE¨štÒE�”ä�t˜^Ô,Ø "ˆDÔä�t˜^Ô,Ø $ˆDÔô �4Ð+Ô,°×1BÑ1BÐ1JØ $× 5Ñ 5ˆDÕð 2KÐ,ùó7 Aùò :ùò MùâEs   Á=H#ÅH)Å+!H.Ç 	H3N)Ú__name__Ú
__module__Ú__qualname__r©   r…   r€   r   r‚   r‚     s   „ ó'6r€   r‚   c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zd„ Zy)ÚGGUFLlamaConverterc                 ó’   — t        |«      | _        | j                  | _        i | _        t	        | j                  dd«      dk7  | _        y )Nr\   r   )r‚   ÚprotoÚoriginal_tokenizerÚadditional_kwargsÚgetattrÚis_llama_3_tokenizer©rœ   Útokenizer_dicts     r   r©   zGGUFLlamaConverter.__init__/  s>   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÔÜ$+¨D¯J©JÐ8HÈ'Ó$RÐV]Ñ$]ˆÕ!r€   c                 óT   — t        t        |j                  |j                  «      «      S ©N©rs   Úzipr]   r^   ©rœ   r°   s     r   r‡   zGGUFLlamaConverter.vocab5  ó   € Ü”C˜Ÿ™ e§l¡lÓ3Ó4Ð4r€   c                 ó   — |j                   S r¸   ©r`   r»   s     r   r`   zGGUFLlamaConverter.merges8  ó   € Ø�|‰|Ðr€   c           
      óÄ  — | j                  | j                  «      }| j                  | j                  «      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}|j                  �|j
                  |j                     nd }t        |dd «      �|j
                  |j                     nd }	t        |dd «      �|j
                  |j                     nd }
t        t        |||dd¬«      «      }g }t        | j                  d«      s^|�|j                  t        |dd¬«      «       |	�|j                  t        |	dd¬«      «       |
�›|j                  t        |
dd¬«      «       n}t        j                  t        j                  | j                  j                   «      dk(  «      d	   }|D ]6  }|j                  t        | j                  j
                  |   dd¬«      «       Œ8 t#        |«      d	k7  r|j%                  |«       t#        | j                  j&                  «      d	k7  r?|j)                  | j                  j&                  D �cg c]  }t        |dd¬«      ‘Œ c}«       || j*                  d
<   |	| j*                  d<   |
| j*                  d<   | j,                  r>d | j*                  d<   d| j*                  d<   d| j*                  d<   d| j.                  _        |S c c}}}w c c}w )Nr6   r7   T)Ú	unk_tokenÚfuse_unkÚbyte_fallbackr_   F©Ú
normalizedÚspecialrh   r   rÁ   Ú	eos_tokenÚ	bos_tokenra   Úclean_up_tokenization_spacesÚlegacy)r‡   r°   r`   r“   r8   r]   r³   r6   r   r	   r’   r—   r   ÚnpÚwherer   r_   rt   Úadd_special_tokensrŽ   Ú
add_tokensr²   r´   r±   rÊ   )rœ   r°   Úvocab_scoresr`   r    ÚwordÚ_scoreÚ	bpe_vocabrÁ   rÈ   rÇ   r5   Úspecial_tokensÚspecial_tokens_idxÚidxÚadded_tokens                   r   r5   zGGUFLlamaConverter.tokenizer;  s¨  € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÓ6M×NÐNÑ!2 ¡N T¨6�T˜1‘WÐNˆ	ÒNà8=×8JÑ8JÐ8V�E—L‘L ×!3Ñ!3Ò4Ð\`ˆ	Ü8?ÀÀ~ÐW[Ó8\Ð8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	Ü8?ÀÀ~ÐW[Ó8\Ð8h�E—L‘L ×!3Ñ!3Ò4Ðnrˆ	äÜØØØ#ØØ"ôó
ˆ	ð ˆä�t—z‘z <Ô0ØÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Ô\àÐ$Ø×%Ñ%¤j°ÀuÐVZÔ&[Õ\ô "$§¡¬"¯(©(°4·:±:×3HÑ3HÓ*IÈQÑ*NÓ!OÐPQÑ!RÐà)ò j�Ø×%Ñ%¤j°·±×1BÑ1BÀ3Ñ1GÐTYÐcgÔ&hÕiðjô ˆ~Ó !Ò#Ø×(Ñ(¨Ô8äˆt�z‰z×&Ñ&Ó'¨1Ò,Ø× Ñ Ø]a×]gÑ]g×]tÑ]tÖuÈk”˜K°EÀ5ÖIÒuôð /8ˆ×Ñ˜{Ñ+Ø.7ˆ×Ñ˜{Ñ+Ø.7ˆ×Ñ˜{Ñ+à×$Ò$Ø9=ˆD×"Ñ"Ð#5Ñ6ØEIˆD×"Ñ"Ð#AÑBà/4ˆD×"Ñ" 8Ñ,Ø-2ˆD×#Ñ#Ô*àÐùôk OùòN vs   ÁKÉKc                 ó0  — t        j                  «       t        j                  «       t        j                  dd«      g}| j                  r|t        j
                  ddd¬«      gz  }|r|t        j                  dd¬«      gz  }t        j                  |«      S )Nõ   â–�r�   FT©ra   Útrim_offsetsÚ	use_regexrf   ©ÚcontentÚleft)r   ÚByteFallbackÚFuseÚReplacer´   Ú	ByteLevelÚStripÚSequence©rœ   Úreplacementra   Úsequences       r   ÚdecoderzGGUFLlamaConverter.decoderu  sƒ   € ä×!Ñ!Ó#Ü�M‰M‹OÜ×Ñ˜U CÓ(ð
ˆð ×$Ò$Øœ×+Ñ+¸UÐQVÐbfÔgÐhÑhˆHáØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r€   c                 ó   — | j                  | j                  «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j
                  }| j                  ||«      }|�||_        | j                  ||«      |_        | j                  «       }|r||_        | j                  r7t        j                  ddd¬«      |_        t        j                  g «      |_        |S )NrØ   Tra   FrÙ   )r5   r°   Ú
normalizerr’   r±   ra   Úpre_tokenizerrè   Úpost_processorr´   r   râ   r   rä   )rœ   r5   rê   ræ   ra   rë   rì   s          r   Ú	convertedzGGUFLlamaConverter.convertedƒ  sò   € à—N‘N 4§:¡:Ó.ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$ð ×$Ò$Ü&4×&>Ñ&>Ø!&°UÀdô'ˆIÔ#ô
 $/×#7Ñ#7¸Ó#;ˆIÔ àÐr€   N)	rª   r«   r¬   r©   r‡   r`   r5   rè   rí   r…   r€   r   r®   r®   .  s"   „ ò^ò5òò8òt+ó!r€   r®   c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFQwen2Converterc                 ó2   — t        |«      | _        i | _        y r¸   ©r‚   r±   r²   rµ   s     r   r©   zGGUFQwen2Converter.__init__¨  ó   € Ü"7¸Ó"GˆÔØ!#ˆÕr€   Úreturnc           	      ó4  •— t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}| j                  j                  }t        ‰| �  ||«      }|j                  t        ddd¬«      t        ddd¬«      t        ddd¬«      g«       |S c c}}w )Nú<|endoftext|>FTrÄ   z<|im_start|>z
<|im_end|>)r“   r±   r]   r`   Úsuperrí   rÍ   r   ©rœ   r    rÐ   r‡   r`   r5   Ú	__class__s         €r   rí   zGGUFQwen2Converter.converted¬  s–   ø€ Ü(1°$×2IÑ2I×2PÑ2PÓ(Q×R™W˜Q ��q‘ÐRˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% e¨VÓ4ˆ	à×$Ñ$ä˜?°uÀdÔKÜ˜>°eÀTÔJÜ˜<°EÀ4ÔHðô	
ð Ðùó Ss   ¤B©rª   r«   r¬   r©   r   rí   Ú__classcell__©rø   s   @r   rï   rï   §  s   ø„ ò$ð˜9÷ ñ r€   rï   c                   ó6   — e Zd Zd„ Zd„ Zd„ Zd„ Zd„ Zdefd„Z	y)	ÚGGUFPhi3Converterc                 óT   — t        |«      | _        | j                  | _        i | _        y r¸   ©r‚   r°   r±   r²   rµ   s     r   r©   zGGUFPhi3Converter.__init__¼  s"   € Ü*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕr€   c                 óT   — t        t        |j                  |j                  «      «      S r¸   r¹   r»   s     r   r‡   zGGUFPhi3Converter.vocabÁ  r¼   r€   c                 ó   — |j                   S r¸   r¾   r»   s     r   r`   zGGUFPhi3Converter.mergesÄ  r¿   r€   c                 ó  — | j                  | j                  «      }| j                  | j                  «      }t        |«      D ���ci c]  \  }\  }}||“Œ }}}}t	        t        ||«      «      }|j                  t        ddddd¬«      t        ddd¬«      t        dddd¬«      t        d	ddd¬«      t        d
ddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      t        dddd¬«      g«       |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |j                  �|j                  |j                     nd | j                  d<   |S c c}}}w )Nú</s>TF)ÚrstripÚlstriprÅ   rÆ   rõ   rÄ   z<|assistant|>)r  rÅ   rÆ   z<|placeholder1|>z<|placeholder2|>z<|placeholder3|>z<|placeholder4|>z
<|system|>z<|end|>z<|placeholder5|>z<|placeholder6|>z<|user|>rÁ   rÇ   rÈ   Ú	pad_token)r‡   r°   r`   r“   r   r	   rÍ   r   r8   r]   r²   r7   r6   r9   )	rœ   r°   rÏ   r`   r    rÐ   rÑ   rÒ   r5   s	            r   r5   zGGUFPhi3Converter.tokenizerÇ  s   € Ø—z‘z $§*¡*Ó-ˆØ—‘˜TŸZ™ZÓ(ˆÜ6?ÀÓ6M×NÐNÑ!2 ¡N T¨6�T˜1‘WÐNˆ	ÒNäœc )¨VÓ4Ó5ˆ	à×$Ñ$ä˜6¨$°uÈÐX\Ô]Ü˜?°uÀdÔKÜ˜?°4ÀEÐSWÔXÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜<°ÀÐPTÔUÜ˜9¨T¸eÈTÔRÜÐ-°dÀuÐVZÔ[ÜÐ-°dÀuÐVZÔ[Ü˜:¨d¸uÈdÔSðô	
ð$ 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð 16×0BÑ0BÐ0NˆE�L‰L˜×+Ñ+Ò,ÐTXð 	×Ñ˜{Ñ+ð ÐùôE Os   ÁG:c                 óà   — t        j                  «       t        j                  «       t        j                  |d«      g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S )Nr�   rf   rÜ   )r   rß   rà   rá   rã   rä   rå   s       r   rè   zGGUFPhi3Converter.decoderî  s\   € ä×!Ñ!Ó#Ü�M‰M‹OÜ×Ñ˜[¨#Ó.ð
ˆñ ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r€   ró   c                 óÊ   — | j                  | j                  «      }d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      |_        |S )NrØ   Tra   )r5   r°   r’   r±   ra   rè   )rœ   r5   ræ   ra   s       r   rí   zGGUFPhi3Converter.convertedù  s\   € Ø—N‘N 4§:¡:Ó.ˆ	àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà ŸL™L¨Ð6FÓGˆ	ÔàÐr€   N)
rª   r«   r¬   r©   r‡   r`   r5   rè   r   rí   r…   r€   r   rý   rý   »  s(   „ ò$ò
5òò%òN	+ð
˜9ô 
r€   rý   c                   ó*   ‡ — e Zd Zd„ Zdefˆ fd„Zˆ xZS )ÚGGUFGPTConverterc                 ó2   — t        |«      | _        i | _        y r¸   rñ   rµ   s     r   r©   zGGUFGPTConverter.__init__  rò   r€   ró   c                 óÄ   •— t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 }}}| j                  j                  }t        ‰| �  ||«      }|S c c}}w r¸   )r“   r±   r]   r`   rö   rí   r÷   s         €r   rí   zGGUFGPTConverter.converted  s]   ø€ Ü(1°$×2IÑ2I×2PÑ2PÓ(Q×R™W˜Q ��q‘ÐRˆÑRØ×(Ñ(×/Ñ/ˆÜ‘GÑ% e¨VÓ4ˆ	ØÐùó Ss   ¤Arù   rû   s   @r   r
  r
    s   ø„ ò$ð˜9÷ ñ r€   r
  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„Zy)ÚGGUFT5Converterc                 óÜ   — dg|d<   t        |«      | _        t        | j                  j                  «      D ��ci c]  \  }}||“Œ
 c}}| _        | j                  | _        i | _        y c c}}w ©Nz
dummy textr`   )r‚   r°   r“   r]   Útoken2idr±   r²   )rœ   r¶   rŸ   rž   s       r   r©   zGGUFT5Converter.__init__  s]   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ü*3°D·J±J×4EÑ4EÓ*F×G¡$ ! Q˜˜A™ÓGˆŒØ"&§*¡*ˆÔØ!#ˆÕùó Hs   ¹A(c                 óT   — t        t        |j                  |j                  «      «      S r¸   r¹   r»   s     r   r‡   zGGUFT5Converter.vocab  r¼   r€   c                 óø   — t        | j                  dd«      rcg }t        | j                  dd«      r|t        j                  d¬«      gz  }|t        j                  dd¬«      gz  }t        j
                  |«      S y )NrÊ   Tra   rØ   )Úprependr�   )ÚpatternrÝ   )r³   r±   r   ÚPrependrá   rä   )rœ   r°   rç   s      r   rê   zGGUFT5Converter.normalizer  sr   € Ü�4×*Ñ*¨H°dÔ;ØˆHÜ�t×.Ñ.Ð0BÀDÔIØœ[×0Ñ0¸Ô?Ð@Ñ@�Øœ×,Ñ,°SÀ%ÔHÐIÑIˆHÜ×'Ñ'¨Ó1Ð1Ør€   c                 óZ   — t        j                  ddgg d¢d| j                  d   fg¬«      S )Nú$Ar  )r  r  z$Br  )ÚsingleÚpairrÓ   )r   ÚTemplateProcessingr  )rœ   s    r   rì   zGGUFT5Converter.post_processor(  s5   € Ü×,Ñ,Ø˜&�>Ú-à˜Ÿ™ vÑ.Ð/ðô
ð 	
r€   ró   c                 óÐ  — | j                  | j                  «      }t        t        || j                  j                  d¬«      «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      }|�||_	        | j                  ||«      |_
        | j                  «       }|r||_        |S )NF©Úunk_idrÃ   rØ   Tra   )r‡   r°   r   r
   r8   rê   r’   r±   ra   rë   rè   rì   )rœ   rÏ   r5   rê   ræ   ra   rë   rì   s           r   rí   zGGUFT5Converter.converted1  sÝ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø#ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#à ŸL™L¨Ð6FÓGˆ	ÔØ×,Ñ,Ó.ˆÙØ'5ˆIÔ$àÐr€   N)	rª   r«   r¬   r©   r‡   rê   rì   r   rí   r…   r€   r   r  r    s"   „ ò$ò5òò
ð˜9ô r€   r  c                   ó0   — e Zd Zd„ Zd„ Zd„ Zd„ Zdefd„Zy)ÚGGUFGemmaConverterc                 ó`   — dg|d<   t        |«      | _        | j                  | _        i | _        y r  rÿ   rµ   s     r   r©   zGGUFGemmaConverter.__init__R  s.   € à$0 >ˆ�xÑ ä*¨>Ó:ˆŒ
Ø"&§*¡*ˆÔØ!#ˆÕr€   c                 óL  — t        t        |j                  |j                  «      «      }g }|D ]s  \  }}|dk(  r|j	                  d|f«       Œd|v r>t        |j                  «       «      dk(  r"dt        |«      z  }|j	                  ||f«       Œa|j	                  ||f«       Œu |S )Nz<0x09>ú	r�   r   rØ   )rs   rº   r]   r^   r—   rt   Ústrip)rœ   r°   Úoriginal_vocabÚupdated_vocabÚtokenÚscoreÚunderscoress          r   r‡   zGGUFGemmaConverter.vocabZ  s¡   € Üœc %§,¡,°·±Ó=Ó>ˆØˆà*ò 	5‰LˆE�5Ø˜Ò Ø×$Ñ$ d¨E ]Õ3Ø˜‘¤# e§k¡k£mÓ"4¸Ò"9Ø#¤c¨%£jÑ0�Ø×$Ñ$ k°5Ð%9Õ:à×$Ñ$ e¨U ^Õ4ð	5ð Ðr€   c                 ó.   — t        j                  dd«      S )Nr�   rØ   )r   rá   r»   s     r   rê   zGGUFGemmaConverter.normalizeri  s   € Ü×"Ñ" 3¨Ó.Ð.r€   c                 óà   — t        j                  dd«      t        j                  «       t        j                  «       g}|r|t        j                  dd¬«      gz  }t        j
                  |«      S )NrØ   r�   rf   rÜ   )r   rá   rß   rà   rã   rä   rå   s       r   rè   zGGUFGemmaConverter.decoderl  s\   € ä×Ñ˜U CÓ(Ü×!Ñ!Ó#Ü�M‰M‹Oð
ˆñ ØœŸ™°¸!Ô<Ð=Ñ=ˆHÜ× Ñ  Ó*Ð*r€   ró   c                 ó²  — | j                  | j                  «      }t        t        || j                  j                  | j
                  ¬«      «      }| j                  | j                  «      }|�||_        d}d}t        | j                  d«      r| j                  j                  }| j                  ||«      |_
        | j                  ||«      }|�||_        |S )Nr  rØ   Tra   )r‡   r°   r   r
   r8   Úhandle_byte_fallbackrê   r’   r±   ra   rè   rë   )rœ   rÏ   r5   rê   ræ   ra   rë   s          r   rí   zGGUFGemmaConverter.convertedw  sÊ   € Ø—z‘z $§*¡*Ó-ˆÜÜØØ—z‘z×.Ñ.Ø"×7Ñ7ôó
ˆ	ð —_‘_ T§Z¡ZÓ0ˆ
ØÐ!Ø#-ˆIÔ àˆØÐÜ�4×*Ñ*Ð,>Ô?Ø#×6Ñ6×GÑGÐà ŸL™L¨Ð6FÓGˆ	ÔØ×*Ñ*¨;Ð8HÓIˆØÐ$Ø&3ˆIÔ#àÐr€   N)	rª   r«   r¬   r©   r‡   rê   rè   r   rí   r…   r€   r   r   r   Q  s"   „ ò$òò/ò	+ð˜9ô r€   r   )r   r.   Ú	qwen2_moer>   r?   r4   rN   rP   rS   rD   rU   rY   r[   ró   c                 ó^   — | }t        |   |«      }|j                  «       }||j                  fS )a6  
    Utilities to convert a slow tokenizer instance in a fast tokenizer instance.

    Args:
        architecture (`str`): The model architecture derived from gguf file.
        transformer_tokenizer ([`~tokenization_utils_base.PreTrainedTokenizer`]):
            Instance of a slow tokenizer to convert in the backend tokenizer for
            [`~tokenization_utils_base.PreTrainedTokenizerFast`].

    Return:
        A instance of [`~tokenizers.Tokenizer`] to be used as the backend tokenizer of a
        [`~tokenization_utils_base.PreTrainedTokenizerFast`]
    )ÚGGUF_TO_FAST_CONVERTERSrí   r²   )r   r¶   Útokenizer_class_nameÚ	converterÚfast_tokenizers        r   Úconvert_gguf_tokenizerr4  £  s9   € ð (ÐÜ'Ð(<Ñ=¸nÓM€IØ×(Ñ(Ó*€NØ˜9×6Ñ6Ð6Ð6r€   )(Ú__doc__r   ÚnumpyrË   Ú
tokenizersr   r   r   r   r   Útokenizers.modelsr	   r
   Ú r   Úconvert_slow_tokenizerr   r   r   r   r   Úutilsr   Úutils.loggingr   Ú
get_loggerrª   r”   ÚGGUF_CONFIG_MAPPINGÚGGUF_TOKENIZER_MAPPINGr{   r‚   r®   rï   rý   r
  r  r   r0  r4  r…   r€   r   ú<module>r@     sÛ  ðñ õ
 ã ß SÕ Sß *å ß oÕ oÝ Ý  ð 
ˆ×	Ñ	˜HÓ	%€ðmØØ$Ø%ñðmð
 Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñðmð$ Ø3Ø*Ø2Ø)à *Ø&Ø 5Ø#8Ø,:Ø"ñð%mð> Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñð?mðV Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"Ø%Ø2ñðWmðr Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðsmðJ Ø+Ø+Ø!/Ø!/ñ	ðKmðV Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,:Ø"ñðWmðn Ø Ø)Ø (Ø"Ø(<ñðomð| 	Ø'Ø#Ø%Ø%Ø &Ø +Ø#8Ø(<Ø,LØ":Ø"ñð}mðV Ø3Ø*Ø2Ø)Ø $Ø 5Ø#8Ø(8Ø"ñ
ðWmðl Ø Ø!Ø$Ø4Ø (Ø(<ñðmmð| Ø*Ø3Ø)Ø2Ø 5Ø#8Ø(6ñð}mðN Ø"Ø3Ø)Ø,@Ø*Ø(Ø&Ø.Ø-ñ
ðOmðd Ø3Ø*Ø2Ø)Ø $Ø&Ø 5Ø#8Ø,6Ø"ñðemð| Ø3Ø*Ø2Ø)Ø $Ø&ð !+Ø 5Ø#8Ø,:Ø"ñð}mÐ ðb 'ØØØ'ØØ+Ø+Ø!/Ø!/Ø!3ñð )Ø"Ø+Ø+Ø!/Ø!/ññÐ ò0÷0(6ñ (6ôVv˜ô vôr˜ô ô(H˜ô HôV	�}ô 	ô<�kô <ô~>˜ô >ðD  ØØ#ØØØØ ØØ"Ø
ØØ Ø ñÐ ð"7¸Iô 7r€   