Ë
    S^(h—  ã                   óä  — d Z ddlZddlZddlmZmZ ddlZddlm	Z	 ddl
mZ  ej                  e«      Zddd	œZi d
d“dd“dd“dd“dd“dd“dd“dd“dd“dd“dd“d d!“d"d#“d$d%“d&d'“d(d)“d*d+“i d,d-“d.d/“d0d1“d2d3“d4d5“d6d7“d8d9“d:d;“d<d=“d>d?“d@dA“dBdC“dDdE“dFdG“dHdI“dJdK“dLdM“¥i dNdO“dPdQ“dRdS“dTdU“dVdW“dXdY“dZd[“d\d]“d^d_“d`da“dbdc“ddde“dfdg“dhdi“djdk“dldm“dndo“¥dpdqdrdsdtœ¥Zdu„ Z G dv„ dwe	«      ZdwgZy)xz)Tokenization classes for Salesforce CTRL.é    N)ÚOptionalÚTupleé   )ÚPreTrainedTokenizer)Úloggingz
vocab.jsonz
merges.txt)Ú
vocab_fileÚmerges_fileÚ	Pregnancyiµ’ ÚChristianityiû  ÚExplaini·Ÿ ÚFitnessiÐ÷  ÚSavingi»ö  ÚAski#j  ÚAssiñv ÚJokeiµ~ Ú	Questionsi6²  ÚThoughtsiÅÁ  ÚRetailivÌ  ÚFeminismiò� ÚWritingiØ.  ÚAtheismiï ÚNetflixiè½  Ú	Computingi×š  ÚOpinioniÍ¨  ÚAlonei§¯  ÚFunnyi%æ  ÚGamingi¦�  ÚHumaniø  ÚIndiai3  ÚJokeriR- ÚDietin�  ÚLegaliS.  ÚNormaniK  ÚTipiñ ÚWeightiwÌ  ÚMoviesiÁ´  ÚRunningi�[  ÚSciencei*  ÚHorrori¡“  Ú
Confessioniœì  ÚFinanceiÚ/  ÚPoliticsiè?  ÚScaryiñí ÚSupportin1  ÚTechnologiesi  ÚTeenageip ÚEventi€  ÚLearnedi„ ÚNotioniòÉ Ú	WikipediaiÏ’  ÚBooksi	  ÚExtracti) ÚConfessionsi-‘ Ú
Conspiracyiœ( ÚLinksiºø  Ú	Narcissusi™K ÚRelationshipiîÕ  ÚRelationshipsiŒ iÇ¢  i   iÄh  i–õ )ÚReviewsÚNewsÚTranslationÚmultilingualc                 óx   — t        «       }| d   }| dd D ]  }|j                  ||f«       |}Œ t        |«      }|S )z…
    Return set of symbol pairs in a word.

    Word is represented as tuple of symbols (symbols being variable-length strings).
    r   é   N)ÚsetÚadd)ÚwordÚpairsÚ	prev_charÚchars       úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/ctrl/tokenization_ctrl.pyÚ	get_pairsrJ   ^   sO   € ô ‹E€EØ�Q‘€IØ�Q�R�ò ˆØ�	‰	�9˜dÐ#Ô$Ø‰	ðô �‹J€EØ€Ló    c                   ó‚   ‡ — e Zd ZdZeZeZdˆ fd„	Ze	d„ «       Z
d„ Zd„ Zd„ Zd„ Zd„ Zd	„ Zdd
edee   dee   fd„Zˆ xZS )ÚCTRLTokenizera`  
    Construct a CTRL tokenizer. Based on Byte-Pair-Encoding.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
    c           
      óV  •— t        |d¬«      5 }t        j                  |«      | _        d d d «       | j                  j	                  «       D ��ci c]  \  }}||“Œ
 c}}| _        t        |d¬«      5 }|j                  «       j                  d«      dd }	d d d «       	D �
cg c]  }
t        |
j                  «       «      ‘Œ }	}
t        t        |	t        t        |	«      «      «      «      | _        i | _        t        ‰| �@  dd|i|¤Ž y # 1 sw Y   ŒßxY wc c}}w # 1 sw Y   Œ†xY wc c}
w )Núutf-8©Úencodingú
rB   éÿÿÿÿÚ	unk_token© )ÚopenÚjsonÚloadÚencoderÚitemsÚdecoderÚreadÚsplitÚtupleÚdictÚzipÚrangeÚlenÚ	bpe_ranksÚcacheÚsuperÚ__init__)Úselfr   r	   rT   ÚkwargsÚvocab_handleÚkÚvÚmerges_handleÚmergesÚmergeÚ	__class__s              €rI   rf   zCTRLTokenizer.__init__‚   sö   ø€ Ü�* wÔ/ð 	3°<ÜŸ9™9 \Ó2ˆDŒL÷	3à)-¯©×);Ñ);Ó)=×>¡  A˜˜1™Ó>ˆŒÜ�+¨Ô0ð 	<°MØ"×'Ñ'Ó)×/Ñ/°Ó5°a¸Ð;ˆF÷	<à4:Ö;¨5”%˜Ÿ™›Õ&Ð;ˆÐ;Üœc &¬%´°F³Ó*<Ó=Ó>ˆŒØˆŒ
Ü‰ÑÑ7 9Ð7°Ó7÷	3ð 	3üã>÷	<ð 	<üâ;s#   �DÁDÁ2#DÂ! D&ÄDÄD#c                 ó,   — t        | j                  «      S ©N)rb   rY   ©rg   s    rI   Ú
vocab_sizezCTRLTokenizer.vocab_size�   s   € ä�4—<‘<Ó Ð rK   c                 óB   — t        | j                  fi | j                  ¤ŽS rq   )r_   rY   Úadded_tokens_encoderrr   s    rI   Ú	get_vocabzCTRLTokenizer.get_vocab‘   s   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>rK   c                 ó$  ‡ — |‰ j                   v r‰ j                   |   S t        |«      }t        t        |d d «      |d   dz   gz   «      }t        |«      }|s|S 	 t	        |ˆ fd„¬«      }|‰ j
                  vrnÎ|\  }}g }d}|t        |«      k  r�	 |j                  ||«      }	|j                  |||	 «       |	}||   |k(  r6|t        |«      dz
  k  r%||dz      |k(  r|j                  ||z   «       |dz  }n|j                  ||   «       |dz  }|t        |«      k  rŒ�t        |«      }|}t        |«      dk(  rnt        |«      }Œídj                  |«      }|d d	 }|‰ j                   |<   |S # t        $ r |j                  ||d  «       Y Œnw xY w)
NrS   z</w>c                 óN   •— ‰j                   j                  | t        d«      «      S )NÚinf)rc   ÚgetÚfloat)Úpairrg   s    €rI   ú<lambda>z#CTRLTokenizer.bpe.<locals>.<lambda>Ÿ   s   ø€ °·±×1CÑ1CÀDÌ%ÐPUË,Ó1W€ rK   ©Úkeyr   rB   é   ú@@ éüÿÿÿ)rd   r^   ÚlistrJ   Úminrc   rb   ÚindexÚextendÚ
ValueErrorÚappendÚjoin)
rg   ÚtokenrE   rF   ÚbigramÚfirstÚsecondÚnew_wordÚiÚjs
   `         rI   ÚbpezCTRLTokenizer.bpe”   s±  ø€ Ø�D—J‘JÑØ—:‘:˜eÑ$Ð$Ü�U‹|ˆÜ”T˜$˜s ˜)“_¨¨R©°6Ñ(9Ð':Ñ:Ó;ˆÜ˜$“ˆáØˆLàÜ˜Ó$WÔXˆFØ˜TŸ^™^Ñ+ØØ"‰MˆE�6ØˆHØˆAØ”c˜$“i’-ðØŸ
™
 5¨!Ó,�Að
 —O‘O D¨¨1 IÔ.Ø�Aà˜‘7˜eÒ#¨¬C°«I¸©MÒ(9¸dÀ1ÀqÁ5¹kÈVÒ>SØ—O‘O E¨F¡NÔ3Ø˜‘F‘Aà—O‘O D¨¡GÔ,Ø˜‘F�Að ”c˜$“i“-ô  ˜X“ˆHØˆDÜ�4‹y˜AŠ~Øä! $›�ð9 ð: �z‰z˜$ÓˆØ�C�RˆyˆØ ˆ�
‰
�5ÑØˆøô- "ò Ø—O‘O D¨¨ HÔ-Ùðús   ÂE/ Å/FÆFc                 ó´   — g }t        j                  d|«      }|D ]:  }|j                  t        | j	                  |«      j                  d«      «      «       Œ< |S )zTokenize a string.z\S+\n?ú )ÚreÚfindallr†   rƒ   r‘   r]   )rg   ÚtextÚsplit_tokensÚwordsrŠ   s        rI   Ú	_tokenizezCTRLTokenizer._tokenizeÀ   sT   € àˆä—
‘
˜9 dÓ+ˆàò 	BˆEØ×Ñ¤ T§X¡X¨e£_×%:Ñ%:¸3Ó%?Ó @ÕAð	BàÐrK   c                 ó€   — | j                   j                  || j                   j                  | j                  «      «      S )z0Converts a token (str) in an id using the vocab.)rY   rz   rT   )rg   rŠ   s     rI   Ú_convert_token_to_idz"CTRLTokenizer._convert_token_to_idÊ   s,   € à�|‰|×Ñ  t§|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHrK   c                 óN   — | j                   j                  || j                  «      S )z=Converts an index (integer) in a token (str) using the vocab.)r[   rz   rT   )rg   r…   s     rI   Ú_convert_id_to_tokenz"CTRLTokenizer._convert_id_to_tokenÎ   s   € à�|‰|×Ñ  t§~¡~Ó6Ð6rK   c                 ód   — dj                  |«      j                  dd«      j                  «       }|S )z:Converts a sequence of tokens (string) in a single string.r“   r�   Ú )r‰   ÚreplaceÚstrip)rg   ÚtokensÚ
out_strings      rI   Úconvert_tokens_to_stringz&CTRLTokenizer.convert_tokens_to_stringÒ   s,   € à—X‘X˜fÓ%×-Ñ-¨e°RÓ8×>Ñ>Ó@ˆ
ØÐrK   Úsave_directoryÚfilename_prefixÚreturnc           	      óP  — t         j                  j                  |«      st        j	                  d|› d�«       y t         j                  j                  ||r|dz   ndt        d   z   «      }t         j                  j                  ||r|dz   ndt        d   z   «      }t        |dd¬	«      5 }|j                  t        j                  | j                  d
dd¬«      dz   «       d d d «       d}t        |dd¬	«      5 }|j                  d«       t        | j                  j                  «       d„ ¬«      D ]M  \  }}	||	k7  rt        j                  d|› d�«       |	}|j                  dj                  |«      dz   «       |dz  }ŒO 	 d d d «       ||fS # 1 sw Y   Œ®xY w# 1 sw Y   ||fS xY w)NzVocabulary path (z) should be a directoryú-rŸ   r   r	   ÚwrO   rP   r€   TF)ÚindentÚ	sort_keysÚensure_asciirR   r   z#version: 0.2
c                 ó   — | d   S )NrB   rU   )Úkvs    rI   r}   z/CTRLTokenizer.save_vocabulary.<locals>.<lambda>è   s   € ÐY[Ð\]ÑY^€ rK   r~   zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!r“   rB   )ÚosÚpathÚisdirÚloggerÚerrorr‰   ÚVOCAB_FILES_NAMESrV   ÚwriterW   ÚdumpsrY   Úsortedrc   rZ   Úwarning)
rg   r¥   r¦   r   Ú
merge_fileÚfr…   ÚwriterÚ
bpe_tokensÚtoken_indexs
             rI   Úsave_vocabularyzCTRLTokenizer.save_vocabulary×   sœ  € Ü�w‰w�}‰}˜^Ô,Ü�L‰LÐ,¨^Ð,<Ð<SÐTÔUØÜ—W‘W—\‘\Ø±o˜_¨sÒ2È2ÔQbÐcoÑQpÑpó
ˆ
ô —W‘W—\‘\Ø±o˜_¨sÒ2È2ÔQbÐcpÑQqÑqó
ˆ
ô �*˜c¨GÔ4ð 	c¸Ø�G‰G”D—J‘J˜tŸ|™|°AÀÐTYÔZÐ]aÑaÔb÷	cð ˆÜ�*˜c¨GÔ4ð 
	¸Ø�L‰LÐ*Ô+Ü+1°$·.±.×2FÑ2FÓ2HÑN^Ô+_ò Ñ'�
˜KØ˜KÒ'Ü—N‘NØ/°
¨|ð <Mð Môð (�EØ—‘˜SŸX™X jÓ1°DÑ8Ô9Ø˜‘
‘ñ÷
	ð ˜:Ð%Ð%÷!	cð 	cú÷
	ð ˜:Ð%Ð%ús   Â*6FÃ8BFÆFÆF%)z<unk>rq   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__rµ   Úvocab_files_namesÚCONTROL_CODESÚcontrol_codesrf   Úpropertyrs   rv   r‘   r™   r›   r�   r¤   Ústrr   r   r¿   Ú__classcell__)ro   s   @rI   rM   rM   n   ss   ø„ ñð  *ÐØ!€Mõ	8ð ñ!ó ð!ò?ò*òXòIò7òñ
&¨cð &ÀHÈSÁMð &Ð]bÐcfÑ]g÷ &rK   rM   )rÃ   rW   r°   Útypingr   r   Úregexr”   Útokenization_utilsr   Úutilsr   Ú
get_loggerrÀ   r³   rµ   rÅ   rJ   rM   Ú__all__rU   rK   rI   ú<module>rÐ      s  ðñ 0ã Û 	ß "ã å 5Ý ð 
ˆ×	Ñ	˜HÓ	%€ð ØñÐ ð8Ø�ð8à�Dð8ð ˆvð8ð ˆuð	8ð
 ˆeð8ð 
ˆ5ð8ð 
ˆ5ð8ð ˆFð8ð �ð8ð �ð8ð ˆeð8ð �ð8ð ˆuð8ð ˆvð8ð ˆuð8ð  �ð!8ð" ˆuñ#8ð$ ˆUð%8ð& ˆUð'8ð( ˆeð)8ð* ˆTð+8ð, ˆTð-8ð. ˆUð/8ð0 ˆEð18ð2 ˆUð38ð4 ˆdð58ð6 
ˆ5ð78ð8 ˆeð98ð: ˆeð;8ð< ˆuð=8ð> ˆtð?8ð@ ˆeðA8ðB �%ðC8ðD ˆuòE8ðF �ðG8ðH ˆVðI8ðJ ˆuðK8ðL �EðM8ðN ˆuðO8ðP ˆUðQ8ðR ˆuðS8ðT ˆfðU8ðV �ðW8ðX ˆTðY8ðZ ˆuð[8ð\ �6ð]8ð^ �%ð_8ð` ˆUða8ðb �ðc8ðd �Eðe8ðf �Vñg8ðh ØØØòo8€òvô D&Ð'ô D&ðZ Ð
�rK   