Ë
    T^(hS;  ã                   óÌ   — d Z ddlZddlZddlZddlZddlmZmZ ddlm	Z	m
Z
mZmZ ddlmZ  ej                  e«      Zddd	œZd
„ Z G d„ d«      Zd„ Zd„ Z G d„ de	«      ZdgZy)z$Tokenization classes for OpenAI GPT.é    N)ÚOptionalÚTupleé   )ÚPreTrainedTokenizerÚ_is_controlÚ_is_punctuationÚ_is_whitespace)Úloggingz
vocab.jsonz
merges.txt)Ú
vocab_fileÚmerges_filec                 óN   — | j                  «       } | sg S | j                  «       }|S )z@Runs basic whitespace cleaning and splitting on a piece of text.)ÚstripÚsplit)ÚtextÚtokenss     úl/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/openai/tokenization_openai.pyÚwhitespace_tokenizer   $   s%   € à�:‰:‹<€DÙØˆ	Ø�Z‰Z‹\€FØ€Mó    c                   óJ   — e Zd ZdZ	 	 	 	 	 d
d„Zdd„Zd„ Zdd„Zd„ Zd„ Z	d	„ Z
y)ÚBasicTokenizeraª  
    Constructs a BasicTokenizer that will run basic tokenization (punctuation splitting, lower casing, etc.).

    Args:
        do_lower_case (`bool`, *optional*, defaults to `True`):
            Whether or not to lowercase the input when tokenizing.
        never_split (`Iterable`, *optional*):
            Collection of tokens which will never be split during tokenization. Only has an effect when
            `do_basic_tokenize=True`
        tokenize_chinese_chars (`bool`, *optional*, defaults to `True`):
            Whether or not to tokenize Chinese characters.

            This should likely be deactivated for Japanese (see this
            [issue](https://github.com/huggingface/transformers/issues/328)).
        strip_accents (`bool`, *optional*):
            Whether or not to strip all accents. If this option is not specified, then it will be determined by the
            value for `lowercase` (as in the original BERT).
        do_split_on_punc (`bool`, *optional*, defaults to `True`):
            In some instances we want to skip the basic punctuation splitting so that later tokenization can capture
            the full context of the words, such as contractions.
    Nc                 ód   — |€g }|| _         t        |«      | _        || _        || _        || _        y ©N)Údo_lower_caseÚsetÚnever_splitÚtokenize_chinese_charsÚstrip_accentsÚdo_split_on_punc)Úselfr   r   r   r   r   s         r   Ú__init__zBasicTokenizer.__init__E   s<   € ð ÐØˆKØ*ˆÔÜ˜{Ó+ˆÔØ&<ˆÔ#Ø*ˆÔØ 0ˆÕr   c                 óJ  — |r$| j                   j                  t        |«      «      n| j                   }| j                  |«      }| j                  r| j                  |«      }t        j                  d|«      }t        |«      }g }|D ]€  }||vrY| j                  r0|j                  «       }| j                  dur/| j                  |«      }n| j                  r| j                  |«      }|j                  | j                  ||«      «       Œ‚ t        dj                  |«      «      }|S )aj  
        Basic Tokenization of a piece of text. For sub-word tokenization, see WordPieceTokenizer.

        Args:
            never_split (`List[str]`, *optional*)
                Kept for backward compatibility purposes. Now implemented directly at the base class level (see
                [`PreTrainedTokenizer.tokenize`]) List of token not to split.
        ÚNFCFú )r   Úunionr   Ú_clean_textr   Ú_tokenize_chinese_charsÚunicodedataÚ	normalizer   r   Úlowerr   Ú_run_strip_accentsÚextendÚ_run_split_on_puncÚjoin)r   r   r   Úunicode_normalized_textÚorig_tokensÚsplit_tokensÚtokenÚoutput_tokenss           r   ÚtokenizezBasicTokenizer.tokenizeU   s  € ñ CN�d×&Ñ&×,Ñ,¬S°Ó-=Ô>ÐSW×ScÑScˆØ×Ñ Ó%ˆð ×&Ò&Ø×/Ñ/°Ó5ˆDä"-×"7Ñ"7¸¸tÓ"DÐÜ)Ð*AÓBˆØˆØ ò 	MˆEØ˜KÑ'Ø×%Ò%Ø!ŸK™K›M�EØ×)Ñ)°Ñ6Ø $× 7Ñ 7¸Ó >™Ø×'Ò'Ø ×3Ñ3°EÓ:�EØ×Ñ × 7Ñ 7¸¸{Ó KÕLð	Mô ,¨C¯H©H°\Ó,BÓCˆØÐr   c                 óº   — t        j                  d|«      }g }|D ].  }t        j                  |«      }|dk(  rŒ|j                  |«       Œ0 dj	                  |«      S )z$Strips accents from a piece of text.ÚNFDÚMnÚ )r'   r(   ÚcategoryÚappendr-   )r   r   ÚoutputÚcharÚcats        r   r*   z!BasicTokenizer._run_strip_accents{   s^   € ä×$Ñ$ U¨DÓ1ˆØˆØò 	 ˆDÜ×&Ñ& tÓ,ˆCØ�dŠ{ØØ�M‰M˜$Õð		 ð
 �w‰w�v‹Ðr   c                 óv  — | j                   r|�||v r|gS t        |«      }d}d}g }|t        |«      k  rb||   }t        |«      r|j	                  |g«       d}n)|r|j	                  g «       d}|d   j	                  |«       |dz  }|t        |«      k  rŒb|D �cg c]  }dj                  |«      ‘Œ c}S c c}w )z&Splits punctuation on a piece of text.r   TFéÿÿÿÿé   r7   )r   ÚlistÚlenr   r9   r-   )	r   r   r   ÚcharsÚiÚstart_new_wordr:   r;   Úxs	            r   r,   z!BasicTokenizer._run_split_on_punc†   s¿   € à×$Ò$¨Ð)@ÀTÈ[ÑEXØ�6ˆMÜ�T“
ˆØˆØˆØˆØ”#�e“*ŠnØ˜‘8ˆDÜ˜tÔ$Ø—‘˜t˜fÔ%Ø!%‘á!Ø—M‘M "Ô%Ø!&�Ø�r‘
×!Ñ! $Ô'Ø�‰FˆAð ”#�e“*‹nð %+Ö+˜q�—‘˜•
Ò+Ð+ùÒ+s   ÂB6c                 óø   — g }|D ]c  }t        |«      }| j                  |«      r4|j                  d«       |j                  |«       |j                  d«       ŒS|j                  |«       Œe dj                  |«      S )z)Adds whitespace around any CJK character.r#   r7   )ÚordÚ_is_chinese_charr9   r-   ©r   r   r:   r;   Úcps        r   r&   z&BasicTokenizer._tokenize_chinese_charsœ   sm   € àˆØò 	$ˆDÜ�T“ˆBØ×$Ñ$ RÔ(Ø—‘˜cÔ"Ø—‘˜dÔ#Ø—‘˜cÕ"à—‘˜dÕ#ð	$ð �w‰w�v‹Ðr   c                 ó¦   — |dk\  r|dk  sF|dk\  r|dk  s<|dk\  r|dk  s2|dk\  r|dk  s(|d	k\  r|d
k  s|dk\  r|dk  s|dk\  r|dk  s
|dk\  r|dk  ryy)z6Checks whether CP is the codepoint of a CJK character.i N  iÿŸ  i 4  i¿M  i   iß¦ i § i?· i@· i¸ i ¸ i¯Î i ù  iÿú  i ø iú TF© )r   rJ   s     r   rH   zBasicTokenizer._is_chinese_char©   sr   € ð �6Š\˜b FšlØ�f’  v¢Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�g’ "¨¢-Ø�f’  v¢Ø�g’ "¨¢-ààr   c                 óÔ   — g }|D ]Q  }t        |«      }|dk(  s|dk(  st        |«      rŒ$t        |«      r|j                  d«       ŒA|j                  |«       ŒS dj	                  |«      S )zBPerforms invalid character removal and whitespace cleanup on text.r   iýÿ  r#   r7   )rG   r   r	   r9   r-   rI   s        r   r%   zBasicTokenizer._clean_textÁ   sf   € àˆØò 	$ˆDÜ�T“ˆBØ�QŠw˜" š,¬+°dÔ*;ØÜ˜dÔ#Ø—‘˜cÕ"à—‘˜dÕ#ð	$ð �w‰w�v‹Ðr   )TNTNTr   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r    r3   r*   r,   r&   rH   r%   rL   r   r   r   r   .   s<   „ ñð0 ØØ#ØØó1ó $òL	ó,ò,òó0r   r   c                 ób   — t        «       }| d   }| dd D ]  }|j                  ||f«       |}Œ |S )zƒ
    Return set of symbol pairs in a word. word is represented as tuple of symbols (symbols being variable-length
    strings)
    r   r?   N)r   Úadd)ÚwordÚpairsÚ	prev_charr;   s       r   Ú	get_pairsrW   Ï   sF   € ô
 ‹E€EØ�Q‘€IØ�Q�R�ò ˆØ�	‰	�9˜dÐ#Ô$Ø‰	ðð €Lr   c                 ó`  — | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } | j                  dd«      } t        j                  d	d
| «      } t        j                  dd| «      } t        j                  dd| «      } | j                  «       S )zm
    fixes some issues the spacy tokenizer had on books corpus also does some whitespace standardization
    u   â€”ú-u   â€“u   â€•u   â€¦z...õ   Â´ú'zD(-+|~+|!+|"+|;+|\?+|\++|,+|\)+|\(+|\\+|\/+|\*+|\[+|\]+|}+|{+|\|+|_+)z \1 z\s*\n\s*z 
 z[^\S\n]+r#   )ÚreplaceÚreÚsubr   )r   s    r   Útext_standardizer_   Ü   s™   € ð �<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜sÓ#€DØ�<‰<˜˜uÓ%€DØ�<‰<˜˜cÓ"€DÜ�6‰6Ð]Ð_fÐhlÓm€DÜ�6‰6�+˜v tÓ,€DÜ�6‰6�+˜s DÓ)€DØ�:‰:‹<Ðr   c                   ó–   ‡ — e Zd ZdZeZddgZdˆ fd„	Zed„ «       Z	ed„ «       Z
d„ Zd„ Zd	„ Zd
„ Zd„ Zd„ Zddedee   dee   fd„Zˆ xZS )ÚOpenAIGPTTokenizera(  
    Construct a GPT Tokenizer. Based on Byte-Pair-Encoding with the following peculiarities:

    - lowercases all inputs,
    - uses `SpaCy` tokenizer and `ftfy` for pre-BPE tokenization if they are installed, fallback to BERT's
      `BasicTokenizer` if not.

    This tokenizer inherits from [`PreTrainedTokenizer`] which contains most of the main methods. Users should refer to
    this superclass for more information regarding those methods.

    Args:
        vocab_file (`str`):
            Path to the vocabulary file.
        merges_file (`str`):
            Path to the merges file.
        unk_token (`str`, *optional*, defaults to `"<unk>"`):
            The unknown token. A token that is not in the vocabulary cannot be converted to an ID and is set to be this
            token instead.
    Ú	input_idsÚattention_maskc           
      ó:  •— 	 dd l }ddlm}  |«       }|j                  | _        |j
                  | _        t        |d¬«      5 }t        j                  |«      | _        d d d «       | j                  j                  «       D �	�
ci c]  \  }	}
|
|	“Œ
 c}
}	| _        t        |d¬«      5 }|j!                  «       j#                  d«      d	d
 }d d d «       D �cg c]  }t%        |j#                  «       «      ‘Œ }}t'        t)        |t+        t-        |«      «      «      «      | _        i | _        t3        ‰| �h  dd|i|¤Ž y # t        $ r1 t        j                  d«       t        d¬«      | _        d | _        Y �Œ?w xY w# 1 sw Y   �ŒxY wc c}
}	w # 1 sw Y   ŒÄxY wc c}w )Nr   )ÚEnglishzQftfy or spacy is not installed using BERT BasicTokenizer instead of SpaCy & ftfy.T)r   úutf-8©Úencodingú
r?   r>   Ú	unk_tokenrL   )ÚftfyÚspacy.lang.enre   Ú	tokenizerÚnlpÚfix_textÚImportErrorÚloggerÚwarningr   ÚopenÚjsonÚloadÚencoderÚitemsÚdecoderÚreadr   ÚtupleÚdictÚzipÚrangerA   Ú	bpe_ranksÚcacheÚsuperr    )r   r   r   rj   Úkwargsrk   re   Ú_nlpÚvocab_handleÚkÚvÚmerges_handleÚmergesÚmergeÚ	__class__s                 €r   r    zOpenAIGPTTokenizer.__init__  sQ  ø€ ð
	!ÛÝ-á“9ˆDØ—~‘~ˆDŒHØ ŸM™MˆDŒMô �* wÔ/ð 	3°<ÜŸ9™9 \Ó2ˆDŒL÷	3à)-¯©×);Ñ);Ó)=×>¡  A˜˜1™Ó>ˆŒÜ�+¨Ô0ð 	<°MØ"×'Ñ'Ó)×/Ñ/°Ó5°a¸Ð;ˆF÷	<à4:Ö;¨5”%˜Ÿ™›Õ&Ð;ˆÐ;Üœc &¬%´°F³Ó*<Ó=Ó>ˆŒØˆŒ
ä‰ÑÑ7 9Ð7°Ó7øô ò 	!Ü�N‰NÐnÔoÜ%°DÔ9ˆDŒHØ ˆD�Mð	!ú÷
	3ñ 	3üã>÷	<ð 	<üâ;s5   ƒ3D< ÁE9ÂFÂ&#FÃ FÄ<6E6Å5E6Å9FÆFc                  ó   — y)NTrL   ©r   s    r   r   z OpenAIGPTTokenizer.do_lower_case  s   € àr   c                 ó,   — t        | j                  «      S r   )rA   rv   r‹   s    r   Ú
vocab_sizezOpenAIGPTTokenizer.vocab_size  s   € ä�4—<‘<Ó Ð r   c                 óB   — t        | j                  fi | j                  ¤ŽS r   )r{   rv   Úadded_tokens_encoderr‹   s    r   Ú	get_vocabzOpenAIGPTTokenizer.get_vocab#  s   € Ü�D—L‘LÑ> D×$=Ñ$=Ñ>Ð>r   c                 ó  ‡ — t        |d d «      |d   dz   fz   }|‰ j                  v r‰ j                  |   S t        |«      }|s|dz   S 	 t        |ˆ fd„¬«      }|‰ j                  vrnÎ|\  }}g }d}|t        |«      k  r�	 |j                  ||«      }	|j                  |||	 «       |	}||   |k(  r6|t        |«      dz
  k  r%||dz      |k(  r|j                  ||z   «       |dz  }n|j                  ||   «       |dz  }|t        |«      k  rŒ�t        |«      }|}t        |«      dk(  rnt        |«      }Œídj                  |«      }|d	k(  rd
}|‰ j                  |<   |S # t        $ r |j                  ||d  «       Y Œpw xY w)Nr>   ú</w>c                 óN   •— ‰j                   j                  | t        d«      «      S )NÚinf)r~   ÚgetÚfloat)Úpairr   s    €r   ú<lambda>z(OpenAIGPTTokenizer.bpe.<locals>.<lambda>0  s   ø€ °·±×1CÑ1CÀDÌ%ÐPUË,Ó1W€ r   ©Úkeyr   r?   é   r#   z
  </w>z
</w>)rz   r   rW   Úminr~   rA   Úindexr+   Ú
ValueErrorr9   r-   )
r   r1   rT   rU   ÚbigramÚfirstÚsecondÚnew_wordrC   Újs
   `         r   ÚbpezOpenAIGPTTokenizer.bpe&  sª  ø€ Ü�U˜3˜B�ZÓ  E¨"¡I°Ñ$6Ð#8Ñ8ˆØ�D—J‘JÑØ—:‘:˜eÑ$Ð$Ü˜$“ˆáØ˜6‘>Ð!àÜ˜Ó$WÔXˆFØ˜TŸ^™^Ñ+ØØ"‰MˆE�6ØˆHØˆAØ”c˜$“i’-ðØŸ
™
 5¨!Ó,�Að
 —O‘O D¨¨1 IÔ.Ø�Aà˜‘7˜eÒ#¨¬C°«I¸©MÒ(9¸dÀ1ÀqÁ5¹kÈVÒ>SØ—O‘O E¨F¡NÔ3Ø˜‘F‘Aà—O‘O D¨¡GÔ,Ø˜‘F�Að ”c˜$“i“-ô  ˜X“ˆHØˆDÜ�4‹y˜AŠ~Øä! $›�ð9 ð: �x‰x˜‹~ˆØ�:ÒØˆDØ ˆ�
‰
�5ÑØˆøô/ "ò Ø—O‘O D¨¨ HÔ-Ùðús   ÂE  Å F Å?F c           	      óÚ  — g }| j                   €\| j                  j                  |«      }|D ]:  }|j                  t	        | j                  |«      j                  d«      «      «       Œ< |S | j                  t        | j                  |«      «      «      }|D ]R  }|j                  t	        | j                  |j                  j                  «       «      j                  d«      «      «       ŒT |S )zTokenize a string.r#   )
ro   rn   r3   r+   r@   r¤   r   r_   r   r)   )r   r   r0   r1   s       r   Ú	_tokenizezOpenAIGPTTokenizer._tokenizeR  sÎ   € àˆØ�=‰=Ð à—8‘8×$Ñ$ TÓ*ˆDØò F�Ø×#Ñ#¤D¨¯©°%«×)>Ñ)>¸sÓ)CÓ$DÕEðFð Ðð —8‘8Ô,¨T¯]©]¸4Ó-@ÓAÓBˆDØò S�Ø×#Ñ#¤D¨¯©°%·*±*×2BÑ2BÓ2DÓ)E×)KÑ)KÈCÓ)PÓ$QÕRðSàÐr   c                 ó€   — | j                   j                  || j                   j                  | j                  «      «      S )z0Converts a token (str) in an id using the vocab.)rv   r•   rj   )r   r1   s     r   Ú_convert_token_to_idz'OpenAIGPTTokenizer._convert_token_to_ida  s,   € à�|‰|×Ñ  t§|¡|×'7Ñ'7¸¿¹Ó'GÓHÐHr   c                 óN   — | j                   j                  || j                  «      S )z0Converts an id in a token (BPE) using the vocab.)rx   r•   rj   )r   r�   s     r   Ú_convert_id_to_tokenz'OpenAIGPTTokenizer._convert_id_to_tokene  s   € à�|‰|×Ñ  t§~¡~Ó6Ð6r   c                 ód   — dj                  |«      j                  dd«      j                  «       }|S )z:Converts a sequence of tokens (string) in a single string.r7   r’   r#   )r-   r\   r   )r   r   Ú
out_strings      r   Úconvert_tokens_to_stringz+OpenAIGPTTokenizer.convert_tokens_to_stringi  s+   € à—W‘W˜V“_×,Ñ,¨V°SÓ9×?Ñ?ÓAˆ
ØÐr   Úsave_directoryÚfilename_prefixÚreturnc           	      óP  — t         j                  j                  |«      st        j	                  d|› d�«       y t         j                  j                  ||r|dz   ndt        d   z   «      }t         j                  j                  ||r|dz   ndt        d   z   «      }t        |dd¬	«      5 }|j                  t        j                  | j                  d
dd¬«      dz   «       d d d «       d}t        |dd¬	«      5 }|j                  d«       t        | j                  j                  «       d„ ¬«      D ]M  \  }}	||	k7  rt        j                  d|› d�«       |	}|j                  dj                  |«      dz   «       |dz  }ŒO 	 d d d «       ||fS # 1 sw Y   Œ®xY w# 1 sw Y   ||fS xY w)NzVocabulary path (z) should be a directoryrY   r7   r   r   Úwrf   rg   r›   TF)ÚindentÚ	sort_keysÚensure_asciiri   r   z#version: 0.2
c                 ó   — | d   S )Nr?   rL   )Úkvs    r   r˜   z4OpenAIGPTTokenizer.save_vocabulary.<locals>.<lambda>  s   € ÐY[Ð\]ÑY^€ r   r™   zSaving vocabulary to zZ: BPE merge indices are not consecutive. Please check that the tokenizer is not corrupted!r#   r?   )ÚosÚpathÚisdirrq   Úerrorr-   ÚVOCAB_FILES_NAMESrs   Úwritert   Údumpsrv   Úsortedr~   rw   rr   )
r   r®   r¯   r   Ú
merge_fileÚfr�   ÚwriterÚ
bpe_tokensÚtoken_indexs
             r   Úsave_vocabularyz"OpenAIGPTTokenizer.save_vocabularyn  sœ  € Ü�w‰w�}‰}˜^Ô,Ü�L‰LÐ,¨^Ð,<Ð<SÐTÔUØÜ—W‘W—\‘\Ø±o˜_¨sÒ2È2ÔQbÐcoÑQpÑpó
ˆ
ô —W‘W—\‘\Ø±o˜_¨sÒ2È2ÔQbÐcpÑQqÑqó
ˆ
ô �*˜c¨GÔ4ð 	c¸Ø�G‰G”D—J‘J˜tŸ|™|°AÀÐTYÔZÐ]aÑaÔb÷	cð ˆÜ�*˜c¨GÔ4ð 
	¸Ø�L‰LÐ*Ô+Ü+1°$·.±.×2FÑ2FÓ2HÑN^Ô+_ò Ñ'�
˜KØ˜KÒ'Ü—N‘NØ/°
¨|ð <Mð Môð (�EØ—‘˜SŸX™X jÓ1°DÑ8Ô9Ø˜‘
‘ñ÷
	ð ˜:Ð%Ð%÷!	cð 	cú÷
	ð ˜:Ð%Ð%ús   Â*6FÃ8BFÆFÆF%)z<unk>r   )rN   rO   rP   rQ   r¼   Úvocab_files_namesÚmodel_input_namesr    Úpropertyr   r�   r�   r¤   r¦   r¨   rª   r­   Ústrr   r   rÅ   Ú__classcell__)r‰   s   @r   ra   ra   ë   sŽ   ø„ ñð( *ÐØ$Ð&6Ð7Ðõ8ð0 ñó ðð ñ!ó ð!ò?ò*òXòIò7òñ
&¨cð &ÀHÈSÁMð &Ð]bÐcfÑ]g÷ &r   ra   )rQ   rt   r¸   r]   r'   Útypingr   r   Útokenization_utilsr   r   r   r	   Úutilsr
   Ú
get_loggerrN   rq   r¼   r   r   rW   r_   ra   Ú__all__rL   r   r   ú<module>rÐ      s|   ðñ +ã Û 	Û 	Û ß "ç cÓ cÝ ð 
ˆ×	Ñ	˜HÓ	%€ð ØñÐ ò÷^ñ ^òB
òô^&Ð,ô ^&ðB  Ð
 �r   