Ë
    l^(hd  ã                  óž   — d dl mZ d dlZd dlZd dlZd dlZd dlZd dlmZ d dl	m
Z
mZ ddlmZmZ  ej                  e«      Z G d„ de«      Zy)	é    )ÚannotationsN)ÚIterable)ÚNLTK_IMPORT_ERRORÚis_nltk_availableé   )ÚENGLISH_STOP_WORDSÚWordTokenizerc                  óf   — e Zd ZdZg edddf	 	 	 	 	 	 	 	 	 dd„Zd„ Zdd„Zdd„Zdd	„Z	e
dd
„«       Zy)ÚPhraseTokenizera~  Tokenizes the text with respect to existent phrases in the vocab.

    This tokenizers respects phrases that are in the vocab. Phrases are separated with 'ngram_separator', for example,
    in Google News word2vec file, ngrams are separated with a _ like New_York. These phrases are detected in text and merged as one special token. (New York is the ... => [New_York, is, the])
    FÚ_é   c                óè   — t        «       s2t        t        j                  | j                  j
                  «      «      ‚t        |«      | _        || _        || _	        || _
        | j                  |«       y ©N)r   ÚImportErrorr   ÚformatÚ	__class__Ú__name__ÚsetÚ
stop_wordsÚdo_lower_caseÚngram_separatorÚmax_ngram_lengthÚ	set_vocab)ÚselfÚvocabr   r   r   r   s         út/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/sentence_transformers/models/tokenizer/PhraseTokenizer.pyÚ__init__zPhraseTokenizer.__init__   sZ   € ô !Ô"ÜÔ/×6Ñ6°t·~±~×7NÑ7NÓOÓPÐPä˜j›/ˆŒØ*ˆÔØ.ˆÔØ 0ˆÔØ�‰�uÕó    c                ó   — | j                   S r   )r   )r   s    r   Ú	get_vocabzPhraseTokenizer.get_vocab)   s   € Ø�z‰zÐr   c                óÈ  — || _         t        j                  t        |«      D ��cg c]	  \  }}||f‘Œ c}}«      | _        t        «       | _        t        «       | _        |D ]ž  }| j                  €Œ| j                  |v sŒ|j                  | j                  «      dz   }| j                  | j                  z   |vsŒY|| j                  k  sŒi| j                  j                  |«       | j                  j                  |«       Œ  t        |«      dkD  rNt        j                  d| j                  › �«       t        j                  dt        | j                  «      › �«       y y c c}}w )Nr   r   z(PhraseTokenizer - Phrase ngram lengths: zPhraseTokenizer - Num phrases: )r   ÚcollectionsÚOrderedDictÚ	enumerateÚword2idxr   Úngram_lookupÚngram_lengthsr   Úcountr   ÚaddÚlenÚloggerÚinfo)r   r   ÚidxÚwordÚngram_counts        r   r   zPhraseTokenizer.set_vocab,   s-  € ØˆŒ
Ü#×/Ñ/ÌiÐX]ÓN^×0_ÁÀÀd°$¸²Ó0_Ó`ˆŒô  ›EˆÔÜ ›UˆÔØò 	8ˆDØ×#Ñ#Ñ/°D×4HÑ4HÈDÒ4Pà"Ÿj™j¨×)=Ñ)=Ó>ÀÑB�Ø×'Ñ'¨$×*>Ñ*>Ñ>ÀdÒJÈ{Ð^b×^sÑ^sÓOsØ×%Ñ%×)Ñ)¨$Ô/Ø×&Ñ&×*Ñ*¨;Õ7ð	8ô ˆu‹:˜Š>Ü�K‰KÐBÀ4×CUÑCUÐBVÐWÔXÜ�K‰KÐ9¼#¸d×>OÑ>OÓ:PÐ9QÐRÕSð ùó 1`s   ¥E
c                ó‚  — ddl m}  ||d¬«      }t        | j                  d¬«      D ]™  }d}|t	        |«      |z
  k  sŒ| j
                  j                  ||||z    «      }|| j                  v r
|g||||z    n3|j                  «       | j                  v r|j                  «       g||||z    |dz  }|t	        |«      |z
  k  rŒƒŒ› g }|D ]ô  }	|	| j                  v rŒ|	| j                  v r|j                  | j                  |	   «       Œ?|	j                  «       }	|	| j                  v rŒ^|	| j                  v r|j                  | j                  |	   «       Œ‹|	j                  t        j                  «      }	|	| j                  v rŒ¹t	        |	«      dkD  sŒÈ|	| j                  v sŒ×|j                  | j                  |	   «       Œö |S )Nr   )Úword_tokenizeT)Úpreserve_line)Úreverser   )Únltkr1   Úsortedr'   r*   r   Újoinr&   Úlowerr   r%   ÚappendÚstripÚstringÚpunctuation)
r   ÚtextÚkwargsr1   ÚtokensÚ	ngram_lenr-   ÚngramÚtokens_filteredÚtokens
             r   ÚtokenizezPhraseTokenizer.tokenize?   s´  € Ý&á˜t°4Ô8ˆô   × 2Ñ 2¸DÔAò 	ˆIØˆCØœ˜V› yÑ0Ó0Ø×,Ñ,×1Ñ1°&¸¸sÀY¹Ð2OÓP�Ø˜D×-Ñ-Ñ-Ø5:°G�F˜3  y¡Ñ1Ø—[‘[“] d×&7Ñ&7Ñ7Ø5:·[±[³]°O�F˜3  y¡Ð1Ø�q‘�ð œ˜V› yÑ0Ô0ð	ð ˆØò 	ˆEØ˜Ÿ™Ñ'ØØ˜$Ÿ-™-Ñ'Ø×&Ñ& t§}¡}°UÑ';Ô<Øà—K‘K“MˆEØ˜Ÿ™Ñ'ØØ˜$Ÿ-™-Ñ'Ø×&Ñ& t§}¡}°UÑ';Ô<Øà—K‘K¤× 2Ñ 2Ó3ˆEØ˜Ÿ™Ñ'ØÜ�U“˜a“ E¨T¯]©]Ò$:Ø×&Ñ& t§}¡}°UÑ';Ô<Øð'	ð* Ðr   c           	     ó^  — t        t        j                  j                  |d«      d«      5 }t	        j
                  t        | j                  j                  «       «      t        | j                  «      | j                  | j                  | j                  dœ|«       d d d «       y # 1 sw Y   y xY w)Núphrasetokenizer_config.jsonÚw)r   r   r   r   r   )ÚopenÚosÚpathr6   ÚjsonÚdumpÚlistr%   Úkeysr   r   r   r   )r   Úoutput_pathÚfOuts      r   ÚsavezPhraseTokenizer.saveh   s†   € Ü”"—'‘'—,‘,˜{Ð,IÓJÈCÓPð 
	ÐTXÜ�I‰Iä! $§-¡-×"4Ñ"4Ó"6Ó7Ü"& t§¡Ó"7Ø%)×%7Ñ%7Ø'+×';Ñ';Ø(,×(=Ñ(=ñð ô	÷
	÷ 
	ñ 
	ús   «A/B#Â#B,c                ó¾   — t        t        j                  j                  | d«      «      5 }t	        j
                  |«      }d d d «       t        di ¤ŽS # 1 sw Y   ŒxY w)NrE   © )rG   rH   rI   r6   rJ   Úloadr   )Ú
input_pathÚfInÚconfigs      r   rS   zPhraseTokenizer.loadu   sN   € ä”"—'‘'—,‘,˜zÐ+HÓIÓJð 	$ÈcÜ—Y‘Y˜s“^ˆF÷	$ô Ñ( Ñ(Ð(÷	$ð 	$ús   ªAÁAN)
r   úIterable[str]r   rW   r   Úboolr   Ústrr   Úint)r   rW   )r<   rY   Úreturnz	list[int])rN   rY   )rT   rY   )r   Ú
__module__Ú__qualname__Ú__doc__r   r   r    r   rC   rP   ÚstaticmethodrS   rR   r   r   r   r      sw   „ ñð  "Ø$6Ø#Ø"Ø !ðàðð "ðð ð	ð
 ðð óò"óTó&'óRð ò)ó ñ)r   r   )Ú
__future__r   r"   rJ   ÚloggingrH   r:   Úcollections.abcr   Útransformers.utils.import_utilsr   r   r	   r   Ú	getLoggerr   r+   r   rR   r   r   ú<module>re      s@   ðÝ "ã Û Û Û 	Û Ý $ç Pç <à	ˆ×	Ñ	˜8Ó	$€ôi)�mõ i)r   