Ë
    T^(h  ã                   óŠ   — d Z ddlZddlZddlmZmZ ddlmZ ddlm	Z	 ddl
mZ  e	j                  e«      Z G d	„ d
«      Zd
gZy)zTokenization classes for RAG.é    N)ÚListÚOptionalé   )ÚBatchEncoding)Úloggingé   )Ú	RagConfigc                   ó    — e Zd Zd„ Zd„ Zed„ «       Zd„ Zd„ Zd„ Z	d„ Z
d„ Z	 	 	 	 	 	 dd
ee   deee      dee   dee   dedee   dedefd„Zy	)ÚRagTokenizerc                 óB   — || _         || _        | j                   | _        y ©N)Úquestion_encoderÚ	generatorÚcurrent_tokenizer)Úselfr   r   s      úf/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/rag/tokenization_rag.pyÚ__init__zRagTokenizer.__init__   s   € Ø 0ˆÔØ"ˆŒØ!%×!6Ñ!6ˆÕó    c                 óz  — t         j                  j                  |«      rt        d|› d�«      ‚t        j                  |d¬«       t         j                  j                  |d«      }t         j                  j                  |d«      }| j                  j                  |«       | j                  j                  |«       y )NzProvided path (z#) should be a directory, not a fileT)Úexist_okÚquestion_encoder_tokenizerÚgenerator_tokenizer)	ÚosÚpathÚisfileÚ
ValueErrorÚmakedirsÚjoinr   Úsave_pretrainedr   )r   Úsave_directoryÚquestion_encoder_pathÚgenerator_paths       r   r   zRagTokenizer.save_pretrained#   sˆ   € Ü�7‰7�>‰>˜.Ô)Ü˜¨~Ð.>Ð>aÐbÓcÐcÜ
�‰�N¨TÕ2Ü "§¡§¡¨^Ð=YÓ ZÐÜŸ™Ÿ™ nÐ6KÓLˆØ×Ñ×-Ñ-Ð.CÔDØ�‰×&Ñ& ~Õ6r   c                 óì   — ddl m} |j                  dd «      }|€t        j                  |«      }|j	                  ||j
                  d¬«      }|j	                  ||j                  d¬«      } | ||¬«      S )Né   )ÚAutoTokenizerÚconfigr   )r&   Ú	subfolderr   )r   r   )Úauto.tokenization_autor%   Úpopr	   Úfrom_pretrainedr   r   )ÚclsÚpretrained_model_name_or_pathÚkwargsr%   r&   r   r   s          r   r*   zRagTokenizer.from_pretrained,   s†   € õ 	;à—‘˜H dÓ+ˆàˆ>Ü×.Ñ.Ð/LÓMˆFà(×8Ñ8Ø)°&×2IÑ2IÐUqð 9ó 
Ðð "×1Ñ1Ø)°&×2BÑ2BÐNcð 2ó 
ˆ	ñ Ð$4À	ÔJÐJr   c                 ó&   —  | j                   |i |¤ŽS r   )r   ©r   Úargsr-   s      r   Ú__call__zRagTokenizer.__call__>   s   € Ø%ˆt×%Ñ% tÐ6¨vÑ6Ð6r   c                 ó:   —  | j                   j                  |i |¤ŽS r   )r   Úbatch_decoder/   s      r   r3   zRagTokenizer.batch_decodeA   s   € Ø*ˆt�~‰~×*Ñ*¨DÐ;°FÑ;Ð;r   c                 ó:   —  | j                   j                  |i |¤ŽS r   )r   Údecoder/   s      r   r5   zRagTokenizer.decodeD   s   € Ø$ˆt�~‰~×$Ñ$ dÐ5¨fÑ5Ð5r   c                 ó&   — | j                   | _        y r   )r   r   ©r   s    r   Ú_switch_to_input_modez"RagTokenizer._switch_to_input_modeG   s   € Ø!%×!6Ñ!6ˆÕr   c                 ó&   — | j                   | _        y r   )r   r   r7   s    r   Ú_switch_to_target_modez#RagTokenizer._switch_to_target_modeJ   s   € Ø!%§¡ˆÕr   NÚ	src_textsÚ	tgt_textsÚ
max_lengthÚmax_target_lengthÚpaddingÚreturn_tensorsÚ
truncationÚreturnc           
      óî   — t        j                  dt        «       |€| j                  j                  } | |fd||||dœ|¤Ž}	|€|	S |€| j                  j                  } | d|d||||dœ|¤Ž}
|
d   |	d<   |	S )Nu4  `prepare_seq2seq_batch` is deprecated and will be removed in version 5 of ðŸ¤— Transformers. Use the regular `__call__` method to prepare your inputs and the tokenizer under the `with_target_tokenizer` context manager to prepare your targets. See the documentation of your specific tokenizer for more detailsT)Úadd_special_tokensr@   r=   r?   rA   )Útext_targetrD   r@   r?   r=   rA   Ú	input_idsÚlabels© )ÚwarningsÚwarnÚFutureWarningr   Úmodel_max_length)r   r;   r<   r=   r>   r?   r@   rA   r-   Úmodel_inputsrG   s              r   Úprepare_seq2seq_batchz"RagTokenizer.prepare_seq2seq_batchM   sÆ   € ô 	�‰ðô ô	
ð ÐØ×/Ñ/×@Ñ@ˆJÙØð
à#Ø)Ø!ØØ!ñ
ð ñ
ˆð ÐØÐàÐ$Ø $× 6Ñ 6× GÑ GÐÙð 
Ø!Ø#Ø)ØØ(Ø!ñ
ð ñ
ˆð "(¨Ñ!4ˆ�XÑØÐr   )NNNÚlongestNT)Ú__name__Ú
__module__Ú__qualname__r   r   Úclassmethodr*   r1   r3   r5   r8   r:   r   Ústrr   ÚintÚboolr   rN   rH   r   r   r   r      s»   „ ò7ò
7ð ñKó ðKò"7ò<ò6ò7ò0ð *.Ø$(Ø+/Ø Ø(,Øñ,à˜‘9ð,ð ˜D ™IÑ&ð,ð ˜S‘Mð	,ð
 $ C™=ð,ð ð,ð ! ™ð,ð ð,ð 
ô,r   r   )Ú__doc__r   rI   Útypingr   r   Útokenization_utils_baser   Úutilsr   Úconfiguration_ragr	   Ú
get_loggerrP   Úloggerr   Ú__all__rH   r   r   ú<module>r_      sG   ðñ $ã 	Û ß !å 4Ý Ý (ð 
ˆ×	Ñ	˜HÓ	%€÷\ñ \ð~ Ð
�r   