Ë
    l^(h®   ã                  óŠ   — d Z ddlmZ ddlZddlZddlZddlmZ ddlm	Z	 ddl
mZ  ej                  e«      Z G d„ de«      Zy)	a@  
This file contains deprecated code that can only be used with the old `model.fit`-style Sentence Transformers v2.X training.
It exists for backwards compatibility with the `model.old_fit` method, but will be removed in a future version.

Nowadays, with Sentence Transformers v3+, it is recommended to use the `SentenceTransformerTrainer` class to train models.
See https://www.sbert.net/docs/sentence_transformer/training_overview.html for more information.

Instead, you should create a `datasets` `Dataset` for training: https://huggingface.co/docs/datasets/create_dataset
é    )ÚannotationsN)ÚDataset)ÚSentenceTransformer)ÚInputExamplec                  ó†   — e Zd ZdZ	 	 d	 	 	 	 	 	 	 dd„Z	 d	 	 	 	 	 	 	 	 	 dd„Z	 	 	 d	 	 	 	 	 	 	 dd„Zd„ Zd„ Zd„ Z	d	„ Z
d
„ Zy)ÚParallelSentencesDatasetu�  
    This dataset reader can be used to read-in parallel sentences, i.e., it reads in a file with tab-seperated sentences with the same
    sentence in different languages. For example, the file can look like this (EN	DE	ES):
    hello world     hallo welt  hola mundo
    second sentence zweiter satz    segunda oraciÃ³n

    The sentence in the first column will be mapped to a sentence embedding using the given the embedder. For example,
    embedder is a mono-lingual sentence embedding method for English. The sentences in the other languages will also be
    mapped to this English sentence embedding.

    When getting a sample from the dataset, we get one sentence with the according sentence embedding for this sentence.

    teacher_model can be any class that implement an encode function. The encode function gets a list of sentences and
    returns a list of sentence embeddings
    c                ó¬   — || _         || _        g | _        g | _        g | _        g | _        g | _        g | _        || _        || _	        i | _
        d| _        y)a+  
        Parallel sentences dataset reader to train student model given a teacher model

        Args:
            student_model (SentenceTransformer): The student sentence embedding model that should be trained.
            teacher_model (SentenceTransformer): The teacher model that provides the sentence embeddings for the first column in the dataset file.
            batch_size (int, optional): The batch size for training. Defaults to 8.
            use_embedding_cache (bool, optional): Whether to use an embedding cache. Defaults to True.
        r   N)Ústudent_modelÚteacher_modelÚdatasetsÚdatasets_iteratorÚdatasets_tokenizedÚdataset_indicesÚcopy_dataset_indicesÚcacheÚ
batch_sizeÚuse_embedding_cacheÚembedding_cacheÚnum_sentences)Úselfr
   r   r   r   s        úu/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/sentence_transformers/datasets/ParallelSentencesDataset.pyÚ__init__z!ParallelSentencesDataset.__init__*   sa   € ð  +ˆÔØ*ˆÔØˆŒØ!#ˆÔØ"$ˆÔØ!ˆÔØ$&ˆÔ!ØˆŒ
Ø$ˆŒØ#6ˆÔ Ø!ˆÔØˆÕó    Nc           
     óø  — t         j                  d|z   «       g }|j                  d«      rt        j                  |dd¬«      nt	        |d¬«      5 }d}|D ]t  }|j                  «       j                  d«      }	|�+|dkD  r&t        |	D �
cg c]  }
t        |
«      ‘Œ c}
«      |kD  rŒO|j                  |	«       |d	z  }|€Œh|dkD  sŒn||k\  sŒt n ddd«       | j                  ||||¬
«       yc c}
w # 1 sw Y   Œ$xY w)aÆ  
        Reads in a tab-seperated .txt/.csv/.tsv or .gz file. The different columns contain the different translations of the sentence in the first column

        Args:
            filepath (str): Filepath to the file.
            weight (int, optional): If more than one dataset is loaded with load_data, specifies the frequency at which data should be sampled from this dataset. Defaults to 100.
            max_sentences (int, optional): Maximum number of lines to be read from the filepath. Defaults to None.
            max_sentence_length (int, optional): Skip the example if one of the sentences has more characters than max_sentence_length. Defaults to 128.

        Returns:
            None
        zLoad z.gzÚrtÚutf8)Úencodingr   ú	Né   )ÚweightÚmax_sentencesÚmax_sentence_length)ÚloggerÚinfoÚendswithÚgzipÚopenÚstripÚsplitÚmaxÚlenÚappendÚadd_dataset)r   Úfilepathr    r!   r"   Úparallel_sentencesÚfInÚcountÚlineÚ	sentencesÚsents              r   Ú	load_dataz"ParallelSentencesDataset.load_dataG   s  € ô  	�‰�G˜hÑ&Ô'ØÐð × Ñ  Ô'ô �I‰I�h ¨vÕ6ä�h¨Ô0ð	ð 58àˆEØò �Ø ŸJ™J›L×.Ñ.¨tÓ4�	à'Ð3Ø+¨aÒ/Ü°9Ö=¨4œS �YÒ=Ó>ÐATÒTàà"×)Ñ)¨)Ô4Ø˜‘
�Ø Ñ,°ÀÓ1BÀuÐP]ÓG]Ùð÷	ð& 	×ÑØ v¸]Ð`sð 	õ 	
ùò >÷	ð 	ús0   Á7C0ÂC+Â#C0Â>C0ÃC0Ã
C0Ã+C0Ã0C9c                ó¨  — i }|D ]y  }|�+|dkD  r&t        |D �cg c]  }t        |«      ‘Œ c}«      |kD  rŒ0|d   }||vrt        «       ||<   |D ]  }||   j                  |«       Œ |€Œd|dkD  sŒjt        |«      |k\  sŒy n t        |«      dk(  ry | xj                  t        |D �cg c]  }t        ||   «      ‘Œ c}«      z  c_        t        | j                  «      }	| j                  j                  t        |j                  «       «      «       | j                  j                  d«       | j                  j                  |	g|z  «       y c c}w c c}w ©Nr   )r*   r+   ÚsetÚaddr   Úsumr   r,   ÚlistÚitemsr   r   Úextend)
r   r/   r    r!   r"   Úsentences_mapr3   r4   Úsource_sentenceÚ
dataset_ids
             r   r-   z$ParallelSentencesDataset.add_datasetq   sI  € ð ˆØ+ò 	ˆIà#Ð/Ø'¨!Ò+Ü¨yÖ9 tœ˜T�Ò9Ó:Ð=PÒPàà'¨™lˆOØ mÑ3Ü14³�˜oÑ.à!ò 9�Ø˜oÑ.×2Ñ2°4Õ8ð9ð Ñ(¨]¸QÓ->Ä3À}ÓCUÐYfÓCfÙð!	ô$ ˆ}Ó Ò"Øà×ÒœcÈÖ"VÀ¤3 }°TÑ':Õ#;Ò"VÓWÑWÕä˜Ÿ™Ó'ˆ
Ø�‰×ÑœT -×"5Ñ"5Ó"7Ó8Ô9Ø×Ñ×%Ñ% aÔ(Ø×Ñ×#Ñ# Z L°6Ñ$9Õ:ùò- :ùò" #Ws   ˜E

Â%Ec                ó~  — g }g }| j                   D ]8  }| j                  |«      \  }}|j                  |«       |j                  |«       Œ: | j                  |«      }t	        ||«      D ]3  \  }}|D ])  }| j
                  j                  t        |g|¬«      «       Œ+ Œ5 t        j                  | j
                  «       y )N)ÚtextsÚlabel)	r   Ú
next_entryr,   Úget_embeddingsÚzipr   r   ÚrandomÚshuffle)	r   Úsource_sentences_listÚtarget_sentences_listÚdata_idxÚsrc_sentenceÚtrg_sentencesÚsrc_embeddingsÚsrc_embeddingÚtrg_sentences	            r   Úgenerate_dataz&ParallelSentencesDataset.generate_data•   sÁ   € Ø "ÐØ "ÐØ×,Ñ,ò 	8ˆHØ*.¯/©/¸(Ó*CÑ'ˆL˜-Ø!×(Ñ(¨Ô6Ø!×(Ñ(¨Õ7ð	8ð ×,Ñ,Ð-BÓCˆä,/°Ð@UÓ,Vò 	[Ñ(ˆM˜=Ø -ò [�Ø—
‘
×!Ñ!¤,°l°^È=Ô"YÕZñ[ð	[ô 	�‰�t—z‘zÕ"r   c                ó.  — | j                   |   | j                  |      \  }}| j                  |xx   dz  cc<   | j                  |   t        | j                   |   «      k\  r1d| j                  |<   t        j                  | j                   |   «       ||fS )Nr   r   )r   r   r+   rG   rH   )r   rK   ÚsourceÚtarget_sentencess       r   rD   z#ParallelSentencesDataset.next_entry¦   s�   € Ø#'§=¡=°Ñ#:¸4×;QÑ;QÐRZÑ;[Ñ#\Ñ ˆÐ à×Ñ˜xÓ(¨AÑ-Ó(Ø×!Ñ! (Ñ+¬s°4·=±=ÀÑ3JÓ/KÒKØ/0ˆD×"Ñ" 8Ñ,Ü�N‰N˜4Ÿ=™=¨Ñ2Ô3àÐ'Ð'Ð'r   c                ó¶  — | j                   s)| j                  j                  || j                  dd¬«      S g }|D ]"  }|| j                  vsŒ|j                  |«       Œ$ t        |«      dkD  rL| j                  j                  || j                  dd¬«      }t        ||«      D ]  \  }}|| j                  |<   Œ |D �cg c]  }| j                  |   ‘Œ c}S c c}w )NFT)r   Úshow_progress_barÚconvert_to_numpyr   )r   r   Úencoder   r   r,   r+   rF   )r   r3   Únew_sentencesr4   Únew_embeddingsÚ	embeddings         r   rE   z'ParallelSentencesDataset.get_embeddings°   sï   € Ø×'Ò'Ø×%Ñ%×,Ñ,Ø d§o¡oÈÐaeð -ó ð ð
 ˆØò 	+ˆDØ˜4×/Ñ/Ò/Ø×$Ñ$ TÕ*ð	+ô ˆ}Ó Ò!Ø!×/Ñ/×6Ñ6Ø¨$¯/©/ÈUÐeið 7ó ˆNô $' }°nÓ#Eò 7‘��iØ-6�×$Ñ$ TÒ*ð7ð 8AÖA¨t�×$Ñ$ TÓ*ÒAÐAùÒAs   Â=Cc                ó   — | j                   S )N)r   )r   s    r   Ú__len__z ParallelSentencesDataset.__len__Å   s   € Ø×!Ñ!Ð!r   c                ó†   — t        | j                  «      dk(  r| j                  «        | j                  j                  «       S r7   )r+   r   rQ   Úpop)r   Úidxs     r   Ú__getitem__z$ParallelSentencesDataset.__getitem__È   s/   € Üˆt�z‰z‹?˜aÒØ×ÑÔ à�z‰z�~‰~ÓÐr   )é   T)r
   r   r   r   r   Úintr   Úbool)éd   Né€   )
r.   Ústrr    rc   r!   rc   r"   rc   ÚreturnÚNone)r/   zlist[list[str]]r    rc   r!   rc   r"   rc   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r5   r-   rQ   rD   rE   r]   ra   © r   r   r   r      sÀ   „ ñð( Ø$(ðà*ðð +ðð ð	ð
 "óð< gjð(
Øð(
Ø%(ð(
Ø?Bð(
Ø`cð(
à	ó(
ðZ Ø!Ø#&ð";à+ð";ð ð";ð ð	";ð
 !ó";òH#ò"(òBò*"ó r   r   )rm   Ú
__future__r   r&   ÚloggingrG   Útorch.utils.datar   Úsentence_transformersr   Úsentence_transformers.readersr   Ú	getLoggerrj   r#   r   rn   r   r   ú<module>ru      sA   ðñõ #ã Û Û å $å 5Ý 6à	ˆ×	Ñ	˜8Ó	$€ôs ˜wõ s r   