Ë
    l^(h-  ã                  óÎ   — d dl mZ d dlmZmZ d dlmZ d dlmZ d dl	m
Z
 d dlZd dlZd dlmZmZ d dlmZmZ d d	lmZ d d
lmZ 	 	 	 	 	 	 	 	 dd„Z G d„ dej.                  «      Zy)é    )Úannotations)ÚIterableÚIterator)Únullcontext)Úpartial)ÚAnyN)ÚTensorÚnn)ÚSentenceTransformerÚutil)ÚRandContext)ÚStaticEmbeddingÚ+CachedMultipleNegativesSymmetricRankingLossc           
     óÂ  — |j                   €J ‚|j                  €J ‚t        j                  «       5  t	        ||j                   |j                  «      D ]u  \  }}}t	        |j                  |dd|¬«      |«      D ]M  \  \  }}}t        j                  |j                  «       |j                  «       «      | z  }	|	j                  «        ŒO Œw 	 ddd«       y# 1 sw Y   yxY w)zOA backward hook to backpropagate the cached gradients mini-batch by mini-batch.NTF)Úsentence_featureÚ	with_gradÚcopy_random_stateÚrandom_states)	Úcacher   ÚtorchÚenable_gradÚzipÚembed_minibatch_iterÚdotÚflattenÚbackward)
Úgrad_outputÚsentence_featuresÚloss_objr   Úgradr   Úreps_mbÚ_Úgrad_mbÚ	surrogates
             ú†/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/sentence_transformers/losses/CachedMultipleNegativesSymmetricRankingLoss.pyÚ_backward_hookr&      så   € ð �>‰>Ð%Ð%Ð%Ø×!Ñ!Ð-Ð-Ð-Ü	×	Ñ	Ó	ñ %Ü58Ð9JÈHÏNÉNÐ\d×\rÑ\rÓ5sò 	%Ñ1Ð˜d MÜ),Ø×-Ñ-Ø%5Ø"Ø&+Ø"/ð	 .ó ð ó*ò 
%Ñ%‘�˜!˜gô "ŸI™I g§o¡oÓ&7¸¿¹Ó9JÓKÈkÑY�	Ø×"Ñ"Õ$ñ
%ñ	%÷%÷ %ñ %ús   ±BCÃCc                  óº   ‡ — e Zd Zdej                  ddf	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Z	 d	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 d	 	 	 	 	 	 	 	 	 dd„Zdd„Zddd„Z	dd	„Z
dd
„Zˆ xZS )r   g      4@é    Fc                óò   •— t         ‰| �  «        t        |d   t        «      rt	        d«      ‚|| _        || _        || _        t        j                  «       | _
        || _        d| _        d| _        || _        y)a  
        Boosted version of :class:`MultipleNegativesSymmetricRankingLoss` (MNSRL) by GradCache (https://arxiv.org/pdf/2101.06983.pdf).

        Given a list of (anchor, positive) pairs, MNSRL sums the following two losses:

        1. Forward loss: Given an anchor, find the sample with the highest similarity out of all positives in the batch.
        2. Backward loss: Given a positive, find the sample with the highest similarity out of all anchors in the batch.

        For example with question-answer pairs, the forward loss finds the answer for a given question and the backward loss
        finds the question for a given answer. This loss is common in symmetric tasks, such as semantic textual similarity.

        The caching modification allows for large batch sizes (which give a better training signal) with constant memory usage,
        allowing you to reach optimal training signal with regular hardware.

        Note: If you pass triplets, the negative entry will be ignored. An anchor is just searched for the positive.

        Args:
            model: SentenceTransformer model
            scale: Output of similarity function is multiplied by scale value
            similarity_fct: similarity function between sentence embeddings. By default, cos_sim.
                Can also be set to dot product (and then set scale to 1)
            mini_batch_size: Mini-batch size for the forward pass, this denotes how much memory is actually used during
                training and evaluation. The larger the mini-batch size, the more memory efficient the training is, but
                the slower the training will be.
            show_progress_bar: If True, shows progress bar during processing

        Requirements:
            1. (anchor, positive) pairs
            2. Should be used with large batch sizes for superior performance, but has slower training time than non-cached versions

        Inputs:
            +---------------------------------------+--------+
            | Texts                                 | Labels |
            +=======================================+========+
            | (anchor, positive) pairs              | none   |
            +---------------------------------------+--------+

        Recommendations:
            - Use ``BatchSamplers.NO_DUPLICATES`` (:class:`docs <sentence_transformers.training_args.BatchSamplers>`) to
              ensure that no in-batch negatives are duplicates of the anchor or positive samples.

        Relations:
            - Like :class:`MultipleNegativesRankingLoss`, but with an additional symmetric loss term and caching mechanism.
            - Inspired by :class:`CachedMultipleNegativesRankingLoss`, adapted for symmetric loss calculation.

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.CachedMultipleNegativesSymmetricRankingLoss(model, mini_batch_size=32)

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()

        References:
            - Efficient Natural Language Response Suggestion for Smart Reply, Section 4.4: https://arxiv.org/pdf/1705.00652.pdf
            - Scaling Deep Contrastive Learning Batch Size under Memory Limited Setup: https://arxiv.org/pdf/2101.06983.pdf
        r   z¸CachedMultipleNegativesSymmetricRankingLoss is not compatible with a SentenceTransformer model based on a StaticEmbedding. Consider using MultipleNegativesSymmetricRankingLoss instead.N)ÚsuperÚ__init__Ú
isinstancer   Ú
ValueErrorÚmodelÚscaleÚsimilarity_fctr
   ÚCrossEntropyLossÚcross_entropy_lossÚmini_batch_sizer   r   Úshow_progress_bar)Úselfr.   r/   r0   r3   r4   Ú	__class__s         €r%   r+   z4CachedMultipleNegativesSymmetricRankingLoss.__init__)   sz   ø€ ôZ 	‰ÑÔÜ�e˜A‘h¤Ô0ÜðPóð ð
 ˆŒ
ØˆŒ
Ø,ˆÔÜ"$×"5Ñ"5Ó"7ˆÔØ.ˆÔØ04ˆŒ
Ø=AˆÔØ!2ˆÕó    c                óv  — |rt         nt        j                  }|€
t        «       n|}|j                  «       D �	�
ci c]  \  }	}
|	|
|| “Œ }}	}
|5   |«       5  |rt	        |j                  «       Ž nd}| j                  |«      d   }ddd«       ddd«       |fS c c}
}	w # 1 sw Y   ŒxY w# 1 sw Y   |fS xY w)z Embed a mini-batch of sentences.NÚsentence_embedding)r   r   Úno_gradÚitemsr   Úvaluesr.   )r5   r   ÚbeginÚendr   r   Úrandom_stateÚgrad_contextÚrandom_state_contextÚkÚvÚsentence_feature_minibatchÚrepss                r%   Úembed_minibatchz;CachedMultipleNegativesSymmetricRankingLoss.embed_minibatch†   sÍ   € ñ '0•{´U·]±]ˆØ0<Ð0Dœ{œ}È,ÐØBR×BXÑBXÓBZ×%[¹$¸!¸Q a¨¨5°¨¡oÐ%[Ð"Ñ%[Ø!ñ 	TÙ“ñ TÙTeœ{Ð,F×,MÑ,MÓ,OÑPÐko�Ø—z‘zÐ"<Ó=Ð>RÑS�÷T÷	Tð �\Ð!Ð!ùó &\÷Tð Tú÷	Tð �\Ð!Ð!ús)   ºBÁB,Á/B ÂB,Â B)	Â%B,Â,B8c           
   #  ó   K  — |d   }|j                   \  }}t        t        j                  d|| j                  d| j
                   ¬«      «      D ];  \  }}	|	| j                  z   }
| j                  ||	|
|||€dn||   ¬«      \  }}||f–— Œ= y­w)z5Iterate over mini-batches of sentences for embedding.Ú	input_idsr   zEmbed mini-batches©ÚdescÚdisableN)r   r=   r>   r   r   r?   )ÚshapeÚ	enumerateÚtqdmÚtranger3   r4   rF   )r5   r   r   r   r   rH   Úbszr"   ÚiÚbÚerE   r?   s                r%   r   z@CachedMultipleNegativesSymmetricRankingLoss.embed_minibatch_iter™   s¸   è ø€ ð -¨[Ñ9ˆ	Ø—‘‰ˆˆQÜÜ�K‰KØØØ×$Ñ$Ø)Ø ×2Ñ2Ð2ôó
ò 	%‰DˆAˆqð �D×(Ñ(Ñ(ˆAØ!%×!5Ñ!5Ø!1ØØØ#Ø"3Ø%2Ð%:™TÀÈaÑ@Pð "6ó "ÑˆD�,ð ˜Ð$Ó$ñ%	%ùs   ‚BBc                óÜ   — | j                  |d¬«      }|j                  «       j                  «       }|D ��cg c]  }|D �cg c]  }|j                  ‘Œ c}‘Œ c}}| _        |S c c}w c c}}w )z1Calculate the symmetric loss and cache gradients.T)Úwith_backward)Úcalculate_lossÚdetachÚrequires_grad_r    r   )r5   rE   ÚlossÚrsÚrs        r%   Ú"calculate_loss_and_cache_gradientszNCachedMultipleNegativesSymmetricRankingLoss.calculate_loss_and_cache_gradients·   sY   € à×"Ñ" 4°tÐ"Ó<ˆØ�{‰{‹}×+Ñ+Ó-ˆà59×:¨r rÖ* !�q—v“vÔ*Ó:ˆŒ
àˆùò +ùÓ:s   ·	A(Á A#ÁA(Á#A(c           	     ó(  — t        j                  |d   «      }t        j                  |dd D �cg c]  }t        j                  |«      ‘Œ c}«      }t        |«      }t        j                  ||j                  ¬«      }g }t        j                  d|| j                  d| j                   ¬«      D ]Ä  }	t        |	| j                  z   |«      }
| j                  ||	|
 |«      | j                  z  }| j                  |||	|
 «      }|dd…|	|
…f   }| j                  |j                  «       |dt        |«       «      }||z   dz  }|r |j                  «        |j                  «       }|j!                  |«       ŒÆ t#        |«      t        |«      z  }|S c c}w )zHCalculate the symmetric loss without caching gradients (for evaluation).r   é   N)ÚdevicezCalculating lossrI   é   )r   ÚcatÚlenÚaranger_   rN   rO   r3   r4   Úminr0   r/   r2   Útr   rW   ÚappendÚsum)r5   rE   rU   Úembeddings_ar[   Úembeddings_bÚ
batch_sizeÚlabelsÚlossesrR   rS   ÚscoresÚforward_lossÚpositive_scoresÚbackward_lossÚloss_mbatchrY   s                    r%   rV   z:CachedMultipleNegativesSymmetricRankingLoss.calculate_lossÀ   s�  € ä—y‘y  a¡Ó)ˆÜ—y‘y¸¸Q¸R¸Ö!A°1¤%§)¡)¨A¥,Ò!AÓBˆä˜Ó&ˆ
Ü—‘˜j°×1DÑ1DÔEˆà%'ˆÜ—‘ØØØ× Ñ Ø#Ø×.Ñ.Ð.ô
ò 	'ˆAô �A˜×,Ñ,Ñ,¨jÓ9ˆAØ!×0Ñ0°¸aÀÐ1BÀLÓQÐTX×T^ÑT^Ñ^ˆFØ)-×)@Ñ)@ÀÈÐPQÐRSÈÓ)UˆLà$¢Q¨¨!¨ V™nˆOØ*.×*AÑ*AÀ/×BSÑBSÓBUÐW]Ð^tÔ`cÐdsÓ`tÐWuÓ*vˆMà'¨-Ñ7¸1Ñ<ˆKÙØ×$Ñ$Ô&Ø)×0Ñ0Ó2�Ø�M‰M˜+Õ&ð%	'ô( �6‹{œS ›[Ñ(ˆØˆùò7 "Bs   ¯Fc                óì  — g }g | _         |D ]Œ  }g }g }| j                  |dd¬«      D ]C  \  }}|j                  |j                  «       j	                  «       «       |j                  |«       ŒE |j                  |«       | j                   j                  |«       ŒŽ t        j                  «       r4| j                  |«      }	|	j                  t        t        || ¬«      «       |	S | j                  |«      }	|	S )z"Forward pass of the loss function.FT)r   r   r   )r   r   )r   r   rf   rW   rX   r   Úis_grad_enabledr\   Úregister_hookr   r&   rV   )
r5   r   rk   rE   r   Úreps_mbsÚrandom_state_mbsr!   r?   rY   s
             r%   Úforwardz3CachedMultipleNegativesSymmetricRankingLoss.forwardà   sü   € àˆØˆÔØ 1ò 	8ÐØˆHØ!ÐØ)-×)BÑ)BØ!1ØØ"&ð *Có *ò 6Ñ%�˜ð
 —‘ §¡Ó 0× ?Ñ ?Ó AÔBØ ×'Ñ'¨Õ5ð6ð �K‰K˜Ô!Ø×Ñ×%Ñ%Ð&6Õ7ð	8ô × Ñ Ô"Ø×:Ñ:¸4Ó@ˆDØ×Ñœw¤~ÐIZÐeiÔjÔkð ˆð ×&Ñ& tÓ,ˆDàˆr7   c                ó^   — | j                   | j                  j                  | j                  dœS )z+Get the configuration of the loss function.)r/   r0   r3   )r/   r0   Ú__name__r3   )r5   s    r%   Úget_config_dictz;CachedMultipleNegativesSymmetricRankingLoss.get_config_dictù   s-   € ð —Z‘ZØ"×1Ñ1×:Ñ:Ø#×3Ñ3ñ
ð 	
r7   )r.   r   r/   Úfloatr0   z"callable[[Tensor, Tensor], Tensor]r3   Úintr4   ÚboolÚreturnÚNone)N)r   údict[str, Tensor]r=   r|   r>   r|   r   r}   r   r}   r?   zRandContext | Noner~   z!tuple[Tensor, RandContext | None])
r   r€   r   r}   r   r}   r   zlist[RandContext] | Noner~   z+Iterator[tuple[Tensor, RandContext | None]])rE   úlist[list[Tensor]]r~   r	   )F)rE   r�   rU   r}   r~   r	   )r   úIterable[dict[str, Tensor]]rk   r	   r~   r	   )r~   zdict[str, Any])ry   Ú
__module__Ú__qualname__r   Úcos_simr+   rF   r   r\   rV   rw   rz   Ú__classcell__)r6   s   @r%   r   r   (   sù   ø„ ð Ø=A¿\¹\Ø!Ø"'ð[3à"ð[3ð ð[3ð ;ð	[3ð
 ð[3ð  ð[3ð 
õ[3ðH ,0ð"à+ð"ð ð"ð ð	"ð
 ð"ð  ð"ð )ð"ð 
+ó"ð0 37ð%à+ð%ð ð%ð  ð	%ð
 0ð%ð 
5ó%ó<ôó@÷2
r7   )r   r	   r   r‚   r   r   r~   r   )Ú
__future__r   Úcollections.abcr   r   Ú
contextlibr   Ú	functoolsr   Útypingr   r   rN   r	   r
   Úsentence_transformersr   r   Ú?sentence_transformers.losses.CachedMultipleNegativesRankingLossr   Úsentence_transformers.modelsr   r&   ÚModuler   © r7   r%   ú<module>r‘      sa   ðÝ "ç .Ý "Ý Ý ã Û ß ç ;Ý WÝ 8ð%Øð%à2ð%ð :ð%ð 
ó	%ô.W
°"·)±)õ W
r7   