Ë
    l^(ha(  ã                  óŠ   — d dl mZ d dlmZ d dlmZmZ d dlZd dlmZm	Z	 d dl
mZmZ d dlmZ  G d„ d	e	j                  «      Zy)
é    )Úannotations)ÚIterable)ÚAnyÚLiteralN)ÚTensorÚnn)ÚStaticEmbeddingÚTransformer)ÚSentenceTransformerc                  óh   ‡ — e Zd Z	 	 	 d	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zdd„Zd	d„Zd
d„Zedd„«       Zˆ xZ	S )ÚGISTEmbedLossc                óf  •— t         ‰| �  «        || _        || _        || _        t        j                  d¬«      | _        t        |d   t        «      rt        |d   t        «      st        d«      ‚|j                  j                  «       |j                  j                  «       k7  xs |j                  |j                  k  | _        | j                  rC| j                  j                  | _        t        | j                  d   t        «      rt        d«      ‚|dvrt        d«      ‚|| _        || _        y)	a|  
        This loss is used to train a SentenceTransformer model using the GISTEmbed algorithm.
        It takes a model and a guide model as input, and uses the guide model to guide the
        in-batch negative sample selection. The cosine similarity is used to compute the loss
        and the temperature parameter is used to scale the cosine similarities.

        You can apply different false-negative filtering strategies to discard hard negatives that are too similar to
        the positive. Two strategies are supported:

            - "absolute": Discards negatives whose similarity score is greater than or equal to ``positive_score - margin``.
            - "relative": Discards negatives whose similarity score is greater than or equal to ``positive_score * (1 - margin)``.

        Args:
            model: SentenceTransformer model based on a `transformers` model.
            guide: SentenceTransformer model to guide the in-batch negative sample selection.
            temperature: Temperature parameter to scale the cosine similarities.
            margin_strategy: Strategy used for false negative filtering. One of {"absolute", "relative"}.
            margin: The margin value for filtering negatives. Defaults to 0.0, together with the "absolute" strategy,
                this only removes negatives that are more similar to the query than the positive is to the query.

        References:
            - For further details, see: https://arxiv.org/abs/2402.16829

        Requirements:
            1. (anchor, positive, negative) triplets
            2. (anchor, positive) pairs

        Inputs:
            +---------------------------------------+--------+
            | Texts                                 | Labels |
            +=======================================+========+
            | (anchor, positive, negative) triplets | none   |
            +---------------------------------------+--------+
            | (anchor, positive) pairs              | none   |
            +---------------------------------------+--------+

        Recommendations:
            - Use ``BatchSamplers.NO_DUPLICATES`` (:class:`docs <sentence_transformers.training_args.BatchSamplers>`) to
              ensure that no in-batch negatives are duplicates of the anchor or positive samples.

        Relations:
            - :class:`MultipleNegativesRankingLoss` is similar to this loss, but it does not use
              a guide model to guide the in-batch negative sample selection. `GISTEmbedLoss` yields
              a stronger training signal at the cost of some training overhead.

        Example:
            ::

                from sentence_transformers import SentenceTransformer, SentenceTransformerTrainer, losses
                from datasets import Dataset

                model = SentenceTransformer("microsoft/mpnet-base")
                guide = SentenceTransformer("all-MiniLM-L6-v2")
                train_dataset = Dataset.from_dict({
                    "anchor": ["It's nice weather outside today.", "He drove to work."],
                    "positive": ["It's so sunny.", "He took the car to the office."],
                })
                loss = losses.GISTEmbedLoss(model, guide)

                trainer = SentenceTransformerTrainer(
                    model=model,
                    train_dataset=train_dataset,
                    loss=loss,
                )
                trainer.train()
        éÿÿÿÿ©Údimr   z_Both the training model and the guiding model must be based on the `transformers` architecture.z”If we must retokenize because the guide model has a different tokenizer, then the Sentence Transformer model must not be based on a StaticEmbedding.)ÚabsoluteÚrelativez1margin_strategy must be 'absolute' or 'relative'.N)ÚsuperÚ__init__ÚmodelÚguideÚtemperaturer   ÚCosineSimilarityÚsimilarity_fctÚ
isinstancer
   Ú
ValueErrorÚ	tokenizerÚ	get_vocabÚmax_seq_lengthÚmust_retokenizer	   Úmargin_strategyÚmargin)Úselfr   r   r   r!   r"   Ú	__class__s         €úh/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/sentence_transformers/losses/GISTEmbedLoss.pyr   zGISTEmbedLoss.__init__   s  ø€ ôT 	‰ÑÔØˆŒ
ØˆŒ
Ø&ˆÔÜ ×1Ñ1°bÔ9ˆÔÜ˜% ™(¤KÔ0¼
À5ÈÁ8Ì[Ô8YÜØqóð ð �O‰O×%Ñ%Ó'¨5¯?©?×+DÑ+DÓ+FÑFÒuÈ%×J^ÑJ^Ðaf×auÑauÑJuð 	Ôð ×ÒØ!ŸZ™Z×1Ñ1ˆDŒNä˜$Ÿ*™* Q™-¬Ô9Ü ðbóð ð
 Ð":Ñ:ÜÐPÓQÐQØ.ˆÔØˆ�ó    c                ób   — | j                  |j                  d«      |j                  d«      «      S )Né   r   )r   Ú	unsqueeze)r#   Úembed1Úembed2s      r%   Ú
sim_matrixzGISTEmbedLoss.sim_matrixr   s+   € Ø×"Ñ" 6×#3Ñ#3°AÓ#6¸×8HÑ8HÈÓ8KÓLÐLr&   c                ó   ‡ ‡— |D �cg c]  }‰ j                  |«      d   ‘Œ }}t        j                  «       5  ‰ j                  r­|D �cg c]"  }‰ j                  j                  |d   d¬«      ‘Œ$ }}|D �cg c]  }‰ j                  j                  |«      ‘Œ }}|D ���cg c]I  }|j                  «       D ��ci c]+  \  }}||j                  ‰ j                  j                  «      “Œ- c}}‘ŒK }}}}|D �cg c]  }‰ j                  |«      d   ‘Œ }	}d d d «       d }
d }t        |«      dk(  r|\  }}	\  }}n2t        |«      dk(  r|\  }}}
	\  }}}nt        dt        |«      › �«      ‚‰ j                  ||«      }‰ j                  ||«      }‰ j                  ||«      }‰ j                  ||«      }‰ j                  ||«      }‰ j                  ||«      }|j                  «       j                  dd	«      Šddˆˆ fd
„}t        j                   |j"                  t        j$                  |j                  dœŽ} ||||¬«      } |||«      } |||«      }|||g}|
�>‰ j                  ||
«      }‰ j                  ||«      } |||«      }|j'                  |«       t        j(                  |d	¬«      ‰ j*                  z  }t        j,                  |j/                  d«      «      j1                  «       j                  |j                  «      } t3        j4                  «       ||«      S c c}w c c}w c c}w c c}}w c c}}}w c c}w # 1 sw Y   �Œ2xY w)NÚsentence_embeddingÚ	input_idsT)Úskip_special_tokensé   é   z Expected 2 or 3 embeddings, got r   r(   c                óÌ   •— ‰j                   dk(  r| ‰‰j                  z
  kD  }n$‰j                   dk(  r| ‰d‰j                  z
  z  kD  }|�| z  }t        j                   |<   |S )Nr   r   r(   )r!   r"   ÚtorchÚinf)Úguided_sim_matÚsim_matÚpositive_maskÚmaskÚ
guided_simr#   s       €€r%   Úmask_false_negativesz3GISTEmbedLoss.forward.<locals>.mask_false_negatives¡   so   ø€ Ø×#Ñ# zÒ1à%¨°d·k±kÑ)AÑB‘Ø×%Ñ%¨Ò3à%¨°q¸4¿;¹;±Ñ)GÑH�àÐ(à˜}˜nÑ,�Ü"ŸY™Y˜JˆG�D‰MØˆNr&   )ÚdtypeÚdevice)r8   r   r   )N)r8   zTensor | None)r   r4   Úno_gradr    r   Úbatch_decoder   ÚtokenizeÚitemsÚtor=   Úlenr   r,   ÚdiagonalÚviewÚeyeÚshapeÚboolÚappendÚcatr   ÚarangeÚsizeÚlongr   ÚCrossEntropyLoss)r#   Úsentence_featuresÚlabelsÚsentence_featureÚ
embeddingsÚdecodedÚ	sentencesÚkeyÚvalueÚguide_embeddingsÚnegativeÚnegative_guideÚanchorÚpositiveÚanchor_guideÚpositive_guideÚap_simÚaa_simÚpp_simÚguided_ap_simÚguided_aa_simÚguided_pp_simr;   r8   ÚscoresÚan_simÚguided_an_simr:   s   `                          @r%   ÚforwardzGISTEmbedLoss.forwardu   s>  ù€ ØarÖsÐM]�d—j‘jÐ!1Ó2Ð3GÓHÐsˆ
ÐsÜ�]‰]‹_ñ 	Ø×#Ò#ð ->öà(ð —N‘N×/Ñ/Ð0@ÀÑ0MÐcgÐ/Õhð�ð ð V]Ö$]È	 T§Z¡Z×%8Ñ%8¸Õ%CÐ$]Ð!Ð$]ð ->÷%ð %à(ð IY×H^ÑH^ÓH`×a¹*¸#¸u�S˜%Ÿ(™( 4§:¡:×#4Ñ#4Ó5Ñ5Õað%Ð!ò %ð \mö ØGW�—
‘
Ð+Ó,Ð-AÓBð Ðð  ÷	ð  ˆØˆäˆz‹?˜aÒØ)ÑˆF�HØ+;Ñ(ˆL™.Ü�‹_ Ò!Ø)3Ñ&ˆF�H˜hØ;KÑ8ˆL˜.©.äÐ?ÄÀJÃÐ?PÐQÓRÐRð —‘ ¨Ó2ˆØ—‘ ¨Ó0ˆØ—‘ ¨8Ó4ˆð Ÿ™¨°nÓEˆØŸ™¨°lÓCˆØŸ™¨¸ÓGˆð #×+Ñ+Ó-×2Ñ2°2°qÓ9ˆ
÷	ô Ÿ	™	 =×#6Ñ#6¼e¿j¹jÐQ^×QeÑQeÒfˆñ & m°VÈ=ÔYˆÙ% m°VÓ<ˆÙ% m°VÓ<ˆà˜& &Ð)ˆð ÐØ—_‘_ V¨XÓ6ˆFØ ŸO™O¨L¸.ÓIˆMÙ)¨-¸Ó@ˆFà�M‰M˜&Ô!ä—‘˜6 qÔ)¨D×,<Ñ,<Ñ<ˆô —‘˜fŸk™k¨!›nÓ-×2Ñ2Ó4×7Ñ7¸¿¹ÓFˆà$Œr×"Ñ"Ó$ V¨VÓ4Ð4ùòc tùòùò %^ùãaùô%ùò
 ÷	ñ 	ús^   ‡L"¸MÁ	'L'Á0MÁ6"L,ÂMÂ L7
Â80L1Ã(L7
Ã.MÃ6L>ÄMÌ'
MÌ1L7
Ì7MÍMc                ó`   — | j                   | j                  | j                  | j                  dœS )N©r   r   r!   r"   ri   ©r#   s    r%   Úget_config_dictzGISTEmbedLoss.get_config_dictÉ   s,   € à—Z‘ZØ×+Ñ+Ø#×3Ñ3Ø—k‘kñ	
ð 	
r&   c                 ó   — y)Na  
@misc{solatorio2024gistembed,
    title={GISTEmbed: Guided In-sample Selection of Training Negatives for Text Embedding Fine-tuning},
    author={Aivin V. Solatorio},
    year={2024},
    eprint={2402.16829},
    archivePrefix={arXiv},
    primaryClass={cs.LG}
}
© rj   s    r%   ÚcitationzGISTEmbedLoss.citationÑ   s   € ð	r&   )g{®Gáz„?r   g        )r   r   r   r   r   Úfloatr!   zLiteral['absolute', 'relative']r"   ro   ÚreturnÚNone)r*   r   r+   r   rp   r   )rO   zIterable[dict[str, Tensor]]rP   r   rp   r   )rp   zdict[str, Any])rp   Ústr)
Ú__name__Ú
__module__Ú__qualname__r   r,   rg   rk   Úpropertyrn   Ú__classcell__)r$   s   @r%   r   r      s{   ø„ ð
 "Ø;EØðbà"ðbð #ðbð ð	bð
 9ðbð ðbð 
õbóHMóR5óh
ð ò
ó ô
r&   r   )Ú
__future__r   Úcollections.abcr   Útypingr   r   r4   r   r   Úsentence_transformers.modelsr	   r
   Ú)sentence_transformers.SentenceTransformerr   ÚModuler   rm   r&   r%   ú<module>r~      s,   ðÝ "å $ß ã ß ç EÝ IôO�B—I‘Iõ Or&   