Ë
    l^(h¹Y  ã                  óÎ   — d dl mZ d dlZd dlZd dlZd dlmZ d dlmZm	Z	 d dl
Zd dlZd dlmZ d dlmZ d dlmZ d dlmZ erd d	lmZ  ej,                  e«      Z G d
„ de«      Zy)é    )ÚannotationsN)Únullcontext)ÚTYPE_CHECKINGÚCallable)ÚTensor)Útrange)ÚSentenceEvaluator)ÚSimilarityFunction)ÚSentenceTransformerc                  ó  ‡ — e Zd ZdZddgdgg d¢g d¢dgdddd	d
d
d
d
d
d
d
f	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd„Zd„ Z	 d	 	 	 	 	 	 	 	 	 dd„Z	 d	 	 	 	 	 dd„Zdd„Zd„ Z	e
d„ «       Zd„ Zˆ xZS )ÚInformationRetrievalEvaluatora�  
    This class evaluates an Information Retrieval (IR) setting.

    Given a set of queries and a large corpus set. It will retrieve for each query the top-k most similar document. It measures
    Mean Reciprocal Rank (MRR), Recall@k, and Normalized Discounted Cumulative Gain (NDCG)

    Args:
        queries (Dict[str, str]): A dictionary mapping query IDs to queries.
        corpus (Dict[str, str]): A dictionary mapping document IDs to documents.
        relevant_docs (Dict[str, Set[str]]): A dictionary mapping query IDs to a set of relevant document IDs.
        corpus_chunk_size (int): The size of each chunk of the corpus. Defaults to 50000.
        mrr_at_k (List[int]): A list of integers representing the values of k for MRR calculation. Defaults to [10].
        ndcg_at_k (List[int]): A list of integers representing the values of k for NDCG calculation. Defaults to [10].
        accuracy_at_k (List[int]): A list of integers representing the values of k for accuracy calculation. Defaults to [1, 3, 5, 10].
        precision_recall_at_k (List[int]): A list of integers representing the values of k for precision and recall calculation. Defaults to [1, 3, 5, 10].
        map_at_k (List[int]): A list of integers representing the values of k for MAP calculation. Defaults to [100].
        show_progress_bar (bool): Whether to show a progress bar during evaluation. Defaults to False.
        batch_size (int): The batch size for evaluation. Defaults to 32.
        name (str): A name for the evaluation. Defaults to "".
        write_csv (bool): Whether to write the evaluation results to a CSV file. Defaults to True.
        truncate_dim (int, optional): The dimension to truncate the embeddings to. Defaults to None.
        score_functions (Dict[str, Callable[[Tensor, Tensor], Tensor]]): A dictionary mapping score function names to score functions. Defaults to the ``similarity`` function from the ``model``.
        main_score_function (Union[str, SimilarityFunction], optional): The main score function to use for evaluation. Defaults to None.
        query_prompt (str, optional): The prompt to be used when encoding the corpus. Defaults to None.
        query_prompt_name (str, optional): The name of the prompt to be used when encoding the corpus. Defaults to None.
        corpus_prompt (str, optional): The prompt to be used when encoding the corpus. Defaults to None.
        corpus_prompt_name (str, optional): The name of the prompt to be used when encoding the corpus. Defaults to None.

    Example:
        ::

            import random
            from sentence_transformers import SentenceTransformer
            from sentence_transformers.evaluation import InformationRetrievalEvaluator
            from datasets import load_dataset

            # Load a model
            model = SentenceTransformer('all-MiniLM-L6-v2')

            # Load the Touche-2020 IR dataset (https://huggingface.co/datasets/BeIR/webis-touche2020, https://huggingface.co/datasets/BeIR/webis-touche2020-qrels)
            corpus = load_dataset("BeIR/webis-touche2020", "corpus", split="corpus")
            queries = load_dataset("BeIR/webis-touche2020", "queries", split="queries")
            relevant_docs_data = load_dataset("BeIR/webis-touche2020-qrels", split="test")

            # For this dataset, we want to concatenate the title and texts for the corpus
            corpus = corpus.map(lambda x: {'text': x['title'] + " " + x['text']}, remove_columns=['title'])

            # Shrink the corpus size heavily to only the relevant documents + 30,000 random documents
            required_corpus_ids = set(map(str, relevant_docs_data["corpus-id"]))
            required_corpus_ids |= set(random.sample(corpus["_id"], k=30_000))
            corpus = corpus.filter(lambda x: x["_id"] in required_corpus_ids)

            # Convert the datasets to dictionaries
            corpus = dict(zip(corpus["_id"], corpus["text"]))  # Our corpus (cid => document)
            queries = dict(zip(queries["_id"], queries["text"]))  # Our queries (qid => question)
            relevant_docs = {}  # Query ID to relevant documents (qid => set([relevant_cids])
            for qid, corpus_ids in zip(relevant_docs_data["query-id"], relevant_docs_data["corpus-id"]):
                qid = str(qid)
                corpus_ids = str(corpus_ids)
                if qid not in relevant_docs:
                    relevant_docs[qid] = set()
                relevant_docs[qid].add(corpus_ids)

            # Given queries, a corpus and a mapping with relevant documents, the InformationRetrievalEvaluator computes different IR metrics.
            ir_evaluator = InformationRetrievalEvaluator(
                queries=queries,
                corpus=corpus,
                relevant_docs=relevant_docs,
                name="BeIR-touche2020-subset-test",
            )
            results = ir_evaluator(model)
            '''
            Information Retrieval Evaluation of the model on the BeIR-touche2020-test dataset:
            Queries: 49
            Corpus: 31923

            Score-Function: cosine
            Accuracy@1: 77.55%
            Accuracy@3: 93.88%
            Accuracy@5: 97.96%
            Accuracy@10: 100.00%
            Precision@1: 77.55%
            Precision@3: 72.11%
            Precision@5: 71.43%
            Precision@10: 62.65%
            Recall@1: 1.72%
            Recall@3: 4.78%
            Recall@5: 7.90%
            Recall@10: 13.86%
            MRR@10: 0.8580
            NDCG@10: 0.6606
            MAP@100: 0.2934
            '''
            print(ir_evaluator.primary_metric)
            # => "BeIR-touche2020-test_cosine_map@100"
            print(results[ir_evaluator.primary_metric])
            # => 0.29335196224364596
    iPÃ  é
   )é   é   é   r   éd   Fé    Ú TNc                ój  •— t         ‰| �  «        g | _        |D ]4  }||v sŒt        ||   «      dkD  sŒ| j                  j	                  |«       Œ6 | j                  D �cg c]  }||   ‘Œ	 c}| _        t        |j                  «       «      | _        | j                  D �cg c]  }||   ‘Œ	 c}| _	        || _
        || _        || _        || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _        || _        || _        || _        || _        |r,t5        t        | j2                  j                  «       «      «      ng | _        |rt9        |«      nd | _        || _        |rd|z   }d|z   dz   | _        ddg| _         | jC                  | j6                  «       y c c}w c c}w )Nr   Ú_z Information-Retrieval_evaluationz_results.csvÚepochÚsteps)"ÚsuperÚ__init__Úqueries_idsÚlenÚappendÚqueriesÚlistÚkeysÚ
corpus_idsÚcorpusÚquery_promptÚquery_prompt_nameÚcorpus_promptÚcorpus_prompt_nameÚrelevant_docsÚcorpus_chunk_sizeÚmrr_at_kÚ	ndcg_at_kÚaccuracy_at_kÚprecision_recall_at_kÚmap_at_kÚshow_progress_barÚ
batch_sizeÚnameÚ	write_csvÚscore_functionsÚsortedÚscore_function_namesr
   Úmain_score_functionÚtruncate_dimÚcsv_fileÚcsv_headersÚ_append_csv_headers)Úselfr   r"   r'   r(   r)   r*   r+   r,   r-   r.   r/   r0   r1   r6   r2   r5   r#   r$   r%   r&   ÚqidÚcidÚ	__class__s                          €ú|/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/sentence_transformers/evaluation/InformationRetrievalEvaluator.pyr   z&InformationRetrievalEvaluator.__init__{   s¤  ø€ ô. 	‰ÑÔØˆÔØò 	-ˆCØ�mÒ#¬¨M¸#Ñ,>Ó(?À!Ó(CØ× Ñ ×'Ñ'¨Õ,ð	-ð 15×0@Ñ0@ÖA¨˜ ›ÒAˆŒä˜vŸ{™{›}Ó-ˆŒØ.2¯o©oÖ> s�v˜c“{Ò>ˆŒà(ˆÔØ!2ˆÔØ*ˆÔØ"4ˆÔà*ˆÔØ!2ˆÔØ ˆŒØ"ˆŒØ*ˆÔØ%:ˆÔ"Ø ˆŒà!2ˆÔØ$ˆŒØˆŒ	Ø"ˆŒØ.ˆÔÙQ`¤F¬4°×0DÑ0D×0IÑ0IÓ0KÓ+LÔ$MÐfhˆÔ!ÙNaÔ#5Ð6IÔ#JÐgkˆÔ Ø(ˆÔáØ˜‘:ˆDà?À$ÑFÈÑWˆŒØ# WÐ-ˆÔà× Ñ  ×!:Ñ!:Õ;ùòC Bùò ?s   ÁF+ÂF0c                ó@  — |D �]  }| j                   D ]"  }| j                  j                  |› d|› �«       Œ$ | j                  D ]B  }| j                  j                  |› d|› �«       | j                  j                  |› d|› �«       ŒD | j                  D ]"  }| j                  j                  |› d|› �«       Œ$ | j
                  D ]"  }| j                  j                  |› d|› �«       Œ$ | j                  D ]"  }| j                  j                  |› d|› �«       Œ$ �Œ y )Nz
-Accuracy@z-Precision@z-Recall@z-MRR@z-NDCG@z-MAP@)r+   r8   r   r,   r)   r*   r-   )r:   r4   Ú
score_nameÚks       r>   r9   z1InformationRetrievalEvaluator._append_csv_headers»   sB  € Ø.ó 	AˆJØ×'Ñ'ò F�Ø× Ñ ×'Ñ'¨:¨,°jÀÀÐ(DÕEðFð ×/Ñ/ò D�Ø× Ñ ×'Ñ'¨:¨,°kÀ!ÀÐ(EÔFØ× Ñ ×'Ñ'¨:¨,°h¸q¸cÐ(BÕCðDð —]‘]ò A�Ø× Ñ ×'Ñ'¨:¨,°e¸A¸3Ð(?Õ@ðAð —^‘^ò B�Ø× Ñ ×'Ñ'¨:¨,°f¸Q¸CÐ(@ÕAðBð —]‘]ò A�Ø× Ñ ×'Ñ'¨:¨,°e¸A¸3Ð(?Õ@òAñ	Aó    c                ó†  — |dk7  r|dk(  rd|› �}nd|› d|› d�}nd}| j                   �|d| j                   › d�z  }t        j                  d	| j                  › d
|› d�«       | j                  €J|j
                  |j                  i| _        |j
                  g| _        | j                  | j                  «        | j                  |g|¢­i |¤Ž}|��| j                  �röt        j                  j                  || j                  «      }	t        j                  j                  |	«      sJt!        |	dd¬«      }
|
j#                  dj                  | j$                  «      «       |
j#                  d«       nt!        |	dd¬«      }
||g}| j                  D ]ó  }| j&                  D ]  }|j)                  ||   d   |   «       Œ | j*                  D ]6  }|j)                  ||   d   |   «       |j)                  ||   d   |   «       Œ8 | j,                  D ]  }|j)                  ||   d   |   «       Œ | j.                  D ]  }|j)                  ||   d   |   «       Œ | j0                  D ]  }|j)                  ||   d   |   «       Œ Œõ |
j#                  dj                  t3        t4        |«      «      «       |
j#                  d«       |
j7                  «        | j8                  s¦| j:                  €gt=        | j                  D �cg c]"  }|||   d   t=        | j.                  «         f‘Œ$ c}d„ ¬«      d   }|› dt=        | j.                  «      › �| _        n3| j:                  j>                  › dt=        | j.                  «      › �| _        |jA                  «       D ������ci c]Y  \  }}|jA                  «       D ]A  \  }}|jA                  «       D ])  \  }}|› d|jC                  ddt5        |«      z   «      › �|“Œ+ ŒC Œ[ }}}}}}}| jE                  || j                  «      }| jG                  ||||«       |S c c}w c c}}}}}}w )Néÿÿÿÿz after epoch z
 in epoch z after z stepsr   z (truncated to ú)z5Information Retrieval Evaluation of the model on the z datasetú:Úwzutf-8)ÚmodeÚencodingú,ú
Úaú
accuracy@kúprecision@kúrecall@kúmrr@kúndcg@kúmap@kc                ó   — | d   S )Nr   © ©Úxs    r>   ú<lambda>z8InformationRetrievalEvaluator.__call__.<locals>.<lambda>  s
   €  ! A¡$€ rB   )Úkeyr   z_ndcg@r   z@kú@)$r6   ÚloggerÚinfor0   r2   Úsimilarity_fn_nameÚ
similarityr4   r9   Úcompute_metricesr1   ÚosÚpathÚjoinr7   ÚisfileÚopenÚwriter8   r+   r   r,   r)   r*   r-   ÚmapÚstrÚcloseÚprimary_metricr5   ÚmaxÚvalueÚitemsÚreplaceÚprefix_name_to_metricsÚ store_metrics_in_model_card_data)r:   ÚmodelÚoutput_pathr   r   ÚargsÚkwargsÚout_txtÚscoresÚcsv_pathÚfOutÚoutput_datar0   rA   Úscore_functionÚvalues_dictÚmetric_nameÚvaluesrj   Úmetricss                       r>   Ú__call__z&InformationRetrievalEvaluator.__call__Í   sM  € ð �BŠ;Ø˜Š{Ø)¨%¨Ð1‘à& u g¨W°U°G¸6ÐB‘àˆGØ×ÑÐ(Ø˜¨×):Ñ):Ð(;¸1Ð=Ñ=ˆGä�‰ÐKÈDÏIÉIÈ;ÐV^Ð_fÐ^gÐghÐiÔjà×ÑÐ'Ø$)×$<Ñ$<¸e×>NÑ>NÐ#OˆDÔ Ø).×)AÑ)AÐ(BˆDÔ%Ø×$Ñ$ T×%>Ñ%>Ô?à&�×&Ñ& uÐ>¨tÒ>°vÑ>ˆð Ñ" t§~£~Ü—w‘w—|‘| K°·±Ó?ˆHÜ—7‘7—>‘> (Ô+Ü˜H¨3¸ÔA�Ø—
‘
˜3Ÿ8™8 D×$4Ñ$4Ó5Ô6Ø—
‘
˜4Õ ô ˜H¨3¸ÔA�à  %˜.ˆKØ×1Ñ1ò A�Ø×+Ñ+ò F�AØ×&Ñ& v¨d¡|°LÑ'AÀ!Ñ'DÕEðFð ×3Ñ3ò D�AØ×&Ñ& v¨d¡|°MÑ'BÀ1Ñ'EÔFØ×&Ñ& v¨d¡|°JÑ'?ÀÑ'BÕCðDð Ÿ™ò A�AØ×&Ñ& v¨d¡|°GÑ'<¸QÑ'?Õ@ðAð Ÿ™ò B�AØ×&Ñ& v¨d¡|°HÑ'=¸aÑ'@ÕAðBð Ÿ™ò A�AØ×&Ñ& v¨d¡|°GÑ'<¸QÑ'?Õ@ñAðAð" �J‰J�s—x‘x¤¤C¨Ó 5Ó6Ô7Ø�J‰J�tÔØ�J‰JŒLà×"Ò"Ø×'Ñ'Ð/Ü!$ØUY×UnÑUnÖoÈT�d˜F 4™L¨Ñ2´3°t·~±~Ó3FÑGÒHÒoÙ&ô"ð ñ"�ð *8Ð(8¸¼sÀ4Ç>Á>Ó?RÐ>SÐ&T�Õ#à)-×)AÑ)A×)GÑ)GÐ(HÈÌsÐSW×SaÑSaÓObÐNcÐ&d�Ô#ð 06¯|©|«~÷
ó 
á+� Ø'2×'8Ñ'8Ó':ò
ñ $�˜VØ"ŸL™L›Nò	
ñ ��5ð Ð˜a × 3Ñ 3°D¸#ÄÀAÃ¹,Ó GÐHÐIÈ5ÑPð
ØIð
ØIð
ˆõ 
ð ×-Ñ-¨g°t·y±yÓAˆØ×-Ñ-¨e°W¸eÀUÔKØˆùò pù÷
s   Ë?'P4ÎAP9c                ó:	  — |€|}t        t        | j                  «      t        | j                  «      t        | j                  «      t        | j                  «      t        | j
                  «      «      }| j                  €
t        «       n|j                  | j                  «      5  |j                  | j                  | j                  | j                  | j                  | j                  d¬«      }d d d «       i }| j                  D ]'  }t!        t#        «      «      D �cg c]  }g ‘Œ c}||<   Œ) t%        dt#        | j&                  «      | j(                  d| j                   ¬«      D �]ï  }	t+        |	| j(                  z   t#        | j&                  «      «      }
|€‡| j                  €
t        «       n|j                  | j                  «      5  |j                  | j&                  |	|
 | j,                  | j.                  | j                  | j                  d¬«      }d d d «       n||	|
 }| j                  j1                  «       D �]  \  }} |«      }t3        j4                  |t+        |t#        |d   «      «      ddd¬«      \  }}|j7                  «       j9                  «       }|j7                  «       j9                  «       }t!        t#        |«      «      D ]  }t;        ||   ||   «      D ]h  \  }}| j<                  |	|z      }t#        ||   |   «      |k  rt?        j@                  ||   |   ||f«       ŒKt?        jB                  ||   |   ||f«       Œj Œ� �Œ �Œò |D ]Y  }t!        t#        ||   «      «      D ]=  }t!        t#        ||   |   «      «      D ]  }||   |   |   \  }}||d	œ||   |   |<   Œ  Œ? Œ[ tD        jG                  d
t#        | j                  «      › �«       tD        jG                  dt#        | j&                  «      › d�«       | j                  D �ci c]  }|| jI                  ||   «      “Œ }}| jJ                  D ].  }tD        jG                  d|› �«       | jM                  ||   «       Œ0 |S # 1 sw Y   �Œ�xY wc c}w # 1 sw Y   �ŒqxY wc c}w )NT)Úprompt_nameÚpromptr/   r.   Úconvert_to_tensorr   zCorpus Chunks)ÚdescÚdisabler   F)ÚdimÚlargestr3   )Ú	corpus_idÚscorez	Queries: zCorpus: rK   zScore-Function: )'ri   r)   r*   r+   r,   r-   r6   r   Útruncate_sentence_embeddingsÚencoder   r$   r#   r/   r.   r2   Úranger   r   r"   r(   Úminr&   r%   rk   ÚtorchÚtopkÚcpuÚtolistÚzipr!   ÚheapqÚheappushÚheappushpoprZ   r[   Úcompute_metricsr4   Úoutput_scores)r:   ro   Úcorpus_modelÚcorpus_embeddingsÚmax_kÚquery_embeddingsÚqueries_result_listr0   r   Úcorpus_start_idxÚcorpus_end_idxÚsub_corpus_embeddingsrx   Úpair_scoresÚpair_scores_top_k_valuesÚpair_scores_top_k_idxÚ	query_itrÚsub_corpus_idr‡   r†   Údoc_itrrt   s                         r>   r^   z.InformationRetrievalEvaluator.compute_metrices  sÈ  € ð ÐØ ˆLäÜ�—‘ÓÜ�—‘ÓÜ�×"Ñ"Ó#Ü�×*Ñ*Ó+Ü�—‘Óó
ˆð #×/Ñ/Ð7Œ[Œ]¸U×=_Ñ=_Ð`d×`qÑ`qÓ=rñ 	Ø$Ÿ|™|Ø—‘Ø ×2Ñ2Ø×(Ñ(ØŸ?™?Ø"&×"8Ñ"8Ø"&ð  ,ó  Ð÷	ð !ÐØ×(Ñ(ò 	SˆDÜ5:¼3Ð?OÓ;PÓ5QÖ(R°ªÒ(RÐ Ò%ð	Sô !'ØŒs�4—;‘;Ó ×!7Ñ!7¸oÐ[_×[qÑ[qÐWqô!
ó 0	hÐô !Ð!1°D×4JÑ4JÑ!JÌCÐPT×P[ÑP[ÓL\Ó]ˆNð !Ð(ð ×(Ñ(Ð0ô  ”Mà%×BÑBÀ4×CTÑCTÓUñð
 -9×,?Ñ,?ØŸ™Ð$4°^ÐDØ$(×$;Ñ$;Ø#×1Ñ1Ø#'§?¡?Ø*.×*@Ñ*@Ø*.ð -@ó -Ð)÷ð ð ):Ð:JÈ>Ð(ZÐ%ð )-×(<Ñ(<×(BÑ(BÓ(Dó hÑ$��nÙ,Ð-=Ð?TÓU�ô CHÇ*Á*Ø¤ U¬C°¸A±Ó,?Ó!@ÀaÐQUÐ^côCÑ?Ð(Ð*?ð ,D×+GÑ+GÓ+I×+PÑ+PÓ+RÐ(Ø(=×(AÑ(AÓ(C×(JÑ(JÓ(LÐ%ä!&¤sÐ+;Ó'<Ó!=ò h�IÜ03Ø-¨iÑ8Ð:RÐS\Ñ:]ó1ò hÑ,˜ uð %)§O¡OÐ4DÀ}Ñ4TÑ$U˜	ô Ð2°4Ñ8¸ÑCÓDÀuÒLä!ŸN™NÐ+>¸tÑ+DÀYÑ+OÐRWÐYbÐQcÕdä!×-Ñ-Ð.AÀ$Ñ.GÈ	Ñ.RÐUZÐ\eÐTfÕgñhòhòhð10	hðd (ò 	mˆDÜ"¤3Ð':¸4Ñ'@Ó#AÓBò m�	Ü$¤SÐ)<¸TÑ)BÀ9Ñ)MÓ%NÓOò m�GØ':¸4Ñ'@ÀÑ'KÈGÑ'TÑ$�E˜9ØR[ÐfkÑDlÐ'¨Ñ-¨iÑ8¸ÒAñmñmð	mô 	�‰�i¤ D§L¡LÓ 1Ð2Ð3Ô4Ü�‰�hœs 4§;¡;Ó/Ð0°Ð3Ô4ð UY×ThÑThÖiÈD�$˜×,Ñ,Ð-@ÀÑ-FÓGÑGÐiˆÐið ×-Ñ-ò 	-ˆDÜ�K‰KÐ*¨4¨&Ð1Ô2Ø×Ñ˜v d™|Õ,ð	-ð ˆ÷e	ñ 	üò )S÷ñ üòj js&   Â$A
Q9Ä	RÇ	ARÐRÑ9RÒR	c           	     óD  — | j                   D �ci c]  }|d“Œ }}| j                  D �ci c]  }|g “Œ }}| j                  D �ci c]  }|g “Œ }}| j                  D �ci c]  }|d“Œ }}| j                  D �ci c]  }|g “Œ }}| j                  D �ci c]  }|g “Œ }}t        t        |«      «      D �]ë  }	| j                  |	   }
t        ||	   d„ d¬«      }| j                  |
   }| j                   D ]"  }|d| D ]  }|d   |v sŒ||xx   dz  cc<    Œ" Œ$ | j                  D ]R  }d}|d| D ]  }|d   |v sŒ|dz  }Œ ||   j                  ||z  «       ||   j                  |t        |«      z  «       ŒT | j                  D ]4  }t        |d| «      D ]!  \  }}|d   |v sŒ||xx   d|dz   z  z  cc<    Œ4 Œ6 | j                  D ]e  }|d| D �cg c]  }|d   |v rdnd‘Œ }}dgt        |«      z  }| j                  ||«      | j                  ||«      z  }||   j                  |«       Œg | j                  D ]`  }d}d}t        |d| «      D ]  \  }}|d   |v sŒ|dz  }|||dz   z  z  }Œ |t        |t        |«      «      z  }||   j                  |«       Œb �Œî |D ]"  }||xx   t        | j                  «      z  cc<   Œ$ |D ]  }t        j                   ||   «      ||<   Œ |D ]  }t        j                   ||   «      ||<   Œ |D ]  }t        j                   ||   «      ||<   Œ |D ]"  }||xx   t        | j                  «      z  cc<   Œ$ |D ]  }t        j                   ||   «      ||<   Œ ||||||dœS c c}w c c}w c c}w c c}w c c}w c c}w c c}w )	Nr   c                ó   — | d   S )Nr‡   rT   rU   s    r>   rW   z?InformationRetrievalEvaluator.compute_metrics.<locals>.<lambda>‰  s
   € ÈAÈgÉJ€ rB   T)rX   Úreverser†   r   g      ð?)rM   rN   rO   rQ   rP   rR   )r+   r,   r)   r*   r-   rŠ   r   r   r3   r'   r   Ú	enumerateÚcompute_dcg_at_kr‹   r   ÚnpÚmean)r:   rš   rA   Únum_hits_at_kÚprecisions_at_kÚrecall_at_kÚMRRÚndcgÚ	AveP_at_kr¡   Úquery_idÚtop_hitsÚquery_relevant_docsÚk_valÚhitÚnum_correctÚrankÚtop_hitÚpredicted_relevanceÚtrue_relevancesÚ
ndcg_valueÚsum_precisionsÚavg_precisions                          r>   r”   z-InformationRetrievalEvaluator.compute_metrics{  sŽ  € à'+×'9Ñ'9Ö: !˜˜A™Ð:ˆÐ:Ø*.×*DÑ*DÖE Q˜1˜b™5ÐEˆÐEØ&*×&@Ñ&@ÖA �q˜"‘uÐAˆÐAØ!Ÿ]™]Ö+˜ˆq�!‰tÐ+ˆÐ+Ø#Ÿ~™~Ö.˜!��2‘Ð.ˆÐ.Ø$(§M¡MÖ2˜q�Q˜‘UÐ2ˆ	Ð2ô œsÐ#6Ó7Ó8ó 6	7ˆIØ×'Ñ'¨	Ñ2ˆHô Ð1°)Ñ<ÑBVÐ`dÔeˆHØ"&×"4Ñ"4°XÑ">Ðð ×+Ñ+ò �Ø# A eÐ,ò �CØ˜;Ñ'Ð+>Ò>Ø% eÓ,°Ñ1Ó,Ùñðð ×3Ñ3ò R�Ø�Ø# A eÐ,ò )�CØ˜;Ñ'Ð+>Ò>Ø# qÑ(™ð)ð   Ñ&×-Ñ-¨k¸EÑ.AÔBØ˜EÑ"×)Ñ)¨+¼Ð<OÓ8PÑ*PÕQðRð Ÿ™ò �Ü!*¨8°A°eÐ+<Ó!=ò ‘I�D˜#Ø˜;Ñ'Ð+>Ò>Ø˜E›
 c¨T°A©XÑ&6Ñ6›
Ùñðð Ÿ™ò 	/�à[cÐdeÐfkÐ[lö'ØPW˜ Ñ-Ð1DÑD‘AÈ!ÑKð'Ð#ð 'ð $% #¬Ð,?Ó(@Ñ"@�à!×2Ñ2Ð3FÈÓNÐQU×QfÑQfØ# UóRñ �
ð �U‘×"Ñ" :Õ.ð	/ð Ÿ™ò 
7�Ø�Ø!"�ä!*¨8°A°eÐ+<Ó!=ò C‘I�D˜#Ø˜;Ñ'Ð+>Ò>Ø# qÑ(˜Ø&¨+¸À¹Ñ*BÑB™ðCð
 !/´°U¼CÐ@SÓ<TÓ1UÑ U�Ø˜%Ñ ×'Ñ'¨Õ6ò
7ðY6	7ðr ò 	2ˆAØ˜!Ó¤ D§L¡LÓ 1Ñ1Ôð	2ð !ò 	=ˆAÜ!#§¡¨¸Ñ);Ó!<ˆO˜AÒð	=ð ò 	5ˆAÜŸW™W [°¡^Ó4ˆK˜ŠNð	5ð ò 	'ˆAÜ—g‘g˜d 1™gÓ&ˆD�ŠGð	'ð ò 	(ˆAØ�‹F”c˜$Ÿ,™,Ó'Ñ'ŒFð	(ð ò 	1ˆAÜŸ7™7 9¨Q¡<Ó0ˆI�aŠLð	1ð (Ø*Ø#ØØØñ
ð 	
ùòg ;ùÚEùÚAùÚ+ùÚ.ùÚ2ùòH's(   �
M?©
NÁ
N	Á
NÁ7
NÂ
NÇNc                ó’  — |d   D ]0  }t         j                  dj                  ||d   |   dz  «      «       Œ2 |d   D ]0  }t         j                  dj                  ||d   |   dz  «      «       Œ2 |d   D ]0  }t         j                  dj                  ||d   |   dz  «      «       Œ2 |d   D ]-  }t         j                  d	j                  ||d   |   «      «       Œ/ |d
   D ]-  }t         j                  dj                  ||d
   |   «      «       Œ/ |d   D ]-  }t         j                  dj                  ||d   |   «      «       Œ/ y )NrM   zAccuracy@{}: {:.2f}%r   rN   zPrecision@{}: {:.2f}%rO   zRecall@{}: {:.2f}%rP   zMRR@{}: {:.4f}rQ   zNDCG@{}: {:.4f}rR   zMAP@{}: {:.4f})rZ   r[   Úformat)r:   rt   rA   s      r>   r•   z+InformationRetrievalEvaluator.output_scoresÙ  sv  € Ø˜Ñ%ò 	YˆAÜ�K‰KÐ.×5Ñ5°a¸ÀÑ9MÈaÑ9PÐSVÑ9VÓWÕXð	Yð ˜Ñ&ò 	[ˆAÜ�K‰KÐ/×6Ñ6°q¸&ÀÑ:OÐPQÑ:RÐUXÑ:XÓYÕZð	[ð ˜
Ñ#ò 	UˆAÜ�K‰KÐ,×3Ñ3°A°v¸jÑ7IÈ!Ñ7LÈsÑ7RÓSÕTð	Uð ˜‘ò 	HˆAÜ�K‰KÐ(×/Ñ/°°6¸'±?À1Ñ3EÓFÕGð	Hð ˜Ñ!ò 	JˆAÜ�K‰KÐ)×0Ñ0°°F¸8Ñ4DÀQÑ4GÓHÕIð	Jð ˜‘ò 	HˆAÜ�K‰KÐ(×/Ñ/°°6¸'±?À1Ñ3EÓFÕGñ	HrB   c                ó’   — d}t        t        t        | «      |«      «      D ]#  }|| |   t        j                  |dz   «      z  z  }Œ% |S )Nr   é   )rŠ   r‹   r   r©   Úlog2)Ú
relevancesrA   ÚdcgÚis       r>   r¨   z.InformationRetrievalEvaluator.compute_dcg_at_kì  sJ   € àˆÜ”sœ3˜z›?¨AÓ.Ó/ò 	2ˆAØ�:˜a‘=¤2§7¡7¨1¨q©5£>Ñ1Ñ1‰Cð	2àˆ
rB   c                óX   — i }g d¢}|D ]  }t        | |«      €Œt        | |«      ||<   Œ  |S )N)r6   r#   r$   r%   r&   )Úgetattr)r:   Úconfig_dictÚconfig_dict_candidate_keysrX   s       r>   Úget_config_dictz-InformationRetrievalEvaluator.get_config_dictó  sF   € Øˆò&
Ð"ð .ò 	6ˆCÜ�t˜SÓ!Ñ-Ü#*¨4°Ó#5�˜CÒ ð	6ð ÐrB   )*r   údict[str, str]r"   rË   r'   zdict[str, set[str]]r(   Úintr)   ú	list[int]r*   rÍ   r+   rÍ   r,   rÍ   r-   rÍ   r.   Úboolr/   rÌ   r0   rf   r1   rÎ   r6   z
int | Noner2   z4dict[str, Callable[[Tensor, Tensor], Tensor]] | Noner5   zstr | SimilarityFunction | Noner#   ú
str | Noner$   rÏ   r%   rÏ   r&   rÏ   ÚreturnÚNone)NrD   rD   )
ro   r   rp   rf   r   rÌ   r   rÌ   rÐ   údict[str, float])NN)ro   r   r—   zTensor | NonerÐ   rÒ   )rš   zlist[object])Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r9   r}   r^   r”   r•   Ústaticmethodr¨   rÊ   Ú__classcell__)r=   s   @r>   r   r      s¬  ø„ ñaðP "'Ø!˜dØ "˜tÚ#0Ú+8Ø"˜eØ"'ØØØØ#'ØPTØ?CØ#'Ø(,Ø$(Ø)-ð+><àð><ð ð><ð +ð	><ð
 ð><ð ð><ð ð><ð !ð><ð  )ð><ð ð><ð  ð><ð ð><ð ð><ð ð><ð !ð><ð  Nð!><ð" =ð#><ð$ !ð%><ð& &ð'><ð( "ð)><ð* 'ð+><ð, 
õ-><ò@Að& bdðIØ(ðIØ7:ðIØJMðIØ[^ðIà	óIðX aeðaØ(ðaØP]ðaà	óaóF\
ò|Hð& ñó ðörB   r   )Ú
__future__r   r‘   Úloggingr_   Ú
contextlibr   Útypingr   r   Únumpyr©   rŒ   r   Útqdmr   Ú2sentence_transformers.evaluation.SentenceEvaluatorr	   Ú*sentence_transformers.similarity_functionsr
   Ú)sentence_transformers.SentenceTransformerr   Ú	getLoggerrÓ   rZ   r   rT   rB   r>   ú<module>rã      sP   ðÝ "ã Û Û 	Ý "ß *ã Û Ý Ý å PÝ IáÝMà	ˆ×	Ñ	˜8Ó	$€ôhÐ$5õ hrB   