Ë
    l^(h³P  ã                  ó(  — d dl mZ d dlZd dlZd dlmZmZmZmZ d dl	Z
d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d d
lmZ erd dlmZ  ej2                  e«      Zed   ZddddddddddddddœZdddddddd d!d"d#d$d%dœZ G d&„ d'e«      Zy)(é    )ÚannotationsN)ÚTYPE_CHECKINGÚAnyÚCallableÚLiteral)ÚTensor)Útqdm)ÚSentenceTransformer)ÚInformationRetrievalEvaluator)ÚSentenceEvaluator)ÚSimilarityFunction)Úis_datasets_available)ÚclimatefeverÚdbpediaÚfeverÚfiqa2018ÚhotpotqaÚmsmarcoÚnfcorpusÚnqÚquoraretrievalÚscidocsÚarguanaÚscifactÚ
touche2020zzeta-alpha-ai/NanoClimateFEVERzzeta-alpha-ai/NanoDBPediazzeta-alpha-ai/NanoFEVERzzeta-alpha-ai/NanoFiQA2018zzeta-alpha-ai/NanoHotpotQAzzeta-alpha-ai/NanoMSMARCOzzeta-alpha-ai/NanoNFCorpuszzeta-alpha-ai/NanoNQz zeta-alpha-ai/NanoQuoraRetrievalzzeta-alpha-ai/NanoSCIDOCSzzeta-alpha-ai/NanoArguAnazzeta-alpha-ai/NanoSciFactzzeta-alpha-ai/NanoTouche2020ÚClimateFEVERÚDBPediaÚFEVERÚFiQA2018ÚHotpotQAÚMSMARCOÚNFCorpusÚNQÚQuoraRetrievalÚSCIDOCSÚArguAnaÚSciFactÚ
Touche2020c                  óî   ‡ — e Zd ZdZddgdgg d¢g d¢dgddddddej
                  d	ddf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dˆ fd
„Zd„ Z	 d	 	 	 	 	 	 	 	 	 dd„Zdd„Z	dd„Z
d„ Zd„ Zdd„Zˆ xZS )ÚNanoBEIREvaluatora¨  
    This class evaluates the performance of a SentenceTransformer Model on the NanoBEIR collection of Information Retrieval datasets.

    The collection is a set of datasets based on the BEIR collection, but with a significantly smaller size, so it can
    be used for quickly evaluating the retrieval performance of a model before commiting to a full evaluation.
    The datasets are available on Hugging Face in the `NanoBEIR collection <https://huggingface.co/collections/zeta-alpha-ai/nanobeir-66e1a0af21dfd93e620cd9f6>`_.
    This evaluator will return the same metrics as the InformationRetrievalEvaluator (i.e., MRR, nDCG, Recall@k), for each dataset and on average.

    Args:
        dataset_names (List[str]): The names of the datasets to evaluate on. Defaults to all datasets.
        mrr_at_k (List[int]): A list of integers representing the values of k for MRR calculation. Defaults to [10].
        ndcg_at_k (List[int]): A list of integers representing the values of k for NDCG calculation. Defaults to [10].
        accuracy_at_k (List[int]): A list of integers representing the values of k for accuracy calculation. Defaults to [1, 3, 5, 10].
        precision_recall_at_k (List[int]): A list of integers representing the values of k for precision and recall calculation. Defaults to [1, 3, 5, 10].
        map_at_k (List[int]): A list of integers representing the values of k for MAP calculation. Defaults to [100].
        show_progress_bar (bool): Whether to show a progress bar during evaluation. Defaults to False.
        batch_size (int): The batch size for evaluation. Defaults to 32.
        write_csv (bool): Whether to write the evaluation results to a CSV file. Defaults to True.
        truncate_dim (int, optional): The dimension to truncate the embeddings to. Defaults to None.
        score_functions (Dict[str, Callable[[Tensor, Tensor], Tensor]]): A dictionary mapping score function names to score functions. Defaults to {SimilarityFunction.COSINE.value: cos_sim, SimilarityFunction.DOT_PRODUCT.value: dot_score}.
        main_score_function (Union[str, SimilarityFunction], optional): The main score function to use for evaluation. Defaults to None.
        aggregate_fn (Callable[[list[float]], float]): The function to aggregate the scores. Defaults to np.mean.
        aggregate_key (str): The key to use for the aggregated score. Defaults to "mean".
        query_prompts (str | dict[str, str], optional): The prompts to add to the queries. If a string, will add the same prompt to all queries. If a dict, expects that all datasets in dataset_names are keys.
        corpus_prompts (str | dict[str, str], optional): The prompts to add to the corpus. If a string, will add the same prompt to all corpus. If a dict, expects that all datasets in dataset_names are keys.

    Example:
        ::

            from sentence_transformers import SentenceTransformer
            from sentence_transformers.evaluation import NanoBEIREvaluator

            model = SentenceTransformer('intfloat/multilingual-e5-large-instruct')

            datasets = ["QuoraRetrieval", "MSMARCO"]
            query_prompts = {
                "QuoraRetrieval": "Instruct: Given a question, retrieve questions that are semantically equivalent to the given question\nQuery: ",
                "MSMARCO": "Instruct: Given a web search query, retrieve relevant passages that answer the query\nQuery: "
            }

            evaluator = NanoBEIREvaluator(
                dataset_names=datasets,
                query_prompts=query_prompts,
            )

            results = evaluator(model)
            '''
            NanoBEIR Evaluation of the model on ['QuoraRetrieval', 'MSMARCO'] dataset:
            Evaluating NanoQuoraRetrieval
            Information Retrieval Evaluation of the model on the NanoQuoraRetrieval dataset:
            Queries: 50
            Corpus: 5046

            Score-Function: cosine
            Accuracy@1: 92.00%
            Accuracy@3: 98.00%
            Accuracy@5: 100.00%
            Accuracy@10: 100.00%
            Precision@1: 92.00%
            Precision@3: 40.67%
            Precision@5: 26.00%
            Precision@10: 14.00%
            Recall@1: 81.73%
            Recall@3: 94.20%
            Recall@5: 97.93%
            Recall@10: 100.00%
            MRR@10: 0.9540
            NDCG@10: 0.9597
            MAP@100: 0.9395

            Evaluating NanoMSMARCO
            Information Retrieval Evaluation of the model on the NanoMSMARCO dataset:
            Queries: 50
            Corpus: 5043

            Score-Function: cosine
            Accuracy@1: 40.00%
            Accuracy@3: 74.00%
            Accuracy@5: 78.00%
            Accuracy@10: 88.00%
            Precision@1: 40.00%
            Precision@3: 24.67%
            Precision@5: 15.60%
            Precision@10: 8.80%
            Recall@1: 40.00%
            Recall@3: 74.00%
            Recall@5: 78.00%
            Recall@10: 88.00%
            MRR@10: 0.5849
            NDCG@10: 0.6572
            MAP@100: 0.5892
            Average Queries: 50.0
            Average Corpus: 5044.5

            Aggregated for Score Function: cosine
            Accuracy@1: 66.00%
            Accuracy@3: 86.00%
            Accuracy@5: 89.00%
            Accuracy@10: 94.00%
            Precision@1: 66.00%
            Recall@1: 60.87%
            Precision@3: 32.67%
            Recall@3: 84.10%
            Precision@5: 20.80%
            Recall@5: 87.97%
            Precision@10: 11.40%
            Recall@10: 94.00%
            MRR@10: 0.7694
            NDCG@10: 0.8085
            '''
            print(evaluator.primary_metric)
            # => "NanoBEIR_mean_cosine_ndcg@10"
            print(results[evaluator.primary_metric])
            # => 0.8084508771660436
    Né
   )é   é   é   r+   éd   Fé    TÚmeanc                óV  •— t         ‰| �  «        |€t        t        j	                  «       «      }|| _        || _        || _        |	| _        || _	        || _
        || _        |	| _        || _        |r,t        t        | j                  j	                  «       «      «      ng | _        || _        |
| _        d|› �| _        | j                   r"| xj"                  d| j                   › �z  c_        || _        || _        || _        || _        || _        | j/                  «        | j1                  «        ||||||||	|
||dœ}t3        | j
                  dd¬«      D �cg c]  } | j4                  |fi |¤Ž‘Œ c}| _        d|› d�| _        d	d
g| _        | j=                  | j                  «       y c c}w )NÚ	NanoBEIR_Ú_)Úmrr_at_kÚ	ndcg_at_kÚaccuracy_at_kÚprecision_recall_at_kÚmap_at_kÚshow_progress_barÚ
batch_sizeÚ	write_csvÚtruncate_dimÚscore_functionsÚmain_score_functionzLoading NanoBEIR datasetsF)ÚdescÚleaveÚNanoBEIR_evaluation_z_results.csvÚepochÚsteps)ÚsuperÚ__init__ÚlistÚdataset_name_to_idÚkeysÚdataset_namesÚaggregate_fnÚaggregate_keyr<   Úquery_promptsÚcorpus_promptsr:   r>   ÚsortedÚscore_function_namesr?   r=   Únamer5   r6   r7   r8   r9   Ú_validate_dataset_namesÚ_validate_promptsr	   Ú_load_datasetÚ
evaluatorsÚcsv_fileÚcsv_headersÚ_append_csv_headers)ÚselfrJ   r5   r6   r7   r8   r9   r:   r;   r<   r=   r>   r?   rK   rL   rM   rN   Úir_evaluator_kwargsrQ   Ú	__class__s                      €úp/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/sentence_transformers/evaluation/NanoBEIREvaluator.pyrF   zNanoBEIREvaluator.__init__½   s¹  ø€ ô& 	‰ÑÔØÐ Ü Ô!3×!8Ñ!8Ó!:Ó;ˆMØ*ˆÔØ(ˆÔØ*ˆÔØ"ˆŒØ*ˆÔØ,ˆÔØ!2ˆÔØ"ˆŒØ.ˆÔÙQ`¤F¬4°×0DÑ0D×0IÑ0IÓ0KÓ+LÔ$MÐfhˆÔ!Ø#6ˆÔ Ø(ˆÔØ ˜Ð/ˆŒ	Ø×ÒØ�IŠI˜1˜T×.Ñ.Ð/Ð0Ñ0�Ià ˆŒØ"ˆŒØ*ˆÔØ%:ˆÔ"Ø ˆŒà×$Ñ$Ô&Ø×ÑÔ ð !Ø"Ø*Ø%:Ø Ø!2Ø$Ø"Ø(Ø.Ø#6ñ
Ðô  ˜T×/Ñ/Ð6QÐY^Ô_ö
àð ˆD×Ñ˜tÑ;Ð':Ó;ò
ˆŒð
  4°M°?À,ÐOˆŒØ# WÐ-ˆÔà× Ñ  ×!:Ñ!:Õ;ùò
s   ÅF&c                ó@  — |D �]  }| j                   D ]"  }| j                  j                  |› d|› �«       Œ$ | j                  D ]B  }| j                  j                  |› d|› �«       | j                  j                  |› d|› �«       ŒD | j                  D ]"  }| j                  j                  |› d|› �«       Œ$ | j
                  D ]"  }| j                  j                  |› d|› �«       Œ$ | j                  D ]"  }| j                  j                  |› d|› �«       Œ$ �Œ y )Nz
-Accuracy@z-Precision@z-Recall@z-MRR@z-NDCG@z-MAP@)r7   rW   Úappendr8   r5   r6   r9   )rY   rP   Ú
score_nameÚks       r\   rX   z%NanoBEIREvaluator._append_csv_headers  sB  € Ø.ó 	AˆJØ×'Ñ'ò F�Ø× Ñ ×'Ñ'¨:¨,°jÀÀÐ(DÕEðFð ×/Ñ/ò D�Ø× Ñ ×'Ñ'¨:¨,°kÀ!ÀÐ(EÔFØ× Ñ ×'Ñ'¨:¨,°h¸q¸cÐ(BÕCðDð —]‘]ò A�Ø× Ñ ×'Ñ'¨:¨,°e¸A¸3Ð(?Õ@ðAð —^‘^ò B�Ø× Ñ ×'Ñ'¨:¨,°f¸Q¸CÐ(@ÕAðBð —]‘]ò A�Ø× Ñ ×'Ñ'¨:¨,°e¸A¸3Ð(?Õ@òAñ	Aó    c                óZ  — i }i }|dk7  r|dk(  rd|› �}	nd|› d|› d�}	nd}	| j                   �|	d| j                   › d�z  }	t        j                  d	| j                  › d
|	› d�«       | j                  €J|j
                  |j                  i| _        |j
                  g| _        | j                  | j                  «       t        | j                  d| j                   ¬«      D ]ž  }
t        j                  d|
j                  › �«        |
||||«      }|D ]j  }| j                   r|j                  dd¬«      \  }}}n|j                  dd¬«      \  }}||vrg ||<   ||   ||dz   |z   <   ||   j                  ||   «       Œl Œ  i }|D ]  }| j                  ||   «      ||<   Œ |��ý| j                   �rðt"        j$                  j'                  || j(                  «      }t"        j$                  j+                  |«      sJt-        |dd¬«      }|j/                  dj'                  | j0                  «      «       |j/                  d«       nt-        |dd¬«      }||g}| j                  D ]í  }| j2                  D ]  }|j                  ||› d|› �   «       Œ | j4                  D ]4  }|j                  ||› d|› �   «       |j                  ||› d|› �   «       Œ6 | j6                  D ]  }|j                  ||› d|› �   «       Œ | j8                  D ]  }|j                  ||› d|› �   «       Œ | j:                  D ]  }|j                  ||› d|› �   «       Œ Œï |j/                  dj'                  t=        t>        |«      «      «       |j/                  d«       |jA                  «        | jB                  s¥| jD                  €ftG        | j                  D �cg c]!  }|||› dtG        | j8                  «      › �   f‘Œ# c}d„ ¬ «      d!   }|› dtG        | j8                  «      › �| _!        n3| jD                  jH                  › dtG        | j8                  «      › �| _!        tK        jL                  | j                  D �
cg c]  }
tO        |
jP                  «      ‘Œ c}
«      }tK        jL                  | j                  D �
cg c]  }
tO        |
jR                  «      ‘Œ c}
«      }t        j                  d"|› �«       t        j                  d#|› d�«       | j                  D �]I  }t        j                  d$|› �«       | j2                  D ]2  }t        j                  d%jU                  |||› d|› �   d&z  «      «       Œ4 | j4                  D ]b  }t        j                  d'jU                  |||› d|› �   d&z  «      «       t        j                  d(jU                  |||› d|› �   d&z  «      «       Œd | j6                  D ]/  }t        j                  d)jU                  |||› d|› �   «      «       Œ1 | j8                  D ]/  }t        j                  d*jU                  |||› d|› �   «      «       Œ1 �ŒL | jW                  || j                  «      }| jY                  ||||«       |j[                  |«       |S c c}w c c}
w c c}
w )+Néÿÿÿÿz after epoch z
 in epoch z after z stepsÚ z (truncated to ú)z$NanoBEIR Evaluation of the model on z datasetú:zEvaluating datasets)r@   ÚdisablezEvaluating r4   é   )Úmaxsplitr,   Úwzutf-8)ÚmodeÚencodingú,ú
Úaz
_accuracy@z_precision@z_recall@z_mrr@z_ndcg@z_map@c                ó   — | d   S )Nr,   © )Úxs    r\   ú<lambda>z,NanoBEIREvaluator.__call__.<locals>.<lambda>`  s
   €  ! A¡$€ ra   )Úkeyr   zAverage Queries: zAverage Corpus: zAggregated for Score Function: zAccuracy@{}: {:.2f}%r/   zPrecision@{}: {:.2f}%zRecall@{}: {:.2f}%zMRR@{}: {:.4f}zNDCG@{}: {:.4f}).r=   ÚloggerÚinforJ   r>   Úsimilarity_fn_nameÚ
similarityrP   rX   r	   rU   r:   rQ   Úsplitr^   rK   r<   ÚosÚpathÚjoinrV   ÚisfileÚopenÚwriterW   r7   r8   r5   r6   r9   ÚmapÚstrÚcloseÚprimary_metricr?   ÚmaxÚvalueÚnpr1   ÚlenÚqueriesÚcorpusÚformatÚprefix_name_to_metricsÚ store_metrics_in_model_card_dataÚupdate)rY   ÚmodelÚoutput_pathrC   rD   ÚargsÚkwargsÚper_metric_resultsÚper_dataset_resultsÚout_txtÚ	evaluatorÚ
evaluationr`   Údatasetr4   ÚmetricÚagg_resultsÚcsv_pathÚfOutÚoutput_datarQ   Úscore_functionÚavg_queriesÚ
avg_corpuss                           r\   Ú__call__zNanoBEIREvaluator.__call__  sÐ  € ð  ÐØ ÐØ�BŠ;Ø˜Š{Ø)¨%¨Ð1‘à& u g¨W°U°G¸6ÐB‘àˆGØ×ÑÐ(Ø˜¨×):Ñ):Ð(;¸1Ð=Ñ=ˆGÜ�‰Ð:¸4×;MÑ;MÐ:NÈhÐW^ÐV_Ð_`ÐaÔbà×ÑÐ'Ø$)×$<Ñ$<¸e×>NÑ>NÐ#OˆDÔ Ø).×)AÑ)AÐ(BˆDÔ%Ø×$Ñ$ T×%>Ñ%>Ô?ä˜dŸo™oÐ4IÐW[×WmÑWmÐSmÔnò 	AˆIÜ�K‰K˜+ i§n¡nÐ%5Ð6Ô7Ù" 5¨+°u¸eÓDˆJØò A�Ø×$Ò$Ø)*¯©°¸q¨Ó)AÑ&�G˜Q¡à&'§g¡g¨c¸A gÓ&>‘O�G˜VØÐ!3Ñ3Ø13Ð& vÑ.Ø>HÈ¹mÐ# G¨c¡M°FÑ$:Ñ;Ø" 6Ñ*×1Ñ1°*¸Q±-Õ@ñAð	Að ˆØ(ò 	PˆFØ"&×"3Ñ"3Ð4FÀvÑ4NÓ"OˆK˜Òð	Pð Ñ" t§~£~Ü—w‘w—|‘| K°·±Ó?ˆHÜ—7‘7—>‘> (Ô+Ü˜H¨3¸ÔA�Ø—
‘
˜3Ÿ8™8 D×$4Ñ$4Ó5Ô6Ø—
‘
˜4Õ ô ˜H¨3¸ÔA�à  %˜.ˆKØ×1Ñ1ò G�Ø×+Ñ+ò L�AØ×&Ñ& {°d°V¸:ÀaÀSÐ3IÑ'JÕKðLð ×3Ñ3ò J�AØ×&Ñ& {°d°V¸;ÀqÀcÐ3JÑ'KÔLØ×&Ñ& {°d°V¸8ÀAÀ3Ð3GÑ'HÕIðJð Ÿ™ò G�AØ×&Ñ& {°d°V¸5ÀÀÐ3DÑ'EÕFðGð Ÿ™ò H�AØ×&Ñ& {°d°V¸6À!ÀÐ3EÑ'FÕGðHð Ÿ™ò G�AØ×&Ñ& {°d°V¸5ÀÀÐ3DÑ'EÕFñGðGð" �J‰J�s—x‘x¤¤C¨Ó 5Ó6Ô7Ø�J‰J�tÔØ�J‰JŒLà×"Ò"Ø×'Ñ'Ð/Ü!$Ø[_×[tÑ[tÖuÐSW�d˜K¨4¨&°´s¸4¿>¹>Ó7JÐ6KÐ(LÑMÒNÒuÙ&ô"ð ñ"�ð *8Ð(8¸¼sÀ4Ç>Á>Ó?RÐ>SÐ&T�Õ#à)-×)AÑ)A×)GÑ)GÐ(HÈÌsÐSW×SaÑSaÓObÐNcÐ&d�Ô#ä—g‘gÀtÇÁÖW¸)œs 9×#4Ñ#4Õ5ÒWÓXˆÜ—W‘WÀTÇ_Á_ÖU¸	œc )×"2Ñ"2Õ3ÒUÓVˆ
Ü�‰Ð'¨ }Ð5Ô6Ü�‰Ð& z l°"Ð5Ô6à×-Ñ-ó 	ZˆDÜ�K‰KÐ9¸$¸Ð@ÔAØ×'Ñ'ò i�Ü—‘Ð2×9Ñ9¸!¸[ÈDÈ6ÐQ[Ð\]Ð[^ÐI_Ñ=`ÐcfÑ=fÓgÕhðið ×/Ñ/ò e�Ü—‘Ð3×:Ñ:¸1¸kÈTÈFÐR]Ð^_Ð]`ÐJaÑ>bÐehÑ>hÓiÔjÜ—‘Ð0×7Ñ7¸¸;È$ÈÈxÐXYÐWZÐG[Ñ;\Ð_bÑ;bÓcÕdðeð —]‘]ò X�Ü—‘Ð,×3Ñ3°A°{ÀdÀVÈ5ÐQRÐPSÐCTÑ7UÓVÕWðXð —^‘^ò Z�Ü—‘Ð-×4Ñ4°Q¸ÀtÀfÈFÐSTÐRUÐDVÑ8WÓXÕYòZð	Zð ×1Ñ1°+¸t¿y¹yÓIˆØ×-Ñ-¨e°[À%ÈÔOà×"Ñ" ;Ô/à"Ð"ùòA vùò XùÚUs   Ï&ZÑ.Z#Ò.Z(c                óv   — dt         |j                  «          › �}| j                  �|d| j                  › �z  }|S )NÚNanor4   )Údataset_name_to_human_readableÚlowerr=   )rY   Údataset_nameÚhuman_readable_names      r\   Ú_get_human_readable_namez*NanoBEIREvaluator._get_human_readable_name�  sJ   € Ø $Ô%CÀL×DVÑDVÓDXÑ%YÐ$ZÐ[ÐØ×ÑÐ(Ø Q t×'8Ñ'8Ð&9Ð#:Ñ:ÐØ"Ð"ra   c                ó¾  — t        «       st        d«      ‚ddlm} t        |j                  «          } ||dd¬«      } ||dd¬«      } ||dd¬«      }|D �ci c]  }t        |d	   «      dkD  sŒ|d
   |d	   “Œ }	}|D �ci c]  }t        |d	   «      dkD  sŒ|d
   |d	   “Œ }
}i }|D ]3  }|d   |vrt        «       ||d   <   ||d      j                  |d   «       Œ5 | j                  �| j                  j                  |d «      |d<   | j                  �| j                  j                  |d «      |d<   | j                  |«      }t        d|
|	||dœ|¤ŽS c c}w c c}w )Nzedatasets is not available. Please install it to use the NanoBEIREvaluator via `pip install datasets`.r   )Úload_datasetr‰   Útrain)ry   rˆ   ÚqrelsÚtextÚ_idzquery-idz	corpus-idÚquery_promptÚcorpus_prompt)rˆ   r‰   Úrelevant_docsrQ   rq   )r   Ú
ValueErrorÚdatasetsr©   rH   r¤   r‡   ÚsetÚaddrM   ÚgetrN   r§   r   )rY   r¥   rZ   r©   Údataset_pathr‰   rˆ   r«   ÚsampleÚcorpus_dictÚqueries_dictÚ
qrels_dictr¦   s                r\   rT   zNanoBEIREvaluator._load_dataset‡  s­  € Ü$Ô&ÜØwóð õ 	*ä)¨,×*<Ñ*<Ó*>Ñ?ˆÙ˜l¨H¸GÔDˆÙ˜|¨Y¸gÔFˆÙ˜\¨7¸'ÔBˆØCIÖe¸ÌSÐQWÐX^ÑQ_ÓM`ÐcdÓMd�v˜e‘} f¨V¡nÑ4ÐeˆÐeØDKÖg¸&ÌsÐSYÐZ`ÑSaÓObÐefÓOf˜˜u™ v¨f¡~Ñ5ÐgˆÐgØˆ
Øò 	DˆFØ�jÑ!¨Ñ3Ü14³�
˜6 *Ñ-Ñ.Ø�v˜jÑ)Ñ*×.Ñ.¨v°kÑ/BÕCð	Dð
 ×ÑÐ)Ø26×2DÑ2D×2HÑ2HÈÐW[Ó2\Ð Ñ/Ø×ÑÐ*Ø37×3FÑ3F×3JÑ3JÈ<ÐY]Ó3^Ð Ñ0Ø"×;Ñ;¸LÓIÐÜ,ð 
Ø ØØ$Ø$ñ	
ð
 "ñ
ð 	
ùò fùÚgs   ÁEÁ/EÂ EÂEc           	     ó  — t        | j                  «      dk(  rt        d«      ‚| j                  D �cg c]  }|j                  «       t        vsŒ|‘Œ c}x}r,t        d|› dt        t        j                  «       «      › �«      ‚y c c}w )Nr   zDdataset_names cannot be empty. Use None to evaluate on all datasets.zDataset(s) z@ not found in the NanoBEIR collection. Valid dataset names are: )r‡   rJ   r±   r¤   rH   rG   rI   )rY   r¥   Úmissing_datasetss      r\   rR   z)NanoBEIREvaluator._validate_dataset_names§  sš   € Üˆt×!Ñ!Ó" aÒ'ÜÐcÓdÐdà-1×-?Ñ-?ö 
Ø)À<×CUÑCUÓCWÔ_qÒCqŠLò 
ð 
Ðð 
ô ØÐ.Ð/ð 0,Ü,0Ô1C×1HÑ1HÓ1JÓ,KÐ+LðNóð ð
ùò  
s   ²BÁBc                óz  — d}| j                   �yt        | j                   t        «      r+| j                  D �ci c]  }|| j                   “Œ c}| _         n4| j                  D �cg c]  }|| j                   vsŒ|‘Œ c}x}r	|d|› d�z  }| j                  �yt        | j                  t        «      r+| j                  D �ci c]  }|| j                  “Œ c}| _        n4| j                  D �cg c]  }|| j                  vsŒ|‘Œ c}x}r	|d|› d�z  }|rt        |j                  «       «      ‚y c c}w c c}w c c}w c c}w )Nrd   z2The following datasets are missing query prompts: rn   z3The following datasets are missing corpus prompts: )rM   Ú
isinstancer�   rJ   rN   r±   Ústrip)rY   Ú	error_msgr¥   Úmissing_query_promptsÚmissing_corpus_promptss        r\   rS   z#NanoBEIREvaluator._validate_prompts²  sQ  € Øˆ	Ø×ÑÐ)Ü˜$×,Ñ,¬cÔ2Ø[_×[mÑ[mÖ%nÈ< l°D×4FÑ4FÑ&FÒ%n�Õ"à15×1CÑ1Cö+Ø!-À|Ð[_×[mÑ[mÒGm’ò+ð Ð&ð ð ÐQÐRgÐQhÐhjÐkÑk�	à×ÑÐ*Ü˜$×-Ñ-¬sÔ3Ø]a×]oÑ]oÖ&pÈ\ |°T×5HÑ5HÑ'HÒ&p�Õ#à15×1CÑ1Cö,Ø!-À|Ð[_×[nÑ[nÒGn’ò,ð Ð'ð ð ÐRÐSiÐRjÐjlÐmÑm�	áÜ˜YŸ_™_Ó.Ó/Ð/ð ùò &oùò+ùò 'qùò,s#   ·D)Á"D.Á6D.Â<D3Ã'D8Ã;D8c                óp   — d| j                   i}g d¢}|D ]  }t        | |«      €Œt        | |«      ||<   Œ  |S )NrJ   )r=   rM   rN   )rJ   Úgetattr)rY   Úconfig_dictÚconfig_dict_candidate_keysrt   s       r\   Úget_config_dictz!NanoBEIREvaluator.get_config_dictÇ  sM   € Ø&¨×(:Ñ(:Ð;ˆÚ%XÐ"Ø-ò 	6ˆCÜ�t˜SÓ!Ñ-Ü#*¨4°Ó#5�˜CÒ ð	6ð Ðra   ) rJ   zlist[DatasetNameType] | Noner5   ú	list[int]r6   rÈ   r7   rÈ   r8   rÈ   r9   rÈ   r:   Úboolr;   Úintr<   rÉ   r=   z
int | Noner>   z-dict[str, Callable[[Tensor, Tensor], Tensor]]r?   zstr | SimilarityFunction | NonerK   zCallable[[list[float]], float]rL   r�   rM   ústr | dict[str, str] | NonerN   rË   )Nrc   rc   )
rŽ   r
   r�   r�   rC   rÊ   rD   rÊ   Úreturnzdict[str, float])r¥   ÚDatasetNameTyperÌ   r�   )r¥   rÍ   rÌ   r   )rÌ   zdict[str, Any])Ú__name__Ú
__module__Ú__qualname__Ú__doc__r†   r1   rF   rX   r    r§   rT   rR   rS   rÇ   Ú__classcell__)r[   s   @r\   r*   r*   H   sW  ø„ ñrðl 7;Ø!˜dØ "˜tÚ#0Ú+8Ø"˜eØ"'ØØØ#'ØIMØ?CØ79·w±wØ#Ø59Ø6:ð#E<à3ðE<ð ðE<ð ð	E<ð
 !ðE<ð  )ðE<ð ðE<ð  ðE<ð ðE<ð ðE<ð !ðE<ð GðE<ð =ðE<ð 5ðE<ð ðE<ð  3ð!E<ð" 4õ#E<òNAð& bdði#Ø(ði#Ø7:ði#ØJMði#Ø[^ði#à	ói#óV#ó
ò@	ò0÷*ra   r*   ) Ú
__future__r   Úloggingrz   Útypingr   r   r   r   Únumpyr†   Útorchr   r	   Úsentence_transformersr
   Ú>sentence_transformers.evaluation.InformationRetrievalEvaluatorr   Ú2sentence_transformers.evaluation.SentenceEvaluatorr   Ú*sentence_transformers.similarity_functionsr   Úsentence_transformers.utilr   Ú)sentence_transformers.SentenceTransformerÚ	getLoggerrÎ   ru   rÍ   rH   r£   r*   rq   ra   r\   ú<module>rß      sÄ   ðÝ "ã Û 	ß 8Ó 8ã Ý Ý å 5Ý hÝ PÝ IÝ <áÝMà	ˆ×	Ñ	˜8Ó	$€àðñ€ð$ 5Ø*Ø&Ø,Ø,Ø*Ø,Ø
 Ø8Ø*Ø*Ø*Ø0ñÐ ð" #ØØØØØØØ
Ø&ØØØØñ"Ð ô"EÐ)õ Era   