Ë
    l^(h÷  ã                  óà  — d dl mZ d dlZd dlZd dlZd dlZd dlmZ d dlm	Z	 d dl
mZmZmZ d dlZd dlZd dlZd dlmZ d dlmZmZ d dlmZ d d	lmZ d d
lmZ d dlmZ d dlmZmZm Z  d dl!m"Z" d dl#m$Z$ d dl%m&Z&m'Z'm(Z( d dl)m*Z*m+Z+m,Z, ddl-m.Z. ddl/m0Z0  e,«       rd dl1m2Z2m3Z3  ejh                  e5«      Z6erd dl7m8Z8 d dl9m:Z:  G d„ de«      Z; G d„ de«      Z< G d„ de«      Z= G d„ d«      Z>y)é    )ÚannotationsN)ÚIterable)ÚPath)ÚTYPE_CHECKINGÚAnyÚCallable)Úversion)ÚTensorÚnn)Ú	Optimizer)ÚLambdaLR)Ú
DataLoader)Útrange)ÚTrainerCallbackÚTrainerControlÚTrainerState)ÚNoDuplicatesDataLoader)ÚSentenceLabelDataset)ÚBatchSamplersÚMultiDatasetBatchSamplersÚ$SentenceTransformerTrainingArguments)Úbatch_to_deviceÚfullnameÚis_datasets_availableé   )ÚSentenceEvaluator)ÚModelCardTemplate)ÚDatasetÚDatasetDict)ÚInputExample)ÚSentenceTransformerc                  óh   ‡ — e Zd ZdZdˆ fd„Zdd„Z	 	 	 	 	 	 	 	 	 	 	 	 dd„Z	 	 	 	 	 	 	 	 	 	 d	d„Zˆ xZS )
ÚSaveModelCallbackaa  A Callback to save the model to the `output_dir`.

    There are two cases:
    1. save_best_model is True and evaluator is defined:
        We save on evaluate, but only if the new model is better than the currently saved one
        according to the evaluator.
    2. If evaluator is not defined:
        We save after the model has been trained.
    c                óZ   •— t         ‰| �  «        || _        || _        || _        d | _        y ©N)ÚsuperÚ__init__Ú
output_dirÚ	evaluatorÚsave_best_modelÚbest_metric)Úselfr(   r)   r*   Ú	__class__s       €ú]/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/sentence_transformers/fit_mixin.pyr'   zSaveModelCallback.__init__7   s,   ø€ Ü‰ÑÔØ$ˆŒØ"ˆŒØ.ˆÔØˆÕó    c                ól   — t        | j                  dd«      r|| j                  kD  S || j                  k  S )NÚgreater_is_betterT)Úgetattrr)   r+   )r,   Ú
new_metrics     r.   Ú	is_betterzSaveModelCallback.is_better>   s5   € Ü�4—>‘>Ð#6¸Ô=Ø × 0Ñ 0Ñ0Ð0Ø˜D×,Ñ,Ñ,Ð,r/   c                ó:  — | j                   ��| j                  r‚t        | j                   dd«      }|j                  «       D ]W  \  }}	|j	                  |«      sŒ| j
                  �| j                  |	«      sŒ6|	| _        |j                  | j                  «       ŒY y y y ©NÚprimary_metricr)   )	r)   r*   r2   ÚitemsÚendswithr+   r4   Úsaver(   )
r,   ÚargsÚstateÚcontrolÚmetricsÚmodelÚkwargsÚ
metric_keyÚkeyÚvalues
             r.   Úon_evaluatezSaveModelCallback.on_evaluateC   s…   € ð �>‰>Ð%¨$×*>Ò*>Ü  §¡Ð1AÀ;ÓOˆJØ%Ÿm™m›oò 4‘
��UØ—<‘< 
Õ+Ø×'Ñ'Ð/°4·>±>À%Õ3HØ+0˜Ô(ØŸ
™
 4§?¡?Õ3ñ	4ð +?Ð%r/   c                óT   — | j                   €|j                  | j                  «       y y r%   )r)   r:   r(   )r,   r;   r<   r=   r?   r@   s         r.   Úon_train_endzSaveModelCallback.on_train_endT   s#   € ð �>‰>Ð!Ø�J‰J�t—‘Õ'ð "r/   )r(   Ústrr)   zSentenceEvaluator | Noner*   ÚboolÚreturnÚNone)r3   ÚfloatrI   rH   )r;   r   r<   r   r=   r   r>   údict[str, Any]r?   r!   rI   rJ   ©
r;   r   r<   r   r=   r   r?   r!   rI   rJ   )	Ú__name__Ú
__module__Ú__qualname__Ú__doc__r'   r4   rD   rF   Ú__classcell__©r-   s   @r.   r#   r#   ,   s†   ø„ ñõ ó-ð
4à2ð4ð ð4ð  ð	4ð
  ð4ð #ð4ð 
ó4ð"	(à2ð	(ð ð	(ð  ð		(ð
 #ð	(ð 
÷	(r/   r#   c                  óB   ‡ — e Zd ZdZddˆ fd„Z	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚEvaluatorCallbackzîThe SentenceTransformers.fit method always ran the evaluator on every epoch,
    in addition to every "evaluation_steps". This callback is responsible for that.

    The `.trainer` must be provided after the trainer has been created.
    c                ó  •— t         ‰| �  «        || _        || _        | j                  �Pt        j
                  j                  | j                  d«      | _        t	        j                  | j                  d¬«       d| _        d | _	        y )NÚevalT©Úexist_ok)
r&   r'   r)   Úoutput_pathÚosÚpathÚjoinÚmakedirsÚmetric_key_prefixÚtrainer)r,   r)   rZ   r-   s      €r.   r'   zEvaluatorCallback.__init__g   si   ø€ Ü‰ÑÔØ"ˆŒØ&ˆÔØ×ÑÐ'Ü!Ÿw™wŸ|™|¨D×,<Ñ,<¸fÓEˆDÔÜ�K‰K˜×(Ñ(°4Õ8à!'ˆÔØˆ�r/   c                ó¾  — | j                  || j                  |j                  |j                  ¬«      }t	        |t
        «      sd|i}t        |j                  «       «      D ]D  }|j                  | j                  › d�«      rŒ"|j                  |«      || j                  › d|› �<   ŒF | j                  �*| j                  j                  j                  ||||¬«       y y )N©rZ   ÚepochÚstepsr)   Ú_)r>   )r)   rZ   rc   Úglobal_stepÚ
isinstanceÚdictÚlistÚkeysÚ
startswithr_   Úpopr`   Úcallback_handlerrD   )r,   r;   r<   r=   r?   r@   Úevaluator_metricsrB   s           r.   Úon_epoch_endzEvaluatorCallback.on_epoch_endr   sâ   € ð !ŸN™NØ˜t×/Ñ/°u·{±{È%×J[ÑJ[ð +ó 
Ðô Ð+¬TÔ2Ø!,Ð.?Ð @Ðô Ð)×.Ñ.Ó0Ó1ò 	bˆCØ—>‘> T×%;Ñ%;Ð$<¸AÐ">Õ?ØGX×G\ÑG\Ð]`ÓGaÐ! T×%;Ñ%;Ð$<¸A¸c¸UÐ"CÒDð	bð �<‰<Ð#Ø�L‰L×)Ñ)×5Ñ5°d¸EÀ7ÐTeÐ5Õfð $r/   r%   )r)   r   rZ   z
str | NonerI   rJ   rM   )rN   rO   rP   rQ   r'   ro   rR   rS   s   @r.   rU   rU   `   sH   ø„ ñö	ðgà2ðgð ðgð  ð	gð
 #ðgð 
÷gr/   rU   c                  ó@   ‡ — e Zd ZdZdˆ fd„Z	 	 	 	 	 	 	 	 	 	 dd„Zˆ xZS )ÚOriginalCallbackzÈA Callback to invoke the original callback function that was provided to SentenceTransformer.fit()

    This callback has the following signature: `(score: float, epoch: int, steps: int) -> None`
    c                ó>   •— t         ‰| �  «        || _        || _        y r%   )r&   r'   Úcallbackr)   )r,   rs   r)   r-   s      €r.   r'   zOriginalCallback.__init__�   s   ø€ Ü‰ÑÔØ ˆŒØ"ˆ�r/   c                óÖ   — t        | j                  dd«      }|j                  «       D ]?  \  }}|j                  |«      sŒ| j	                  ||j
                  |j                  «      c S  y r6   )r2   r)   r8   r9   rs   rc   rf   )	r,   r;   r<   r=   r>   r@   rA   rB   rC   s	            r.   rD   zOriginalCallback.on_evaluate”   s[   € ô ˜TŸ^™^Ð-=¸{ÓKˆ
Ø!Ÿ-™-›/ò 	L‰JˆC�Ø�|‰|˜JÕ'Ø—}‘} U¨E¯K©K¸×9JÑ9JÓKÒKñ	Lr/   )rs   ú!Callable[[float, int, int], None]r)   r   rI   rJ   )
r;   ztransformers.TrainingArgumentsr<   r   r=   r   r>   rL   rI   rJ   )rN   rO   rP   rQ   r'   rD   rR   rS   s   @r.   rq   rq   ‰   sH   ø„ ñõ
#ð
Là,ðLð ðLð  ð	Lð
  ðLð 
÷Lr/   rq   c                  óŠ  — e Zd ZdZdddddej
                  j                  ddidd	dd
dddd
ddd	df	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zedd„«       Z	dd„Z
	 dddddej
                  j                  ddidd	dd
dddd
ddd	f	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd„Zdd„Zdd„Zy)ÚFitMixinzYMixin class for injecting the `fit` and `old_fit` methods into SentenceTransformer modelsNr   ÚWarmupLineari'  Úlrgñhãˆµøô>g{®Gáz„?r   TFiô  c                ó’
  ‡2— t        «       st        d«      ‚ddlm} t	        |Ž \  }}d„ }|D ]	  }||_        Œ d}t        j                  }i }t        |d¬«      D �]  \  }}t        |t        «      rt        j                  }n6t        |d«      r*t        |j                  t        «      rt        j                  }t!        |d	|«      }g }g }|D ];  } t	        | D �!cg c]  }!|!j"                  |!j$                  f‘Œ c}!Ž \  }"}#||"z  }||#z  }Œ= t'        j(                  t        t	        |Ž «      D �$�%ci c]  \  }$}%d
|$› �|%“Œ c}%}$«      }&d}'	 t+        |«      dhk(  rd}'|'r|&j/                  d|«      }&|&|d|› �<   �Œ t1        |«      }d*d„}(t        |d¬«      D �$�)ci c]  \  }$})d|$› �|)“Œ }*}$})d}+|�$|dkD  r|dk(  r|}+nt2        j5                  d«       d}t7        j8                  t:        j<                  «      t7        j8                  d«      k\  rdnd},t?        d+|xs  |(«       |t@        jB                  ||||+dœ|,|
�|
dkD  rdndi¤|
||| |�dnd||dœ¤Ž}-|�|dk(  r3tE        |jG                  «       D �.cg c]  }.tI        |.«      |z  ‘Œ c}.«      }tK        ||z  «      }/tM        | jO                  «       «      }0g d¢}1|0D �2‡2�3cg c]  \  Š2}3tQ        ˆ2fd„|1D «       «      rŒ|3‘Œ c}3}2|	dœ|0D �2‡2�3cg c]  \  Š2}3tQ        ˆ2fd„|1D «       «      sŒ|3‘Œ c}3}2ddœg}4 ||4fi |¤Ž}5| jS                  |5|||/¬«      }6g }7|�8|7jU                  tW        ||«      «       |�|7jU                  tY        ||«      «        || |-|d|*||5|6f|7¬ «      }8|8jZ                  j\                  D ]  }t        |tV        «      sŒ|8|_/        Œ |�|8ja                  tc        |||«      «       |��,|�r)td        jf                  ji                  |«      rðtd        jf                  jk                  |«      rÑt2        jm                  d!|› �«       te        jn                  |«      D �9cg c]7  }9|9jq                  d"«      r$|9js                  d#«      d   ju                  «       r|9‘Œ9 }:}9|:rGtw        |:d$„ ¬%«      };td        jf                  jy                  ||;«      }t2        jm                  d&|› �«       n5t2        j5                  d'|› �«       d}nt2        j5                  d(|› �«       d}|8j{                  |¬)«       yc c}!w c c}%}$w # t,        $ r Y �ŒÑw xY wc c})}$w c c}.w c c}3}2w c c}3}2w c c}9w ),aq  
        Deprecated training method from before Sentence Transformers v3.0, it is recommended to use
        :class:`~sentence_transformers.trainer.SentenceTransformerTrainer` instead. This method uses
        :class:`~sentence_transformers.trainer.SentenceTransformerTrainer` behind the scenes, but does
        not provide as much flexibility as the Trainer itself.

        This training approach uses a list of DataLoaders and Loss functions to train the model. Each DataLoader
        is sampled in turn for one batch. We sample only as many batches from each DataLoader as there are in the
        smallest one to make sure of equal training with each dataset, i.e. round robin sampling.

        This method should produce equivalent results in v3.0+ as before v3.0, but if you encounter any issues
        with your existing training scripts, then you may wish to use
        :meth:`SentenceTransformer.old_fit <sentence_transformers.SentenceTransformer.old_fit>` instead.
        That uses the old training method from before v3.0.

        Args:
            train_objectives: Tuples of (DataLoader, LossFunction). Pass
                more than one for multi-task learning
            evaluator: An evaluator (sentence_transformers.evaluation)
                evaluates the model performance during training on held-
                out dev data. It is used to determine the best model
                that is saved to disk.
            epochs: Number of epochs for training
            steps_per_epoch: Number of training steps per epoch. If set
                to None (default), one epoch is equal the DataLoader
                size from train_objectives.
            scheduler: Learning rate scheduler. Available schedulers:
                constantlr, warmupconstant, warmuplinear, warmupcosine,
                warmupcosinewithhardrestarts
            warmup_steps: Behavior depends on the scheduler. For
                WarmupLinear (default), the learning rate is increased
                from o up to the maximal learning rate. After these many
                training steps, the learning rate is decreased linearly
                back to zero.
            optimizer_class: Optimizer
            optimizer_params: Optimizer parameters
            weight_decay: Weight decay for model parameters
            evaluation_steps: If > 0, evaluate the model using evaluator
                after each number of training steps
            output_path: Storage path for the model and evaluation files
            save_best_model: If true, the best model (according to
                evaluator) is stored at output_path
            max_grad_norm: Used for gradient normalization.
            use_amp: Use Automatic Mixed Precision (AMP). Only for
                Pytorch >= 1.6.0
            callback: Callback function that is invoked after each
                evaluation. It must accept the following three
                parameters in this order: `score`, `epoch`, `steps`
            show_progress_bar: If True, output a tqdm progress bar
            checkpoint_path: Folder to save checkpoints during training
            checkpoint_save_steps: Will save a checkpoint after so many
                steps
            checkpoint_save_total_limit: Total number of checkpoints to
                store
            resume_from_checkpoint: If true, searches for checkpoints
                to continue training from.
        zGPlease install `datasets` to use this function: `pip install datasets`.r   )ÚSentenceTransformerTrainerc                ó   — | S r%   © )Úbatchs    r.   ÚidentityzFitMixin.fit.<locals>.identityþ   s   € ØˆLr/   é   r   )ÚstartÚdatasetÚ
batch_sizeÚ	sentence_TFÚlabelÚ	_dataset_c                 ó@  — d} d}t        | «      j                  «       r€t        t        t        | «      j	                  «       «      «      dk7  rRd|› �} |dz  }t        | «      j                  «       r/t        t        t        | «      j	                  «       «      «      dk7  rŒR| S )Nzcheckpoints/modelr   r   zcheckpoints/model_)r   ÚexistsÚlenri   Úiterdir)Údir_nameÚidxs     r.   Ú_default_checkpoint_dirz-FitMixin.fit.<locals>._default_checkpoint_dir$  sˆ   € Ø*ˆHØˆCÜ�x“.×'Ñ'Ô)¬c´$´t¸H³~×7MÑ7MÓ7OÓ2PÓ.QÐUVÒ.VØ/°¨uÐ5�Ø�q‘�ô �x“.×'Ñ'Ô)¬c´$´t¸H³~×7MÑ7MÓ7OÓ2PÓ.QÐUVÓ.Vð ˆOr/   éÿÿÿÿNzqSetting `steps_per_epoch` alongside `epochs` > 1 no longer works. We will train with the full datasets per epoch.z4.41.0Úeval_strategyÚevaluation_strategy)r(   Úbatch_samplerÚmulti_dataset_batch_samplerÚper_device_train_batch_sizeÚper_device_eval_batch_sizeÚnum_train_epochsÚ	max_stepsrd   Úno)Ú
eval_stepsÚmax_grad_normÚfp16Údisable_tqdmÚsave_strategyÚ
save_stepsÚsave_total_limit©ÚbiaszLayerNorm.biaszLayerNorm.weightc              3  ó&   •K  — | ]  }|‰v –— Œ
 y ­wr%   r}   ©Ú.0ÚndÚns     €r.   ú	<genexpr>zFitMixin.fit.<locals>.<genexpr>_  s   øè ø€ ÒC_ÐPRÀBÈ!ÄGÑC_ùó   ƒ©ÚparamsÚweight_decayc              3  ó&   •K  — | ]  }|‰v –— Œ
 y ­wr%   r}   r¢   s     €r.   r¦   zFitMixin.fit.<locals>.<genexpr>b  s   øè ø€ Ò<XÈ¸RÀ1¼WÑ<Xùr§   ç        ©Ú	schedulerÚwarmup_stepsÚt_total)r?   r;   Útrain_datasetÚeval_datasetÚlossr)   Ú
optimizersÚ	callbackszLooking for checkpoints in: zcheckpoint-ú-c                ó<   — t        | j                  d«      d   «      S )Nr¶   rŽ   )ÚintÚsplit©Úxs    r.   ú<lambda>zFitMixin.fit.<locals>.<lambda>Ž  s   € Ì3ÈqÏwÉwÐWZË|Ð\^ÑO_ÓK`€ r/   ©rB   z!Resuming from latest checkpoint: z.No checkpoints found in checkpoint directory: z;Checkpoint directory does not exist or is not a directory: )Úresume_from_checkpoint)rI   rG   r}   )>r   ÚImportErrorÚsentence_transformers.trainerr{   ÚzipÚ
collate_fnr   ÚBATCH_SAMPLERÚ	enumeraterg   r   ÚNO_DUPLICATESÚhasattrr‚   r   ÚGROUP_BY_LABELr2   Útextsr…   r   Ú	from_dictÚsetÚ	TypeErrorÚ
add_columnr   ÚloggerÚwarningr	   ÚparseÚtransformersÚ__version__r   r   ÚROUND_ROBINÚminÚvaluesr‰   r¸   ri   Únamed_parametersÚanyÚ_get_schedulerÚappendrU   rq   rm   rµ   r`   Úadd_callbackr#   r[   r\   rˆ   ÚisdirÚinfoÚlistdirrk   r¹   ÚisdigitÚmaxr]   Útrain)<r,   Útrain_objectivesr)   ÚepochsÚsteps_per_epochr®   r¯   Úoptimizer_classÚoptimizer_paramsrª   Úevaluation_stepsrZ   r*   r™   Úuse_amprs   Úshow_progress_barÚcheckpoint_pathÚcheckpoint_save_stepsÚcheckpoint_save_total_limitr¾   r{   Údata_loadersÚloss_fnsr   Údata_loaderrƒ   r‘   Útrain_dataset_dictÚ
loader_idxrÈ   Úlabelsr~   ÚexampleÚbatch_textsÚbatch_labelsrŒ   Útextr‚   Úadd_label_columnr�   Úloss_fnÚloss_fn_dictr–   Úeval_strategy_keyr;   r±   Únum_train_stepsÚparam_optimizerÚno_decayr¥   ÚpÚoptimizer_grouped_parametersÚ	optimizerÚscheduler_objrµ   r`   Ú
checkpointÚall_checkpointsÚlatest_checkpoints<                                                     `         r.   ÚfitzFitMixin.fit¥   sÉ  ø€ ô` %Ô&ÜÐgÓhÐhõ 	Mä!$Ð&6Ð!7Ñˆ�hò	ð (ò 	.ˆKØ%-ˆKÕ"ð	.ð ˆ
Ü%×3Ñ3ˆð  ÐÜ'0°ÀQÔ'Gó 	CÑ#ˆJ˜Ü˜+Ô'=Ô>Ü -× ;Ñ ;‘Ü˜ iÔ0´ZÀ×@SÑ@SÔUiÔ5jÜ -× <Ñ <�ä  ¨l¸JÓGˆJØˆEØˆFØ$ò '�Ü,/Ð`eÖ1fÐU\°7·=±=À'Ç-Á-Ò2PÒ1fÐ,gÑ)�˜\Ø˜Ñ$�Ø˜,Ñ&‘ð'ô ×'Ñ'ÔR[Ô\_ÐafÐ\gÓRh×(iÁYÀSÈ$¨9°S°EÐ):¸DÑ)@Ó(iÓjˆGà#ÐðÜ�v“; 1 #Ò%Ø',Ð$ñ  Ø!×,Ñ,¨W°fÓ=�Ø;BÐ ¨:¨,Ð7Ó8ð/	Cô2 )Ð);Ó<Ðó	ô HQÐQYÐabÔGc×d±|°s¸G˜) C 5Ð)¨7Ñ2ÐdˆÑdð ˆ	ØÐ&¨?¸QÒ+>Ø˜Š{Ø+‘	ä—‘ðFôð #'�ô
 �}‰}œ\×5Ñ5Ó6¼'¿-¹-ÈÓ:QÒQñ à&ð 	ô
 4ð 
Ø&ÒCÑ*AÓ*CØ'Ü(A×(MÑ(MØ(2Ø'1Ø#Øñ
ð "Ð.>Ð.JÐO_ÐbcÒOc¡7Ðimðð
ð (Ø'ØØ.Ð.Ø%4Ð%@™'ÀdØ,Ø8ó#
ˆð( Ð" o¸Ò&:Ü!ÐYk×YrÑYrÓYtÖ"uÈ¤3 }Ó#5¸Ó#CÒ"uÓvˆOÜ˜o°Ñ6Ó7ˆô ˜t×4Ñ4Ó6Ó7ˆâAˆð *9×`Ð`¡  AÄÓC_ÐV^ÔC_Õ@_š1Ó`Ø ,ñð '6×YÐY™d˜a ¼Ó<XÈxÔ<XÕ9XšÓYÐknÑoð(
Ð$ñ $Ð$@ÑUÐDTÑUˆ	Ø×+Ñ+Ø ¸Èð ,ó 
ˆð
 ˆ	ØÐ Ø×ÑÔ.¨y¸+ÓFÔGØÐ#Ø× Ñ Ô!1°(¸IÓ!FÔGá,ØØØ,ØØØØ! =Ð1Øô	
ˆð  ×0Ñ0×:Ñ:ò 	+ˆHÜ˜(Ô$5Õ6Ø#*�Õ ð	+ð Ð"Ø× Ñ Ô!2°;À	È?Ó![Ô\àÑ&Ò+AÜ�w‰w�~‰~˜oÔ.´2·7±7·=±=ÀÔ3QÜ—‘Ð:¸?Ð:KÐLÔMô ')§j¡j°Ó&Aö#à"Ø!×,Ñ,¨]Ô;À
×@PÑ@PÐQTÓ@UÐVXÑ@Y×@aÑ@aÔ@cò ð#�ð #ñ #Ü(+¨OÑA`Ô(aÐ%Ü-/¯W©W¯\©\¸/ÐK\Ó-]Ð*Ü—K‘KÐ"CÐDZÐC[Ð \Õ]ä—N‘NÐ%SÐTcÐSdÐ#eÔfØ-1Ñ*ä—‘Ð!\Ð]lÐ\mÐnÔoØ)-Ð&à�‰Ð-CˆÕDùòK 2gùó )jøô ò Úðüó" eùòR #vùó aùó ZùòJ#sH   ÃTÄ,TÅTÆT-É0T3Ê=T8ËT8Ë*T>ÌT>Ñ<UÔ	T*Ô)T*c                óX  — |j                  «       }|dk(  rt        j                  | «      S |dk(  rt        j                  | |¬«      S |dk(  rt        j                  | ||¬«      S |dk(  rt        j
                  | ||¬«      S |dk(  rt        j                  | ||¬«      S t        d|› �«      ‚)	zÛ
        Returns the correct learning rate scheduler. Available scheduler:

        - constantlr,
        - warmupconstant,
        - warmuplinear,
        - warmupcosine,
        - warmupcosinewithhardrestarts
        Ú
constantlrÚwarmupconstant)Únum_warmup_stepsÚwarmuplinear)r  Únum_training_stepsÚwarmupcosineÚwarmupcosinewithhardrestartszUnknown scheduler )ÚlowerrÐ   Úget_constant_scheduleÚ!get_constant_schedule_with_warmupÚget_linear_schedule_with_warmupÚget_cosine_schedule_with_warmupÚ2get_cosine_with_hard_restarts_schedule_with_warmupÚ
ValueError)rþ   r®   r¯   r°   s       r.   r×   zFitMixin._get_schedulerš  sÆ   € ð —O‘OÓ%ˆ	Ø˜Ò$Ü×5Ñ5°iÓ@Ð@ØÐ*Ò*Ü×AÑAÀ)Ð^jÔkÐkØ˜.Ò(Ü×?Ñ?Ø¨LÈWôð ð ˜.Ò(Ü×?Ñ?Ø¨LÈWôð ð Ð8Ò8Ü×RÑRØ¨LÈWôð ô Ð1°)°Ð=Ó>Ð>r/   c                ó”  — |D �cg c]  }|j                   ‘Œ }}t        |Ž D �cg c]  }| j                  |«      ‘Œ }}|D �cg c]  }|j                  ‘Œ }}|rIt	        |d   t
        j                  «      r,t        j                  t        j                  |«      «      }||fS t        j                  |«      }||fS c c}w c c}w c c}w )a;  
        Transforms a batch from a SmartBatchingDataset to a batch of tensors for the model
        Here, batch is a list of InputExample instances: [InputExample(...), ...]

        Args:
            batch: a batch from a SmartBatchingDataset

        Returns:
            a batch of tensors for the model
        r   )rÈ   rÁ   Útokenizer…   rg   ÚnpÚndarrayÚtorchÚ
from_numpyÚstackÚtensor)r,   r~   rñ   rÈ   ÚsentenceÚsentence_featuresrð   Úlabels_tensors           r.   Úsmart_batching_collatezFitMixin.smart_batching_collate¹  sµ   € ð /4Ö4 7�—“Ð4ˆÐ4ÜEHÈ%À[ÖQ¸˜TŸ]™]¨8Õ4ÐQÐÐQØ/4Ö5 G�'—-“-Ð5ˆÐ5ñ ”j ¨¡¬B¯J©JÔ7Ü!×,Ñ,¬R¯X©X°fÓ-=Ó>ˆMð ! -Ð/Ð/ô "ŸL™L¨Ó0ˆMà  -Ð/Ð/ùò 5ùÚQùÚ5s   …B;¤C ÁCc                ó¦  ‡ ‡$— g }|D ]*  \  }}|j                  t        j                  ||«      «       Œ, dj                  |D �cg c]  }|‘Œ c}«      }t	        j
                  t        |«      ||||t        |«      ||	|
|dœ
dd¬«      }d‰ _        t        j                  j                  d|«      j                  d|«      ‰ j                  d	<   |r.d
dlm} t        j                  j                   j#                  «       }‰ j%                  ‰ j&                  «       |D ��cg c]  \  }}|‘Œ	 }}}|D ]  }‰ j(                  |_        Œ |D ��cg c]  \  }}|‘Œ	 }}}|D ]  }|j%                  ‰ j&                  «       Œ d‰ _        |�|d
k(  r"t/        |D �cg c]  }t1        |«      ‘Œ c}«      }t3        ||z  «      }g } g }!|D ]»  }t5        |j7                  «       «      }"g d¢}#|"D �$‡$�%cg c]  \  Š$}%t9        ˆ$fd„|#D «       «      rŒ|%‘Œ c}%}$|	dœ|"D �$‡$�%cg c]  \  Š$}%t9        ˆ$fd„|#D «       «      sŒ|%‘Œ c}%}$ddœg}& ||&fi |¤Ž}'‰ j;                  |'|||¬«      }(| j=                  |'«       |!j=                  |(«       Œ½ d
})|D �cg c]  }t?        |«      ‘Œ }*}t1        |«      }+d},tA        |d| ¬«      D �]�  }-d
}.|D ]"  }|jC                  «        |jE                  «        Œ$ tA        |dd| ¬«      D �]:  }tG        |+«      D �]¨  }/||/   }| |/   }'|!|/   }|*|/   }0	 tI        |0«      }1|1\  }2}3|3j%                  ‰ j&                  «      }3t5        tM        ˆ fd„|2«      «      }2|rÆ «       5   ||2|3«      }4ddd«       jO                  «       }5|jQ                  4«      jS                  «        |jU                  |'«       t        jV                  jX                  j[                  |j]                  «       |«       |j_                  |'«       |ja                  «        |jO                  «       |5k7  },na ||2|3«      }4|4jS                  «        t        jV                  jX                  j[                  |j]                  «       |«       |'j_                  «        |'jC                  «        |,r�Œ™|j_                  «        �Œ« |.dz  }.|)dz  })|
d
kD  rE|.|
z  d
k(  r=‰ jc                  ||||-|.|«       |D ]"  }|jC                  «        |jE                  «        Œ$ |€�Œ|€�Œ|d
kD  s�Œ|)|z  d
k(  s�Œ(‰ je                  |||)«       �Œ= ‰ jc                  ||||-d|«       �Œ“ |€|�‰ jg                  |«       |�‰ je                  |||)«       yyc c}w c c}}w c c}}w c c}w c c}%}$w c c}%}$w c c}w # tJ        $ r" t?        ||/   «      }0|0|*|/<   tI        |0«      }1Y �Œ¡w xY w# 1 sw Y   �Œ\xY w)ak  
        Deprecated training method from before Sentence Transformers v3.0, it is recommended to use
        :class:`sentence_transformers.trainer.SentenceTransformerTrainer` instead. This method should
        only be used if you encounter issues with your existing training scripts after upgrading to v3.0+.

        This training approach uses a list of DataLoaders and Loss functions to train the model. Each DataLoader
        is sampled in turn for one batch. We sample only as many batches from each DataLoader as there are in the
        smallest one to make sure of equal training with each dataset, i.e. round robin sampling.

        Args:
            train_objectives: Tuples of (DataLoader, LossFunction). Pass
                more than one for multi-task learning
            evaluator: An evaluator (sentence_transformers.evaluation)
                evaluates the model performance during training on held-
                out dev data. It is used to determine the best model
                that is saved to disc.
            epochs: Number of epochs for training
            steps_per_epoch: Number of training steps per epoch. If set
                to None (default), one epoch is equal the DataLoader
                size from train_objectives.
            scheduler: Learning rate scheduler. Available schedulers:
                constantlr, warmupconstant, warmuplinear, warmupcosine,
                warmupcosinewithhardrestarts
            warmup_steps: Behavior depends on the scheduler. For
                WarmupLinear (default), the learning rate is increased
                from o up to the maximal learning rate. After these many
                training steps, the learning rate is decreased linearly
                back to zero.
            optimizer_class: Optimizer
            optimizer_params: Optimizer parameters
            weight_decay: Weight decay for model parameters
            evaluation_steps: If > 0, evaluate the model using evaluator
                after each number of training steps
            output_path: Storage path for the model and evaluation files
            save_best_model: If true, the best model (according to
                evaluator) is stored at output_path
            max_grad_norm: Used for gradient normalization.
            use_amp: Use Automatic Mixed Precision (AMP). Only for
                Pytorch >= 1.6.0
            callback: Callback function that is invoked after each
                evaluation. It must accept the following three
                parameters in this order: `score`, `epoch`, `steps`
            show_progress_bar: If True, output a tqdm progress bar
            checkpoint_path: Folder to save checkpoints during training
            checkpoint_save_steps: Will save a checkpoint after so many
                steps
            checkpoint_save_total_limit: Total number of checkpoints to
                store
        z

)
r)   rá   râ   r®   r¯   rã   rä   rª   rå   r™   é   T)ÚindentÚ	sort_keysNz{LOSS_FUNCTIONS}z{FIT_PARAMETERS}z{TRAINING_SECTION}r   )Úautocasti�igÿrŸ   c              3  ó&   •K  — | ]  }|‰v –— Œ
 y ­wr%   r}   r¢   s     €r.   r¦   z#FitMixin.old_fit.<locals>.<genexpr>Z  s   øè ø€ ÒGcÐTVÈÈaÌÑGcùr§   r¨   c              3  ó&   •K  — | ]  }|‰v –— Œ
 y ­wr%   r}   r¢   s     €r.   r¦   z#FitMixin.old_fit.<locals>.<genexpr>]  s   øè ø€ Ò@\ÈRÀÀqÄÑ@\ùr§   r¬   r­   FÚEpoch)ÚdescÚdisableÚ	Iterationgš™™™™™©?)r'  Ú	smoothingr(  c                ó0   •— t        | ‰j                  «      S r%   )r   Údevice)r~   r,   s    €r.   r¼   z"FitMixin.old_fit.<locals>.<lambda>…  s   ø€ ´oÀeÈTÏ[É[Ó6Y€ r/   r   rŽ   )4Úextendr   Úget_train_objective_infor]   ÚjsonÚdumpsr   rG   Ú_model_card_textÚ__TRAINING_SECTION__ÚreplaceÚ_model_card_varsÚtorch.cuda.ampr#  r  ÚcudaÚampÚ
GradScalerÚtor,  r  rÂ   Ú
best_scorerÓ   r‰   r¸   ri   rÕ   rÖ   r×   rØ   Úiterr   Ú	zero_gradrß   ÚrangeÚnextÚStopIterationÚmapÚ	get_scaleÚscaleÚbackwardÚunscale_r   ÚutilsÚclip_grad_norm_Ú
parametersÚstepÚupdateÚ_eval_during_trainingÚ_save_checkpointr:   )6r,   rà   r)   rá   râ   r®   r¯   rã   rä   rª   rå   rZ   r*   r™   ræ   rs   rç   rè   ré   rê   Úinfo_loss_functionsÚ
dataloaderr³   rô   Úinfo_fit_parametersr#  Úscalerre   ÚdataloadersÚloss_modelsÚ
loss_modelrù   r´   Ú
schedulersrú   rû   r¥   rü   rý   rþ   rÿ   rf   Údata_iteratorsÚnum_train_objectivesÚskip_schedulerrc   Útraining_stepsÚ	train_idxÚdata_iteratorÚdataÚfeaturesrð   Ú
loss_valueÚscale_before_steps6   `                                   `                 r.   Úold_fitzFitMixin.old_fitÕ  s%  ù€ ðT !ÐØ 0ò 	eÑˆJ˜Ø×&Ñ&Ô'8×'QÑ'QÐR\Ð^bÓ'cÕdð	eà$Ÿk™kÐ<OÖ*P°Dª4Ò*PÓQÐä"Ÿj™jä% iÓ0Ø Ø#2Ø&Ø ,Ü#& Ó#7Ø$4Ø ,Ø$4Ø!.ñð Øô
Ðð  !%ˆÔÜ6G×6\Ñ6\×6dÑ6dØÐ 3ó7
ç
‰'Ð$Ð&9Ó
:ð 	×ÑÐ2Ñ3ñ Ý/ä—Z‘Z—^‘^×.Ñ.Ó0ˆFà�‰�—‘Ôà7G×H¡m j°!’zÐHˆÑHð &ò 	@ˆJØ$(×$?Ñ$?ˆJÕ!ð	@ð ,<×<¡  4’tÐ<ˆÑ<Ø%ò 	'ˆJØ�M‰M˜$Ÿ+™+Õ&ð	'ð #ˆŒàÐ" o¸Ò&:Ü!À[Ö"Q°z¤3 z¥?Ò"QÓRˆOä˜o°Ñ6Ó7ˆð ˆ
Øˆ
Ø%ò 	-ˆJÜ" :×#>Ñ#>Ó#@ÓAˆOâEˆHð .=×dÐd¡T Q¨ÄCÓGcÐZbÔGcÕDcšqÓdØ$0ñð +:×]Ð]¡$ ! Q¼SÓ@\ÐS[Ô@\Õ=\šAÓ]ÐorÑsð,Ð(ñ (Ð(DÑYÐHXÑYˆIØ ×/Ñ/Ø Y¸\ÐSbð 0ó ˆMð ×Ñ˜iÔ(Ø×Ñ˜mÕ,ð%	-ð( ˆØ=HÖI¨zœ$˜zÕ*ÐIˆÐIä"Ð#3Ó4ÐàˆÜ˜F¨Ð>OÐ:OÔPó D	eˆEØˆNà)ò #�
Ø×$Ñ$Ô&Ø× Ñ Õ"ð#ô ˜O°+ÈÐ[lÐWlÔmó ;e�Ü!&Ð';Ó!<ó &)�IØ!,¨YÑ!7�JØ *¨9Ñ 5�IØ *¨9Ñ 5�IØ$2°9Ñ$=�Mð3Ü# MÓ2˜ð (,Ñ$�H˜fØ#ŸY™Y t§{¡{Ó3�FÜ#¤CÓ(YÐ[cÓ$dÓe�HáÙ%›Zñ FÙ)3°H¸fÓ)E˜J÷Fð -3×,<Ñ,<Ó,>Ð)ØŸ™ ZÓ0×9Ñ9Ô;ØŸ™¨	Ô2ÜŸ™Ÿ™×6Ñ6°z×7LÑ7LÓ7NÐP]Ô^ØŸ™ IÔ.ØŸ™œà)/×)9Ñ)9Ó);Ð?PÑ)P™á%/°¸&Ó%A˜
Ø"×+Ñ+Ô-ÜŸ™Ÿ™×6Ñ6°z×7LÑ7LÓ7NÐP]Ô^Ø!Ÿ™Ô(à×'Ñ'Ô)ã)Ø!Ÿ™Ö(ðM&)ðP  !Ñ#�Ø˜qÑ �à# aÒ'¨NÐ=MÑ,MÐQRÒ,RØ×.Ñ.Ø! ;°ÀÈÐX`ôð '2ò +˜
Ø"×,Ñ,Ô.Ø"×(Ñ(Õ*ð+ð
 $Ò/Ø-Ò9Ø-°Ô1Ø#Ð&;Ñ;¸qÔ@à×)Ñ)¨/Ð;VÐXcÖdðw;eðz ×&Ñ& y°+¸ÐPUÐWYÐ[cÖdðID	eðL Ð Ð!8Ø�I‰I�kÔ"àÐ&Ø×!Ñ! /Ð3NÐP[Õ\ð 'ùòk +Qùó< Iùó =ùò #Rùó eùó ^ùò Jøô* )ò 3Ü(,¨[¸Ñ-CÓ(D˜Ø4A˜ yÑ1Ü# MÓ2›ð3ú÷Fñ FúsT   Á	U1ÄU6ÅU<ÆVÇ#VÈ VÈV
È-V
ÊVÌVÍ"
W	Ö'W	×W	×Wc                ó4  — |}|�Nt        j                  |d¬«       t         j                  j                  |d«      }t        j                  |d¬«       |�D || |||¬«      }|�
 ||||«       || j                  kD  r|| _        |r| j                  |«       yyyy)z#Runs evaluation during the trainingNTrX   rW   rb   )r[   r^   r\   r]   r:  r:   )	r,   r)   rZ   r*   rc   rd   rs   Ú	eval_pathÚscores	            r.   rJ  zFitMixin._eval_during_trainingº  s–   € àˆ	ØÐ"Ü�K‰K˜¨dÕ3ÜŸ™Ÿ™ [°&Ó9ˆIÜ�K‰K˜	¨DÕ1àÐ Ù˜d°	ÀÈeÔTˆEØÐ#Ù˜  uÔ-Ø�t—‘Ò&Ø"'�”Ù"Ø—I‘I˜kÕ*ð #ð 'ð	 !r/   c                óÆ  — | j                  t        j                  j                  |t	        |«      «      «       |�§|dkD  r¡g }t        j
                  |«      D ]N  }|j                  «       sŒ|j                  t        |«      t        j                  j                  ||«      dœ«       ŒP t        |«      |kD  r*t        |d„ ¬«      }t        j                  |d   d   «       y y y y )Nr   )rH  r\   c                ó   — | d   S )NrH  r}   rº   s    r.   r¼   z+FitMixin._save_checkpoint.<locals>.<lambda>×  s
   € ÈÈ&É	€ r/   r½   r\   )r:   r[   r\   r]   rG   rÜ   rÝ   rØ   r¸   r‰   ÚsortedÚshutilÚrmtree)r,   rè   rê   rH  Úold_checkpointsÚsubdirs         r.   rK  zFitMixin._save_checkpointË  sÅ   € à�	‰	”"—'‘'—,‘,˜´°D³	Ó:Ô;ð 'Ð2Ð7RÐUVÒ7VØ ˆOÜŸ*™* _Ó5ò q�Ø—>‘>Õ#Ø#×*Ñ*´C¸³KÌÏÉÏÉÐVeÐgmÓInÑ+oÕpðqô �?Ó#Ð&AÒAÜ"(¨Ñ>QÔ"R�Ü—‘˜o¨aÑ0°Ñ8Õ9ð Bð 8WÐ2r/   )(rà   ú&Iterable[tuple[DataLoader, nn.Module]]r)   r   rá   r¸   r®   rG   r¯   r¸   rã   útype[Optimizer]rä   údict[str, object]rª   rK   rå   r¸   rZ   rG   r*   rH   r™   rK   ræ   rH   rs   ru   rç   rH   rè   rG   ré   r¸   rê   r¸   r¾   rH   rI   rJ   )r®   rG   r¯   r¸   r°   r¸   rI   r   )r~   zlist[InputExample]rI   z&tuple[list[dict[str, Tensor]], Tensor])&rà   ri  r)   r   rá   r¸   r®   rG   r¯   r¸   rã   rj  rä   rk  rª   rK   rå   r¸   rZ   rG   r*   rH   r™   rK   ræ   rH   rs   ru   rç   rH   rè   rG   ré   r¸   rê   r¸   rI   rJ   )rI   rJ   )rN   rO   rP   rQ   r  ÚoptimÚAdamWr  Ústaticmethodr×   r  r^  rJ  rK  r}   r/   r.   rw   rw   ¢   sœ  „ Ùcð
 (,ØØØ'Ø!Ø+0¯;©;×+<Ñ+<Ø/3°T¨lØ"Ø !ØØ $Ø ØØ6:Ø"&Ø#Ø%(Ø+,Ø',ð+sEà@ðsEð %ðsEð ð	sEð ðsEð ðsEð )ðsEð ,ðsEð ðsEð ðsEð ðsEð ðsEð ðsEð ðsEð  4ð!sEð"  ð#sEð$ ð%sEð&  #ð'sEð( &)ð)sEð* !%ð+sEð, 
ó-sEðj ò?ó ð?ó<0ð0ð (,ØØØ'Ø!Ø+0¯;©;×+<Ñ+<Ø/3°T¨lØ"Ø !ØØ $Ø ØØ6:Ø"&Ø#Ø%(Ø+,ð)c]à@ðc]ð %ðc]ð ð	c]ð ðc]ð ðc]ð )ðc]ð ,ðc]ð ðc]ð ðc]ð ðc]ð ðc]ð ðc]ð ðc]ð  4ð!c]ð"  ð#c]ð$ ð%c]ð&  #ð'c]ð( &)ð)c]ð* 
ó+c]óJ+ô":r/   rw   )?Ú
__future__r   r/  Úloggingr[   re  Úcollections.abcr   Úpathlibr   Útypingr   r   r   Únumpyr  r  rÐ   Ú	packagingr	   r
   r   Útorch.optimr   Útorch.optim.lr_schedulerr   Útorch.utils.datar   Útqdm.autonotebookr   r   r   r   Ú5sentence_transformers.datasets.NoDuplicatesDataLoaderr   Ú3sentence_transformers.datasets.SentenceLabelDatasetr   Ú#sentence_transformers.training_argsr   r   r   Úsentence_transformers.utilr   r   r   Ú
evaluationr   Úmodel_card_templatesr   Údatasetsr   r   Ú	getLoggerrN   rÍ   Ú*sentence_transformers.readers.InputExampler    Ú)sentence_transformers.SentenceTransformerr!   r#   rU   rq   rw   r}   r/   r.   ú<module>r„     sº   ðÝ "ã Û Û 	Û Ý $Ý ß /Ñ /ã Û Û Ý ß Ý !Ý -Ý 'Ý $ß FÑ Få XÝ T÷ñ ÷
 XÑ Wå )Ý 3áÔß-à	ˆ×	Ñ	˜8Ó	$€áÝGÝMô1(˜ô 1(ôh&g˜ô &gôRL�ô L÷2v:ò v:r/   