Ë
    S^(h$  ã                   ó^  — d dl Z d dlZd dlmZmZ d dlmZ d dlmZm	Z	m
Z
mZ d dlZd dlmZ d dlmZ ddlmZ dd	lmZ dd
lmZ ddlmZmZmZmZ  ej8                  e«      Z e ej@                  «       «      Z! e"d„ e!D «       «      Z#e G d„ d«      «       Z$ G d„ de«      Z% G d„ de«      Z&y)é    N)Ú	dataclassÚfield)ÚEnum)ÚDictÚListÚOptionalÚUnion)ÚFileLock)ÚDataseté   )Ú$MODEL_FOR_QUESTION_ANSWERING_MAPPING)ÚPreTrainedTokenizer)Úloggingé   )ÚSquadFeaturesÚSquadV1ProcessorÚSquadV2ProcessorÚ"squad_convert_examples_to_featuresc              #   ó4   K  — | ]  }|j                   –— Œ y ­w©N)Ú
model_type)Ú.0Úconfs     ú^/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/data/datasets/squad.pyú	<genexpr>r   "   s   è ø€ ÒE¨�D—O•OÑEùs   ‚c                   óÎ  — e Zd ZU dZ eddddj                  e«      z   i¬«      Zee	d<    edddi¬«      Z
ee	d	<    ed
ddi¬«      Zee	d<    ed
ddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    edddi¬«      Zee	d<    eddd i¬«      Zee	d!<    ed"dd#i¬«      Zee	d$<   y)%ÚSquadDataTrainingArgumentszb
    Arguments pertaining to what data we are going to input our model for training and eval.
    NÚhelpz!Model type selected in the list: z, )ÚdefaultÚmetadatar   zFThe input data dir. Should contain the .json files for the SQuAD task.Údata_diré€   z‹The maximum total input sequence length after tokenization. Sequences longer than this will be truncated, sequences shorter will be padded.Úmax_seq_lengthzVWhen splitting up a long document into chunks, how much stride to take between chunks.Ú
doc_strideé@   zkThe maximum number of tokens for the question. Questions longer than this will be truncated to this length.Úmax_query_lengthé   z�The maximum length of an answer that can be generated. This is needed because the start and end predictions are not conditioned on one another.Úmax_answer_lengthFz1Overwrite the cached training and evaluation setsÚoverwrite_cachezDIf true, the SQuAD examples contain some that do not have an answer.Úversion_2_with_negativeg        zIIf null_score - best_non_null is greater than the threshold predict null.Únull_score_diff_thresholdé   Ún_best_sizer   zjlanguage id of input for language-specific xlm models (see tokenization_xlm.PRETRAINED_INIT_CONFIGURATION)Úlang_idé   z3multiple threads for converting example to featuresÚthreads)Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   ÚjoinÚMODEL_TYPESr   ÚstrÚ__annotations__r!   r#   Úintr$   r&   r(   r)   Úboolr*   r+   Úfloatr-   r.   r0   © ó    r   r   r   %   s•  … ññ Ø Ð(KÈdÏiÉiÐXcÓNdÑ(dÐeô€J�ó ñ Ø Ð(pÐqô€Hˆcó ñ  ØàðQð
ô€N�Có ñ ØØÐrÐsô€J�ó ñ "Øàð/ð
ôÐ�có ñ #ØàðJð
ôÐ�só ñ "Ø Ð)\Ð ]ô€O�Tó ñ %*Ø Ð)oÐ pô%Ð˜Tó ñ (-Ø˜vÐ'rÐsô(Ð˜uó ñ Ø˜fÐ&qÐrô€K�ó ñ ØàðCð
ô€GˆSó ñ  ¨fÐ6kÐ-lÔm€GˆSÔmr=   r   c                   ó   — e Zd ZdZdZy)ÚSplitÚtrainÚdevN)r1   r2   r3   r@   rA   r<   r=   r   r?   r?   h   s   „ Ø€EØ
�Cr=   r?   c                   óÖ   — e Zd ZU dZeed<   ee   ed<   eed<   e	ed<   dej                  dddfded	ed
ee   deeef   dee	   dee   dee   fd„Zd„ Zdeeej(                  f   fd„Zy)ÚSquadDatasetzH
    This will be superseded by a framework-agnostic approach soon.
    ÚargsÚfeaturesÚmodeÚis_language_sensitiveNFÚptÚ	tokenizerÚlimit_lengthÚ	cache_dirÚdataset_formatc                 óÄ  — || _         || _        |j                  r
t        «       n	t	        «       | _        t        |t        «      r
	 t        |   }|| _
        |j                  rdnd}t        j                  j                  |�|n|j                  d|j                  › d|j                   j"                  › d|j$                  › d|› �«      }	|	dz   }
t'        |
«      5  t        j                  j)                  |	«      rõ|j*                  sét-        j,                  «       }t/        j0                  |	d¬«      | _        | j2                  d	   | _        | j2                  j7                  d
d «      | _        | j2                  j7                  dd «      | _        t<        j?                  d|	› d�t-        j,                  «       |z
  «       | j8                  �| j:                  �€dt<        jA                  d|	› d�«       �nI|t        jB                  k(  r+| j
                  jE                  |j                  «      | _        n*| j
                  jG                  |j                  «      | _        tI        | j:                  ||j$                  |jJ                  |jL                  |t        jN                  k(  |jP                  |¬«      \  | _        | _        t-        j,                  «       }t/        jR                  | j4                  | j8                  | j:                  dœ|	«       t<        j?                  d|	› dt-        j,                  «       |z
  d›d�«       d d d «       y # t        $ r t        d«      ‚w xY w# 1 sw Y   y xY w)Nzmode is not a valid split nameÚv2Úv1Úcached_Ú_z.lockT)Úweights_onlyrE   ÚdatasetÚexamplesz"Loading features from cached file z [took %.3f s]zDeleting cached file z; will allow dataset and examples to be cached in future run)rT   rI   r#   r$   r&   Úis_trainingr0   Úreturn_dataset)rE   rS   rT   z!Saving features into cached file z [took z.3fz s])*rD   rG   r*   r   r   Ú	processorÚ
isinstancer7   r?   ÚKeyErrorrF   ÚosÚpathr5   r!   ÚvalueÚ	__class__r1   r#   r
   Úexistsr)   ÚtimeÚtorchÚloadÚold_featuresrE   ÚgetrS   rT   ÚloggerÚinfoÚwarningrA   Úget_dev_examplesÚget_train_examplesr   r$   r&   r@   r0   Úsave)ÚselfrD   rI   rJ   rF   rG   rK   rL   Úversion_tagÚcached_features_fileÚ	lock_pathÚstarts               r   Ú__init__zSquadDataset.__init__w   sß  € ð ˆŒ	Ø%:ˆÔ"Ø/3×/KÒ/KÔ)Ô+ÔQaÓQcˆŒÜ�dœCÔ ðAÜ˜T‘{�ð ˆŒ	à"×:Ò:‘dÀˆÜ!Ÿw™wŸ|™|Ø"Ð.‰I°D·M±MØ�d—j‘j�\  9×#6Ñ#6×#?Ñ#?Ð"@ÀÀ$×BUÑBUÐAVÐVWÐXcÐWdÐeó 
Ðð )¨7Ñ2ˆ	Ü�iÓ ñ ,	Ü�w‰w�~‰~Ð2Ô3¸D×<PÒ<PÜŸ	™	›�Ü$)§J¡JÐ/CÐRVÔ$W�Ô!ð !%× 1Ñ 1°*Ñ =�”Ø#×0Ñ0×4Ñ4°YÀÓE�”Ø $× 1Ñ 1× 5Ñ 5°jÀ$Ó G�”Ü—‘Ø8Ð9MÐ8NÈnÐ]Ô_c×_hÑ_hÓ_jÐmrÑ_rôð —<‘<Ð'¨4¯=©=Ñ+@Ü—N‘NØ/Ð0DÐ/Eð F&ð &öð
 œ5Ÿ9™9Ò$Ø$(§N¡N×$CÑ$CÀDÇMÁMÓ$R�D•Mà$(§N¡N×$EÑ$EÀdÇmÁmÓ$T�D”Mä.PØ!Ÿ]™]Ø'Ø#'×#6Ñ#6Ø#Ÿ™Ø%)×%:Ñ%:Ø $¬¯©Ñ 3Ø ŸL™LØ#1ô	/Ñ+�”˜tœ|ô Ÿ	™	›�Ü—
‘
Ø!%§¡¸4¿<¹<ÐUY×UbÑUbÑcØ(ôô
 —‘Ø7Ð8LÐ7MÈWÔUY×U^ÑU^ÓU`ÐchÑUhÐilÐTmÐmpÐqô÷U,	ð ,	øô ò AÜÐ?Ó@Ð@ðAú÷,	ð ,	ús   Á	L> ÃIMÌ>MÍMc                 ó,   — t        | j                  «      S r   )ÚlenrE   )rj   s    r   Ú__len__zSquadDataset.__len__Â   s   € Ü�4—=‘=Ó!Ð!r=   Úreturnc                 ó(  — | j                   |   }t        j                  |j                  t        j                  ¬«      }t        j                  |j
                  t        j                  ¬«      }t        j                  |j                  t        j                  ¬«      }t        j                  |j                  t        j                  ¬«      }t        j                  |j                  t        j                  ¬«      }t        j                  |j                  t        j                  ¬«      }|||dœ}	| j                  j                  dv r|	d= | j                  j                  dv r |	j                  ||dœ«       | j                  j                  r|	j                  d|i«       | j                  rW|	j                  dt        j                   |j"                  t        j$                  ¬«      | j                  j&                  z  i«       | j(                  t*        j,                  k(  rrt        j                  |j.                  t        j                  ¬«      }
t        j                  |j0                  t        j                  ¬«      }|	j                  |
|d	œ«       |	S )
N)Údtype)Ú	input_idsÚattention_maskÚtoken_type_ids)ÚxlmÚrobertaÚ
distilbertÚ	camembertrx   )Úxlnetry   )Ú	cls_indexÚp_maskÚis_impossibleÚlangs)Ústart_positionsÚend_positions)rE   r`   Útensorrv   Úlongrw   rx   r~   r   r;   r€   rD   r   Úupdater*   rG   ÚonesÚshapeÚint64r.   rF   r?   r@   Ústart_positionÚend_position)rj   ÚiÚfeaturerv   rw   rx   r~   r   r€   Úinputsr‚   rƒ   s               r   Ú__getitem__zSquadDataset.__getitem__Å   s¶  € à—-‘- Ñ"ˆä—L‘L ×!2Ñ!2¼%¿*¹*ÔEˆ	ÜŸ™ g×&<Ñ&<ÄEÇJÁJÔOˆÜŸ™ g×&<Ñ&<ÄEÇJÁJÔOˆÜ—L‘L ×!2Ñ!2¼%¿*¹*ÔEˆ	Ü—‘˜gŸn™n´E·K±KÔ@ˆÜŸ™ W×%:Ñ%:Ä%Ç+Á+ÔNˆð #Ø,Ø,ñ
ˆð �9‰9×ÑÐ#PÑPØÐ'Ð(à�9‰9×ÑÐ#3Ñ3Ø�M‰M¨	¸VÑDÔEØ�y‰y×0Ò0Ø—‘˜°Ð>Ô?Ø×)Ò)Ø—‘˜w¬¯©°I·O±OÌ5Ï;É;Ô)WÐZ^×ZcÑZc×ZkÑZkÑ)kÐmÔnà�9‰9œŸ™Ò#Ü#Ÿl™l¨7×+AÑ+AÌÏÉÔTˆOÜ!ŸL™L¨×)=Ñ)=ÄUÇZÁZÔPˆMØ�M‰M¨oÐP]Ñ^Ô_àˆr=   )r1   r2   r3   r4   r   r8   r   r   r?   r:   r@   r   r   r9   r	   r7   ro   rr   r   r`   ÚTensorr�   r<   r=   r   rC   rC   m   sÄ   … ñð %Ó$Ø�=Ñ!Ó!Ø
ƒKØÓð '+Ø"'§+¡+Ø05Ø#'Ø(,ñIà(ðIð 'ðIð ˜s‘mð	Ið
 �C˜�JÑðIð  (¨™~ðIð ˜C‘=ðIð ! ™óIòV"ð   S¨%¯,©,Ð%6Ñ 7ô  r=   rC   )'rZ   r_   Údataclassesr   r   Úenumr   Útypingr   r   r   r	   r`   Úfilelockr
   Útorch.utils.datar   Úmodels.auto.modeling_autor   Útokenization_utilsr   Úutilsr   Úprocessors.squadr   r   r   r   Ú
get_loggerr1   rd   ÚlistÚkeysÚMODEL_CONFIG_CLASSESÚtupler6   r   r?   rC   r<   r=   r   ú<module>rŸ      s¥   ðó 
Û ß (Ý ß .Ó .ã Ý Ý $å MÝ 5Ý ß tÓ tð 
ˆ×	Ñ	˜HÓ	%€áÐEÐ@×EÑEÓGÓHÐ ÙÑEÐ0DÔEÓE€ð ÷?nð ?nó ð?nôDˆDô ô
x�7õ xr=   