Ë
    S^(hÀ�  ã                   ó¬  — d dl Z d dlZd dlmZ d dlmZmZ d dlmZ d dl	Z
d dlmZ ddlmZ ddlmZmZmZ dd	lmZmZmZ d
dlmZ h d£Z e«       r
d dlZd dlmZ  e«       rd dlZ ej:                  e«      Zd„ Z d„ Z!d„ Z"d„ Z#d„ Z$defd„Z%	 	 	 	 d"d„Z& G d„ de«      Z' G d„ de'«      Z( G d„ de'«      Z) G d„ d«      Z* G d„ d«      Z+ G d „ d!«      Z,y)#é    N)Úpartial)ÚPoolÚ	cpu_count)ÚOptional)Útqdmé   )Úwhitespace_tokenize)ÚBatchEncodingÚPreTrainedTokenizerBaseÚTruncationStrategy)Úis_tf_availableÚis_torch_availableÚloggingé   )ÚDataProcessor>   ÚbartÚmpnetÚrobertaÚ	camembert)ÚTensorDatasetc                 óä   — dj                  |j                  |«      «      }t        ||dz   «      D ];  }t        ||dz
  d«      D ]&  }dj                  | ||dz    «      }||k(  sŒ ||fc c S  Œ= ||fS )zFReturns tokenized answer spans that better match the annotated answer.ú r   éÿÿÿÿ)ÚjoinÚtokenizeÚrange)	Ú
doc_tokensÚinput_startÚ	input_endÚ	tokenizerÚorig_answer_textÚtok_answer_textÚ	new_startÚnew_endÚ	text_spans	            ú`/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/data/processors/squad.pyÚ_improve_answer_spanr'   ,   s‹   € à—h‘h˜y×1Ñ1Ð2BÓCÓD€Oä˜;¨	°A©Ó6ò ,ˆ	Ü˜Y¨	°A©°rÓ:ò 	,ˆGØŸ™ ¨I¸À1¹Ð!FÓGˆIØ˜OÓ+Ø! 7Ð+Ô+ñ	,ð,ð ˜Ð#Ð#ó    c                 ó  — d}d}t        | «      D ]s  \  }}|j                  |j                  z   dz
  }||j                  k  rŒ2||kD  rŒ8||j                  z
  }||z
  }	t        ||	«      d|j                  z  z   }
|�|
|kD  sŒp|
}|}Œu ||k(  S )ú:Check if this is the 'max context' doc span for the token.Nr   ç{®Gáz„?)Ú	enumerateÚstartÚlengthÚmin©Ú	doc_spansÚcur_span_indexÚpositionÚ
best_scoreÚbest_span_indexÚ
span_indexÚdoc_spanÚendÚnum_left_contextÚnum_right_contextÚscores              r&   Ú_check_is_max_contextr<   9   s®   € à€JØ€OÜ )¨)Ó 4ò )Ñˆ
�HØ�n‰n˜xŸ™Ñ.°Ñ2ˆØ�h—n‘nÒ$ØØ�cŠ>ØØ# h§n¡nÑ4ÐØ (™NÐÜÐ$Ð&7Ó8¸4À(Ç/Á/Ñ;QÑQˆØÐ ¨Ó!3ØˆJØ(‰Oð)ð ˜_Ñ,Ð,r(   c                 óÐ   — d}d}t        | «      D ]P  \  }}|d   |d   z   dz
  }||d   k  rŒ||kD  rŒ#||d   z
  }||z
  }	t        ||	«      d|d   z  z   }
|�|
|kD  sŒM|
}|}ŒR ||k(  S )r*   Nr-   r.   r   r+   )r,   r/   r0   s              r&   Ú_new_check_is_max_contextr>   M   sµ   € ð €JØ€OÜ )¨)Ó 4ò )Ñˆ
�HØ�wÑ (¨8Ñ"4Ñ4°qÑ8ˆØ�h˜wÑ'Ò'ØØ�cŠ>ØØ# h¨wÑ&7Ñ7ÐØ (™NÐÜÐ$Ð&7Ó8¸4À(È8ÑBTÑ;TÑTˆØÐ ¨Ó!3ØˆJØ(‰Oð)ð ˜_Ñ,Ð,r(   c                 óJ   — | dk(  s| dk(  s| dk(  s| dk(  st        | «      dk(  ryy)Nr   ú	úú
i/   TF)Úord)Úcs    r&   Ú_is_whitespacerE   c   s,   € ØˆC‚x�1˜’9  T¢	¨Q°$ªY¼#¸a»&ÀFÒ:JØØr(   c                 ór  — g }|r›| j                   s�| j                  }| j                  }dj                  | j                  ||dz    «      }	dj                  t        | j                  «      «      }
|	j                  |
«      dk(  rt        j                  d|	› d|
› d�«       g S g }g }g }t        | j                  «      D ]‘  \  }}|j                  t        |«      «       t        j                  j                  dv rt        j!                  |d¬	«      }nt        j!                  |«      }|D ]$  }|j                  |«       |j                  |«       Œ& Œ“ |r„| j                   sx|| j                     }| j                  t        | j                  «      dz
  k  r|| j                  dz      dz
  }nt        |«      dz
  }t#        |||t        | j                  «      \  }}g }t        j%                  | j&                  d
d|¬«      }t)        t        «      j                  j+                  dd«      j-                  «       }|t.        v r$t        j0                  t        j2                  z
  dz   n t        j0                  t        j2                  z
  }t        j0                  t        j4                  z
  }|}t        |«      |z  t        |«      k  �r9t        j6                  dk(  r|}|}t8        j:                  j<                  }n|}|}t8        j>                  j<                  }t        jA                  |||||d||z
  t        |«      z
  |z
  d¬«      }tC        t        |«      t        |«      |z  z
  |t        |«      z
  |z
  «      }t        jD                  |d   v r‚t        j6                  dk(  r)|d   d |d   jG                  t        jD                  «       }nKt        |d   «      dz
  |d   d d d…   jG                  t        jD                  «      z
  } |d   | dz   d  }n|d   }t        jI                  |«      }!i }"tK        |«      D ]?  }t        j6                  dk(  rt        |«      |z   |z   n|}#|t        |«      |z  |z      |"|#<   ŒA ||d<   |!|d<   |"|d<   t        |«      |z   |d<   i |d<   t        |«      |z  |d<   ||d<   |j                  |«       d|vsd|v rt        |d   «      dk(  rn!|d   }t        |«      |z  t        |«      k  r�Œ9tK        t        |«      «      D ]V  }$tK        ||$   d   «      D ]@  }%tM        ||$|$|z  |%z   «      }&t        j6                  dk(  r|%n
||$   d   |%z   }#|&||$   d   |#<   ŒB ŒX |D �]Ï  }'|'d   jG                  t        jN                  «      }(tQ        jR                  |'d   «      })t        j6                  dk(  rd|)t        |«      |z   d  nd|)t        |'d   «       t        |«      |z     tQ        jT                  tQ        jV                  |'d   t        jD                  k(  «      «      }*tQ        jX                  t        j[                  |'d   d¬«      «      j]                  «       }+d|)|*<   d|)|+<   d|)|(<   | j                   },d}d}|r`|,s^|'d   }-|'d   |'d   z   dz
  }.d
}/|-k\  r|.k  sd}/|/r|(}|(}d},n4t        j6                  dk(  rd}0nt        |«      |z   }0||-z
  |0z   }|-z
  |0z   }|j                  t_        |'d   |'d   |'d   |(|)ja                  «       dd|'d   |'d   |'d   |'d   |||,| jb                  ¬«      «       �ŒÒ |S )Nr   r   r   zCould not find answer: 'z' vs. 'ú')ÚRobertaTokenizerÚLongformerTokenizerÚBartTokenizerÚRobertaTokenizerFastÚLongformerTokenizerFastÚBartTokenizerFastT)Úadd_prefix_spaceF)Úadd_special_tokensÚ
truncationÚ
max_lengthÚ	TokenizerÚ Úright)rP   ÚpaddingrQ   Úreturn_overflowing_tokensÚstrideÚreturn_token_type_idsÚ	input_idsÚparagraph_lenÚtokensÚtoken_to_orig_mapÚ*truncated_query_with_special_tokens_lengthÚtoken_is_max_contextr-   r.   Úoverflowing_tokensr   ÚleftÚtoken_type_ids)Úalready_has_special_tokensÚattention_mask)
Úexample_indexÚ	unique_idrZ   r^   r[   r\   Ústart_positionÚend_positionÚis_impossibleÚqas_id)2rh   rf   rg   r   r   r	   Úanswer_textÚfindÚloggerÚwarningr,   ÚappendÚlenr    Ú	__class__Ú__name__r   r'   ÚencodeÚquestion_textÚtypeÚreplaceÚlowerÚMULTI_SEP_TOKENS_TOKENIZERS_SETÚmodel_max_lengthÚmax_len_single_sentenceÚmax_len_sentences_pairÚpadding_sider   ÚONLY_SECONDÚvalueÚ
ONLY_FIRSTÚencode_plusr/   Úpad_token_idÚindexÚconvert_ids_to_tokensr   r>   Úcls_token_idÚnpÚ	ones_likeÚwhereÚ
atleast_1dÚasarrayÚget_special_tokens_maskÚnonzeroÚSquadFeaturesÚtolistri   )1ÚexampleÚmax_seq_lengthÚ
doc_strideÚmax_query_lengthÚpadding_strategyÚis_trainingÚfeaturesrf   rg   Úactual_textÚcleaned_answer_textÚtok_to_orig_indexÚorig_to_tok_indexÚall_doc_tokensÚiÚtokenÚ
sub_tokensÚ	sub_tokenÚtok_start_positionÚtok_end_positionÚspansÚtruncated_queryÚtokenizer_typeÚsequence_added_tokensÚsequence_pair_added_tokensÚspan_doc_tokensÚtextsÚpairsrP   Úencoded_dictrZ   Únon_padded_idsÚlast_padding_id_positionr[   r\   r�   Údoc_span_indexÚjÚis_max_contextÚspanÚ	cls_indexÚp_maskÚpad_token_indicesÚspecial_token_indicesÚspan_is_impossibleÚ	doc_startÚdoc_endÚout_of_spanÚ
doc_offsets1                                                    r&   Ú!squad_convert_example_to_featuresr·   i   s  € ð €HÙ˜7×0Ò0à ×/Ñ/ˆØ×+Ñ+ˆð —h‘h˜w×1Ñ1°.ÀLÐSTÑDTÐVÓWˆØ!Ÿh™hÔ':¸7×;NÑ;NÓ'OÓPÐØ×ÑÐ/Ó0°BÒ6Ü�N‰NÐ5°k°]À'ÐJ]ÐI^Ð^_Ð`ÔaØˆIàÐØÐØ€NÜ˜g×0Ñ0Ó1ò -‰ˆˆ5Ø× Ñ ¤ ^Ó!4Ô5Ü×Ñ×'Ñ'ð ,
ñ 
ô #×+Ñ+¨EÀDÐ+ÓI‰Jä"×+Ñ+¨EÓ2ˆJØ#ò 	-ˆIØ×$Ñ$ QÔ'Ø×!Ñ! )Õ,ñ	-ð-ñ" ˜7×0Ò0Ø.¨w×/EÑ/EÑFÐØ×Ñ¤# g×&8Ñ&8Ó"9¸AÑ"=Ò=Ø0°×1EÑ1EÈÑ1IÑJÈQÑNÑä" >Ó2°QÑ6Ðä1EØÐ.Ð0@Ä)ÈW×M`ÑM`ó2
Ñ.Ð	Ð-ð €Eä×&Ñ&Ø×Ñ°%ÀDÐUeð 'ó €Oô œ)“_×-Ñ-×5Ñ5°kÀ2ÓF×LÑLÓN€Nð Ô<Ñ<ô 	×"Ñ"¤Y×%FÑ%FÑFÈÒJä×'Ñ'¬)×*KÑ*KÑKð ô
 "+×!;Ñ!;¼i×>^Ñ>^Ñ!^Ðà$€OÜ
ˆe‹*�zÑ
!¤C¨Ó$7Ó
7ä×!Ñ! WÒ,Ø#ˆEØ#ˆEÜ+×7Ñ7×=Ñ=‰Jà#ˆEØ#ˆEÜ+×6Ñ6×<Ñ<ˆJä ×,Ñ,ØØØ!Ø$Ø%Ø&*Ø! JÑ.´°_Ó1EÑEÐHbÑbØ"&ð -ó 	
ˆô Ü�Ó¤# e£*¨zÑ"9Ñ9ØœS Ó1Ñ1Ð4NÑNó
ˆô
 ×!Ñ! \°+Ñ%>Ñ>Ü×%Ñ%¨Ò0Ø!-¨kÑ!:Ð;t¸\È+Ñ=V×=\Ñ=\Ô]f×]sÑ]sÓ=tÐ!u‘ô ˜ [Ñ1Ó2°QÑ6¸ÀkÑ9RÑSWÐUWÐSWÑ9X×9^Ñ9^Ô_h×_uÑ_uÓ9vÑvð )ð ".¨kÑ!:Ð;SÐVWÑ;WÐ;YÐ!Z‘ð *¨+Ñ6ˆNä×0Ñ0°Ó@ˆàÐÜ�}Ó%ò 	VˆAÜHQ×H^ÑH^ÐbiÒHi”C˜Ó(Ð+@Ñ@À1ÒDÐopˆEØ'8¼¸U»ÀjÑ9PÐSTÑ9TÑ'UÐ˜eÒ$ð	Vð )6ˆ�_Ñ%Ø!'ˆ�XÑØ,=ˆÐ(Ñ)ÜEHÈÓEYÐ\qÑEqˆÐAÑBØ/1ˆÐ+Ñ,Ü # E£
¨ZÑ 7ˆ�WÑØ!.ˆ�XÑà�‰�\Ô"à |Ñ3Ø  LÑ0´S¸ÐFZÑ9[Ó5\Ð`aÒ5aàØ&Ð';Ñ<ˆôy ˆe‹*�zÑ
!¤C¨Ó$7Ô
7ô|  ¤ E£
Ó+ò RˆÜ�u˜^Ñ,¨_Ñ=Ó>ò 	RˆAÜ6°u¸nÈnÐ_iÑNiÐlmÑNmÓnˆNô ×)Ñ)¨VÒ3ñ à˜>Ñ*Ð+WÑXÐ[\Ñ\ð ð
 DRˆE�.Ñ!Ð"8Ñ9¸%Ò@ñ	RðRð ó C
ˆà˜Ñ%×+Ñ+¬I×,BÑ,BÓCˆ	ô —‘˜dÐ#3Ñ4Ó5ˆÜ×!Ñ! WÒ,ØEFˆF”3�Ó'Ð*?Ñ?ÐAÑBà]^ˆF”C˜˜X™Ó'Ð'¬C°Ó,@ÐCXÑ,XÐ*YÐZäŸH™H¤R§]¡]°4¸Ñ3DÌ	×H^ÑH^Ñ3^Ó%_Ó`ÐÜ "§
¡
Ü×-Ñ-¨d°;Ñ.?Ð\`Ð-Óaó!
ç
‰'‹)ð 	ð %&ˆÐ Ñ!Ø()ˆÐ$Ñ%ð ˆˆyÑà$×2Ñ2ÐØˆØˆÙÑ1ð ˜W™ˆIØ˜7‘m d¨8¡nÑ4°qÑ8ˆGØˆKà&¨)Ò3Ð8HÈGÒ8SØ"�áØ!*�Ø(�Ø%)Ñ"ä×)Ñ)¨VÒ3Ø!"‘Jä!$ _Ó!5Ð8MÑ!M�Jà!3°iÑ!?À*Ñ!L�Ø/°)Ñ;¸jÑH�à�‰ÜØ�[Ñ!ØÐ%Ñ&ØÐ%Ñ&ØØ—‘“ØØØ" ?Ñ3Ø%)Ð*@Ñ%AØ˜H‘~Ø"&Ð':Ñ";Ø-Ø)Ø0Ø—~‘~ôö	
ðcC
ðH €Or(   Útokenizer_for_convertc                 ó   — | a y ©N)r    )r¸   s    r&   Ú&squad_convert_example_to_features_initr»   8  s   € à%�Ir(   c
           
      ó
  ‡— g Št        |t        «       «      }t        |t        |f¬«      5 }
t	        t
        |||||¬«      }t        t        |
j                  || d¬«      t        | «      d|	 ¬«      «      Šddd«       g }d}d	}t        ‰t        ‰«      d
|	 ¬«      D ]5  }|sŒ|D ]&  }||_
        ||_        |j                  |«       |dz  }Œ( |dz  }Œ7 |Š~|dk(  �r]t        «       st        d«      ‚t        j                   ‰D �cg c]  }|j"                  ‘Œ c}t        j$                  ¬«      }t        j                   ‰D �cg c]  }|j&                  ‘Œ c}t        j$                  ¬«      }t        j                   ‰D �cg c]  }|j(                  ‘Œ c}t        j$                  ¬«      }t        j                   ‰D �cg c]  }|j*                  ‘Œ c}t        j$                  ¬«      }t        j                   ‰D �cg c]  }|j,                  ‘Œ c}t        j.                  ¬«      }t        j                   ‰D �cg c]  }|j0                  ‘Œ c}t        j.                  ¬«      }|sHt        j2                  |j5                  d	«      t        j$                  ¬«      }t7        ||||||«      }‰|fS t        j                   ‰D �cg c]  }|j8                  ‘Œ c}t        j$                  ¬«      }t        j                   ‰D �cg c]  }|j:                  ‘Œ c}t        j$                  ¬«      }t7        ||||||||«      }‰|fS |dk(  �rt=        «       st        d«      ‚ˆfd„}d|j>                  v �rpt@        jB                  t@        jB                  t@        jB                  t@        jD                  t@        jF                  dœt@        jD                  t@        jD                  t@        jD                  t@        jB                  t@        jB                  dœf}tA        jH                  dg«      tA        jH                  dg«      tA        jH                  dg«      tA        jH                  g «      tA        jH                  g «      dœtA        jH                  g «      tA        jH                  g «      tA        jH                  g «      tA        jH                  dg«      tA        jH                  g «      dœf}�nJt@        jB                  t@        jB                  t@        jD                  t@        jF                  dœt@        jD                  t@        jD                  t@        jD                  t@        jB                  t@        jB                  dœf}tA        jH                  dg«      tA        jH                  dg«      tA        jH                  g «      tA        jH                  g «      dœtA        jH                  g «      tA        jH                  g «      tA        jH                  g «      tA        jH                  dg«      tA        jH                  g «      dœf}t@        jJ                  jL                  jO                  |||«      S ‰S # 1 sw Y   �ŒÝxY wc c}w c c}w c c}w c c}w c c}w c c}w c c}w c c}w )a—  
    Converts a list of examples into a list of features that can be directly given as input to a model. It is
    model-dependant and takes advantage of many of the tokenizer's features to create the model's inputs.

    Args:
        examples: list of [`~data.processors.squad.SquadExample`]
        tokenizer: an instance of a child of [`PreTrainedTokenizer`]
        max_seq_length: The maximum sequence length of the inputs.
        doc_stride: The stride used when the context is too large and is split across several features.
        max_query_length: The maximum length of the query.
        is_training: whether to create features for model evaluation or model training.
        padding_strategy: Default to "max_length". Which padding strategy to use
        return_dataset: Default False. Either 'pt' or 'tf'.
            if 'pt': returns a torch.data.TensorDataset, if 'tf': returns a tf.data.Dataset
        threads: multiple processing threads.


    Returns:
        list of [`~data.processors.squad.SquadFeatures`]

    Example:

    ```python
    processor = SquadV2Processor()
    examples = processor.get_dev_examples(data_dir)

    features = squad_convert_examples_to_features(
        examples=examples,
        tokenizer=tokenizer,
        max_seq_length=args.max_seq_length,
        doc_stride=args.doc_stride,
        max_query_length=args.max_query_length,
        is_training=not evaluate,
    )
    ```)ÚinitializerÚinitargs)rŽ   r�   r�   r‘   r’   é    )Ú	chunksizez"convert squad examples to features)ÚtotalÚdescÚdisableNi Êš;r   zadd example index and unique idr   Úptz6PyTorch must be installed to return a PyTorch dataset.)ÚdtypeÚtfz<TensorFlow must be installed to return a TensorFlow dataset.c               3   óè  •K  — t        ‰«      D ]ß  \  } }|j                  €b|j                  |j                  | |j                  dœ|j
                  |j                  |j                  |j                  |j                  dœf–— Œt|j                  |j                  |j                  | |j                  dœ|j
                  |j                  |j                  |j                  |j                  dœf–— Œá y ­w)N©rY   rc   Úfeature_indexri   ©Ústart_positionsÚend_positionsr®   r¯   rh   ©rY   rc   ra   rÉ   ri   )
r,   ra   rY   rc   ri   rf   rg   r®   r¯   rh   )r™   Úexr“   s     €r&   Úgenz/squad_convert_examples_to_features.<locals>.gen¶  sæ   øè ø€ Ü" 8Ó,ò !‘��2Ø×$Ñ$Ð,ð *,¯©Ø.0×.?Ñ.?Ø-.Ø&(§i¡iñ	ð 02×/@Ñ/@Ø-/¯_©_Ø)+¯©Ø&(§i¡iØ-/×-=Ñ-=ñðó ð$ *,¯©Ø.0×.?Ñ.?Ø.0×.?Ñ.?Ø-.Ø&(§i¡iñð 02×/@Ñ/@Ø-/¯_©_Ø)+¯©Ø&(§i¡iØ-/×-=Ñ-=ñðó ñ%!ùs   ƒC/C2ra   rÍ   rÊ   rÈ   )(r/   r   r   r»   r   r·   Úlistr   Úimapro   rd   re   rn   r   ÚRuntimeErrorÚtorchÚtensorrY   Úlongrc   ra   r®   r¯   Úfloatrh   ÚarangeÚsizer   rf   rg   r   Úmodel_input_namesrÆ   Úint32Úint64ÚstringÚTensorShapeÚdataÚDatasetÚfrom_generator) Úexamplesr    rŽ   r�   r�   r’   r‘   Úreturn_datasetÚthreadsÚtqdm_enabledÚpÚ	annotate_Únew_featuresre   rd   Úexample_featuresÚexample_featureÚfÚall_input_idsÚall_attention_masksÚall_token_type_idsÚall_cls_indexÚ
all_p_maskÚall_is_impossibleÚall_feature_indexÚdatasetÚall_start_positionsÚall_end_positionsrÏ   Útrain_typesÚtrain_shapesr“   s                                   @r&   Ú"squad_convert_examples_to_featuresr÷   =  s(  ø€ ð` €Hä�'œ9›;Ó'€GÜ	ˆgÔ#IÐU^ÐT`Ô	að 
ÐefÜÜ-Ø)Ø!Ø-Ø-Ø#ô
ˆ	ô ÜØ—‘�y (°b�Ó9Ü˜(“mØ9Ø(Ð(ô	ó
ˆ÷
ð$ €LØ€IØ€MÜ Øœ˜H›Ð,MÐ[gÐWgôò 
Ðñ  ØØ/ò 	ˆOØ,9ˆOÔ)Ø(1ˆOÔ%Ø×Ñ Ô0Ø˜‰N‰Ið		ð
 	˜Ñ‰ð
ð €HØØ˜ÓÜ!Ô#ÜÐWÓXÐXô Ÿ™¸8Ö%D°a a§k£kÒ%DÌEÏJÉJÔWˆÜ#Ÿl™lÀhÖ+OÀ¨A×,<Ó,<Ò+OÔW\×WaÑWaÔbÐÜ"Ÿ\™\ÀXÖ*NÀ¨1×+;Ó+;Ò*NÔV[×V`ÑV`ÔaÐÜŸ™¸8Ö%D°a a§k£kÒ%DÌEÏJÉJÔWˆÜ—\‘\°XÖ">° 1§8£8Ò">ÄeÇkÁkÔRˆ
Ü!ŸL™LÀ8Ö)L¸a¨!¯/«/Ò)LÔTY×T_ÑT_Ô`ÐáÜ %§¡¨]×-?Ñ-?ÀÓ-BÌ%Ï*É*Ô UÐÜ#ØÐ2Ð4FÐHYÐ[hÐjtóˆGð" ˜Ð Ð ô #(§,¡,È(Ö/SÀQ°×0@Ó0@Ò/SÔ[`×[eÑ[eÔ"fÐÜ %§¡ÀhÖ-OÀ¨a¯n«nÒ-OÔW\×WaÑWaÔ bÐÜ#ØØ#Ø"Ø#Ø!ØØØ!ó	ˆGð ˜Ð Ð Ø	˜4Ó	ÜÔ ÜÐ]Ó^Ð^ô"	ðJ ˜y×:Ñ:Ò:ô "$§¡Ü&(§h¡hÜ&(§h¡hÜ%'§X¡XÜ Ÿi™iñô (*§x¡xÜ%'§X¡XÜ!#§¡Ü Ÿh™hÜ%'§X¡XñðˆKô& "$§¡°°Ó!7Ü&(§n¡n°d°VÓ&<Ü&(§n¡n°d°VÓ&<Ü%'§^¡^°BÓ%7Ü Ÿn™n¨RÓ0ñô (*§~¡~°bÓ'9Ü%'§^¡^°BÓ%7Ü!#§¡°Ó!3Ü Ÿn™n¨d¨VÓ4Ü%'§^¡^°BÓ%7ñðŠLô$ !Ÿh™h¼"¿(¹(ÔUW×U]ÑU]Ôik×irÑirÑsä')§x¡xÜ%'§X¡XÜ!#§¡Ü Ÿh™hÜ%'§X¡Xñð	ˆKô "$§¡°°Ó!7Ü&(§n¡n°d°VÓ&<Ü%'§^¡^°BÓ%7Ü Ÿn™n¨RÓ0ñ	ô (*§~¡~°bÓ'9Ü%'§^¡^°BÓ%7Ü!#§¡°Ó!3Ü Ÿn™n¨d¨VÓ4Ü%'§^¡^°BÓ%7ñðˆLô  �w‰w�‰×-Ñ-¨c°;ÀÓMÐMàˆ÷W
ñ 
üòN &EùÚ+OùÚ*NùÚ%DùÚ">ùÚ)Lùò 0TùÚ-Os<   «A
YÄYÅY"Å?Y'Æ<Y,Ç9Y1È6Y6Ê=Y;Ë:Z ÙYc                   ó>   — e Zd ZdZdZdZdd„Zdd„Zd	d„Zd	d„Z	d„ Z
y)
ÚSquadProcessorz¤
    Processor for the SQuAD data set. overridden by SquadV1Processor and SquadV2Processor, used by the version 1.1 and
    version 2.0 of SQuAD, respectively.
    Nc           
      óv  — |sD|d   d   d   j                  «       j                  d«      }|d   d   d   j                  «       }g }n\t        |d   d   |d   d   «      D ��cg c]5  \  }}|j                  «       |j                  «       j                  d«      dœ‘Œ7 }}}d }d }t        |d   j                  «       j                  d«      |d   j                  «       j                  d«      |d	   j                  «       j                  d«      |||d
   j                  «       j                  d«      |¬«      S c c}}w )NÚanswersÚtextr   úutf-8Úanswer_start)rþ   rü   ÚidÚquestionÚcontextÚtitle)ri   rs   Úcontext_textrj   Ústart_position_characterr  rû   )ÚnumpyÚdecodeÚzipÚSquadExample)ÚselfÚtensor_dictÚevaluateÚanswerrþ   rû   r-   rü   s           r&   Ú_get_example_from_tensor_dictz,SquadProcessor._get_example_from_tensor_dict'  sB  € ÙØ  Ñ+¨FÑ3°AÑ6×<Ñ<Ó>×EÑEÀgÓNˆFØ& yÑ1°.ÑAÀ!ÑD×JÑJÓLˆLØ‰Gô $' {°9Ñ'=¸nÑ'MÈ{Ð[dÑOeÐflÑOmÓ#n÷á�E˜4ð "'§¡£¸¿
¹
»×8KÑ8KÈGÓ8TÓUðˆGñ ð
 ˆFØˆLäØ˜tÑ$×*Ñ*Ó,×3Ñ3°GÓ<Ø% jÑ1×7Ñ7Ó9×@Ñ@ÀÓIØ$ YÑ/×5Ñ5Ó7×>Ñ>¸wÓGØØ%1Ø˜gÑ&×,Ñ,Ó.×5Ñ5°gÓ>Øô
ð 	
ùós   Á":D5c                 óˆ   — |r|d   }n|d   }g }t        |«      D ]$  }|j                  | j                  ||¬«      «       Œ& |S )av  
        Creates a list of [`~data.processors.squad.SquadExample`] using a TFDS dataset.

        Args:
            dataset: The tfds dataset loaded from *tensorflow_datasets.load("squad")*
            evaluate: Boolean specifying if in evaluation mode or in training mode

        Returns:
            List of SquadExample

        Examples:

        ```python
        >>> import tensorflow_datasets as tfds

        >>> dataset = tfds.load("squad")

        >>> training_examples = get_examples_from_dataset(dataset, evaluate=False)
        >>> evaluation_examples = get_examples_from_dataset(dataset, evaluate=True)
        ```Ú
validationÚtrain)r  )r   rn   r  )r	  rò   r  rá   r
  s        r&   Úget_examples_from_datasetz(SquadProcessor.get_examples_from_dataset?  sY   € ñ, Ø˜lÑ+‰Gà˜gÑ&ˆGàˆÜ ›=ò 	`ˆKØ�O‰O˜D×>Ñ>¸{ÐU]Ð>Ó^Õ_ð	`ð ˆr(   c                 ó*  — |€d}| j                   €t        d«      ‚t        t        j                  j                  ||€| j                   n|«      dd¬«      5 }t        j                  |«      d   }ddd«       | j                  d«      S # 1 sw Y   ŒxY w)	a–  
        Returns the training examples from the data directory.

        Args:
            data_dir: Directory containing the data files used for training and evaluating.
            filename: None by default, specify this if the training file has a different name than the original one
                which is `train-v1.1.json` and `train-v2.0.json` for squad versions 1.1 and 2.0 respectively.

        NrS   úNSquadProcessor should be instantiated via SquadV1Processor or SquadV2ProcessorÚrrý   ©ÚencodingrÞ   r  )	Ú
train_fileÚ
ValueErrorÚopenÚosÚpathr   ÚjsonÚloadÚ_create_examples©r	  Údata_dirÚfilenameÚreaderÚ
input_datas        r&   Úget_train_examplesz!SquadProcessor.get_train_examples`  s�   € ð ÐØˆHà�?‰?Ð"ÜÐmÓnÐnäÜ�G‰G�L‰L˜°hÐ6F 4§?¢?ÈHÓUÐWZÐelô
ð 	3àÜŸ™ 6Ó*¨6Ñ2ˆJ÷	3ð ×$Ñ$ Z°Ó9Ð9÷		3ð 	3úó   ÁB	Â	Bc                 ó*  — |€d}| j                   €t        d«      ‚t        t        j                  j                  ||€| j                   n|«      dd¬«      5 }t        j                  |«      d   }ddd«       | j                  d«      S # 1 sw Y   ŒxY w)	a”  
        Returns the evaluation example from the data directory.

        Args:
            data_dir: Directory containing the data files used for training and evaluating.
            filename: None by default, specify this if the evaluation file has a different name than the original one
                which is `dev-v1.1.json` and `dev-v2.0.json` for squad versions 1.1 and 2.0 respectively.
        NrS   r  r  rý   r  rÞ   Údev)	Údev_filer  r  r  r  r   r  r  r  r  s        r&   Úget_dev_exampleszSquadProcessor.get_dev_examplesv  s�   € ð ÐØˆHà�=‰=Ð ÜÐmÓnÐnäÜ�G‰G�L‰L˜°HÐ4D 4§=¢=È(ÓSÐUXÐcjô
ð 	3àÜŸ™ 6Ó*¨6Ñ2ˆJ÷	3ð ×$Ñ$ Z°Ó7Ð7÷		3ð 	3úr%  c                 ó4  — |dk(  }g }t        |«      D ]‚  }|d   }|d   D ]s  }|d   }|d   D ]d  }	|	d   }
|	d   }d }d }g }|	j                  dd	«      }|s|r|	d
   d   }|d   }|d   }n|	d
   }t        |
|||||||¬«      }|j                  |«       Œf Œu Œ„ |S )Nr  r  Ú
paragraphsr  Úqasrÿ   r   rh   Frû   r   rü   rþ   )ri   rs   r  rj   r  r  rh   rû   )r   Úgetr  rn   )r	  r#  Úset_typer’   rá   Úentryr  Ú	paragraphr  Úqari   rs   r  rj   rû   rh   r  r�   s                     r&   r  zSquadProcessor._create_examples‹  sú   € Ø 'Ñ)ˆØˆÜ˜*Ó%ò 	-ˆEØ˜'‘NˆEØ" <Ñ0ò -�	Ø(¨Ñ3�Ø# EÑ*ò -�BØ ™X�FØ$& z¡N�MØ/3Ð,Ø"&�KØ �Gà$&§F¡F¨?¸EÓ$B�MÙ(Ù&Ø%'¨	¡]°1Ñ%5˜FØ*0°©.˜KØ7=¸nÑ7MÑ4à&(¨¡m˜Gä*Ø%Ø&3Ø%1Ø$/Ø1IØ#Ø&3Ø 'ô	�Gð —O‘O GÕ,ñ5-ñ-ð	-ð> ˆr(   )Frº   )rq   Ú
__module__Ú__qualname__Ú__doc__r  r(  r  r  r$  r)  r  © r(   r&   rù   rù     s-   „ ñð
 €JØ€Hó
ó0óB:ó,8ó*"r(   rù   c                   ó   — e Zd ZdZdZy)ÚSquadV1Processorztrain-v1.1.jsonzdev-v1.1.jsonN©rq   r2  r3  r  r(  r5  r(   r&   r7  r7  °  ó   „ Ø"€JØ�Hr(   r7  c                   ó   — e Zd ZdZdZy)ÚSquadV2Processorztrain-v2.0.jsonzdev-v2.0.jsonNr8  r5  r(   r&   r;  r;  µ  r9  r(   r;  c                   ó   — e Zd ZdZg dfd„Zy)r  aT  
    A single training/test example for the Squad dataset, as loaded from disk.

    Args:
        qas_id: The example's unique identifier
        question_text: The question string
        context_text: The context string
        answer_text: The answer string
        start_position_character: The character position of the start of the answer
        title: The title of the example
        answers: None by default, this is used during evaluation. Holds answers as well as their start positions.
        is_impossible: False by default, set to True if the example has no possible answer.
    Fc	                 óè  — || _         || _        || _        || _        || _        || _        || _        d\  | _        | _        g }	g }
d}| j                  D ]P  }t        |«      rd}n#|r|	j                  |«       n|	dxx   |z  cc<   d}|
j                  t        |	«      dz
  «       ŒR |	| _        |
| _        |�=|s:|
|   | _        |
t        |t        |«      z   dz
  t        |
«      dz
  «         | _        y y y )N)r   r   Tr   Fr   )ri   rs   r  rj   r  rh   rû   rf   rg   rE   rn   ro   r   Úchar_to_word_offsetr/   )r	  ri   rs   r  rj   r  r  rû   rh   r   r>  Úprev_is_whitespacerD   s                r&   Ú__init__zSquadExample.__init__É  s   € ð ˆŒØ*ˆÔØ(ˆÔØ&ˆÔØˆŒ
Ø*ˆÔØˆŒà15Ñ.ˆÔ˜TÔ.àˆ
Ø ÐØ!Ðð ×"Ñ"ò 		<ˆAÜ˜aÔ Ø%)Ñ"á%Ø×%Ñ% aÕ(à˜r“N aÑ'“NØ%*Ð"Ø×&Ñ&¤s¨:£¸Ñ':Õ;ð		<ð %ˆŒØ#6ˆÔ ð $Ð/¹Ø"5Ð6NÑ"OˆDÔØ 3ÜÐ,¬s°;Ó/?Ñ?À!ÑCÄSÐI\ÓE]Ð`aÑEaÓbñ!ˆDÕð 9FÐ/r(   N©rq   r2  r3  r4  r@  r5  r(   r&   r  r  º  s   „ ñð, Øô-r(   r  c                   ó2   — e Zd ZdZ	 	 ddee   dee   fd„Zy)r‹   aƒ  
    Single squad example features to be fed to a model. Those features are model-specific and can be crafted from
    [`~data.processors.squad.SquadExample`] using the
    :method:*~transformers.data.processors.squad.squad_convert_examples_to_features* method.

    Args:
        input_ids: Indices of input sequence tokens in the vocabulary.
        attention_mask: Mask to avoid performing attention on padding token indices.
        token_type_ids: Segment token indices to indicate first and second portions of the inputs.
        cls_index: the index of the CLS token.
        p_mask: Mask identifying tokens that can be answers vs. tokens that cannot.
            Mask with 1 for tokens than cannot be in the answer and 0 for token that can be in an answer
        example_index: the index of the example
        unique_id: The unique Feature identifier
        paragraph_len: The length of the context
        token_is_max_context:
            List of booleans identifying which tokens have their maximum context in this feature object. If a token
            does not have their maximum context in this feature object, it means that another feature object has more
            information related to that token and should be prioritized over this feature for that token.
        tokens: list of tokens corresponding to the input ids
        token_to_orig_map: mapping between the tokens and the original text, needed in order to identify the answer.
        start_position: start of the answer token index
        end_position: end of the answer token index
        encoding: optionally store the BatchEncoding with the fast-tokenizer alignment methods.
    Nri   r  c                 óä   — || _         || _        || _        || _        || _        || _        || _        || _        |	| _        |
| _	        || _
        || _        || _        || _        || _        || _        y rº   )rY   rc   ra   r®   r¯   rd   re   rZ   r^   r[   r\   rf   rg   rh   ri   r  )r	  rY   rc   ra   r®   r¯   rd   re   rZ   r^   r[   r\   rf   rg   rh   ri   r  s                    r&   r@  zSquadFeatures.__init__  s}   € ð& #ˆŒØ,ˆÔØ,ˆÔØ"ˆŒØˆŒà*ˆÔØ"ˆŒØ*ˆÔØ$8ˆÔ!ØˆŒØ!2ˆÔà,ˆÔØ(ˆÔØ*ˆÔØˆŒà ˆ�r(   )NN)rq   r2  r3  r4  r   Ústrr
   r@  r5  r(   r&   r‹   r‹   ù  s2   „ ñðT !%Ø,0ñ#%!ð  ˜‘ð!%!ð" ˜=Ñ)ô#%!r(   r‹   c                   ó   — e Zd ZdZdd„Zy)ÚSquadResultaJ  
    Constructs a SquadResult which can be used to evaluate a model's output on the SQuAD dataset.

    Args:
        unique_id: The unique identifier corresponding to that example.
        start_logits: The logits corresponding to the start of the answer
        end_logits: The logits corresponding to the end of the answer
    Nc                 ó^   — || _         || _        || _        |r|| _        || _        || _        y y rº   )Ústart_logitsÚ
end_logitsre   Ústart_top_indexÚend_top_indexÚ
cls_logits)r	  re   rH  rI  rJ  rK  rL  s          r&   r@  zSquadResult.__init__F  s7   € Ø(ˆÔØ$ˆŒØ"ˆŒáØ#2ˆDÔ Ø!.ˆDÔØ(ˆD�Oð r(   )NNNrA  r5  r(   r&   rF  rF  <  s   „ ñô)r(   rF  )rQ   Fr   T)-r  r  Ú	functoolsr   Úmultiprocessingr   r   Útypingr   r  r„   r   Úmodels.bert.tokenization_bertr	   Útokenization_utils_baser
   r   r   Úutilsr   r   r   r   rw   rÓ   Útorch.utils.datar   Ú
tensorflowrÆ   Ú
get_loggerrq   rl   r'   r<   r>   rE   r·   r»   r÷   rù   r7  r;  r  r‹   rF  r5  r(   r&   ú<module>rV     sæ   ðó Û 	Ý ß +Ý ã Ý å @ß aÑ aß AÑ AÝ  ò #LÐ ñ ÔÛÝ.áÔÛà	ˆ×	Ñ	˜HÓ	%€ò
$ò-ò(-ò,òLð^&ÐBYó &ð "ØØØó^ôBO�]ô Oôd�~ô ô
�~ô ÷
<ñ <÷~@!ñ @!÷F)ò )r(   