Ë
    S^(h6  ã                   óè   — d dl Z d dlZd dlZd dlmZ d dlmZmZmZ ddlm	Z	m
Z
mZ  ej                  e«      Ze G d„ d«      «       Z ed¬	«       G d
„ d«      «       Z G d„ d«      Z G d„ de«      Zy)é    N)Ú	dataclass)ÚListÚOptionalÚUnioné   )Úis_tf_availableÚis_torch_availableÚloggingc                   óT   — e Zd ZU dZeed<   eed<   dZee   ed<   dZee   ed<   d„ Z	y)ÚInputExamplea5  
    A single training/test example for simple sequence classification.

    Args:
        guid: Unique id for the example.
        text_a: string. The untokenized text of the first sequence. For single
            sequence tasks, only this sequence must be specified.
        text_b: (Optional) string. The untokenized text of the second sequence.
            Only must be specified for sequence pair tasks.
        label: (Optional) string. The label of the example. This should be
            specified for train and dev examples, but not for test examples.
    ÚguidÚtext_aNÚtext_bÚlabelc                 ó\   — t        j                  t        j                  | «      d¬«      dz   S )ú*Serializes this instance to a JSON string.é   )Úindentú
©ÚjsonÚdumpsÚdataclassesÚasdict©Úselfs    ú`/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/data/processors/utils.pyÚto_json_stringzInputExample.to_json_string1   s#   € ä�z‰zœ+×,Ñ,¨TÓ2¸1Ô=ÀÑDÐDó    )
Ú__name__Ú
__module__Ú__qualname__Ú__doc__ÚstrÚ__annotations__r   r   r   r   © r   r   r   r      s5   … ñð ƒIØƒKØ €FˆH�S‰MÓ Ø€Eˆ8�C‰=ÓóEr   r   T)Úfrozenc                   óz   — e Zd ZU dZee   ed<   dZeee      ed<   dZ	eee      ed<   dZ
eeeef      ed<   d„ Zy)ÚInputFeaturesa¿  
    A single set of features of data. Property names are the same names as the corresponding inputs to a model.

    Args:
        input_ids: Indices of input sequence tokens in the vocabulary.
        attention_mask: Mask to avoid performing attention on padding token indices.
            Mask values selected in `[0, 1]`: Usually `1` for tokens that are NOT MASKED, `0` for MASKED (padded)
            tokens.
        token_type_ids: (Optional) Segment token indices to indicate first and second
            portions of the inputs. Only some models use them.
        label: (Optional) Label corresponding to the input. Int for classification problems,
            float for regression problems.
    Ú	input_idsNÚattention_maskÚtoken_type_idsr   c                 óX   — t        j                  t        j                  | «      «      dz   S )r   r   r   r   s    r   r   zInputFeatures.to_json_stringK   s!   € ä�z‰zœ+×,Ñ,¨TÓ2Ó3°dÑ:Ð:r   )r    r!   r"   r#   r   Úintr%   r+   r   r,   r   r   Úfloatr   r&   r   r   r)   r)   6   sV   … ñð �C‰yÓØ*.€N�H˜T #™YÑ'Ó.Ø*.€N�H˜T #™YÑ'Ó.Ø)-€Eˆ8�E˜#˜u˜*Ñ%Ñ&Ó-ó;r   r)   c                   óF   — e Zd ZdZd„ Zd„ Zd„ Zd„ Zd„ Zd„ Z	e
d
d	„«       Zy)ÚDataProcessorzEBase class for data converters for sequence classification data sets.c                 ó   — t        «       ‚)zÏ
        Gets an example from a dict with tensorflow tensors.

        Args:
            tensor_dict: Keys and values should match the corresponding Glue
                tensorflow_dataset examples.
        ©ÚNotImplementedError)r   Útensor_dicts     r   Úget_example_from_tensor_dictz*DataProcessor.get_example_from_tensor_dictS   s   € ô "Ó#Ð#r   c                 ó   — t        «       ‚)z8Gets a collection of [`InputExample`] for the train set.r3   ©r   Údata_dirs     r   Úget_train_examplesz DataProcessor.get_train_examples]   ó   € ä!Ó#Ð#r   c                 ó   — t        «       ‚)z6Gets a collection of [`InputExample`] for the dev set.r3   r8   s     r   Úget_dev_exampleszDataProcessor.get_dev_examplesa   r;   r   c                 ó   — t        «       ‚)z7Gets a collection of [`InputExample`] for the test set.r3   r8   s     r   Úget_test_exampleszDataProcessor.get_test_examplese   r;   r   c                 ó   — t        «       ‚)z*Gets the list of labels for this data set.r3   r   s    r   Ú
get_labelszDataProcessor.get_labelsi   r;   r   c                 ó”   — t        | j                  «       «      dkD  r+| j                  «       t        |j                  «         |_        |S )z¦
        Some tensorflow_datasets datasets are not formatted the same way the GLUE datasets are. This method converts
        examples to the correct format.
        é   )ÚlenrA   r.   r   )r   Úexamples     r   Útfds_mapzDataProcessor.tfds_mapm   s9   € ô
 ˆt�‰Ó Ó! AÒ%Ø ŸO™OÓ-¬c°'·-±-Ó.@ÑAˆGŒMØˆr   Nc                 óŒ   — t        |dd¬«      5 }t        t        j                  |d|¬«      «      cddd«       S # 1 sw Y   yxY w)z!Reads a tab separated value file.Úrz	utf-8-sig)Úencodingú	)Ú	delimiterÚ	quotecharN)ÚopenÚlistÚcsvÚreader)ÚclsÚ
input_filerL   Úfs       r   Ú	_read_tsvzDataProcessor._read_tsvv   s@   € ô �*˜c¨KÔ8ð 	L¸AÜœŸ
™
 1°À	ÔJÓK÷	L÷ 	Lò 	Lús	   �!:ºA©N)r    r!   r"   r#   r6   r:   r=   r?   rA   rF   ÚclassmethodrT   r&   r   r   r1   r1   P   s9   „ ÙOò$ò$ò$ò$ò$òð òLó ñLr   r1   c                   ó|   — e Zd ZdZdd„Zd„ Zd„ Ze	 dd„«       Zedd„«       Z		 	 	 	 	 	 	 dd„Z
	 dd	„Z	 	 	 	 	 dd
„Zy)Ú%SingleSentenceClassificationProcessorz@Generic processor for a single sentence classification data set.Nc                 óL   — |€g n|| _         |€g n|| _        || _        || _        y rU   )ÚlabelsÚexamplesÚmodeÚverbose)r   rZ   r[   r\   r]   s        r   Ú__init__z.SingleSentenceClassificationProcessor.__init__€   s+   € Ø"˜N‘b°ˆŒØ&Ð.™°HˆŒØˆŒ	Øˆ�r   c                 ó,   — t        | j                  «      S rU   )rD   r[   r   s    r   Ú__len__z-SingleSentenceClassificationProcessor.__len__†   s   € Ü�4—=‘=Ó!Ð!r   c                 óˆ   — t        |t        «      r$t        | j                  | j                  |   ¬«      S | j                  |   S )N)rZ   r[   )Ú
isinstanceÚslicerX   rZ   r[   )r   Úidxs     r   Ú__getitem__z1SingleSentenceClassificationProcessor.__getitem__‰   s9   € Ü�cœ5Ô!Ü8ÀÇÁÐVZ×VcÑVcÐdgÑVhÔiÐiØ�}‰}˜SÑ!Ð!r   c           
      óH   —  | di |¤Ž}|j                  ||||||dd¬«       |S )NT)Ú
split_nameÚcolumn_labelÚcolumn_textÚ	column_idÚskip_first_rowÚoverwrite_labelsÚoverwrite_examplesr&   )Úadd_examples_from_csv)	rQ   Ú	file_namerg   rh   ri   rj   rk   ÚkwargsÚ	processors	            r   Úcreate_from_csvz5SingleSentenceClassificationProcessor.create_from_csvŽ   sB   € ñ ‘M˜&‘Mˆ	Ø×'Ñ'ØØ!Ø%Ø#ØØ)Ø!Ø#ð 	(ô 		
ð Ðr   c                 ó<   —  | di |¤Ž}|j                  ||¬«       |S )N)rZ   r&   )Úadd_examples)rQ   Útexts_or_text_and_labelsrZ   rp   rq   s        r   Úcreate_from_examplesz:SingleSentenceClassificationProcessor.create_from_examplesŸ   s'   € á‘M˜&‘Mˆ	Ø×ÑÐ7ÀÐÔGØÐr   c	                 óX  — | j                  |«      }	|r|	dd  }	g }
g }g }t        |	«      D ]i  \  }}|
j                  ||   «       |j                  ||   «       |�|j                  ||   «       ŒE|r|› d|› �n
t        |«      }|j                  |«       Œk | j	                  |
||||¬«      S )NrC   ú-)rl   rm   )rT   Ú	enumerateÚappendr$   rt   )r   ro   rg   rh   ri   rj   rk   rl   rm   ÚlinesÚtextsrZ   ÚidsÚiÚliner   s                   r   rn   z;SingleSentenceClassificationProcessor.add_examples_from_csv¥   sÊ   € ð —‘˜yÓ)ˆÙØ˜!˜"�IˆEØˆØˆØˆÜ  Ó'ò 	!‰GˆAˆtØ�L‰L˜˜kÑ*Ô+Ø�M‰M˜$˜|Ñ,Ô-ØÐ$Ø—
‘
˜4 	™?Õ+á.8˜*˜ Q q cÑ*¼cÀ!»f�Ø—
‘
˜4Õ ð	!ð × Ñ Ø�6˜3Ð1AÐVhð !ó 
ð 	
r   c           	      ó  — |�:t        |«      t        |«      k7  r#t        dt        |«      › dt        |«      › �«      ‚|�:t        |«      t        |«      k7  r#t        dt        |«      › dt        |«      › �«      ‚|€d gt        |«      z  }|€d gt        |«      z  }g }t        «       }t        |||«      D ]U  \  }}	}
t	        |t
        t        f«      r|	€|\  }}	n|}|j                  |	«       |j                  t        |
|d |	¬«      «       ŒW |r|| _
        n| j                  j                  |«       |rt        |«      | _        | j                  S t        t        | j                  «      j                  |«      «      | _        | j                  S )Nz(Text and labels have mismatched lengths z and z%Text and ids have mismatched lengths )r   r   r   r   )rD   Ú
ValueErrorÚsetÚziprb   ÚtuplerN   Úaddrz   r   r[   ÚextendrZ   Úunion)r   ru   rZ   r}   rl   rm   r[   Úadded_labelsÚtext_or_text_and_labelr   r   Útexts               r   rt   z2SingleSentenceClassificationProcessor.add_examplesÃ   s–  € ð Ð¤#Ð&>Ó"?Ä3ÀvÃ;Ò"NÜØ:¼3Ð?WÓ;XÐ:YÐY^Ô_bÐciÓ_jÐ^kÐlóð ð ˆ?œsÐ#;Ó<ÄÀCÃÒHÜÐDÄSÐIaÓEbÐDcÐchÔilÐmpÓiqÐhrÐsÓtÐtØˆ;Ø�&œ3Ð7Ó8Ñ8ˆCØˆ>Ø�VœcÐ":Ó;Ñ;ˆFØˆÜ“uˆÜ36Ð7OÐQWÐY\Ó3]ò 	\Ñ/Ð" E¨4ÜÐ0´5¼$°-Ô@ÀUÀ]Ø4‘�‘eà-�Ø×Ñ˜UÔ#Ø�O‰OœL¨d¸4ÈÐTYÔZÕ[ð	\ñ Ø$ˆD�Mà�M‰M× Ñ  Ô*ñ Ü˜|Ó,ˆDŒKð �}‰}Ðô œs 4§;¡;Ó/×5Ñ5°lÓCÓDˆDŒKà�}‰}Ðr   c                 óT
  ‡— |€|j                   }t        | j                  «      D ��ci c]  \  }}||“Œ
 }	}}g }
t        | j                  «      D ]h  \  }}|dz  dk(  rt        j                  d|› �«       |j                  |j                  dt        ||j                   «      ¬«      }|
j                  |«       Œj t        d„ |
D «       «      }g Št        t        |
| j                  «      «      D �]?  \  }\  }}|dz  dk(  r.t        j                  d|› d	t        | j                  «      › �«       |rd
ndgt        |«      z  }|t        |«      z
  }|r|g|z  |z   }|rdnd
g|z  |z   }n||g|z  z   }||rdnd
g|z  z   }t        |«      |k7  rt        dt        |«      › d|› �«      ‚t        |«      |k7  rt        dt        |«      › d|› �«      ‚| j                  dk(  r|	|j                     }n:| j                  dk(  rt!        |j                  «      }nt        | j                  «      ‚|dk  rå| j"                  rÙt        j                  d«       t        j                  d|j$                  › �«       t        j                  ddj'                  |D �cg c]  }t)        |«      ‘Œ c}«      › �«       t        j                  ddj'                  |D �cg c]  }t)        |«      ‘Œ c}«      › �«       t        j                  d|j                  › d|› d�«       ‰j                  t+        |||¬«      «       �ŒB |€‰S |dk(  ržt-        «       st/        d«      ‚ddl}ˆfd„}|j2                  j4                  j7                  ||j8                  |j8                  dœ|j:                  f|j=                  dg«      |j=                  dg«      dœ|j=                  g «      f«      }|S |dk(  �rt?        «       st/        d«      ‚ddl }ddl!m"} |jG                  ‰D �cg c]  }|jH                  ‘Œ c}|jJ                  ¬ «      }
|jG                  ‰D �cg c]  }|jL                  ‘Œ c}|jJ                  ¬ «      }| j                  dk(  r6|jG                  ‰D �cg c]  }|j                  ‘Œ c}|jJ                  ¬ «      }nD| j                  dk(  r5|jG                  ‰D �cg c]  }|j                  ‘Œ c}|j                   ¬ «      } ||
|«      }|S t        d!«      ‚c c}}w c c}w c c}w c c}w c c}w c c}w c c}w )"a�  
        Convert examples in a list of `InputFeatures`

        Args:
            tokenizer: Instance of a tokenizer that will tokenize the examples
            max_length: Maximum example length
            pad_on_left: If set to `True`, the examples will be padded on the left rather than on the right (default)
            pad_token: Padding token
            mask_padding_with_zero: If set to `True`, the attention mask will be filled by `1` for actual values
                and by `0` for padded values. If set to `False`, inverts it (`1` for padded values, `0` for actual
                values)

        Returns:
            If the `examples` input is a `tf.data.Dataset`, will return a `tf.data.Dataset` containing the
            task-specific features. If the input is a list of `InputExamples`, will return a list of task-specific
            `InputFeatures` which can be fed to the model.

        Ni'  r   zTokenizing example T)Úadd_special_tokensÚ
max_lengthc              3   ó2   K  — | ]  }t        |«      –— Œ y ­wrU   )rD   )Ú.0r*   s     r   ú	<genexpr>zESingleSentenceClassificationProcessor.get_features.<locals>.<genexpr>  s   è ø€ ÒI¨iœ3˜yŸ>ÑIùs   ‚zWriting example ú/rC   zError with input length z vs ÚclassificationÚ
regressioné   z*** Example ***zguid: zinput_ids: ú zattention_mask: zlabel: z (id = ú)©r*   r+   r   Útfz?return_tensors set to 'tf' but TensorFlow 2.0 can't be importedc               3   ój   •K  — ‰D ])  } | j                   | j                  dœ| j                  f–— Œ+ y ­w)N©r*   r+   r—   )ÚexÚfeaturess    €r   Úgenz?SingleSentenceClassificationProcessor.get_features.<locals>.genC  s9   øè ø€ Ø"ò g�BØ)+¯©È×IZÑIZÑ[Ð]_×]eÑ]eÐfÓfñgùs   ƒ03rš   Úptz8return_tensors set to 'pt' but PyTorch can't be imported)ÚTensorDataset)Údtypez,return_tensors should be one of 'tf' or 'pt')'Úmax_lenry   rZ   r[   ÚloggerÚinfoÚencoder   Úminrz   Úmaxrƒ   rD   r�   r\   r   r/   r]   r   Újoinr$   r)   r   ÚRuntimeErrorÚ
tensorflowÚdataÚDatasetÚfrom_generatorÚint32Úint64ÚTensorShaper	   ÚtorchÚtorch.utils.datarŸ   Útensorr*   Úlongr+   )r   Ú	tokenizerr�   Úpad_on_leftÚ	pad_tokenÚmask_padding_with_zeroÚreturn_tensorsr~   r   Ú	label_mapÚall_input_idsÚex_indexrE   r*   Úbatch_lengthr+   Úpadding_lengthÚxr˜   r�   Údatasetr°   rŸ   rS   Úall_attention_maskÚ
all_labelsrœ   s                             @r   Úget_featuresz2SingleSentenceClassificationProcessor.get_featuresè   s¿  ø€ ð6 ÐØ"×*Ñ*ˆJä.7¸¿¹Ó.D×E¡( ! U�U˜A‘XÐEˆ	ÑEàˆÜ!*¨4¯=©=Ó!9ò 		,ÑˆH�gØ˜%Ñ 1Ò$Ü—‘Ð1°(°Ð<Ô=à!×(Ñ(Ø—‘Ø#'Ü˜z¨9×+<Ñ+<Ó=ð )ó ˆIð
 × Ñ  Õ+ð		,ô ÑI¸=ÔIÓIˆàˆÜ.7¼¸MÈ4Ï=É=Ó8YÓ.Zó #	lÑ*ˆHÑ*�y 'Ø˜%Ñ 1Ò$Ü—‘Ð.¨x¨j¸¼#¸d¿m¹mÓ:LÐ9MÐNÔOñ $:™a¸qÐAÄCÈ	ÃNÑRˆNð *¬C°	«NÑ:ˆNÙØ'˜[¨>Ñ9¸YÑF�	Ù(>¡1ÀAÐ"FÈÑ"WÐ[iÑ!i‘à%¨)¨°~Ñ)EÑF�	Ø!/Ñ9O±AÐUVÐ3WÐZhÑ3hÑ!i�ä�9‹~ Ò-Ü Ð#;¼CÀ	»NÐ;KÈ4ÐP\È~Ð!^Ó_Ð_Ü�>Ó" lÒ2Ü Ð#;¼CÀÓ<OÐ;PÐPTÐUaÐTbÐ!cÓdÐdà�y‰yÐ,Ò,Ø! '§-¡-Ñ0‘Ø—‘˜lÒ*Ü˜gŸm™mÓ,‘ä  §¡Ó+Ð+à˜!Š| §¢Ü—‘Ð-Ô.Ü—‘˜f W§\¡\ NÐ3Ô4Ü—‘˜k¨#¯(©(ÀIÖ3N¸q´C¸µFÒ3NÓ*OÐ)PÐQÔRÜ—‘Ð.¨s¯x©xÈÖ8XÀA¼¸Q½Ò8XÓ/YÐ.ZÐ[Ô\Ü—‘˜g g§m¡m _°G¸E¸7À!ÐDÔEà�O‰OœM°IÈnÐdiÔjÖkðG#	lðJ Ð!ØˆOØ˜tÒ#Ü"Ô$Ü"Ð#dÓeÐeÛ#ôgð —g‘g—o‘o×4Ñ4ØØ!Ÿx™x¸2¿8¹8ÑDÀbÇhÁhÐOØ!Ÿ~™~¨t¨fÓ5ÈÏÉÐY]ÐX^ÓI_Ñ`Ðbd×bpÑbpÐqsÓbtÐuóˆGð
 ˆNØ˜tÓ#Ü%Ô'Ü"Ð#]Ó^Ð^ÛÝ6à!ŸL™L¸xÖ)H¸!¨!¯+«+Ò)HÐPU×PZÑPZ˜LÓ[ˆMØ!&§¡ÈÖ.RÀA¨q×/?Ó/?Ò.RÐZ_×ZdÑZd Ó!eÐØ�y‰yÐ,Ò,Ø"Ÿ\™\¸HÖ*E°q¨1¯7«7Ò*EÈUÏZÉZ˜\ÓX‘
Ø—‘˜lÒ*Ø"Ÿ\™\¸HÖ*E°q¨1¯7«7Ò*EÈUÏ[É[˜\ÓY�
á# MÐ3EÀzÓRˆGØˆNäÐKÓLÐLùóo Fùò` 4OùÚ8Xùò8 *IùÚ.Rùâ*Eùâ*Es)   ¨TÊTËTÐTÑ TÒT Ó	T%)NNr’   F)Ú r   rC   NFrU   )rÃ   r   rC   NFFF)NNFF)NFr   TN)r    r!   r"   r#   r^   r`   re   rV   rr   rv   rn   rt   rÂ   r&   r   r   rX   rX   }   s€   „ ÙJóò"ò"ð
 àejòó ðð  òó ðð ØØØØØØ ó
ð> kpó#ðP ØØØ#ØôuMr   rX   )rO   r   r   r   Útypingr   r   r   Úutilsr   r	   r
   Ú
get_loggerr    r¢   r   r)   r1   rX   r&   r   r   ú<module>rÇ      sŠ   ðó" Û Û Ý !ß (Ñ (ç AÑ Að 
ˆ×	Ñ	˜HÓ	%€ð ÷Eð Eó ðEñ0 �$Ô÷;ð ;ó ð;÷2*Lñ *LôZ`M¨Mõ `Mr   