T^(hbdZddlmZddlZddlZddlmZddlmZm Z m Z m Z ddl Z ddlZddlmZddlmZmZmZmZmZdd lmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$dd l%m&Z&m'Z'm(Z(dd l)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/d d l0m1Z1e/jde3Z4dZ5dZ6dZ7d3dZ8Gdde"jrjtZ;Gdde"jrjtZeGdde+Z?dZ@dZAe-de@Gdd e>ZBGd!d"e"jrjtZCe-d#e@Gd$d%e>ZDe-d&e@Gd'd(e>eZEe-d)e@Gd*d+e>eZFe-d,e@Gd-d.e>e ZGe-d/e@Gd0d1e>eZHgd2ZIy)4z TF 2.0 XLM model. ) annotationsN) dataclass)DictOptionalTupleUnion)get_tf_activation)TFBaseModelOutputTFMultipleChoiceModelOutputTFQuestionAnsweringModelOutputTFSequenceClassifierOutputTFTokenClassifierOutput) TFModelInputTypeTFMultipleChoiceLossTFPreTrainedModelTFQuestionAnsweringLossTFSequenceClassificationLossTFSequenceSummaryTFSharedEmbeddingsTFTokenClassificationLossget_initializerkeraskeras_serializable unpack_inputs)check_embeddings_within_bounds shape_liststable_softmax)MULTIPLE_CHOICE_DUMMY_INPUTS ModelOutputadd_code_sample_docstringsadd_start_docstrings%add_start_docstrings_to_model_forwardlogging) XLMConfigzFacebookAI/xlm-mlm-en-2048r&ctjt|Dcgc];}t|Dcgc]$}|tjdd|dzz|z z &c}=c}}}t j tj |dddddf|dddddf<t j tj|dddddf|dddddf<ycc}wcc}}w)Ni'rr%)nparrayrangepowertfconstantsincos)n_posdimoutposj position_encs e/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/xlm/modeling_tf_xlm.pycreate_sinusoidal_embeddingsr8Es88hmnshtuadQVWZQ[\AcBHHUAaL34F$GG\uvL;;rvvl1add7&;<=C14a4L;;rvvl1add7&;<=C14a4L]usC$ )CC$ C$ c t|d}||}n;tj||j}|tj|dk}|r\tj tj tjdd|f||dftj|d|df}n|}tjjt|||g|r,tjjt||||g||fS)zH Generate hidden states mask, and optionally an attention mask. rdtyper%axis) rr-r+r; expand_dims less_equaltilereshape debugging assert_equal)slenlengthscausal padding_maskbsmaskalen attn_masks r7 get_masksrLKs G Q BxxGMM2bnnW155MM GGBJJtaD\2RqM BBJJtVWY]_`UaDb  LLj.T ;  !!*Y"7"dD9IJ ?cVeZdZejZfdZdZddZddZ xZ S)TFXLMMultiHeadAttentionc 2t|di|ttj|_||_||_|j|_|j |jzdk(sJtjj|t|jd|_tjj|t|jd|_tjj|t|jd|_tjj|t|jd|_tjj%|j&|_t+|_||_y)Nrq_linkernel_initializernamek_linv_linout_lin)super__init__nextrONEW_IDlayer_idr2n_headsoutput_attentionsrlayersDenserinit_stdrQrUrVrWDropoutattention_dropoutdropoutset pruned_heads)selfr^r2configkwargs __class__s r7rZz TFXLMMultiHeadAttention.__init__ks2 "6"4;;<  !'!9!9xx$,,&!+++\\''PVP_P_@`gn'o \\''PVP_P_@`gn'o \\''PVP_P_@`gn'o ||))#/RXRaRaBbir)s ||++F,D,DE ErMctNNotImplementedError)rhheadss r7 prune_headsz#TFXLMMultiHeadAttention.prune_heads{s!!rMc\t|\}} | ||n|d|z} nt|d} jjztt|dk(rd|| fndd| f} fd} fd} | j |}|/| j |}| j |}nB|j|vr2|x}}| j |}| j |}|xj|vrY|E|j\}}tj|gd}tj|gd}n|j\}}f|j<tj|j}tj|tjj|}tj|j}tj||d }tj || }tj||j}|d d |z zz }t#|d }j%||}|||z}tj|}| |}j'|f}|r||fz}|S)zd Self-attention (if kv is None) or attention over source sentence (provided by kv). rDr%r c vtjtj|djfdS) projectionrr(r%r perm)r- transposerAr^xrH dim_per_headrhs r7shapez+TFXLMMultiHeadAttention.call..shapes-<< 1r2t||\.R SZfg grMcztjtj|ddjzfS)zcompute contextrvrwru)r-rAryr^rzs r7unshapez-TFXLMMultiHeadAttention.call..unshapes0::bll1<@2r4<)Fg?rutraining)rr2r^lenrQrUrVr]r-concatcastr;multiplymathrsqrtmatmulrArrerW)rhinputrIkvcache head_maskr_rqlenr2klen mask_reshaper}rqkvk_v_f_dim_per_headscoresweightscontextoutputsrHr|s` @@r7callzTFXLMMultiHeadAttention.call~sx #5) D# : =4eFmd.BDb>!$Dxx4<</ .1*T2B.Cq.HAtT*rSTVWY]N^  h i $**U# $ :djj'(Adjj'(A ]dmm58JAdjjm$Adjjm$A  }}%:"4==1FB 2q'2A 2q'2A /DAq$%q6E$-- QWW= KK277==8 9 GGAQWW %1aT2zz$ -wwt6<<0$#*-- b1,,w,:   )G))GQ''"<<(*  *GrMc|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYAxYw#1swYxYw#1swYxYw#1swYyxYw)NTrQrUrVrW) builtgetattrr- name_scoperQrTbuildr2rUrVrWrh input_shapes r7rzTFXLMMultiHeadAttention.buildss ::  4$ ' 3tzz/ 9   $dhh!78 9 4$ ' 3tzz/ 9   $dhh!78 9 4$ ' 3tzz/ 9   $dhh!78 9 4D ) 5t||001 ; ""D$#9: ; ; 6 9 9 9 9 9 9 ; ;s0 )F32)G)G )G3F=G  GG!Frm) __name__ __module__ __qualname__ itertoolscountr\rZrqrr __classcell__rks@r7rOrOhs' Y__ F "DL;rMrOc.eZdZfdZddZddZxZS)TFXLMTransformerFFNc t|di|tjj |t |j d|_tjj |t |j d|_|jr tdn td|_ tjj|j|_ ||_||_y)Nlin1rRlin2gelurelurX)rYrZrr`rarrbrrgelu_activationr actrcrein_dim dim_hidden)rhrrout_dimrirjrks r7rZzTFXLMTransformerFFN.__init__s "6"LL&&zoV\VeVeFfms&t LL&&w?SYSbSbCcjp&q 060F0F$V,L]^dLe||++FNN;  $rMc|j|}|j|}|j|}|j||}|S)Nr)rrrre)rhrrr{s r7rzTFXLMTransformerFFN.callsA IIe  HHQK IIaL LLXL .rMc|jryd|_t|ddZtj|jj 5|jj dd|jgdddt|dd[tj|jj 5|jj dd|jgdddyy#1swYrxYw#1swYyxYw)NTrr) rrr-rrrTrrrrrs r7rzTFXLMTransformerFFN.builds ::  4 & 2tyy~~. ; tT[[ 9: ; 4 & 2tyy~~. ? tT__ => ? ? 3 ; ; ? ?s )C%2)C1%C.1C:rrm)rrrrZrrrrs@r7rrs% ?rMrcleZdZeZfdZddZdZdZdZ e d d dZ xZ S) TFXLMMainLayerc t|di|||_|j|_|j|_|j |_|j|_|j |_|jr td|j|_ |j|_ |j|_ |j|_ |j|_|j|_|j |_|j"dz|_|j&|_|j(|_|j*|_|j,|_|j"|j&zdk7r t/dt0j2j5|j6|_t0j2j5|j8|_|j:rtt=|j|j"|j,d|_t0j2jA|jBd|_"g|_#g|_$g|_%g|_&tO|j(D]}|jFjQtS|j&|j"|d | |jHjQt0j2jA|jBd ||jJjQtU|j"|j$|j"|d | |jLjQt0j2jA|jBd |tW|dr|jXj[j]}i|_,|D]h\}}|jFt_|j&|j&k(s6|jat_|tctet^|ijyy)Nz,Currently XLM can only be used as an encoderrz-transformer dim must be a multiple of n_heads embeddingsinitializer_rangerTlayer_norm_emb)epsilonrTz attentions_._)rirTzlayer_norm1_._zffns_._zlayer_norm2_._rgrX)3rYrZrioutput_hidden_statesr_use_return_dict return_dict is_encoder is_decoderrorFn_langs use_lang_embn_words eos_index pad_indexemb_dimr2 hidden_dimr^n_layersmax_position_embeddingsembed_init_std ValueErrorrr`rcrerdsinusoidal_embeddingsrrLayerNormalizationlayer_norm_epsr attentions layer_norm1ffns layer_norm2r+appendrOrhasattrrgcopyitemsintrqlistmap)rhrirjirglayerrprks r7rZzTFXLMMainLayer.__init__sr "6" $*$?$?!!'!9!9!11!++$/// ??%&TU Umm ~~ "//~~ ))))>>((Q,~~  '-'E'E$$33 88dll "a 'LM M||++FNN; !&!5!5f6N6N!O  ' '% %- LL$((f6K6KR^ $ll==fF[F[br=s  t}}% A OO " "' dhhvVcdecfTgh     # # //8M8MVdefdgTh/i  II  #DHHdootxxPV_fghfi]jk     # # //8M8MVdefdgTh/i  " 6> *!..335;;=L"$F  , J u??3u:.66&..H$$c%j$s32G%HI J +rMc|jryd|_tjd5|jd|j|j gt |j|_ddd|jdkDrl|jr`tjd5|jd|j|j gt |j|_ dddt|ddMtj|jj5|jjddddt|ddZtj|j j5|j jdd|j gddd|j"D];}tj|j5|jdddd=|j$D]H}tj|j5|jdd|j gdddJ|j&D];}tj|j5|jdddd=|j(D]H}tj|j5|jdd|j gdddJy#1swYxYw#1swYxYw#1swYxYw#1swYtxYw#1swYuxYw#1swY8xYw#1swYxYw#1swYxYw)NTposition_embeddingsr)rTr} initializerr%lang_embeddingsr)rr-r add_weightrr2rrrrrrrrrTrrrrrr)rhrrs r7rzTFXLMMainLayer.buildPs ::  ]]0 1 '+!33TXX>+D,?,?@(7(D $  <L LL%(L2 5L? M M c|jSrm)rrhs r7get_input_embeddingsz#TFXLMMainLayer.get_input_embeddingsu rMc`||j_t|d|j_yNr)rweightr vocab_sizerhvalues r7set_input_embeddingsz#TFXLMMainLayer.set_input_embeddingsxs$!&%/%6q%9"rMct)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel rn)rhheads_to_prunes r7 _prune_headszTFXLMMainLayer._prune_heads|s "!rMc | | td|t|\}}n| t| dd\}}n td|p|Utjtjtj ||j |jd}ntj|g|z}tjjt|d|dt|dd |d ft|||j| \}}|Btjtj|d}tj||df}tjjt|||gd t|d ||gd f|?tjjt|||gdt|d ||gd f|t dg|j"z}|H|F||dz }|dd| df}|dd| df}| |dd| df}|dd| df}|dd| df}| 1t%||j&j(|j'|} | tj*|j,|z}|>|j.r2|j0dkDr#|tj*|j2|z}|||j'|z}|j5|}|j7|| }tj||j}|tj|dz}| rdnd}| rdnd}t|j"D]}| r||fz}|j8|||d|||| | }|d}| r ||dfz}|j7|| }||z}|j:||}||j<||z}|j>||}|tj|dz}| r||fz}||dxx|jAdz cc<| stCd|||fDStE|||S)NzDYou cannot specify both input_ids and inputs_embeds at the same timer(z5You have to specify either input_ids or inputs_embedsr:r%r<rzExpected batch size z and received batch size z mismatched)rGzPosition id shape z and input shape z Lang shape rDrrurXc3&K|] }|| ywrmrX).0rs r7 z&TFXLMMainLayer.call..'sYq1=Ys)last_hidden_state hidden_statesr)#rrr- reduce_sumr not_equalrr;convert_to_tensorrBrCrLrFr>r+r@rorrrrgatherrrrrrrerrrrsizetupler )rh input_idsattention_masklangstoken_type_ids position_idsrErr inputs_embedsr_rrrrHrDrIrK_slentensorrrr attn_outputsattns r7rzTFXLMMainLayer.calls&  ]%>cd d  "!),HB  &!-0!4HBTU U ?$--GGBLLDNNC9??[bc..v{; LL % %j&9!&>"((4.qAL77<"a9L LL % %j&>T K L!9 ::KRQUJ TensorFlow models and layers in `transformers` accept two formats as input: - having all inputs as keyword arguments (like PyTorch models), or - having all inputs as a list, tuple or dict in the first positional argument. The reason the second format is supported is that Keras methods prefer this format when passing inputs to models and layers. Because of this support, when using methods like `model.fit()` things should "just work" for you - just pass your inputs and labels in any format that `model.fit()` supports! If, however, you want to use the second format outside of Keras methods like `fit()` and `predict()`, such as when creating your own layers or models with the Keras `Functional` API, there are three possibilities you can use to gather all the input Tensors in the first positional argument: - a single Tensor with `input_ids` only and nothing else: `model(input_ids)` - a list of varying length with one or several input Tensors IN THE ORDER given in the docstring: `model([input_ids, attention_mask])` or `model([input_ids, attention_mask, token_type_ids])` - a dictionary with one or several input Tensors associated to the input names given in the docstring: `model({"input_ids": input_ids, "token_type_ids": token_type_ids})` Note that when creating models and layers with [subclassing](https://keras.io/guides/making_new_layers_and_models_via_subclassing/) then you don't need to worry about any of this, as you can just pass inputs like you would to any other Python function! Parameters: config ([`XLMConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`Numpy array` or `tf.Tensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.__call__`] and [`PreTrainedTokenizer.encode`] for details. [What are input IDs?](../glossary#input-ids) attention_mask (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: - 1 for tokens that are **not masked**, - 0 for tokens that are **masked**. [What are attention masks?](../glossary#attention-mask) langs (`tf.Tensor` or `Numpy array` of shape `({0})`, *optional*): A parallel sequence of tokens to be used to indicate the language of each token in the input. Indices are languages ids which can be obtained from the language names by using two conversion mappings provided in the configuration of the model (only provided for multilingual models). More precisely, the *language name to language id* mapping is in `model.config.lang2id` (which is a dictionary string to int) and the *language id to language name* mapping is in `model.config.id2lang` (dictionary int to string). See usage examples detailed in the [multilingual documentation](../multilingual). token_type_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`Numpy array` or `tf.Tensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) lengths (`tf.Tensor` or `Numpy array` of shape `(batch_size,)`, *optional*): Length of each sentence that can be used to avoid performing attention on padding token indices. You can also use *attention_mask* for the same result (see above), kept here for compatibility. Indices selected in `[0, ..., input_ids.size(-1)]`. cache (`Dict[str, tf.Tensor]`, *optional*): Dictionary string to `tf.Tensor` that contains precomputed hidden states (key and values in the attention blocks) as computed by the model (see `cache` output below). Can be used to speed up sequential decoding. The dictionary object will be modified in-place during the forward pass to add newly computed hidden-states. head_mask (`Numpy array` or `tf.Tensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: - 1 indicates the head is **not masked**, - 0 indicates the head is **masked**. inputs_embeds (`tf.Tensor` of shape `({0}, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. This argument can be used only in eager mode, in graph mode the value in the config will be used instead. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. This argument can be used in eager mode, in graph mode the value will always be set to True. training (`bool`, *optional*, defaults to `False`): Whether or not to use the model in training mode (some modules like dropout modules have different behaviors between training and evaluation). z]The bare XLM Model transformer outputting raw hidden-states without any specific head on top.ceZdZfdZeeejdee e e  d ddZ ddZ xZS) TFXLMModelcPt||g|i|t|d|_y)NrrT)rYrZrrrhriinputsrjrks r7rZzTFXLMModel.__init__s) 3&3F3)&}ErMbatch_size, sequence_length checkpoint output_typer cB|j||||||||| | | | |  }|S)N rrrrrrErrrr_rrr)r)rhrrrrrrErrrr_rrrrs r7rzTFXLMModel.callsG.""))%'/!5##  rMc|jryd|_t|ddNtj|jj 5|jj ddddyy#1swYyxYw)NTr)rrr-rrrTrrs r7rzTFXLMModel.buildsm ::  4 - 9t//445 -  &&t, - - : - -s A11A:r )rTFModelInputType | Nonertf.Tensor | Nonerr.rr.rr.rEr.rzDict[str, tf.Tensor] | Nonerr.rr.r_ bool | Nonerr/rr/rboolr z$TFBaseModelOutput | Tuple[tf.Tensor]rm)rrrrZrr#XLM_INPUTS_DOCSTRINGformatr!_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOCrrrrs@r7r!r!s F*+?+F+FGd+ef&%$.2+/"&+/)-$(-1&**.)-,0#' * )   )  ' " + $ ( ' * !   .  g D-rMr!cJeZdZdZfdZfdZdZdZdZdZ dZ xZ S) TFXLMPredLayerz? Prediction layer (cross_entropy or adaptive_softmax). c t|di||j|_|j|_|j|_|jdur||_yt )NFrX)rYrZasmrrinput_embeddingsro)rhrir9rjrks r7rZzTFXLMPredLayer.__init__sP "6"::~~ )) :: $4D !% %rMcn|j|jfddd|_t||y)NzerosTbias)r}r trainablerT)rrr<rYr)rhrrks r7rzTFXLMPredLayer.build&s/OO4< 1` a classification loss is computed (Cross-Entropy). r+rNr%lossrrr)rrjhf_compute_lossrrr)rhrrrrrrErrrr_rrlabelsrr`rarrns r7rz#TFXLMForSequenceClassification.calls<#..))%'/!5#/ %Q'&&v.~t4+?+?+OY!4QR!88F)-)9TGf$ Ev E)-;;*55   rMc|jryd|_t|ddMtj|jj 5|jj ddddt|ddNtj|jj 5|jj ddddyy#1swYexYw#1swYyxYw)NTrrj)rrr-rrrTrrjrs r7rz$TFXLMForSequenceClassification.builds ::  4 - 9t//445 -  &&t, - 4+T 2 >t4499: 2%%++D1 2 2 ? - - 2 2rcNNNNNNNNNNNNNF)rr-rrdrrdrrdrrdrErdrrerrdrrdr_rfrrfrrfrprdrr0r z3Union[TFSequenceClassifierOutput, Tuple[tf.Tensor]]rm)rrrrZrr#r1r2r!r3rr4rrrrs@r7rhrhsv*+?+F+FGd+ef&.$.28?~t4+?+?+X%'*=ab*AAF)-)9TGf$ Ev E*"-;;*55   rMc|jryd|_t|ddMtj|jj 5|jj ddddt|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swYxYw#1swYxYw#1swYyxYw)NTrrjrv) rrr-rrrTrrjrvrirkrs r7rzTFXLMForMultipleChoice.buildrs$ ::  4 - 9t//445 -  &&t, - 4+T 2 >t4499: 2%%++D1 2 4 - 9t//445 M  &&dDKK4J4J'KL M M :  - - 2 2 M Ms$ D<%E?3E<EEErr)rr-rrdrrdrrdrrdrErdrrerrdrrdr_rfrrfrrfrprdrr0r z4Union[TFMultipleChoiceModelOutput, Tuple[tf.Tensor]]rm)rrrrZrrrr#r1r2r!r3r r4rrrrs@r7rtrts9$*+?+F+FGq+rs&/$.28!I tI V MrMrtz XLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for Named-Entity-Recognition (NER) tasks. ceZdZfdZeeejdee e e  d ddZ ddZ xZS) TFXLMForTokenClassificationcdt||g|i||j|_t|d|_t j j|j|_t j j|jt|jd|_ ||_ y)Nrr# classifierrR)rYrZrkrrrr`rcrerarrbrrir$s r7rZz$TFXLMForTokenClassification.__init__s 3&3F3 ++)&}E||++FNN; ,,,,   /&//2RYe-  rMr&r'c,|j||||||||| | | | | }|d}|j||}|j|}| dn|j| |}| s|f|ddz}||f|zS|St |||j |j S)z labels (`tf.Tensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. r+rrNr%rm)rrerrorrr)rhrrrrrrErrrr_rrrprr`sequence_outputrrnras r7rz TFXLMForTokenClassification.calls8#..))%'/!5#/ .a0,,,J1~t4+?+?+OY!4QR!88F)-)9TGf$ Ev E&-;;*55   rMc|jryd|_t|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTrr) rrr-rrrTrrri hidden_sizers r7rz!TFXLMForTokenClassification.build ::  4 - 9t//445 -  &&t, - 4t , 8t334 M%%tT4;;3J3J&KL M M 9 - - M M C"%3C."C+.C7rr)rr-rrdrrdrrdrrdrErdrrerrdrrdr_rfrrfrrfrprdrr0r z0Union[TFTokenClassifierOutput, Tuple[tf.Tensor]]rm)rrrrZrr#r1r2r!r3rr4rrrrs@r7rrs *+?+F+FGd+ef&+$.28 "''z0JKD"J/2Eab2IIF)-)9TGf$ Ev E-%!-;;*55   rMc|jryd|_t|ddMtj|jj 5|jj ddddt|ddetj|jj 5|jj dd|jjgdddyy#1swY|xYw#1swYyxYw)NTrr) rrr-rrrTrrrirrs r7rz%TFXLMForQuestionAnsweringSimple.build8rr)NNNNNNNNNNNNNNF) rr-rrdrrdrrdrrdrErdrrerrdrrdr_rfrrfrrfrrdrrdrr0r z7Union[TFQuestionAnsweringModelOutput, Tuple[tf.Tensor]]rm)rrrrZrr#r1r2r!r3r r4rrrrs@r7rrs4*+?+F+FGd+ef&2$.28rsd#!///    SR)   H %2> :k;ell00k;\?%,,,,?>soU\\''sosol L,L2 44 42(TGTc4-%4- 4-n0U\\''0f W,/W,W,t O2%9;WO2O2d |M13G|M|M~ RM"68QRMRMj ]M&: