T^(h>dZddlZddlZddlmZmZmZddlZddlZddlm Z ddl m Z m Z m Z ddlmZddlmZmZmZdd lmZmZmZmZmZmZdd lmZmZmZmZdd l m!Z!d d l"m#Z#e!jHe%Z&dZ'dZ(Gdde jRZ*d=dZ+Gdde jRZ,Gdde jRZ-Gdde jRZ.Gdde jRZ/Gdde jRZ0Gdde jRZ1Gdd e jRZ2Gd!d"e jRZ3Gd#d$e jRZ4d%e4iZ5Gd&d'e jRZ6Gd(d)e jRZ7Gd*d+e jRZ8Gd,d-eZ9d.Z:d/Z;ed0e:Gd1d2e9Z<ed3e:Gd4d5e9Z=ed6e:Gd7d8e9Z>ed9e:Gd:d;e9Z?gd<Z@y)>zPyTorch MarkupLM model.N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN)add_start_docstrings%add_start_docstrings_to_model_forwardreplace_return_docstrings))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModelapply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)logging)MarkupLMConfigzmicrosoft/markuplm-baserc*eZdZdZfdZddZxZS)XPathEmbeddingszConstruct the embeddings from xpath tags and subscripts. We drop tree-id in this version, as its info can be covered by xpath. c tt| |j|_t j |j |jz|j|_t j|j|_ t j|_ t j |j |jzd|jz|_t j d|jz|j|_t j t#|jDcgc],}t j$|j&|j .c}|_t j t#|jDcgc],}t j$|j*|j .c}|_ycc}wcc}w)N)superr__init__ max_depthrLinearxpath_unit_hidden_size hidden_sizexpath_unitseq2_embeddingsDropouthidden_dropout_probdropoutReLU activationxpath_unitseq2_inner inner2emb ModuleListrange Embeddingmax_xpath_tag_unit_embeddingsxpath_tag_sub_embeddingsmax_xpath_subs_unit_embeddingsxpath_subs_sub_embeddingsselfconfig_ __class__s l/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/markuplm/modeling_markuplm.pyr!zXPathEmbeddings.__init__>s\ ot-/)))+63P3PSWSaSa3acicucu)v&zz&"<"<= '')$&IIf.K.Kdnn.\^_bhbtbt^t$u!1v'9'9#96;M;MN(* t~~.  VAA6C`C`a ) %*,t~~.  VBBFDaDab * &   s 51G1Gc g}g}t|jD]^}|j|j||dddd|f|j|j||dddd|f`t j |d}t j |d}||z}|j|j|j|j|}|S)Ndim) r/r"appendr2r4torchcatr-r)r+r,)r6xpath_tags_seqxpath_subs_seqxpath_tags_embeddingsxpath_subs_embeddingsixpath_embeddingss r:forwardzXPathEmbeddings.forwardXs " "t~~& eA ! ( ()I)F)Fq)I.YZ\]_`Y`Ja)b c ! ( ()J)G)G)J>Z[]^`aZaKb)c d e!& *?R H % *?R H03HH>>$,,ttG`G`aqGr7s*tu)NN)__name__ __module__ __qualname____doc__r!rH __classcell__r9s@r:rr8s  4 rIrc|j|j}tj|dj ||z|z}|j |zS)a Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols are ignored. This is modified from fairseq's `utils.make_positions`. Args: x: torch.Tensor x: Returns: torch.Tensor rr=)neintr@cumsumtype_aslong) input_ids padding_idxpast_key_values_lengthmaskincremental_indicess r:"create_position_ids_from_input_idsr[ksW << $ ( ( *D <<!4<eZdZdZfdZdZ ddZxZS)MarkupLMEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.cltt| ||_t j |j |j|j|_ t j |j|j|_ |j|_ t||_t j |j|j|_t j"|j|j$|_t j&|j(|_|j-dt/j0|jj3dd|j|_t j |j|j|j4|_ y)N)rWeps position_ids)rr<F) persistent)r r]r!r7rr0 vocab_sizer% pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsr"rrGtype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsr'r(r)register_bufferr@arangeexpandrWr6r7r9s r:r!zMarkupLMEmbeddings.__init__~s=  $02 !||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c )) / 7%'\\&2H2H&J\J\%]"f&8&8f>S>STzz&"<"<=   ELL)G)GHOOPWXej  "..#%<<  * *F,>,>DL\L\$  rIc|jdd}|d}tj|jdz||jzdztj|j }|j dj|S)z We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids. Args: inputs_embeds: torch.Tensor Returns: torch.Tensor Nr<rdtypedevicer)sizer@rmrWrUrs unsqueezern)r6 inputs_embeds input_shapesequence_lengthras r:&create_position_ids_from_inputs_embedsz9MarkupLMEmbeddings.create_position_ids_from_inputs_embedss$((*3B/ %a.||   q /D4D4D"Dq"HPUPZPZcpcwcw %%a(// <>*::;5::V\7N  !![[44uzzd;'4>>*::;5::V\8N)"66|D $ : :> J00P%(;;>SSVff ^^J/ \\*- rI)NNNNNNr)rJrKrLrMr!ryrHrNrOs@r:r]r]{s,Q 2=& 2rIr]cneZdZfdZdej dej dej fdZxZS)MarkupLMSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNr_) r r!rr#r%denserjrkr'r(r)ros r:r!zMarkupLMSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= rI hidden_states input_tensorreturncr|j|}|j|}|j||z}|SNrr)rjr6rrs r:rHzMarkupLMSelfOutput.forward7 =1  ]3 }|'CD rIrJrKrLr!r@TensorrHrNrOs@r:rr1> U\\RWR^R^rIrcVeZdZfdZdej dej fdZxZS)MarkupLMIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r r!rr#r%intermediate_sizer isinstance hidden_actstrr intermediate_act_fnros r:r!zMarkupLMIntermediate.__init__s] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $rIrrcJ|j|}|j|}|Sr)rrr6rs r:rHzMarkupLMIntermediate.forwards& =1 00? rIrrOs@r:rrs#9U\\ellrIrcneZdZfdZdej dej dej fdZxZS)MarkupLMOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r r!rr#rr%rrjrkr'r(r)ros r:r!zMarkupLMOutput.__init__s` YYv779K9KL f&8&8f>S>STzz&"<"<= rIrrrcr|j|}|j|}|j||z}|Srrrs r:rHzMarkupLMOutput.forwardrrIrrOs@r:rrrrIrcVeZdZfdZdej dej fdZxZS)MarkupLMPoolerct|tj|j|j|_tj |_yr)r r!rr#r%rTanhr+ros r:r!zMarkupLMPooler.__init__ s9 YYv1163E3EF '')rIrrc\|dddf}|j|}|j|}|S)Nr)rr+)r6rfirst_token_tensor pooled_outputs r:rHzMarkupLMPooler.forwards6+1a40 #56  6 rIrrOs@r:rr s#$ U\\ellrIrcVeZdZfdZdej dej fdZxZS)MarkupLMPredictionHeadTransformcht|tj|j|j|_t |jtrt|j|_ n|j|_ tj|j|j|_ yr) r r!rr#r%rrrrr transform_act_fnrjrkros r:r!z(MarkupLMPredictionHeadTransform.__init__s{ YYv1163E3EF f'' -$*6+<+<$=D !$*$5$5D !f&8&8f>S>STrIrrcl|j|}|j|}|j|}|Sr)rrrjrs r:rHz'MarkupLMPredictionHeadTransform.forward&s4 =1 --m< }5 rIrrOs@r:rrs$UU\\ellrIrc*eZdZfdZdZdZxZS)MarkupLMLMPredictionHeadcHt|t||_t j |j |jd|_t jtj|j|_ |j|j_ y)NF)bias) r r!r transformrr#r%rcdecoder Parameterr@r{rros r:r!z!MarkupLMLMPredictionHead.__init__/sm 8@yy!3!3V5F5FUS LLV->->!?@ !II rIc:|j|j_yr)rrr6s r: _tie_weightsz%MarkupLMLMPredictionHead._tie_weights<s II rIcJ|j|}|j|}|Sr)rrrs r:rHz MarkupLMLMPredictionHead.forward?s$}5  ]3 rI)rJrKrLr!rrHrNrOs@r:rr.s &&rIrcVeZdZfdZdej dej fdZxZS)MarkupLMOnlyMLMHeadcBt|t||_yr)r r!r predictionsros r:r!zMarkupLMOnlyMLMHead.__init__Gs 3F;rIsequence_outputrc(|j|}|Sr)r)r6rprediction_scoress r:rHzMarkupLMOnlyMLMHead.forwardKs ,,_=  rIrrOs@r:rrFs#<!u||! !rIrcPeZdZd fd Zdej dej fdZ ddej deejdeejdeejd eejd ee e ejd ee de ej fd Z xZ S)MarkupLMSelfAttentionct||j|jzdk7r2t |ds&t d|jd|jd|j|_t |j|jz |_|j|jz|_tj|j|j|_ tj|j|j|_ tj|j|j|_ tj|j|_|xs t#|dd|_|j$dk(s|j$d k(rF|j&|_tj(d |j&zd z |j|_|j,|_y) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()position_embedding_typeabsolute relative_keyrelative_key_queryr)r r!r%num_attention_headshasattr ValueErrorrRattention_head_size all_head_sizerr#querykeyvaluer'attention_probs_dropout_probr)getattrrrfr0distance_embedding is_decoderr6r7rr9s r:r!zMarkupLMSelfAttention.__init__Rs     : : :a ?PVXhHi#F$6$6#78 445Q8  $*#=#= #&v'9'9F( ' -zC $  ' '> 9T=Y=Y]q=q+1+I+ID (&(ll1v7U7U3UXY3Y[_[s[s&tD # ++rIxrc|jdd|j|jfz}|j|}|j ddddS)Nr<rrrr )rtrrviewpermute)r6r new_x_shapes r:transpose_for_scoresz*MarkupLMSelfAttention.transpose_for_scoreslsLffhsmt'?'?AYAY&ZZ FF; yyAq!$$rIrattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsc$|j|}|du} | r||d} |d} |}n| rC|j|j|} |j|j|} |}n|y|j|j|} |j|j|} t j |d| gd} t j |d| gd} n@|j|j|} |j|j|} |j|} |du} |j r| | f}t j| | jdd}|jdk(s|jdk(r| jd| jd}}| rDt j|dz tj|j jdd}n@t j|tj|j jdd}t j|tj|j jdd}||z }|j!||j"zdz }|j%| j& }|jdk(rt j(d | |}||z}nE|jdk(r6t j(d | |}t j(d | |}||z|z}|t+j,|j.z }|||z}t0j2j5|d}|j7|}|||z}t j|| }|j9dddd j;}|j=dd|j>fz}|j|}|r||fn|f}|j r||fz}|S)Nrrrr=r<rrrqrrzbhld,lrd->bhlrzbhrd,lrd->bhlrr ) rrrrr@rArmatmul transposershapetensorrUrsrrmrrftorreinsummathsqrtrr functionalsoftmaxr)r contiguousrtr)r6rrrrrrrmixed_query_layeris_cross_attention key_layer value_layer query_layer use_cacheattention_scores query_length key_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probs context_layernew_context_layer_shapeoutputss r:rHzMarkupLMSelfAttention.forwardqs!JJ}5 3$> ."<&q)I(+K3N 11$((;P2QRI33DJJ?T4UVK3N  '11$((=2IJI33DJJ}4MNK >!#4i"@aHI))^A%6 $D!LK11$((=2IJI33DJJ}4MNK//0AB "$. ??(5N!<< Y5H5HR5PQ  ' '> 9T=Y=Y]q=q'2'8'8';Y__Q=O*L!&j1nEJJWdWkWk!l!q!q""'l%**UbUiUi!j!o!oprtu!v"\\*EJJ}OcOcdiijkmopN%6H#'#:#:8dFbFb;bef;f#g #7#:#:ARAR#:#S ++~=+0<<8H+Wk+l(#36N#N --1EE16>NP[]q1r./4||S"S%**+BC 6G=/2mM] ?? 11GrIrNNNNNF) rJrKrLr!r@rrr FloatTensorrboolrHrNrOs@r:rrQs,4%ell%u||%7;15=A>BDH,1c||c!!2!23cE--. c ((9(9: c !)):): ; c!uU->->'?!@Ac$D>c u|| crIreagerceZdZd fd ZdZ d dej deejdeejdeejdeejdee e ejd ee d e ej fd Z xZ S)MarkupLMAttentionct|t|j|||_t ||_t|_y)Nr) r r!MARKUPLM_SELF_ATTENTION_CLASSES_attn_implementationr6routputset pruned_headsrs r:r!zMarkupLMAttention.__init__sC 3F4O4OP ,C  )0 ErIc>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)Nrrr=)lenrr6rrrrrrrrrrunion)r6headsindexs r: prune_headszMarkupLMAttention.prune_headss u:? 7 49900$))2O2OQUQbQb u -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:rIrrrrrrrrc p|j|||||||}|j|d|} | f|ddz} | S)Nrr)r6r) r6rrrrrrr self_outputsattention_outputrs r:rHzMarkupLMAttention.forwardsWyy    ! "    ;;|A F#% QR(88rIrr) rJrKrLr!rr@rrrrrrHrNrOs@r:r r s";*7;15=A>BDH,1||!!2!23E--.  ((9(9:  !)):): ; !uU->->'?!@A$D> u|| rIr ceZdZfdZ d dej deejdeejdeejdeejdeeeejdee d eej fd Z d Z xZ S) MarkupLMLayercft||j|_d|_t ||_|j |_|j|_|jr,|j st|dt |d|_ t||_ t||_ y)Nrz> should be used as a decoder model if cross attention is addedrr )r r!chunk_size_feed_forward seq_len_dimr  attentionradd_cross_attentionrcrossattentionr intermediaterrros r:r!zMarkupLMLayer.__init__s '-'E'E$*62 ++#)#=#=  # #?? D6)g!hii"3FT^"_D 08$V, rIrrrrrrrrc ||ddnd}|j|||||} | d} |jr | dd} | d} n| dd} d} |jrT|Rt|dstd|d||d dnd}|j | ||||||}|d} | |ddz} |d}  | z} t |j |j|j| }|f| z} |jr|  fz} | S) Nr)rrrrr<r"z'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) r rrrr"rfeed_forward_chunkrr)r6rrrrrrrself_attn_past_key_valueself_attention_outputsrrpresent_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputs layer_outputs r:rHzMarkupLMLayer.forward s}:H9S>"1#5Y] !%   /3 "0" 2!4 ??,Qr2G 6r : ,QR0G'+$ ??4@4!12 =dVDDD @N?Yrs(;_c %&*&9&9 %&)!' # 7q9  7" ==G,C2+F ( 14P P 0  # #T%A%A4CSCSUe  /G+ ??!2 44GrIcL|j|}|j||}|Sr)r#r)r6rintermediate_outputr,s r:r%z MarkupLMLayer.feed_forward_chunkas,"//0@A{{#68HI rIr) rJrKrLr!r@rrrrrrHr%rNrOs@r:rrs -"7;15=A>BDH,1?||?!!2!23?E--. ? ((9(9: ? !)):): ; ?!uU->->'?!@A?$D>? u|| ?BrIrcDeZdZfdZ ddej deejdeejdeejdeejdeeeejdee d ee d ee d ee d e eej e ffd Z xZ S)MarkupLMEncoderct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}w)NF) r r!r7rr.r/num_hidden_layersrlayergradient_checkpointingr5s r:r!zMarkupLMEncoder.__init__isN  ]]5IaIaCb#caM&$9#cd &+#$dsA#rrrrrpast_key_valuesrroutput_hidden_states return_dictrc | rdnd} |rdnd} |r|jjrdnd} |jr%|jr|rtj dd}|rdnd}t |jD]\}}| r| |fz} |||nd}|||nd}|jr/|jr#|j|j|||||||}n||||||||}|d}|r ||dfz }|s|| |dfz} |jjs| |dfz} | r| |fz} | std||| | | fDSt||| | | S) NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frr<rrc3$K|]}|| ywrr9).0vs r: z*MarkupLMEncoder.forward..s = s)last_hidden_stater5r attentionscross_attentions) r7r!r4traininglogger warning_once enumerater3_gradient_checkpointing_func__call__r~r)r6rrrrrr5rrr6r7all_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cacherF layer_modulelayer_head_maskr layer_outputss r:rHzMarkupLMEncoder.forwardos#7BD$5b4%64;;;Z;Zr`d  & &4==##p" #,R$(4# VOA|#$58H$H!.7.CilO3B3N_Q/TXN**t}} $ A A ))!"#)*"% ! !-!"#)*"%! *!,M"}R'8&::" &9]1=MBEI$(,1/4&*S ||S !!2!23S E--. S ((9(9: S !)):): ; S "%e.?.?(@"ABS D>S $D>S 'tnS d^S  uU\\"$MM NS rIr0cdeZdZdZeZdZdZede e e e jfffd ZxZS)MarkupLMPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. markuplmclt|tjrm|jjj d|j j|j%|jjjyyt|tjrz|jjj d|j j|j2|jj|jjyyt|tjrJ|jjj|jjjdyt|tr%|jjjyy)zInitialize the weightsg)meanstdN?)rrr#weightdatanormal_r7initializer_rangerzero_r0rWrjfill_r)r6modules r: _init_weightsz%MarkupLMPreTrainedModel._init_weightss' fbii ( MM   & &CT[[5R5R & S{{&   &&('  - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  - KK   " " $ MM   $ $S )  8 9 KK   " " $:rIpretrained_model_name_or_pathc2tt| |g|i|Sr)r rOfrom_pretrained)clsr] model_argskwargsr9s r:r_z'MarkupLMPreTrainedModel.from_pretraineds+,cB ) ,6 :@  rI)rJrKrLrMr config_classbase_model_prefixr\ classmethodrrrosPathLiker_rNrOs@r:rOrOsK "L"%$ HU3PRP[P[K[E\<]  rIrOaK This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`MarkupLMConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) xpath_tags_seq (`torch.LongTensor` of shape `({0}, config.max_depth)`, *optional*): Tag IDs for each token in the input sequence, padded up to config.max_depth. xpath_subs_seq (`torch.LongTensor` of shape `({0}, config.max_depth)`, *optional*): Subscript IDs for each token in the input sequence, padded up to config.max_depth. attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: `1` for tokens that are NOT MASKED, `0` for MASKED tokens. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: `0` corresponds to a *sentence A* token, `1` corresponds to a *sentence B* token [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: `1` indicates the head is **not masked**, `0` indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): If set to `True`, the attentions tensors of all attention layers are returned. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): If set to `True`, the hidden states of all layers are returned. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): If set to `True`, the model will return a [`~file_utils.ModelOutput`] instead of a plain tuple. zbThe bare MarkupLM Model transformer outputting raw hidden-states without any specific head on top.ceZdZdfd ZdZdZdZeejde e e  dde ejde ejd e ejd e ej d e ejd e ejd e ej de ej de ede ede edeee ffdZdZxZS) MarkupLMModelct||||_t||_t ||_|r t|nd|_|jyr) r r!r7r]rr0encoderrpooler post_init)r6r7add_pooling_layerr9s r:r!zMarkupLMModel.__init__*sK   ,V4&v. 0AnV,t  rIc.|jjSrrrers r:get_input_embeddingsz"MarkupLMModel.get_input_embeddings6s...rIc&||j_yrrp)r6rs r:set_input_embeddingsz"MarkupLMModel.set_input_embeddings9s*/'rIc|jD]7\}}|jj|jj |9y)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrkr3r r)r6heads_to_pruner3rs r: _prune_headszMarkupLMModel._prune_heads<sE +002 CLE5 LL  u % / / ; ;E B CrIbatch_size, sequence_length output_typercrVrBrCrrrarrvrr6r7rc | | n|jj} | | n|jj} | | n|jj} | | t d|#|j |||j } n!||j dd} n t d| |jn |j} |tj| | }|&tj| tj| }|jdjd}|j|j }d |z d z}||jdk(rh|jd jd jdjd}|j!|jj"dddd}nB|jdk(r/|jdjdjd}|jt%|j'j }ndg|jj"z}|j)|||||| }|j+|||| | | }|d }|j,|j-|nd}| s ||f|ddzSt/|||j0|j2|j4S)a` Returns: Examples: ```python >>> from transformers import AutoProcessor, MarkupLMModel >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> model = MarkupLMModel.from_pretrained("microsoft/markuplm-base") >>> html_string = " Page Title " >>> encoding = processor(html_string, return_tensors="pt") >>> outputs = model(**encoding) >>> last_hidden_states = outputs.last_hidden_state >>> list(last_hidden_states.shape) [1, 4, 768] ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer<z5You have to specify either input_ids or inputs_embeds)rsrqrrrrTgr)rVrBrCrarrv)rrr6r7)r> pooler_outputrr?r@)r7rr6use_return_dictr%warn_if_padding_and_no_attention_maskrtrsr@r}r{rUrurrrr>rnr2next parametersrrkrlrrr?r@)r6rVrBrCrrrarrvrr6r7rwrsextended_attention_maskembedding_outputencoder_outputsrrs r:rHzMarkupLMModel.forwardDsH2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y. Q#..*K  &',,.s3KTU U%.%:!! @T@T  !"ZZ FCN  !"[[EJJvVN"0":":1"="G"G"J"9"<"<4::"<"N#&)@#@H"L  }}!#%//2<!>>I??))%)' + ,,  #/!5# ' *!,8< 8O O4UY #]3oab6II I;-')77&11,==   rIcJd}|D]}|tfd|Dfz }|S)Nr9c3tK|]/}|jdj|j1yw)rN) index_selectrrs)r; past_statebeam_idxs r:r=z/MarkupLMModel._reorder_cache..s.nU_j--aZ=N=N1OPns58)r~)r6r5rreordered_past layer_pasts ` r:_reorder_cachezMarkupLMModel._reorder_caches=) J ncmnn N rI)T) NNNNNNNNNNN)rJrKrLr!rqrsrwr MARKUPLM_INPUTS_DOCSTRINGformatrr_CONFIG_FOR_DOCrr@ LongTensorrrrrrHrrNrOs@r:riri$sm /0C++D+K+KLi+jk+Wfuv1559596:59371559,0/3&*e E,,-e !!1!12e !!1!12 e !!2!23 e !!1!12 e u//0e E--.e   1 12e $D>e 'tne d^e  uBB Ce wle PrIriz MarkupLM Model with a span classification head on top for extractive question-answering tasks like SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`). c!eZdZfdZeej deee  dde e jde e jde e jde e jde e jd e e jd e e jd e e jd e e jd e e jde e de e de e deee jeffdZxZS)MarkupLMForQuestionAnsweringct|||j|_t|d|_t j |j|j|_|jyNF)rn) r r! num_labelsrirPrr#r% qa_outputsrmros r:r!z%MarkupLMForQuestionAnswering.__init__sU   ++%fF ))F$6$68I8IJ rIrxryrVrBrCrrrarrvstart_positions end_positionsrr6r7rc ,| | n|jj} |j||||||||| | |  }|d}|j|}|j dd\}}|j dj }|j dj }d}| | t| jdkDr| j d} t| jdkDr| j d} |jd}| jd|| jd|t|}||| }||| }||zdz }| s||f|ddz}||f|zS|St||||j|j S) a start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. Returns: Examples: ```python >>> from transformers import AutoProcessor, MarkupLMForQuestionAnswering >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base-finetuned-websrc") >>> model = MarkupLMForQuestionAnswering.from_pretrained("microsoft/markuplm-base-finetuned-websrc") >>> html_string = " My name is Niels " >>> question = "What's his name?" >>> encoding = processor(html_string, questions=question, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> answer_start_index = outputs.start_logits.argmax() >>> answer_end_index = outputs.end_logits.argmax() >>> predict_answer_tokens = encoding.input_ids[0, answer_start_index : answer_end_index + 1] >>> processor.decode(predict_answer_tokens).strip() 'Niels' ```N rBrCrrrarrvrr6r7rrr<r=) ignore_indexr)loss start_logits end_logitsrr?)r7r}rPrsplitsqueezerrrtclamp_rrrr?)r6rVrBrCrrrarrvrrrr6r7rrlogitsrr total_loss ignored_indexloss_fct start_lossend_lossrs r:rHz$MarkupLMForQuestionAnswering.forwardsl&1%f 'tnf d^f  uU\\"$@@ Af glf rIrz9MarkupLM Model with a `token_classification` head on top.ceZdZfdZeej deee  dde e jde e jde e jde e jde e jd e e jd e e jd e e jd e e jd e e de e de e deee jeffdZxZS)MarkupLMForTokenClassificationcdt|||j|_t|d|_|j |j n |j }tj||_ tj|j|j|_ |jyr)r r!rrirPclassifier_dropoutr(rr'r)r#r% classifierrmr6r7rr9s r:r!z'MarkupLMForTokenClassification.__init__8s   ++%fF )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rIrxryrVrBrCrrrarrvlabelsrr6r7rc | | n|jj} |j||||||||| | |  } | d}|j|}d}| Ft }||j d|jj | j d}| s|f| ddz}||f|zS|St||| j| jS)a  labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. Returns: Examples: ```python >>> from transformers import AutoProcessor, AutoModelForTokenClassification >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> processor.parse_html = False >>> model = AutoModelForTokenClassification.from_pretrained("microsoft/markuplm-base", num_labels=7) >>> nodes = ["hello", "world"] >>> xpaths = ["/html/body/div/li[1]/div/span", "/html/body/div/li[1]/div/span"] >>> node_labels = [1, 2] >>> encoding = processor(nodes=nodes, xpaths=xpaths, node_labels=node_labels, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> loss = outputs.loss >>> logits = outputs.logits ```Nrrr<rrrrr?) r7r}rPrrrrrrr?)r6rVrBrCrrrarrvrrr6r7rrrrrrs r:rHz&MarkupLMForTokenClassification.forwardFsX&1%X-115151515/3,004)-,0/3&*N ELL)N !.N !. N !. N !. N u||,N ELL)N   -N &N $D>N 'tnN d^N  uU\\"N2 3N YlN rIrz MarkupLM Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. ceZdZfdZeej deee  dde e jde e jde e jde e jde e jd e e jd e e jd e e jd e e jd e e de e de e deee jeffdZxZS)!MarkupLMForSequenceClassificationcnt|||j|_||_t ||_|j |j n |j}tj||_ tj|j|j|_ |jyr)r r!rr7rirPrr(rr'r)r#r%rrmrs r:r!z*MarkupLMForSequenceClassification.__init__s   ++ %f- )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rIrxryrVrBrCrrrarrvrrr6r7rc D| | n|jj} |j||||||||| | |  } | d}|j|}|j |}d}| |jj |j dk(rd|j_nl|j dkDrL| jtjk(s| jtjk(rd|j_nd|j_|jj dk(rIt}|j dk(r&||j| j}n||| }n|jj dk(r=t}||jd|j | jd}n,|jj dk(rt}||| }| s|f| ddz}||f|zS|St!||| j"| j$ S) a& labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Returns: Examples: ```python >>> from transformers import AutoProcessor, AutoModelForSequenceClassification >>> import torch >>> processor = AutoProcessor.from_pretrained("microsoft/markuplm-base") >>> model = AutoModelForSequenceClassification.from_pretrained("microsoft/markuplm-base", num_labels=7) >>> html_string = " Page Title " >>> encoding = processor(html_string, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**encoding) >>> loss = outputs.loss >>> logits = outputs.logits ```Nrr regressionsingle_label_classificationmulti_label_classificationr<rr)r7r}rPr)r problem_typerrrr@rUrRr rrrrrrr?)r6rVrBrCrrrarrvrrr6r7rrrrrrs r:rHz)MarkupLMForSequenceClassification.forwardsV&1%] 'tn] d^]  uU\\"$<< =] cl] rIr)rrrrirO)r)ArMrrftypingrrrr@torch.utils.checkpointrtorch.nnrrr activationsr file_utilsr r rmodeling_outputsrrrrrrmodeling_utilsrrrrutilsrconfiguration_markuplmr get_loggerrJrB_CHECKPOINT_FOR_DOCrModulerr[r]rrrrrrrrr r rr0rOMARKUPLM_START_DOCSTRINGrrirrr__all__r9rIr:rs8 )) AA!  2   H %/"/ bii/ f 4 __F   299  RYY  RYY  bii$ryy0!"))!CBIICN "# 0 0hSBIISnZ biiZ z  o  F .bhL+L L^ t #:t t nUWop` %<` q` F p (?p p f rI