T^(hdZddlZddlmZmZmZddlZddlZddlmZddl m Z m Z m Z ddl mZddlmZmZmZmZmZmZdd lmZdd lmZmZmZdd lmZmZmZm Z d d l!m"Z"ejFe$Z%dZ&dZ'ejPZ)GddejTZ+GddejTZ,GddejTZ-de,iZ.GddejTZ/GddejTZ0GddejTZ1GddejTZ2Gdd ejTZ3Gd!d"ejTZ4Gd#d$ejTZ5Gd%d&ejTZ6Gd'd(ejTZ7Gd)d*eZ8d+Z9d,Z:ed-e9Gd.d/e8Z;ed0e9Gd1d2e8Z<ed3e9Gd4d5e8Z=ed6e9Gd7d8e8Z>ed9e9Gd:d;e8Z?gd<Z@y)=zPyTorch LayoutLM model.N)OptionalTupleUnion)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)ACT2FN))BaseModelOutputWithPastAndCrossAttentions,BaseModelOutputWithPoolingAndCrossAttentionsMaskedLMOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)PreTrainedModel)apply_chunking_to_forward find_pruneable_heads_and_indicesprune_linear_layer)add_start_docstrings%add_start_docstrings_to_model_forwardloggingreplace_return_docstrings)LayoutLMConfigrzmicrosoft/layoutlm-base-uncasedc4eZdZdZfdZ ddZxZS)LayoutLMEmbeddingszGConstruct the embeddings from word, position and token_type embeddings.ctt| tj|j |j |j|_tj|j|j |_ tj|j|j |_ tj|j|j |_ tj|j|j |_tj|j|j |_tj|j |j |_t%|j |j&|_tj*|j,|_|j1dt3j4|jj7ddy)N) padding_idxeps position_ids)rF) persistent)superr__init__r Embedding vocab_size hidden_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingsmax_2d_position_embeddingsx_position_embeddingsy_position_embeddingsh_position_embeddingsw_position_embeddingstype_vocab_sizetoken_type_embeddingsLayoutLMLayerNormlayer_norm_eps LayerNormDropouthidden_dropout_probdropoutregister_buffertorcharangeexpandselfconfig __class__s l/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/layoutlm/modeling_layoutlm.pyr&zLayoutLMEmbeddings.__init__4s^  $02!||F,=,=v?Q?Q_e_r_rs#%<<0N0NPVPbPb#c %'\\&2S2SU[UgUg%h"%'\\&2S2SU[UgUg%h"%'\\&2S2SU[UgUg%h"%'\\&2S2SU[UgUg%h"%'\\&2H2H&J\J\%]"*6+=+=6CXCXYzz&"<"<=   ELL)G)GHOOPWXej  c||j}n|jdd}|d}| |jn |j}||jddd|f}|&tj|tj |}||j |}|} |j|} |j|dddddf} |j|dddddf} |j|dddddf} |j|dddddf}|j|dddddf|dddddfz }|j|dddddf|dddddfz }|j|}| | z| z| z| z|z|z|z|z}|j|}|j|}|S#t$r}td|d}~wwxYw)Nr#rdtypedevicerr z:The `bbox`coordinate values should be within 0-1000 range.)sizerHr"r<zeroslongr+r-r/r0 IndexErrorr1r2r4r7r:)r@ input_idsbboxtoken_type_idsr" inputs_embeds input_shape seq_lengthrHwords_embeddingsr-left_position_embeddingsupper_position_embeddingsright_position_embeddingslower_position_embeddingser1r2r4 embeddingss rCforwardzLayoutLMEmbeddings.forwardEs1  #..*K',,.s3K ^ %.%:!! @T@T  ,,Q  ^ J ! "& '( ((  ( (  ( $  $$ $$ $ ^^J/ \\*- ) bYZ`a a bs,A,F77 G G  G)NNNNN)__name__ __module__ __qualname____doc__r&r[ __classcell__rBs@rCrr1s!Q &  5rDrcPeZdZd fd Zdej dej fdZ ddej deejdeejdeejd eejd ee e ejd ee de ej fd Z xZ S)LayoutLMSelfAttentionct||j|jzdk7r2t |ds&t d|jd|jd|j|_t |j|jz |_|j|jz|_tj|j|j|_ tj|j|j|_ tj|j|j|_ tj|j|_|xs t#|dd|_|j$dk(s|j$d k(rF|j&|_tj(d |j&zd z |j|_|j,|_y) Nrembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads ()position_embedding_typeabsolute relative_keyrelative_key_queryrIr)r%r&r)num_attention_headshasattr ValueErrorintattention_head_size all_head_sizerLinearquerykeyvaluer8attention_probs_dropout_probr:getattrrgr,r'distance_embedding is_decoderr@rArgrBs rCr&zLayoutLMSelfAttention.__init__s     : : :a ?PVXhHi#F$6$6#78 445Q8  $*#=#= #&v'9'9F( ' -zC $  ' '> 9T=Y=Y]q=q+1+I+ID (&(ll1v7U7U3UXY3Y[_[s[s&tD # ++rDxreturnc|jdd|j|jfz}|j|}|j ddddS)Nr#rrIrr )rJrkroviewpermute)r@rz new_x_shapes rCtranspose_for_scoresz*LayoutLMSelfAttention.transpose_for_scoressLffhsmt'?'?AYAY&ZZ FF; yyAq!$$rD hidden_statesattention_mask head_maskencoder_hidden_statesencoder_attention_maskpast_key_valueoutput_attentionsc$|j|}|du} | r||d} |d} |}n| rC|j|j|} |j|j|} |}n|y|j|j|} |j|j|} t j |d| gd} t j |d| gd} n@|j|j|} |j|j|} |j|} |du} |j r| | f}t j| | jdd}|jdk(s|jdk(r| jd| jd}}| rDt j|dz tj|j jdd}n@t j|tj|j jdd}t j|tj|j jdd}||z }|j!||j"zdz }|j%| j& }|jdk(rt j(d | |}||z}nE|jdk(r6t j(d | |}t j(d | |}||z|z}|t+j,|j.z }|||z}t0j2j5|d}|j7|}|||z}t j|| }|j9dddd j;}|j=dd|j>fz}|j|}|r||fn|f}|j r||fz}|S)NrrrIdimr#rirjrFrGzbhld,lrd->bhlrzbhrd,lrd->bhlrr ) rrrrsrtr<catrxmatmul transposergshapetensorrLrHr}r=rwr,torGeinsummathsqrtror functionalsoftmaxr:r~ contiguousrJrp)r@rrrrrrrmixed_query_layeris_cross_attention key_layer value_layer query_layer use_cacheattention_scores query_length key_lengthposition_ids_lposition_ids_rdistancepositional_embeddingrelative_position_scoresrelative_position_scores_queryrelative_position_scores_keyattention_probs context_layernew_context_layer_shapeoutputss rCr[zLayoutLMSelfAttention.forwards!JJ}5 3$> ."<&q)I(+K3N 11$((;P2QRI33DJJ?T4UVK3N  '11$((=2IJI33DJJ}4MNK >!#4i"@aHI))^A%6 $D!LK11$((=2IJI33DJJ}4MNK//0AB "$. ??(5N!<< Y5H5HR5PQ  ' '> 9T=Y=Y]q=q'2'8'8';Y__Q=O*L!&j1nEJJWdWkWk!l!q!q""'l%**UbUiUi!j!o!oprtu!v"\\*EJJ}OcOcdiijkmopN%6H#'#:#:8dFbFb;bef;f#g #7#:#:ARAR#:#S ++~=+0<<8H+Wk+l(#36N#N --1EE16>NP[]q1r./4||S"S%**+BC 6G=/2mM] ?? 11GrDNNNNNNF) r\r]r^r&r<Tensorrr FloatTensorrboolr[r`ras@rCrcrc~s,4%ell%u||%7;15=A>BDH,1c||c!!2!23cE--. c ((9(9: c !)):): ; c!uU->->'?!@Ac$D>c u|| crDrccneZdZfdZdej dej dej fdZxZS)LayoutLMSelfOutputc(t|tj|j|j|_tj |j|j|_tj|j|_ yNr ) r%r&rrqr)denser7r6r8r9r:r?s rCr&zLayoutLMSelfOutput.__init__s` YYv1163E3EF f&8&8f>S>STzz&"<"<= rDr input_tensorr{cr|j|}|j|}|j||z}|Srrr:r7r@rrs rCr[zLayoutLMSelfOutput.forward 7 =1  ]3 }|'CD rDr\r]r^r&r<rr[r`ras@rCrr1> U\\RWR^R^rDreagerceZdZd fd ZdZ d dej deejdeejdeejdeejdee e ejd ee d e ej fd Z xZ S)LayoutLMAttentionct|t|j|||_t ||_t|_y)Nrg) r%r&LAYOUTLM_SELF_ATTENTION_CLASSES_attn_implementationr@routputset pruned_headsrys rCr&zLayoutLMAttention.__init__sC 3F4O4OP ,C  )0 ErDc>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)Nrrr)lenrr@rkrorrrrrsrtrrrpunion)r@headsindexs rC prune_headszLayoutLMAttention.prune_heads"s u:? 7 49900$))2O2OQUQbQb u -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:rDrrrrrrrr{c p|j|||||||}|j|d|} | f|ddz} | S)Nrr)r@r) r@rrrrrrr self_outputsattention_outputrs rCr[zLayoutLMAttention.forward4sWyy    ! "    ;;|A F#% QR(88rDrr) r\r]r^r&rr<rrrrrr[r`ras@rCrrs";*7;15=A>BDH,1||!!2!23E--.  ((9(9:  !)):): ; !uU->->'?!@A$D> u|| rDrcVeZdZfdZdej dej fdZxZS)LayoutLMIntermediatect|tj|j|j |_t|jtrt|j|_ y|j|_ yr) r%r&rrqr)intermediate_sizer isinstance hidden_actstrr intermediate_act_fnr?s rCr&zLayoutLMIntermediate.__init__Ns] YYv1163K3KL f'' -'-f.?.?'@D $'-'8'8D $rDrr{cJ|j|}|j|}|Sr)rrr@rs rCr[zLayoutLMIntermediate.forwardVs& =1 00? rDrras@rCrrMs#9U\\ellrDrcneZdZfdZdej dej dej fdZxZS)LayoutLMOutputc(t|tj|j|j |_tj|j |j|_tj|j|_ yr) r%r&rrqrr)rr7r6r8r9r:r?s rCr&zLayoutLMOutput.__init__^s` YYv779K9KL f&8&8f>S>STzz&"<"<= rDrrr{cr|j|}|j|}|j||z}|Srrrs rCr[zLayoutLMOutput.forwarddrrDrras@rCrr]rrDrceZdZfdZ d dej deejdeejdeejdeejdeeeejdee d eej fd Z d Z xZ S) LayoutLMLayercft||j|_d|_t ||_|j |_|j|_|jr,|j st|dt |d|_ t||_ t||_ y)Nrz> should be used as a decoder model if cross attention is addedrhr)r%r&chunk_size_feed_forward seq_len_dimr attentionrxadd_cross_attentionrmcrossattentionr intermediaterrr?s rCr&zLayoutLMLayer.__init__ms '-'E'E$*62 ++#)#=#=  # #?? D6)g!hii"3FT^"_D 08$V, rDrrrrrrrr{c ||ddnd}|j|||||} | d} |jr | dd} | d} n| dd} d} |jrT|Rt|dstd|d||d dnd}|j | ||||||}|d} | |ddz} |d}  | z} t |j |j|j| }|f| z} |jr|  fz} | S) NrI)rrrrr#rz'If `encoder_hidden_states` are passed, z` has to be instantiated with cross-attention layers by setting `config.add_cross_attention=True`r) rrxrlrmrrfeed_forward_chunkrr)r@rrrrrrrself_attn_past_key_valueself_attention_outputsrrpresent_key_valuecross_attn_present_key_valuecross_attn_past_key_valuecross_attention_outputs layer_outputs rCr[zLayoutLMLayer.forward{s}:H9S>"1#5Y] !%   /3 "0" 2!4 ??,Qr2G 6r : ,QR0G'+$ ??4@4!12 =dVDDD @N?Yrs(;_c %&*&9&9 %&)!' # 7q9  7" ==G,C2+F ( 14P P 0  # #T%A%A4CSCSUe  /G+ ??!2 44GrDcL|j|}|j||}|Sr)rr)r@rintermediate_outputrs rCrz LayoutLMLayer.feed_forward_chunks,"//0@A{{#68HI rDr) r\r]r^r&r<rrrrrr[rr`ras@rCrrls -"7;15=A>BDH,1?||?!!2!23?E--. ? ((9(9: ? !)):): ; ?!uU->->'?!@A?$D>? u|| ?BrDrcDeZdZfdZ ddej deejdeejdeejdeejdeeeejdee d ee d ee d ee d e eej e ffd Z xZ S)LayoutLMEncoderct|||_tjt |j Dcgc] }t|c}|_d|_ ycc}w)NF) r%r&rAr ModuleListrangenum_hidden_layersrlayergradient_checkpointing)r@rA_rBs rCr&zLayoutLMEncoder.__init__sN  ]]5IaIaCb#caM&$9#cd &+#$dsA#rrrrrpast_key_valuesrroutput_hidden_states return_dictr{c | rdnd} |rdnd} |r|jjrdnd} |jr%|jr|rtj dd}|rdnd}t |jD]\}}| r| |fz} |||nd}|||nd}|jr/|jr#|j|j|||||||}n||||||||}|d}|r ||dfz }|s|| |dfz} |jjs| |dfz} | r| |fz} | std||| | | fDSt||| | | S) NzZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...Frr#rrIc3$K|]}|| ywrr).0vs rC z*LayoutLMEncoder.forward.. s = s)last_hidden_staterr attentionscross_attentions) rArrtraininglogger warning_once enumerater_gradient_checkpointing_func__call__tupler )r@rrrrrrrrrrall_hidden_statesall_self_attentionsall_cross_attentionsnext_decoder_cachei layer_modulelayer_head_maskr layer_outputss rCr[zLayoutLMEncoder.forwards#7BD$5b4%64;;;Z;Zr`d  & &4==##p" #,R$(4# VOA|#$58H$H!.7.CilO3B3N_Q/TXN**t}} $ A A ))!"#)*"% ! !-!"#)*"%! *!,M"}R'8&::" &9]1=MBEI$(,1/4&*S ||S !!2!23S E--. S ((9(9: S !)):): ; S "%e.?.?(@"ABS D>S $D>S 'tnS d^S  uU\\"$MM NS rDrcVeZdZfdZdej dej fdZxZS)LayoutLMPoolerct|tj|j|j|_tj |_yr)r%r&rrqr)rTanh activationr?s rCr&zLayoutLMPooler.__init__"s9 YYv1163E3EF '')rDrr{c\|dddf}|j|}|j|}|S)Nr)rr)r@rfirst_token_tensor pooled_outputs rCr[zLayoutLMPooler.forward's6+1a40 #56  6 rDrras@rCrr!s#$ U\\ellrDrcVeZdZfdZdej dej fdZxZS)LayoutLMPredictionHeadTransformcht|tj|j|j|_t |jtrt|j|_ n|j|_ tj|j|j|_ yr) r%r&rrqr)rrrrr transform_act_fnr7r6r?s rCr&z(LayoutLMPredictionHeadTransform.__init__2s{ YYv1163E3EF f'' -$*6+<+<$=D !$*$5$5D !f&8&8f>S>STrDrr{cl|j|}|j|}|j|}|Sr)rr"r7rs rCr[z'LayoutLMPredictionHeadTransform.forward;s4 =1 --m< }5 rDrras@rCr r 1s$UU\\ellrDr c*eZdZfdZdZdZxZS)LayoutLMLMPredictionHeadcHt|t||_t j |j |jd|_t jtj|j|_ |j|j_ y)NF)bias) r%r&r  transformrrqr)r(decoder Parameterr<rKr'r?s rCr&z!LayoutLMLMPredictionHead.__init__Dsm 8@yy!3!3V5F5FUS LLV->->!?@ !II rDc:|j|j_yr)r'r)r@s rC _tie_weightsz%LayoutLMLMPredictionHead._tie_weightsQs II rDcJ|j|}|j|}|Sr)r(r)rs rCr[z LayoutLMLMPredictionHead.forwardTs$}5  ]3 rD)r\r]r^r&r-r[r`ras@rCr%r%Cs &&rDr%cVeZdZfdZdej dej fdZxZS)LayoutLMOnlyMLMHeadcBt|t||_yr)r%r&r% predictionsr?s rCr&zLayoutLMOnlyMLMHead.__init__\s 3F;rDsequence_outputr{c(|j|}|Sr)r2)r@r3prediction_scoress rCr[zLayoutLMOnlyMLMHead.forward`s ,,_=  rDrras@rCr0r0[s#<!u||! !rDr0c"eZdZdZeZdZdZdZy)LayoutLMPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. layoutlmTcXt|tjrm|jjj d|j j|j%|jjjyyt|tjrz|jjj d|j j|j2|jj|jjyyt|trJ|jjj|jjjdyt|tr%|jjjyy)zInitialize the weightsg)meanstdN?)rrrqweightdatanormal_rAinitializer_ranger'zero_r'rr5fill_r%)r@modules rC _init_weightsz%LayoutLMPreTrainedModel._init_weightsos$ fbii ( MM   & &CT[[5R5R & S{{&   &&('  - MM   & &CT[[5R5R & S!!- ""6#5#56<<>.  1 2 KK   " " $ MM   $ $S )  8 9 KK   " " $:rDN) r\r]r^r_r config_classbase_model_prefixsupports_gradient_checkpointingrDrrDrCr7r7es "L"&*#%rDr7a4 The LayoutLM model was proposed in [LayoutLM: Pre-training of Text and Layout for Document Image Understanding](https://arxiv.org/abs/1912.13318) by Yiheng Xu, Minghao Li, Lei Cui, Shaohan Huang, Furu Wei and Ming Zhou. This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`LayoutLMConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a Args: input_ids (`torch.LongTensor` of shape `({0})`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) bbox (`torch.LongTensor` of shape `({0}, 4)`, *optional*): Bounding boxes of each input sequence tokens. Selected in the range `[0, config.max_2d_position_embeddings-1]`. Each bounding box should be a normalized version in (x0, y0, x1, y1) format, where (x0, y0) corresponds to the position of the upper left corner in the bounding box, and (x1, y1) represents the position of the lower right corner. See [Overview](#Overview) for normalization. attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*): Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`: `1` for tokens that are NOT MASKED, `0` for MASKED tokens. [What are attention masks?](../glossary#attention-mask) token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: `0` corresponds to a *sentence A* token, `1` corresponds to a *sentence B* token [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `({0})`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) head_mask (`torch.FloatTensor` of shape `(num_heads,)` or `(num_layers, num_heads)`, *optional*): Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`: `1` indicates the head is **not masked**, `0` indicates the head is **masked**. inputs_embeds (`torch.FloatTensor` of shape `(batch_size, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert *input_ids* indices into associated vectors than the model's internal embedding lookup matrix. output_attentions (`bool`, *optional*): If set to `True`, the attentions tensors of all attention layers are returned. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): If set to `True`, the hidden states of all layers are returned. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): If set to `True`, the model will return a [`~utils.ModelOutput`] instead of a plain tuple. zbThe bare LayoutLM Model transformer outputting raw hidden-states without any specific head on top.ceZdZfdZdZdZdZeejde e e  dde ejde ejd e ej d e ejd e ejd e ej d e ej de ej de ej de ede ede edeee ffdZxZS) LayoutLMModelctt| |||_t ||_t ||_t||_ |jyr) r%rIr&rArrZrencoderrpooler post_initr?s rCr&zLayoutLMModel.__init__sI mT+F3 ,V4&v. $V,  rDc.|jjSrrZr+r,s rCget_input_embeddingsz"LayoutLMModel.get_input_embeddingss...rDc&||j_yrrO)r@rts rCset_input_embeddingsz"LayoutLMModel.set_input_embeddingss*/'rDc|jD]7\}}|jj|jj |9y)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrKrrr)r@heads_to_prunerrs rC _prune_headszLayoutLMModel._prune_headssE +002 CLE5 LL  u % / / ; ;E B CrDbatch_size, sequence_length output_typerErNrOrrPr"rrQrrrrrr{c >| | n|jj} | | n|jj} | | n|jj} | | t d|#|j |||j } n!||j dd} n t d| |jn |j}|tj| |}|&tj| tj|}|)tj| dztj|}|jdjd }|j|j }d |z tj|jj z}||j#dk(rh|jd jd jdjd}|j%|jj&dddd}nB|j#d k(r/|jdjdjd}|jt)|j+j }ndg|jj&z}|j-||||| }|j/|||| | | }|d }|j1|}| s ||f|ddzSt3|||j4|j6|j8S)a  Returns: Examples: ```python >>> from transformers import AutoTokenizer, LayoutLMModel >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased") >>> model = LayoutLMModel.from_pretrained("microsoft/layoutlm-base-uncased") >>> words = ["Hello", "world"] >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782] >>> token_boxes = [] >>> for word, box in zip(words, normalized_word_boxes): ... word_tokens = tokenizer.tokenize(word) ... token_boxes.extend([box] * len(word_tokens)) >>> # add bounding boxes of cls + sep tokens >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]] >>> encoding = tokenizer(" ".join(words), return_tensors="pt") >>> input_ids = encoding["input_ids"] >>> attention_mask = encoding["attention_mask"] >>> token_type_ids = encoding["token_type_ids"] >>> bbox = torch.tensor([token_boxes]) >>> outputs = model( ... input_ids=input_ids, bbox=bbox, attention_mask=attention_mask, token_type_ids=token_type_ids ... ) >>> last_hidden_states = outputs.last_hidden_state ```NzDYou cannot specify both input_ids and inputs_embeds at the same timer#z5You have to specify either input_ids or inputs_embeds)rHrF)rrIrr<r)rNrOr"rPrQ)rrrr)r pooler_outputrrr)rArruse_return_dictrm%warn_if_padding_and_no_attention_maskrJrHr<onesrKrL unsqueezerrGfinfominrr>rnext parametersrZrKrLr rrr)r@rNrOrrPr"rrQrrrrrrRrHextended_attention_maskembedding_outputencoder_outputsr3rs rCr[zLayoutLMModel.forwardsf2C1N-TXT_T_TqTq$8$D $++JjJj &1%cd d  "  6 6y. Q#..*K  &',,.s3KTU U%.%:!! @T@T  !"ZZ FCN  !"[[EJJvVN <;;{T1FSD"0":":1"="G"G"J"9"<"<4::"<"N#&)@#@EKKPTPZPZD[D_D_"_  }}!#%//2<!>>I??%)' + ,,  #/!5# ' *!, O4 #]3oab6II I;-')77&11,==   rD NNNNNNNNNNNN)r\r]r^r&rPrRrVrLAYOUTLM_INPUTS_DOCSTRINGformatrr _CONFIG_FOR_DOCrr< LongTensorrrrrr[r`ras@rCrIrIs /0C++D+K+KLi+jk+Wfuv15+/6:59371559=A>B,0/3&*u E,,-u u''(u !!2!23 u !!1!12 u u//0 u E--.u   1 12u  ((9(9:u !)):): ;u $D>u 'tnu d^u  uBB Cu wlu rDrIz6LayoutLM Model with a `language modeling` head on top.c!eZdZddgZfdZdZdZdZee jde e e  dd eej d eej d eej"d eej d eej deej"deej"deej deej"deej"deedeedeedeee ffdZxZS)LayoutLMForMaskedLMzcls.predictions.decoder.biaszcls.predictions.decoder.weightct||t||_t ||_|j yr)r%r&rIr8r0clsrMr?s rCr&zLayoutLMForMaskedLM.__init__\s4  %f- &v. rDcB|jjjSrr8rZr+r,s rCrPz(LayoutLMForMaskedLM.get_input_embeddingse}}''777rDcB|jjjSr)rpr2r)r,s rCget_output_embeddingsz)LayoutLMForMaskedLM.get_output_embeddingshsxx##+++rDc||jj_|j|jj_yr)rpr2r)r')r@new_embeddingss rCset_output_embeddingsz)LayoutLMForMaskedLM.set_output_embeddingsks,'5$$2$7$7!rDrWrXrNrOrrPr"rrQlabelsrrrrrr{c| | n|jj} |j|||||||| | | | |  }|d}|j|}d}|Ft }||j d|jj |j d}| s|f|ddz}||f|zS|St|||j|jS)a6 labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Returns: Examples: ```python >>> from transformers import AutoTokenizer, LayoutLMForMaskedLM >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased") >>> model = LayoutLMForMaskedLM.from_pretrained("microsoft/layoutlm-base-uncased") >>> words = ["Hello", "[MASK]"] >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782] >>> token_boxes = [] >>> for word, box in zip(words, normalized_word_boxes): ... word_tokens = tokenizer.tokenize(word) ... token_boxes.extend([box] * len(word_tokens)) >>> # add bounding boxes of cls + sep tokens >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]] >>> encoding = tokenizer(" ".join(words), return_tensors="pt") >>> input_ids = encoding["input_ids"] >>> attention_mask = encoding["attention_mask"] >>> token_type_ids = encoding["token_type_ids"] >>> bbox = torch.tensor([token_boxes]) >>> labels = tokenizer("Hello world", return_tensors="pt")["input_ids"] >>> outputs = model( ... input_ids=input_ids, ... bbox=bbox, ... attention_mask=attention_mask, ... token_type_ids=token_type_ids, ... labels=labels, ... ) >>> loss = outputs.loss ```N) rrPr"rrQrrrrrrr#rIlosslogitsrr) rAr]r8rprr}r(rrr)r@rNrOrrPr"rrQryrrrrrrr3r5masked_lm_lossloss_fctrs rCr[zLayoutLMForMaskedLM.forwardos~&1%X15+/6:59371559-1=A>B,0/3&*b E,,-b u''(b !!2!23 b !!1!12 b u//0 b E--.b   1 12b ))*b  ((9(9:b !)):): ;b $D>b 'tnb d^b  un$ %b Ylb rDrnz LayoutLM Model with a sequence classification head on top (a linear layer on top of the pooled output) e.g. for document image classification tasks such as the [RVL-CDIP](https://www.cs.cmu.edu/~aharley/rvl-cdip/) dataset. ceZdZfdZdZeejdee e  dde e jde e jde e jde e jd e e jd e e jd e e jd e e jd e ede ede edeee ffdZxZS)!LayoutLMForSequenceClassificationc,t|||j|_t||_t j |j|_t j|j|j|_ |jyr r%r& num_labelsrIr8rr8r9r:rqr) classifierrMr?s rCr&z*LayoutLMForSequenceClassification.__init__i   ++%f- zz&"<"<= ))F$6$68I8IJ rDcB|jjjSrrrr,s rCrPz6LayoutLMForSequenceClassification.get_input_embeddingsrsrDrWrXrNrOrrPr"rrQryrrrr{c B| | n|jj} |j|||||||| | |  } | d} |j| } |j | }d}||jj |j dk(rd|j_nl|j dkDrL|jtjk(s|jtjk(rd|j_nd|j_|jj dk(rIt}|j dk(r&||j|j}n|||}n|jj dk(r=t}||jd|j |jd}n,|jj dk(rt}|||}| s|f| ddz}||f|zS|St!||| j"| j$ S) aF labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Returns: Examples: ```python >>> from transformers import AutoTokenizer, LayoutLMForSequenceClassification >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased") >>> model = LayoutLMForSequenceClassification.from_pretrained("microsoft/layoutlm-base-uncased") >>> words = ["Hello", "world"] >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782] >>> token_boxes = [] >>> for word, box in zip(words, normalized_word_boxes): ... word_tokens = tokenizer.tokenize(word) ... token_boxes.extend([box] * len(word_tokens)) >>> # add bounding boxes of cls + sep tokens >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]] >>> encoding = tokenizer(" ".join(words), return_tensors="pt") >>> input_ids = encoding["input_ids"] >>> attention_mask = encoding["attention_mask"] >>> token_type_ids = encoding["token_type_ids"] >>> bbox = torch.tensor([token_boxes]) >>> sequence_label = torch.tensor([1]) >>> outputs = model( ... input_ids=input_ids, ... bbox=bbox, ... attention_mask=attention_mask, ... token_type_ids=token_type_ids, ... labels=sequence_label, ... ) >>> loss = outputs.loss >>> logits = outputs.logits ```N rNrOrrPr"rrQrrrr regressionsingle_label_classificationmulti_label_classificationr#rIr{)rAr]r8r:r problem_typerrGr<rLrnr squeezerr}rrrr)r@rNrOrrPr"rrQryrrrrrr}r|rrs rCr[z)LayoutLMForSequenceClassification.forwardsz&1%n 'tnn d^n  u.. /n cln rDra3 LayoutLM Model with a token classification head on top (a linear layer on top of the hidden-states output) e.g. for sequence labeling (information extraction) tasks such as the [FUNSD](https://guillaumejaume.github.io/FUNSD/) dataset and the [SROIE](https://rrc.cvc.uab.es/?ch=13) dataset. ceZdZfdZdZeejdee e  dde e jde e jde e jde e jd e e jd e e jd e e jd e e jd e ede ede edeee ffdZxZS)LayoutLMForTokenClassificationc,t|||j|_t||_t j |j|_t j|j|j|_ |jyrrr?s rCr&z'LayoutLMForTokenClassification.__init__grrDcB|jjjSrrrr,s rCrPz3LayoutLMForTokenClassification.get_input_embeddingsqrsrDrWrXrNrOrrPr"rrQryrrrr{c | | n|jj} |j|||||||| | |  } | d} |j| } |j | }d}|>> from transformers import AutoTokenizer, LayoutLMForTokenClassification >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("microsoft/layoutlm-base-uncased") >>> model = LayoutLMForTokenClassification.from_pretrained("microsoft/layoutlm-base-uncased") >>> words = ["Hello", "world"] >>> normalized_word_boxes = [637, 773, 693, 782], [698, 773, 733, 782] >>> token_boxes = [] >>> for word, box in zip(words, normalized_word_boxes): ... word_tokens = tokenizer.tokenize(word) ... token_boxes.extend([box] * len(word_tokens)) >>> # add bounding boxes of cls + sep tokens >>> token_boxes = [[0, 0, 0, 0]] + token_boxes + [[1000, 1000, 1000, 1000]] >>> encoding = tokenizer(" ".join(words), return_tensors="pt") >>> input_ids = encoding["input_ids"] >>> attention_mask = encoding["attention_mask"] >>> token_type_ids = encoding["token_type_ids"] >>> bbox = torch.tensor([token_boxes]) >>> token_labels = torch.tensor([1, 1, 0, 0]).unsqueeze(0) # batch size of 1 >>> outputs = model( ... input_ids=input_ids, ... bbox=bbox, ... attention_mask=attention_mask, ... token_type_ids=token_type_ids, ... labels=token_labels, ... ) >>> loss = outputs.loss >>> logits = outputs.logits ```Nrrr#rIr{) rAr]r8r:rrr}rrrr)r@rNrOrrPr"rrQryrrrrr3r}r|rrs rCr[z&LayoutLMForTokenClassification.forwardtsv&1%[ 'tn[ d^[  u++ ,[ `l[ rDra LayoutLM Model with a span classification head on top for extractive question-answering tasks such as [DocVQA](https://rrc.cvc.uab.es/?ch=17) (a linear layer on top of the final hidden-states output to compute `span start logits` and `span end logits`). ceZdZdfd ZdZeee ddee jdee jdee jdee jdee jd ee jd ee jd ee jd ee jd ee dee dee de eeffdZxZS)LayoutLMForQuestionAnsweringct|||j|_t||_t j |j|j|_|jyr) r%r&rrIr8rrqr) qa_outputsrM)r@rAhas_visual_segment_embeddingrBs rCr&z%LayoutLMForQuestionAnswering.__init__sS   ++%f- ))F$6$68I8IJ rDcB|jjjSrrrr,s rCrPz1LayoutLMForQuestionAnswering.get_input_embeddingsrsrDrXrNrOrrPr"rrQstart_positions end_positionsrrrr{c *| | n|jj} |j|||||||| | |  } | d}|j|}|j dd\}}|j dj }|j dj }d}|| t|jdkDr|j d}t| jdkDr| j d} |jd}|jd|}| jd|} t|}|||}||| }||zdz }| s||f| ddz}||f|zS|St|||| j| j S) aJ start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the start of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for position (index) of the end of the labelled span for computing the token classification loss. Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence are not taken into account for computing the loss. Returns: Example: In the example below, we prepare a question + context pair for the LayoutLM model. It will give us a prediction of what it thinks the answer is (the span of the answer within the texts parsed from the image). ```python >>> from transformers import AutoTokenizer, LayoutLMForQuestionAnswering >>> from datasets import load_dataset >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("impira/layoutlm-document-qa", add_prefix_space=True) >>> model = LayoutLMForQuestionAnswering.from_pretrained("impira/layoutlm-document-qa", revision="1e3ebac") >>> dataset = load_dataset("nielsr/funsd", split="train", trust_remote_code=True) >>> example = dataset[0] >>> question = "what's his name?" >>> words = example["words"] >>> boxes = example["bboxes"] >>> encoding = tokenizer( ... question.split(), words, is_split_into_words=True, return_token_type_ids=True, return_tensors="pt" ... ) >>> bbox = [] >>> for i, s, w in zip(encoding.input_ids[0], encoding.sequence_ids(0), encoding.word_ids(0)): ... if s == 1: ... bbox.append(boxes[w]) ... elif i == tokenizer.sep_token_id: ... bbox.append([1000] * 4) ... else: ... bbox.append([0] * 4) >>> encoding["bbox"] = torch.tensor([bbox]) >>> word_ids = encoding.word_ids(0) >>> outputs = model(**encoding) >>> loss = outputs.loss >>> start_scores = outputs.start_logits >>> end_scores = outputs.end_logits >>> start, end = word_ids[start_scores.argmax(-1)], word_ids[end_scores.argmax(-1)] >>> print(" ".join(words[start : end + 1])) M. Hamann P. Harper, P. Martinez ```Nrrrr#r) ignore_indexrI)r| start_logits end_logitsrr)rAr]r8rsplitrrrrJclamprrrr)r@rNrOrrPr"rrQrrrrrrr3r}rr total_loss ignored_indexr start_lossend_lossrs rCr[z$LayoutLMForQuestionAnswering.forwardsL&1%v 'tnv d^v  u22 3v gv rDr)rnrrrrIr7)Ar_rtypingrrrr<torch.utils.checkpointrtorch.nnrrr activationsr modeling_outputsr r rrrrmodeling_utilsr pytorch_utilsrrrutilsrrrrconfiguration_layoutlmr get_loggerr\r rk_CHECKPOINT_FOR_DOCr7r5Modulerrcrrrrrrrrr r%r0r7LAYOUTLM_START_DOCSTRINGrirIrnrrr__all__rrDrCrsQ )) AA!.lltt2   H %"7LLIIZCBIICN   "# 0 0h 299  RYY SBIISnZ biiZ | RYY  bii$ryy0!"))!%o%: ,^hQ +Q Q hRTlmz 1z nz z ~ (?~ ~ B  k %<k k \  E #:E E P rD