T^(hKo dZddlZddlZddlmZmZmZmZddlZddl m cm Z ddl Zddlm Z ddlmZmZmZddlmZddlmZmZdd lmZdd lmZmZdd lmZmZmZm Z d d l!m"Z"e jFe$Z%dZ&dZ'gdZ(dZ)dZ*d2dejVde,de-dejVfdZ.Gdde j^Z0Gdde j^Z1Gdde j^Z2Gdde j^Z3Gdd e j^Z4Gd!d"e j^Z5Gd#d$e j^Z6Gd%d&e j^Z7Gd'd(eZ8d)Z9d*Z:ed+e9Gd,d-e8Z;ed.e9Gd/d0e8Z$>? r, embeddingsheightwidthrc||z}tjjs<||jj|jjzk(r |j S|j d||djdddd}tj|||fd}|j dd||zjddd}|S)Nrrr bilinear)sizemode) r#jit is_tracingrGrHrZreshapepermuteF interpolate)r3rdrerfrWinterpolated_embeddingss r*interpolate_pos_encodingz+PvtPatchEmbeddings.interpolate_pos_encoding|sun yy##%+9O9ORVR]R]RhRh9h*h++ +''65"=EEaAqQ "#-- &%Wa"b"9"A"A!RRW"X"`"`abdegh"i&&r, pixel_valuesc|j\}}}}||jk7r td|j|}|j^}}}|j dj dd}|j |}|j|jj|dd} tj| |fd}|j|jddddf||} tj|jddddf| fd} n|j|j||} |j|| z}|||fS)NzeMake sure that the channel dimension of the pixel values match with the one set in the configuration.rirrhdim)r!rK ValueErrorr]flatten transposer`rMexpandr#catrtrZrc) r3ru batch_sizerKrerf patch_embed_rdrMrZs r*r8zPvtPatchEmbeddings.forwardsM2>2D2D/ L&% 4,, ,w ool3 '--FE!))!,66q!< __[1 >> %--j"bAIIz#:BJ"&"?"?@X@XYZ\]\^Y^@_agin"o "'))T-E-Ea!e-LNa,bhi"j "&"?"?@X@XZ`bg"h \\*/B"BC 65((r,F)r<r=r>r?rrintrboolr2r#rArtrr8rCrDs@r*rFrFYs @@#x},-@#x},- @  @  @@@8 '5<< ' 'UX ']b]i]i ')ELL)U5<<c;Q5R)r,rFc`eZdZdedeffd ZdejdejfdZxZ S) PvtSelfOutputrGrLct|tj|||_tj |j |_yr0)r1r2rLineardenserarbrc)r3rGrLr4s r*r2zPvtSelfOutput.__init__s6 YY{K8 zz&"<"<= r,r5rcJ|j|}|j|}|Sr0)rrcr7s r*r8zPvtSelfOutput.forwards$ =1  ]3 r,) r<r=r>rrr2r#rAr8rCrDs@r*rrs1>y>s> U\\ellr,rc eZdZdZdedededeffd Zdedejfd Z ddejd ed ed e de ejf d Z xZS)PvtEfficientSelfAttentionzpEfficient self-attention mechanism with reduction of the sequence [PvT paper](https://arxiv.org/abs/2102.12122).rGrLnum_attention_headssequences_reduction_ratioct|||_||_|j|jzdk7r&t d|jd|jdt |j|jz |_|j|j z|_tj|j|j|j|_ tj|j|j|j|_ tj|j|j|j|_ tj|j|_||_|dkDrEtj$|||||_tj(||j*|_yy) NrzThe hidden size (z6) is not a multiple of the number of attention heads ())biasrrOrQ)r1r2rLrryrattention_head_size all_head_sizerrqkv_biasquerykeyvalueraattention_probs_dropout_probrcrr\sequence_reductionr^r_r`r3rGrLrrr4s r*r2z"PvtEfficientSelfAttention.__init__sr &#6   d66 6! ;#D$4$4#5622316  $'t'7'7$:R:R'R#S !558P8PPYYt//1C1C&//Z 99T--t/A/AXYYt//1C1C&//Z zz&"E"EF )B& $q (&(iiXq'D #!ll;FS"S%**+BC 6G=/2O\M]r,r)r<r=r>r?rrr@r2r#rArrrr8rCrDs@r*rrszSS.1SHKShmS:1#1%,,1#( *||** *  * u||  *r,rc eZdZdedededeffd ZdZ d dejded ed e d e ejf d Z xZ S) PvtAttentionrGrLrrct|t|||||_t |||_t |_y)N)rLrr)rL)r1r2rr3rr)set pruned_headsrs r*r2zPvtAttention.__init__sB - # 3&?   $F D Er,c>t|dk(ryt||jj|jj|j \}}t |jj||j_t |jj||j_t |jj||j_ t |jj|d|j_ |jjt|z |j_|jj|jjz|j_ |j j||_y)Nrrrw)lenrr3rrrrrrrr)rrunion)r3headsindexs r* prune_headszPvtAttention.prune_heads s u:? 7 49900$))2O2OQUQbQb u -TYY__eD *499==%@ ,TYY__eD .t{{/@/@%QO )- (E(EE (R %"&))"?"?$))B_B_"_  --33E:r,r5rerfrrch|j||||}|j|d}|f|ddz}|S)Nrr)r3r))r3r5rerfr self_outputsattention_outputrs r*r8zPvtAttention.forwardsEyy?PQ ;;|A7#% QR(88r,r)r<r=r>rrr@r2rr#rArrr8rCrDs@r*rrsn " ".1 "HK "hm ";&_d"\\36?BW[ u|| r,rc zeZdZ d dededeedeeffd ZdejdejfdZ xZ S) PvtFFNrG in_featureshidden_features out_featurescjt|||n|}tj|||_t |j trt|j |_ n|j |_ tj|||_ tj|j|_ yr0)r1r2rrdense1rT hidden_actrBr intermediate_act_fndense2rarbrc)r3rGrrrr4s r*r2zPvtFFN.__init__&s '3'?|[ ii _= f'' -'-f.?.?'@D $'-'8'8D $ii> zz&"<"<= r,r5rc|j|}|j|}|j|}|j|}|j|}|Sr0)rrrcrr7s r*r8zPvtFFN.forward7sP M2 00?  ]3  M2  ]3 r,)NN) r<r=r>rrrr2r#rAr8rCrDs@r*rr%sY *.&* >>>"# > sm >"U\\ellr,rc feZdZdedededededef fd Zd dejd ed ed e fd Z xZ S)PvtLayerrGrLrr+r mlp_ratiocvt|tj||j|_t |||||_|dkDr t|ntj|_ tj||j|_ t||z}t||||_y)NrQ)rGrLrrr)rGrr)r1r2rr^r_ layer_norm_1r attentionr.Identityr+ layer_norm_2rrmlp) r3rGrLrr+rrmlp_hidden_sizer4s r*r2zPvtLayer.__init__As LL&:O:OP%# 3&?   4=s?Y/ LL&:O:OPkI56[Rabr,r5rerfrc|j|j||||}|d}|dd}|j|}||z}|j|j |}|j|}||z} | f|z}|S)N)r5rerfrrr)rrr+rr) r3r5rerfrself_attention_outputsrr mlp_output layer_outputs r*r8zPvtLayer.forwardWs!%++M:/ "0"  2!4(,>>*:;(=8 XXd// >? ^^J/ $z1 /G+r,r) r<r=r>rrr@r2r#rArr8rCrDs@r*rr@soccc! c  c $) cc,U\\3s_cr,rc xeZdZdeffd Z d dej deedeedeede e e ff dZ xZ S) PvtEncoderrGct |||_tjd|j t |jj}g}t|jD]}|jt||dk(r |jn|jjd|dzzz|j||j||dk(r |j n|j"|dz |j"|||jdz k(t%j&||_g}d}t|jD]}g}|dk7r||j|dz z }t|j|D]\}|jt+||j"||j,||||z|j.||j0|^|jt%j&|t%j&||_t%j4|j"d|j6|_y)Nrrir)rGrHrIrJrKrLrM)rGrLrr+rrrhrQ)r1r2rGr#linspacedrop_path_ratesumdepthstolistrangenum_encoder_blocksappendrFrH patch_sizesstridesrK hidden_sizesr ModuleListpatch_embeddingsrrsequence_reduction_ratios mlp_ratiosblockr^r_r`) r3rGdrop_path_decaysrdiblockscurlayersjr4s r*r2zPvtEncoder.__init__os*  !>>!V-B-BC DVW^^` v001 A   "!45Fv00 @V@V[\abefaf[g@h%11!4!>>!,89Q!4!4FDWDWXY\]X]D^ & 3 3A 66#<#FDYDYZr,ruroutput_hidden_states return_dictrc2|rdnd}|rdnd}|jd}t|j}|} tt |j |jD]|\} \} } | | \} } }| D]&}|| | ||}|d} |r ||dfz}|s!|| fz}(| |dz k7sI| j || |djddddj} ~|j| } |r|| fz}|std| ||fDSt| ||S) Nrrrhr ric3&K|] }|| ywr0r).0vs r* z%PvtEncoder.forward..smq_`_lmslast_hidden_stater5 attentions) r!rr enumerateziprrorprr`tupler )r3rurrrall_hidden_statesall_self_attentionsr~ num_blocksr5idxembedding_layer block_layerrerfr layer_outputss r*r8zPvtEncoder.forwardsi#7BD$5b4!''* _ $ 3rr2r# FloatTensorrrrrr r8rCrDs@r*rrnsn0[y0[j-2/4&* # ''# $D># 'tn # d^ # uo% & # r,rcxeZdZdZeZdZdZgZde e je je jfddfdZy)PvtPreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. pvtrumodulerNct|tjrtjj |j j d|jj|j _|j%|jj jyyt|tjrJ|jj j|j j jdyt|trtjj |jj d|jj|j_|jZtjj |jj d|jj|j_yyy)zInitialize the weightsr)meanstdNg?)rTrrinit trunc_normal_weightdatarGinitializer_rangerzero_r^fill_rFrZrM)r3r s r* _init_weightsz PvtPreTrainedModel._init_weightssS fbii ("$!6!6v}}7I7IPSY]YdYdYvYv!6!wFMM {{&   &&('  - KK   " " $ MM   $ $S )  2 3.0gg.C.C**//KK11/D/F & & + +(*(=(=$$)) 55)>)  %, 4r,)r<r=r>r?r config_classbase_model_prefixmain_input_name_no_split_modulesrrrr\r^rrr,r*rrsK L$OE"))RYY *L$MRVr,raG This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`~PvtConfig`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. a  Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`PvtImageProcessor.__call__`] for details. output_attentions (`bool`, *optional*): Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned tensors for more detail. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zSThe bare Pvt encoder outputting raw hidden-states without any specific head on top.ceZdZdeffd ZdZeejde e e e de  d dejdeed eed eed eee ff d ZxZS)PvtModelrGcrt||||_t||_|j yr0)r1r2rGrencoder post_initr3rGr4s r*r2zPvtModel.__init__ s1   "&)  r,c|jD]7\}}|jj|jj |9y)z Prunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsrlayerrr)r3heads_to_pruner!rs r* _prune_headszPvtModel._prune_headssE +002 CLE5 LL  u % / / ; ;E B Cr,%(batch_size, channels, height, width)vision) checkpoint output_typermodalityexpected_outputrurrrrc,||n|jj}||n|jj}||n|jj}|j ||||}|d}|s |f|ddzSt ||j |jS)Nrurrrrrr)rGrruse_return_dictrr r5r)r3rurrrencoder_outputssequence_outputs r*r8zPvtModel.forwards2C1N-TXT_T_TqTq$8$D $++JjJj &1%rr2r#rPVT_INPUTS_DOCSTRINGr:r_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPEr#rrrrrr8rCrDs@r*rrs yC++?+F+FGn+op&#$. -1/3&*  '' $D> 'tn  d^  uo% &  q r,rz Pvt Model transformer with an image classification head on top (a linear layer on top of the final hidden state of the [CLS] token) e.g. for ImageNet. ceZdZdeddffd Zeejdee e e e  d de ejde ejd e ed e ed e edeee ff d ZxZS)PvtForImageClassificationrGrNc0t|||j|_t||_|jdkDr-t j |jd|jnt j|_ |jy)Nrrh) r1r2 num_labelsrrrrrr classifierrrs r*r2z"PvtForImageClassification.__init__Lsy   ++F#FLEVEVYZEZBIIf))"-v/@/@ A`b`k`k`m  r,r$)r&r'rr)rulabelsrrrc(||n|jj}|j||||}|d}|j|dddddf}d} ||jj|j dk(rd|j_nl|j dkDrL|j tjk(s|j tjk(rd|j_nd|j_|jjdk(rIt} |j dk(r&| |j|j} n| ||} n|jjdk(r=t} | |jd|j |jd} n,|jjdk(rt} | ||} |s|f|ddz} | | f| zS| St| ||j |j" S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr+rr regressionsingle_label_classificationmulti_label_classificationrh)losslogitsr5r)rGr,rr7 problem_typer6rr#longrr squeezer rrrr5r) r3rur8rrrrr.r>r=loss_fctr)s r*r8z!PvtForImageClassification.forwardZs*&1%rr2rr/r:r_IMAGE_CLASS_CHECKPOINTrr1_IMAGE_CLASS_EXPECTED_OUTPUTrr#rArrrr8rCrDs@r*r4r4Ds y T ++?+F+FGn+op*)$4 *.,0/3&* ; u||,; &; $D> ; 'tn ; d^ ;  u++ ,; q; r,r4)r4rr)rF)>r?rUrtypingrrrrr#torch.nn.functionalrrrqtorch.utils.checkpointtorch.nnrr r activationsr modeling_outputsr rmodeling_utilsr pytorch_utilsrrutilsrrrrconfiguration_pvtr get_loggerr<loggerr1r0r2rCrDrAr@rr+Moduler.rFrrrrrrrPVT_START_DOCSTRINGr/rr4__all__rr,r*rTs" 33 AA!F-Q )   H %-%11U\\eTV[VbVb* -")) -A)A)H BII O Od'299'TRYY6+ryy+\V V r!!H   Y7 !7 7 t Q 2Q Q h Jr,