T^(hڕ dZddlmZmZmZddlZddlZddlmZddlm Z m Z m Z ddl m Z ddlmZmZmZmZdd lmZdd lmZmZmZmZmZd d lmZej:eZd Z dZ!gdZ"dZ#dZ$dAde%de%dee%de%fdZ&e'de'dfde'de'de'de'fdZ(GddejRZ*GddejRZ+Gdd ejRZ,Gd!d"ejRZ-Gd#d$ejRZ.Gd%d&ejRZ/Gd'd(ejRZ0Gd)d*ejRZ1Gd+d,ejRZ2Gd-d.eZ3d/Z4d0Z5ed1e4Gd2d3e3Z6ed4e4Gd5d6e3Z7Gd7d8ejRZ8Gd9d:ejRZ9Gd;dd?e3Z;gd@Zz-infinfmin_valmax_valc.t|t||SN)r minrr'r(s r#clipr-Ks wGU+ ,,r%ceZdZ ddedededededededed ed eeefd d ffd Zde jd e jfdZ xZ S)MobileViTV2ConvLayerconfig in_channels out_channels kernel_sizestridegroupsbiasdilationuse_normalizationuse_activationrNc $t |t|dz dz |z} ||zdk7rtd|d|d||zdk7rtd|d|dt j ||||| |||d |_| r t j|d d d d |_nd|_| rdt| trt| |_ yt|jtrt|j|_ y|j|_ yd|_ y)NrrrzInput channels (z) are not divisible by z groups.zOutput channels (zeros) r1r2r3r4paddingr7r5r6 padding_modegh㈵>g?T) num_featuresepsmomentumaffinetrack_running_stats)super__init__r! ValueErrorrConv2d convolution BatchNorm2d normalization isinstancestrr activation hidden_act) selfr0r1r2r3r4r5r6r7r8r9r< __class__s r#rDzMobileViTV2ConvLayer.__init__Qs* {Q!+,x7  1 $/ }UV\U]]efg g99#%#    !#)$( "D "&D  .#."("8F--s3"():):";"("3"3"DOr%featuresc|j|}|j|j|}|j|j|}|Sr*)rGrIrL)rNrPs r#forwardzMobileViTV2ConvLayer.forwardsK##H-    )))(3H ?? &x0Hr%)rrFrTT) __name__ __module__ __qualname__rr!boolrrKrDtorchTensorrR __classcell__rOs@r#r/r/Ps"&+/4#!4#4# 4#  4#  4#4#4#4# 4#dCi(4# 4#l r%r/c xeZdZdZ d dedededededdf fd Zd ejdejfd Z xZ S) MobileViTV2InvertedResidualzQ Inverted residual block (MobileNetv2): https://arxiv.org/abs/1801.04381 r0r1r2r4r7rNc @t|ttt ||j zd}|dvrt d|d|dk(xr||k(|_t|||d|_ t|||d||||_ t|||dd |_ y) Nr)rrzInvalid stride .r)r1r2r3r )r1r2r3r4r5r7Fr1r2r3r9) rCrDr$r!round expand_ratiorE use_residualr/ expand_1x1conv_3x3 reduce_1x1)rNr0r1r2r4r7expanded_channelsrOs r#rDz$MobileViTV2InvertedResidual.__init__s *3u[6CVCV5V/W+XZ[\  vha89 9#q[K{l/J.  :KYZ - )*$  / )%  r%rPc|}|j|}|j|}|j|}|jr||zS|Sr*)rcrdrerb)rNrPresiduals r#rRz#MobileViTV2InvertedResidual.forwardsI??8,==*??8,&*&7&7x("EXEr%)r rSrTrU__doc__rr!rDrWrXrRrYrZs@r#r\r\sc lm ' 69 IL VY eh  BF FFr%r\c teZdZ d dedededededdf fd Zd ejdejfd ZxZ S) MobileViTV2MobileNetLayerr0r1r2r4 num_stagesrNct|tj|_t |D]5}t ||||dk(r|nd}|jj||}7y)Nrr)r1r2r4)rCrDr ModuleListlayerranger\append) rNr0r1r2r4rmirprOs r#rDz"MobileViTV2MobileNetLayer.__init__sh ]]_ z" 'A/')!"avQ E JJ  e $&K 'r%rPc8|jD] }||} |Sr*rp)rNrP layer_modules r#rRz!MobileViTV2MobileNetLayer.forwards$ JJ .L#H-H .r%)rr rSrTrUrr!rDrWrXrRrYrZs@r#rlrlsVqr'''69'IL'VY'kn' '  r%rlcheZdZdZdededdffd ZdejdejfdZ xZ S) MobileViTV2LinearSelfAttentionaq This layer applies a self-attention with linear complexity, as described in MobileViTV2 paper: https://arxiv.org/abs/2206.02680 Args: config (`MobileVitv2Config`): Model configuration object embed_dim (`int`): `input_channels` from an expected input of size :math:`(batch_size, input_channels, height, width)` r0 embed_dimrNc t|t||dd|zzdddd|_t j |j |_t|||dddd|_||_y)NrrTF)r0r1r2r6r3r8r9p) rCrDr/qkv_projrDropout attn_dropoutout_projrz)rNr0rzrOs r#rDz'MobileViTV2LinearSelfAttention.__init__s{ ,!a)m,#   JJ)<)<=,!"#   #r% hidden_statesc|j|}tj|d|j|jgd\}}}tjj j |d}|j|}||z}tj|dd}tjj j||j|z}|j|}|S)Nr)split_size_or_sectionsdimrTrkeepdim) r~rWsplitrzr functionalsoftmaxrsumrelu expand_asr) rNrqkvquerykeyrcontext_scorescontext_vectorouts r#rRz&MobileViTV2LinearSelfAttention.forwardsmmM* "KKQX\XfXfDgmnosE,,44U4C**>:~->r4Hhh!!&&u-0H0H0OOmmC  r%rirZs@r#ryrys> #0#S#T#2U\\ellr%ryc peZdZ d dededededdf fd Zdejdejfd Z xZ S) MobileViTV2FFNr0rzffn_latent_dim ffn_dropoutrNc t|t|||ddddd|_t j ||_t|||ddddd|_t j ||_y)NrTF)r0r1r2r3r4r6r8r9) rCrDr/conv1rrdropout1conv2dropout2)rNr0rzrrrOs r#rDzMobileViTV2FFN.__init__s| )!'#    ;/ )&"#     ;/ r%rc|j|}|j|}|j|}|j|}|Sr*)rrrr)rNrs r#rRzMobileViTV2FFN.forward9s@ =1  m4  =1  m4 r% rSrTrUrr!floatrDrWrXrRrYrZs@r#rrsY ! 0!00 0  0  0@U\\ellr%rc peZdZ d dededededdf fd Zdejdejfd Z xZ S) MobileViTV2TransformerLayerr0rzrdropoutrNcPt|tjd||j|_t |||_tj||_ tjd||j|_ t||||j|_ y)Nr num_groups num_channelsr?r|)rCrDr GroupNormlayer_norm_epslayernorm_beforery attentionrrlayernorm_afterrrffn)rNr0rzrrrOs r#rDz$MobileViTV2TransformerLayer.__init__Bs~  "  W]WlWl m7 J W- !||qyV\VkVkl!&)^VEWEWXr%rc|j|}|j|}||z}|j|}|j|}||z}|Sr*)rrrr)rNrlayernorm_1_outattention_output layer_outputs r#rRz#MobileViTV2TransformerLayer.forwardPsY// >>>/:(=8 ++M: xx - #m3 r%rrrZs@r#rrAs^  Y! Y Y Y  Y  Y U\\ ell r%rcheZdZdedededdffd ZdejdejfdZxZ S) MobileViTV2Transformerr0n_layersd_modelrNc<t ||j}||zg|z}|Dcgc]}t|dzdz}}t j |_t|D].}t||||}|j j|0ycc}w)N)rzr) rCrDffn_multiplierr!rrorprqrrr) rNr0rrrffn_dimsd block_idxtransformer_layerrOs r#rDzMobileViTV2Transformer.__init__]s .."W,-82::ACbB'::]]_ x 1I ;'(9:M!  JJ  / 0  1;sBrc8|jD] }||} |Sr*ru)rNrrvs r#rRzMobileViTV2Transformer.forwardns% JJ 8L(7M 8r%rwrZs@r#rr\sA101C1#1RV1"U\\ellr%rceZdZdZ ddededededededed d ffd Zd ejd e eje eefffd Z dejde eefd ejfdZ dejd ejfdZ xZ S)MobileViTV2Layerz= MobileViTV2 layer: https://arxiv.org/abs/2206.02680 r0r1r2 attn_unit_dim n_attn_blocksr7r4rNct ||j|_|j|_|}|dk(r)t ||||dk(r|nd|dkDr|dznd|_|}nd|_t||||j||_ t|||ddd|_ t||||_ tjd||j|_t|||dd d|_y) Nrr)r1r2r4r7)r1r2r3r5F)r1r2r3r8r9)rrrT)rCrD patch_size patch_width patch_heightr\downsampling_layerr/conv_kernel_sizeconv_kxkconv_1x1r transformerrrr layernormconv_projection) rNr0r1r2rrr7r4 cnn_out_dimrOs r#rDzMobileViTV2Layer.__init__ys  !,,"--# Q;&A')!)QvA*2Q,QA 'D #'K&*D #- #$//   - #$#   2&-ZghTZTiTij 4 #$"  r% feature_mapc"|j\}}}}tjj||j|j f|j|j f}|j |||j|j zd}|||ffS)N)r3r4r)shaperrunfoldrrreshape)rNr batch_sizer1 img_height img_widthpatchess r# unfoldingzMobileViTV2Layer.unfoldings9D9J9J6 KY--&& **D,<,<=%%t'7'78'  //*k4;L;LtO_O_;_acdY///r%r output_sizec|j\}}}}|j|||z|}tjj |||j |j f|j |j f}|S)N)rr3r4)rrrrfoldrr)rNrrrin_dimr n_patchesrs r#foldingzMobileViTV2Layer.foldingsz4;MM1 FJ //*fz.A9Mmm(( #**D,<,<=%%t'7'78 ) r%rPc6|jr|j|}|j|}|j|}|j|\}}|j |}|j |}|j ||}|j|}|Sr*)rrrrrrrr)rNrPrrs r#rRzMobileViTV2Layer.forwards  " "..x8H==*==* $~~h7""7+..)<<5''1r%)rrr)rSrTrUrjrr!rDrWrXrrrrRrYrZs@r#rrts; !; ;  ;  ;  ; ; ;  ; z 0U\\ 0eELL%PSUXPX/z-MobileViTV2Encoder.forward..MsXq!-Xs)last_hidden_stater) enumeraterprtraining_gradient_checkpointing_func__call__tupler )rNrrrall_hidden_statesrsrvs r#rRzMobileViTV2Encoder.forward8s #7BD(4 IOA|**t}} $ A A ))!! !-] ; #$58H$H! IX]4E$FXX X- ]noor%)FT) rSrTrUrrDrWrXrVrr r rRrYrZs@r#rrsbO#0O#TO#h&+ p||p#p p u44 5 pr%rc~eZdZdZeZdZdZdZdgZ de e je je jfddfd Zy) MobileViTV2PreTrainedModelz An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained models. mobilevitv2 pixel_valuesTrmodulerNct|tjtjfrm|jj j d|jj|j%|jj jyyt|tjrJ|jj j|jj jdyy)zInitialize the weightsr)meanstdNg?) rJrLinearrFweightdatanormal_r0initializer_ranger6zero_ LayerNormfill_)rNrs r# _init_weightsz(MobileViTV2PreTrainedModel._init_weights_s fryy"))4 5 MM   & &CT[[5R5R & S{{&   &&('  - KK   " " $ MM   $ $S ).r%)rSrTrUrjr config_classbase_model_prefixmain_input_namesupports_gradient_checkpointing_no_split_modulesrrrrFrrrr%r#r r SsT %L%$O&*#+, *E"))RYY *L$M *RV *r%r aM This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) subclass. Use it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and behavior. Parameters: config ([`MobileViTV2Config`]): Model configuration class with all the parameters of the model. Initializing with a config file does not load the weights associated with the model, only the configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights. aF Args: pixel_values (`torch.FloatTensor` of shape `(batch_size, num_channels, height, width)`): Pixel values. Pixel values can be obtained using [`AutoImageProcessor`]. See [`MobileViTImageProcessor.__call__`] for details. output_hidden_states (`bool`, *optional*): Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for more detail. return_dict (`bool`, *optional*): Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple. zYThe bare MobileViTV2 model outputting raw hidden-states without any specific head on top.c eZdZd dedeffd ZdZeee e e e de  d deej deed eed eee ffd ZxZS)MobileViTV2Modelr0 expand_outputc t||||_||_t t d|j zdddd}t||j|ddd d |_ t||_ |jy) Nrrrr,rrr rTr1r2r3r4r8r9) rCrDr0r#r$r-rr/r conv_stemrencoder post_init)rNr0r#rrOs r#rDzMobileViTV2Model.__init__s   *$ rF333R LVWce . ++$" *&1  r%c|jD]e\}}|jj|}t|ts0|j jD]}|j j|gy)zPrunes heads of the model. heads_to_prune: dict of {layer_num: list of heads to prune in this layer} See base class PreTrainedModel N)itemsr'rprJrrr prune_heads)rNheads_to_prune layer_indexheadsmobilevitv2_layerrs r# _prune_headszMobileViTV2Model._prune_headssv#1"6"6"8 C K $ 2 2; ? +-=>):)F)F)L)LC%%//;;EBC Cr%vision) checkpoint output_typermodalityexpected_outputrrrrc||n|jj}||n|jj}| td|j |}|j |||}|j r |d}tj|ddgd}n|d}d}|s|||fn|f}||ddzSt|||j S) Nz You have to specify pixel_valuesrrrrFrr)r pooler_outputr) r0ruse_return_dictrEr&r'r#rWrr r) rNrrrembedding_outputencoder_outputsr pooled_outputoutputs r#rRzMobileViTV2Model.forwards%9$D $++JjJj &1%>,7,, !5#'     / 2 "JJ'8r2hPUVM / 2  M;H;T'7[lZnFOAB// /7/')77  r%)T)NNN)rSrTrUrrVrDr0rMOBILEVITV2_INPUTS_DOCSTRINGr_CHECKPOINT_FOR_DOCr _CONFIG_FOR_DOC_EXPECTED_OUTPUT_SHAPErrWrXrr rRrYrZs@r#r"r"s 0.C++GH&<$. 04/3&* ' u||,' 'tn' d^ ' u>> ? ' I' r%r"z MobileViTV2 model with an image classification head on top (a linear layer on top of the pooled features), e.g. for ImageNet. ceZdZdeddffd Zeeeee e e  d de e jde ede e jd e edeee ff d ZxZS) !MobileViTV2ForImageClassificationr0rNcLt|||j|_t||_t d|j zd}|jdkDr!tj||jntj|_ |jy)Nrrrr) in_features out_features) rCrD num_labelsr"r r$rrrIdentity classifierr()rNr0r2rOs r#rDz*MobileViTV2ForImageClassification.__init__s   +++F3%cF,C,C&CQO   1$ II,V=N=N O  r%)r2r3rr5rrlabelsrc||n|jj}|j|||}|r |jn|d}|j |}d}||jj |j dk(rd|j_nl|j dkDrL|jtjk(s|jtjk(rd|j_nd|j_|jj dk(rIt} |j dk(r&| |j|j}n| ||}n|jj dk(r=t} | |jd|j |jd}n,|jj dk(rt} | ||}|s|f|ddz} ||f| zS| St!|||j" S) a labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the image classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss). If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). Nr7r regressionsingle_label_classificationmulti_label_classificationrr)losslogitsr)r0r:r r9rJ problem_typerHdtyperWlongr!r squeezerviewrrr) rNrrrKroutputsr=rQrPloss_fctr>s r#rRz)MobileViTV2ForImageClassification.forwards(&1%, #%"!  r%rPc|jdd}|j|}|j|}tjj ||dd}|S)Nr8bilinearFsizemode align_corners)rr`rrr interpolate)rNrP spatial_sizes r#rRzMobileViTV2ASPPPooling.forwardDsS~~bc* ##H-==*==,,XLzin,or%rwrZs@r#r]r]4sB  0  s  RU  Z^   r%r]cdeZdZdZdeddffd ZdejdejfdZxZ S)MobileViTV2ASPPzs ASPP module defined in DeepLab papers: https://arxiv.org/abs/1606.00915, https://arxiv.org/abs/1706.05587 r0rNc t|td|jzd}|}|j}t |j dk7r tdtj|_ t|||dd}|jj||jj|j Dcgc]}t|||d|d c}t|||}|jj|t|d |z|dd|_tj |j" |_ycc}w) Nrrrr z"Expected 3 values for atrous_ratesrrr_)r1r2r3r7r9r|)rCrDr$raspp_out_channelslen atrous_ratesrErroconvsr/rrextendr]projectraspp_dropout_probr) rNr0encoder_out_channelsr1r2 in_projectionrate pool_layerrOs r#rDzMobileViTV2ASPP.__init__Qs6 -cF4K4K.KUVW* // v"" #q (AB B]]_ , #%!   -( #// % +!- !!#)   ,FKN  *%+ L 0|YZkq  zzF$<$<= ) sErPcg}|jD]}|j||tj|d}|j |}|j |}|S)Nrr)rprrrWcatrrr)rNrPpyramidconvpooled_featuress r#rRzMobileViTV2ASPP.forward}s\JJ +D NN4> * +))G+,,w/,,7r% rSrTrUrjrrDrWrXrRrYrZs@r#rjrjLs8*>0*>T*>X r%rjcdeZdZdZdeddffd ZdejdejfdZxZ S)MobileViTV2DeepLabV3zB DeepLabv3 architecture: https://arxiv.org/abs/1706.05587 r0rNc t|t||_t j |j |_t||j|jdddd|_ y)NrFT)r1r2r3r8r9r6) rCrDrjasppr Dropout2dclassifier_dropout_probrr/rmrHrJrNr0rOs r#rDzMobileViTV2DeepLabV3.__init__s] #F+ ||F$B$BC . 00**#  r%rcr|j|d}|j|}|j|}|S)Nr)rrrJ)rNrrPs r#rRzMobileViTV2DeepLabV3.forwards699]2./<<)??8,r%r}rZs@r#rrs7 0 T U\\ellr%rzZ MobileViTV2 model with a semantic segmentation head on top, e.g. for Pascal VOC. ceZdZdeddffd Zeeeee  d de e jde e jde e d e e deeeff d ZxZS) "MobileViTV2ForSemanticSegmentationr0rNct|||j|_t|d|_t ||_|jy)NF)r#)rCrDrHr"r rsegmentation_headr(rs r#rDz+MobileViTV2ForSemanticSegmentation.__init__sE   +++F%H!5f!= r%)r3rrrKrrch||n|jj}||n|jj}|$|jjdk(r t d|j |d|}|r |j n|d}|j|}d}|Ytjj||jdddd } t|jj } | | |}|s|r |f|ddz} n |f|d dz} ||f| zS| St|||r|j d Sdd S) a labels (`torch.LongTensor` of shape `(batch_size, height, width)`, *optional*): Ground truth semantic segmentation maps for computing the loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels > 1`, a classification loss is computed (Cross-Entropy). Returns: Examples: ```python >>> import requests >>> import torch >>> from PIL import Image >>> from transformers import AutoImageProcessor, MobileViTV2ForSemanticSegmentation >>> url = "http://images.cocodataset.org/val2017/000000039769.jpg" >>> image = Image.open(requests.get(url, stream=True).raw) >>> image_processor = AutoImageProcessor.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") >>> model = MobileViTV2ForSemanticSegmentation.from_pretrained("apple/mobilevitv2-1.0-imagenet1k-256") >>> inputs = image_processor(images=image, return_tensors="pt") >>> with torch.no_grad(): ... outputs = model(**inputs) >>> # logits are of shape (batch_size, num_labels, height, width) >>> logits = outputs.logits ```Nrz/The number of labels should be greater than oneTr7r8rbFrc) ignore_indexr)rPrQr attentions)r0rr:rHrEr rrrrrgrrsemantic_loss_ignore_indexr) rNrrKrrrWencoder_hidden_statesrQrPupsampled_logitsrXr>s r#rRz*MobileViTV2ForSemanticSegmentation.forwardssN%9$D $++JjJj &1%  !}}88V\\"#.Zu 9  (T[[5[5[\H,f5D# WQR[0 WQR[0)-)9TGf$ Ev E&3G'//   NR   r%rY)rSrTrUrrDrr?rrrArrWrXrVrr rRrYrZs@r#rrs0T++GH+BQ`a04)-/3&* K u||,K &K 'tn K d^ K u-- . K bIK r%r)rDrr"r )rN)=rjtypingrrrrWtorch.utils.checkpointrtorch.nnrrr activationsr modeling_outputsr r rrmodeling_utilsrutilsrrrrrconfiguration_mobilevitv2r get_loggerrSloggerrAr@rBrZr[r!r$rr-Moduler/r\rlryrrrrrr MOBILEVITV2_START_DOCSTRINGr?r"rDr]rjrr__all__rr%r#rsJ"!)) AA! .9   H %&='A1 #  HSM UX ).f e ----Y^- =299=B-F"))-Fb .<RYY<~&RYY&R"))6RYY0oryyodipipZ**2  _Q 1Q Q h M (BM M bRYY09bii9z2998  X )CX  X v r%