Ë
    S^(hKC  ã                   ó¤  — d dl mZ d dlZd dlmZ ddlmZ  e«       rd dlmZm	Z	 d dl
mZmZ  G d„ d	ej                  j                  «      Z	 	 dd
edeej                      deej                      fd„Z G d„ dej                  j                  «      Z G d„ dej                  j                  «      Z G d„ dej                  j                  «      Zy)é    )ÚOptionalN)ÚGenerationConfigé   )Úis_torch_available)ÚPreTrainedModelÚStaticCache)Úis_torch_greater_or_equalÚ"is_torch_greater_or_equal_than_2_3c            	       óÒ   ‡ — e Zd ZdZdefˆ fd„Zdej                  dej                  fd„Ze	dej                  j                  dej                  d	ed
ej                  fd„«       Zˆ xZS )Ú$TorchExportableModuleWithStaticCacheaÊ  
    A wrapper module designed to make a `PreTrainedModel` exportable with `torch.export`,
    specifically for use with static caching. This module ensures that the exported model
    is compatible with further lowering and execution in `ExecuTorch`.

    Note:
        This class is specifically designed to support export process using `torch.export`
        in a way that ensures the model can be further lowered and run efficiently in `ExecuTorch`.
    Úmodelc                 óÒ  •— t         ‰| �  «        |j                  €t        d«      ‚|j                  j                  st        d«      ‚|j                  j
                  dk7  rt        d«      ‚|| _        t        | j                  j                  | j                  j                  j                  j                  | j                  j                  j                  j                  | j                  j                  j                  j                  | j                  j                  ¬«      | _        t        t!        | j                  j"                  «      «      D ]^  }| j%                  d|› �| j                  j"                  |   d¬	«       | j%                  d
|› �| j                  j&                  |   d¬	«       Œ` t)        d„ | j                  j                  j*                  D «       «      | _        | j,                  rvt/        j0                  t/        j2                  | j                  j                  | j                  j                  t.        j4                  ¬«      «      }| j%                  d|d¬	«       yy)a¿  
        Initializes the wrapper module with the pretrained model.

        Args:
            model (`PreTrainedModel`): The pretrained model to wrap. The model must have caching
            enabled and use a 'static' caching implementation.

        Raises:
            AssertionError: If the pretrained model does not have caching enabled or if it does
            not use a 'static' caching implementation in `model.generation_config`.
        NzkThe model must have a generation config to be exported with static caching. Please set `generation_config`.zvThe model must have caching enabled to be exported with static caching. Please set `generation_config.use_cache=True`.Ústaticz–The model must use a 'static' caching implementation to be exported with static caching. Please set `generation_config.cache_implementation='static'`.©ÚconfigÚmax_batch_sizeÚmax_cache_lenÚdeviceÚdtypeÚ
key_cache_F©Ú
persistentÚvalue_cache_c              3   ó$   K  — | ]  }d |v –— Œ
 y­w)ÚCausalLMN© )Ú.0Úarchs     úb/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/integrations/executorch.pyú	<genexpr>z@TorchExportableModuleWithStaticCache.__init__.<locals>.<genexpr>S   s   è ø€ Ò\°D˜Z¨4Ô/Ñ\ùs   ‚©r   Úmask)ÚsuperÚ__init__Úgeneration_configÚAssertionErrorÚ	use_cacheÚcache_implementationr   r   r   Úcache_configÚ
batch_sizer   r   r   Ústatic_cacheÚrangeÚlenÚ	key_cacheÚregister_bufferÚvalue_cacheÚanyÚarchitecturesÚ	is_causalÚtorchÚtrilÚonesÚbool)Úselfr   ÚiÚcausal_maskÚ	__class__s       €r   r$   z-TorchExportableModuleWithStaticCache.__init__&   s  ø€ ô 	‰ÑÔð ×"Ñ"Ð*Ü ð2óð ð
 ×&Ñ&×0Ò0Ü ðAóð ð
 ×"Ñ"×7Ñ7¸8ÒCÜ ðPóð ð
 ˆŒ
Ü'Ø—:‘:×$Ñ$ØŸ:™:×7Ñ7×DÑD×OÑOØŸ*™*×6Ñ6×CÑC×QÑQØ—:‘:×/Ñ/×<Ñ<×CÑCØ—*‘*×"Ñ"ô
ˆÔô ”s˜4×,Ñ,×6Ñ6Ó7Ó8ò 	iˆAØ× Ñ  :¨a¨SÐ!1°4×3DÑ3D×3NÑ3NÈqÑ3QÐ^cÐ ÔdØ× Ñ  <°¨sÐ!3°T×5FÑ5F×5RÑ5RÐSTÑ5UÐbgÐ Õhð	iô Ñ\¸D¿J¹J×<MÑ<M×<[Ñ<[Ô\Ó\ˆŒØ�>Š>ÜŸ*™*Ü—
‘
Ø×%Ñ%×3Ñ3Ø×%Ñ%×3Ñ3ÜŸ*™*ôóˆKð × Ñ  ¨ÀÐ ÕGð ó    Ú	input_idsÚcache_positionc                 óâ   — |j                   \  }}| j                  r| j                  |d|…f   nd}|j                  d«      }| j                  }| j                  |||||d¬«      }|j                  S )aÐ  
        Forward pass of the module, which is compatible with the ExecuTorch runtime.

        Args:
            input_ids (`torch.Tensor`): Tensor representing current input token id to the module.
            cache_position (`torch.Tensor`): Tensor representing current input position in the cache.

        Returns:
            torch.Tensor: Logits output from the model.

        This forward adapter serves two primary purposes:

        1. **Making the Model `torch.export`-Compatible**:
            The adapter hides unsupported objects, such as the `Cache`, from the graph inputs and outputs,
            enabling the model to be exportable using `torch.export` without encountering issues.

        2. **Ensuring Compatibility with `ExecuTorch` runtime**:
            The adapter matches the model's forward signature with that in `executorch/extension/llm/runner`,
            ensuring that the exported model can be executed in `ExecuTorch` out-of-the-box.
        Nr   T)r=   Úattention_maskÚposition_idsr>   Úpast_key_valuesr'   )Úshaper3   r"   Ú	unsqueezer+   r   Úlogits)	r8   r=   r>   Ú_ÚseqlenÚ	attn_maskrA   rB   Úoutss	            r   Úforwardz,TorchExportableModuleWithStaticCache.forward^   s|   € ð* —O‘O‰	ˆˆ6Ø:>¿.º.�D—I‘I˜n¨g¨v¨gÐ5Ò6Èdˆ	Ø%×/Ñ/°Ó2ˆØ×+Ñ+ˆà�z‰zØØ$Ø%Ø)Ø+Øð ó 
ˆð �{‰{Ðr<   Úexported_programÚprompt_token_idsÚmax_new_tokensÚreturnc                 óF  — |j                   d   }||z   }| j                  «       D ]3  \  }}|j                  d«      sŒ|j                   d   }t        ||«      } n g }t	        t        ||«      «      D ]x  }	| j                  «       j                  |dd…|	|	dz   …f   t        j                  |	gt        j                  ¬«      ¬«      }
|j                  |d   |	   j                  «       «       Œz t        j                  
dd…ddd…f   d¬	«      j                  «       }|j                  |«       t        |«      |k  rÂ| j                  «       j                  t        j                  |ggt        j                  ¬«      t        j                  t        |«      gt        j                  ¬«      ¬«      }
t        j                  |
dd…ddd…f   d¬	«      j                  «       }|j                  |«       t        |«      |k  rŒÂt        j                  |gt        j                  ¬«      S )
aà  
        Generate a sequence of tokens using an exported program.

        This util function is designed to test exported models by simulating the generation process.
        It processes the input prompt tokens sequentially (no parallel prefill).
        This generate function is not intended to replace the original `generate` method, and the support
        for leveraging the original `generate` is potentially planed!

        Args:
            exported_program (`torch.export.ExportedProgram`): The exported program generated via `torch.export`.
            prompt_token_ids (`torch.Tensor`): Tensor representing the input prompt token IDs.
            max_new_tokens (`int`): Maximum number of new tokens to generate. Note that the total generation
                length is limited by both `max_new_tokens` and the model's cache size.

        Returns:
            torch.Tensor: A tensor containing the generated sequence of token IDs, including the original prompt tokens.
        éÿÿÿÿr.   r   Né   r!   )r=   r>   r   ©Údim)rC   Únamed_buffersÚ
startswithÚminr,   ÚmodulerJ   r4   ÚtensorÚlongÚappendÚitemÚargmaxr-   )rK   rL   rM   Úprompt_token_lenÚmax_generation_lengthÚbuffer_nameÚbufferr   Úresponse_tokensÚ	input_posÚresultÚcurrent_tokens               r   Úgeneratez-TorchExportableModuleWithStaticCache.generate‚   sê  € ð* ,×1Ñ1°"Ñ5ÐØ 0°>Ñ AÐØ#3×#AÑ#AÓ#Cò 	ÑˆK˜Ø×%Ñ% kÕ2Ø &§¡¨Q¡�Ü(+Ð,AÀ=Ó(QÐ%Ùð		ð ˆÜœsÐ#8Ð:JÓKÓLò 	JˆIØ%×,Ñ,Ó.×6Ñ6Ø*ª1¨i¸)Àa¹-Ð.GÐ+GÑHÜ$Ÿ|™|¨Y¨K¼u¿z¹zÔJð 7ó ˆFð ×"Ñ"Ð#3°AÑ#6°yÑ#A×#FÑ#FÓ#HÕIð	Jô Ÿ™ VªA¨r²1¨HÑ%5¸2Ô>×CÑCÓEˆØ×Ñ˜}Ô-ä�/Ó"Ð%:Ò:Ø%×,Ñ,Ó.×6Ñ6ÜŸ,™,¨¨Ð'8ÄÇ
Á
ÔKÜ$Ÿ|™|¬S°Ó-AÐ,BÌ%Ï*É*ÔUð 7ó ˆFô "ŸL™L¨²°2²q°Ñ)9¸rÔB×GÑGÓIˆMØ×"Ñ" =Ô1ô �/Ó"Ð%:Ó:ô �|‰|˜_Ð-´U·Z±ZÔ@Ð@r<   )Ú__name__Ú
__module__Ú__qualname__Ú__doc__r   r$   r4   ÚTensorrJ   ÚstaticmethodÚexportÚExportedProgramÚintre   Ú__classcell__©r;   s   @r   r   r      s   ø„ ñð6H˜oõ 6Hðp" §¡ð "¸u¿|¹|ó "ðH ð/AØŸ,™,×6Ñ6ð/AØJOÏ,É,ð/AØhkð/Aà	�‰ò/Aó ô/Ar<   r   r   Úexample_input_idsÚexample_cache_positionc                 óæ  — t         st        d«      ‚ddl} |j                  «       5  |�|n |j                  dgg|j
                  ¬«      }|�|n |j                  dg|j
                  ¬«      }t        d«      r,|j                  j                  t        | «      |fd|id¬	«      }n6|j                  j                  j                  t        | «      |fd|id
d¬«      }|cddd«       S # 1 sw Y   yxY w)a6  
    Convert a `PreTrainedModel` into an exportable module and export it using `torch.export`,
    ensuring the exported model is compatible with `ExecuTorch`.

    Args:
        model (`PreTrainedModel`): The pretrained model to be exported.
        example_input_ids (`torch.Tensor`): Example input token id used by `torch.export`.
        example_cache_position (`torch.Tensor`): Example current cache position used by `torch.export`.

    Returns:
        Exported program (`torch.export.ExportedProgram`): The exported program generated via `torch.export`.
    ztorch >= 2.3 is required.r   NrQ   r!   z2.5.0r>   T)ÚargsÚkwargsÚstrictF)rt   ru   Úpre_dispatchrv   )r
   ÚImportErrorÚtorch.export._traceÚno_gradrX   rY   r	   rl   r   Ú_traceÚ_export)r   rq   rr   r4   rK   s        r   Úconvert_and_export_with_cacher}   µ   s  € õ" .ÜÐ5Ó6Ð6ãà	ˆ�‰‹ñ  ð "3Ð!>ÑÀLÀEÇLÁLÐSTÐRUÐQVÐ^c×^hÑ^hÔDið 	ð '=Ð&HÑ"ÈlÈeÏlÉlÐ\]Ð[^Ðfk×fpÑfpÔNqð 	ô % WÔ-Ø$Ÿ|™|×2Ñ2Ü4°UÓ;Ø'Ð)Ø(Ð*@ÐAØð	  3ó  Ñð  %Ÿ|™|×2Ñ2×:Ñ:Ü4°UÓ;Ø'Ð)Ø(Ð*@ÐAØ"Øð  ;ó  Ðð  ÷9 ÷  ò  ús   §B6C'Ã'C0c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )Ú Seq2SeqLMEncoderExportableModulez·
    A wrapper module designed to make a Seq2Seq LM encoder exportable with `torch.export`.
    This module ensures that the exported encoder model is compatible with ExecuTorch.
    c                 ó0   •— t         ‰| �  «        || _        y )N)r#   r$   Úencoder)r8   Úencoder_modelr;   s     €r   r$   z)Seq2SeqLMEncoderExportableModule.__init__ð   s   ø€ Ü‰ÑÔØ$ˆ�r<   c                 ó:   — | j                  |¬«      j                  S )N)r=   )r�   Úlast_hidden_state)r8   r=   s     r   rJ   z(Seq2SeqLMEncoderExportableModule.forwardô   s   € Ø�|‰| iˆ|Ó0×BÑBÐBr<   ©rf   rg   rh   ri   r$   rJ   ro   rp   s   @r   r   r   ê   s   ø„ ñô
%öCr<   r   c                   ó(   ‡ — e Zd ZdZˆ fd„Zd„ Zˆ xZS )Ú/Seq2SeqLMDecoderExportableModuleWithStaticCachezÚ
    A wrapper module designed to make a Seq2Seq LM decoder exportable with `torch.export`,
    specifically for use with static caching. This module ensures the exported decoder
    is compatible with ExecuTorch.
    c                 óü  •— t         ‰| �  «        |j                  «       | _        |j                  | _        |j
                  | _        t        | j
                  ||dt        j                  ¬«      | _	        t        t        | j                  j                  «      «      D ]^  }| j                  d|› �| j                  j                  |   d¬«       | j                  d|› �| j                  j                  |   d¬«       Œ` y )NÚcpur   r   Fr   r   )r#   r$   Úget_decoderÚdecoderÚlm_headr   r   r4   Úfloat32r+   r,   r-   r.   r/   r0   )r8   r   Úmax_static_cache_lengthr*   r9   r;   s        €r   r$   z8Seq2SeqLMDecoderExportableModuleWithStaticCache.__init__ÿ   sâ   ø€ Ü‰ÑÔð ×(Ñ(Ó*ˆŒØ—}‘}ˆŒØ—l‘lˆŒô (Ø—;‘;Ø%Ø1ØÜ—-‘-ô
ˆÔô ”s˜4×,Ñ,×6Ñ6Ó7Ó8ò 	iˆAØ× Ñ  :¨a¨SÐ!1°4×3DÑ3D×3NÑ3NÈqÑ3QÐ^cÐ ÔdØ× Ñ  <°¨sÐ!3°T×5FÑ5F×5RÑ5RÐSTÑ5UÐbgÐ Õhñ	ir<   c                 ón   — | j                  ||| j                  d|¬«      }| j                  |d   «      }|S )NT)r=   Úencoder_hidden_statesrB   r'   r>   r   )r‹   r+   rŒ   )r8   Údecoder_input_idsr�   r>   ÚoutputsÚ	lm_logitss         r   rJ   z7Seq2SeqLMDecoderExportableModuleWithStaticCache.forward  sD   € à—,‘,Ø'Ø"7Ø ×-Ñ-ØØ)ð ó 
ˆð —L‘L ¨¡Ó,ˆ	àÐr<   r…   rp   s   @r   r‡   r‡   ø   s   ø„ ñôiö,r<   r‡   c                   ó<   ‡ — e Zd Z	 dˆ fd„	Zd„ Zd„ Zdd„Zd„ Zˆ xZS )ÚSeq2SeqLMExportableModulec                 óÔ   •— t         ‰| �  «        || _        |j                  «       | _        |j
                  | _        || _        t        d||||dœ¬«      | _        d | _	        d | _
        y )NT)r*   r   )r'   Ú
max_lengthr(   r)   )r#   r$   Ú
full_modelÚget_encoderr�   r   Úmax_hidden_seq_lengthr   r%   Úexported_encoderÚexported_decoder)r8   r   r*   rš   r(   Úmax_cache_lengthr;   s         €r   r$   z"Seq2SeqLMExportableModule.__init__&  sm   ø€ ô 	‰ÑÔàˆŒØ×(Ñ(Ó*ˆŒØ—l‘lˆŒØ%:ˆÔ"Ü!1ØØ'Ø!5à(Ø!1ñô	"
ˆÔð !%ˆÔØ $ˆÕr<   c                 ód  — t        | j                  «      j                  d«      j                  «       }t        j
                  j                  d| j                  ¬«      }t	        j                  «       5  t        j
                  j                  ||fdd|iid¬«      }d d d «       |S # 1 sw Y   S xY w)Nr‰   Úencoder_seq_length©Úmaxr=   rQ   T©Údynamic_shapesrv   )	r   r�   ÚtoÚevalr4   rl   ÚDimrš   rz   )r8   Úencoder_input_idsÚwrapped_encoderÚseq_len_dimr›   s        r   Ú_export_encoderz)Seq2SeqLMExportableModule._export_encoder;  s¥   € Ü:¸4¿<¹<ÓH×KÑKÈEÓR×WÑWÓYˆô —l‘l×&Ñ&Ð';À×A[ÑA[Ð&Ó\ˆô �]‰]‹_ñ 	Ü$Ÿ|™|×2Ñ2ØÐ"3Ð!5À{ÐUVÐXcÐTdÐFeÐnrð  3ó  Ð÷	ð
  Ð÷	ð
  Ðús   Á2)B%Â%B/c           	      óê  — t        | j                  | j                  j                  j                  | j                  j                  j
                  ¬«      j                  d«      j                  «       }t        j                  j                  d| j                  ¬«      }t        j                  «       5  t        j                  j                  ||||fd d|id dœd¬«      }d d d «       |S # 1 sw Y   S xY w)	N)r   rŽ   r*   r‰   Úencoder_hidden_seq_lengthr    rQ   )r‘   r�   r>   Tr¢   )r‡   r˜   r%   r)   r   r*   r¤   r¥   r4   rl   r¦   rš   rz   )r8   r‘   r�   r>   Úwrapped_decoderÚencoder_seq_len_dimrœ   s          r   Ú_export_decoderz)Seq2SeqLMExportableModule._export_decoderI  sâ   € ä;Ø—o‘oØ(,×(>Ñ(>×(KÑ(K×(YÑ(YØ×1Ñ1×>Ñ>×IÑIô÷
 ‰R�‹Yß‰T‹Vð 	ô $Ÿl™l×.Ñ.Ð/JÐPT×PjÑPjÐ.ÓkÐô �]‰]‹_ñ 
	Ü$Ÿ|™|×2Ñ2ØØ"Ð$9¸>ÐJà)-Ø./Ð1DÐ-EØ&*ñ ð
 ð  3ó 	 Ð÷
	ð  Ð÷
	ð  Ðús   Â1-C(Ã(C2c                 ó  — |�|n$t        j                  dt         j                  ¬«      }|�|n&t        j                  dggt         j                  ¬«      }|�|n%t        j                  dgt         j                  ¬«      }|�|nYt        j                  | j
                  j                  j                  d| j                  j                  ft         j                  ¬«      }| j                  |«      | _        | j                  |||«      | _        | S )N)rQ   é
   r!   r   r±   )r4   r6   rY   rX   Úzerosr%   r)   r*   r   Úd_modelr�   rª   r›   r¯   rœ   )	r8   r§   r‘   r�   r>   Úexample_encoder_input_idsÚexample_decoder_input_idsrr   Úexample_encoder_hidden_statess	            r   rl   z Seq2SeqLMExportableModule.exportf  sþ   € à!2Ð!>ÑÄEÇJÁJÈwÔ^c×^hÑ^hÔDið 	"ð "3Ð!>ÑÄEÇLÁLÐSTÐRUÐQVÔ^c×^hÑ^hÔDið 	"ð 4BÐ3M¡ÔSX×S_ÑS_ÐabÐ`cÔkp×kuÑkuÔSvÐð %Ð0ñ "ä—‘Ø×'Ñ'×4Ñ4×?Ñ?ÀÀTÇ[Á[×EXÑEXÐYÔaf×anÑanôð 	&ð !%× 4Ñ 4Ð5NÓ OˆÔØ $× 4Ñ 4Ø%Ð'DÐF\ó!
ˆÔð
 ˆr<   c           
      ó¤  — t        j                  «       5   | j                  j                  «       |«      }t        j                  dggt         j
                  ¬«      }dg}t        |dz
  «      D ]Ê  } | j                  j                  «       ||t        j                  |gt         j
                  ¬«      «      }t        j                  |d d …dd d …f   d¬«      j                  «       }|j                  |«       t        j                  |ggt         j
                  ¬«      }|| j                  j                  k(  sŒÊ n |cd d d «       S # 1 sw Y   y xY w)Nr   r!   rQ   rP   rR   )r4   rz   r›   rW   rX   rY   r,   rœ   r\   r[   rZ   r   Úeos_token_id)	r8   rL   rM   Úencoder_outputr‘   Úgenerated_idsr9   rE   Ú
next_tokens	            r   re   z"Seq2SeqLMExportableModule.generate}  s  € Ü�]‰]‹_ñ 	!à;˜T×2Ñ2×9Ñ9Ó;Ð<LÓMˆNô !&§¡¨q¨c¨U¼%¿*¹*Ô EÐØ˜CˆMô ˜>¨AÑ-Ó.ò �à7˜×.Ñ.×5Ñ5Ó7Ø% ~´u·|±|ÀQÀCÌuÏzÉzÔ7Zó�ô
 #Ÿ\™\¨&²°Bº°Ñ*:ÀÔC×HÑHÓJ�
Ø×$Ñ$ ZÔ0ô %*§L¡L°:°,°ÄuÇzÁzÔ$RÐ!ð  §¡×!9Ñ!9Ó9Ùðð" !÷5	!÷ 	!ò 	!ús   •D"EÄ8EÅE)rQ   i   r   i   )NNNN)	rf   rg   rh   r$   rª   r¯   rl   re   ro   rp   s   @r   r•   r•   %  s   ø„ àosõ%ò* ò ó:ö.!r<   r•   )NN)Útypingr   r4   Ú+transformers.generation.configuration_utilsr   Úutils.import_utilsr   Útransformersr   r   Útransformers.pytorch_utilsr	   r
   ÚnnÚModuler   rj   r}   r   r‡   r•   r   r<   r   ú<module>rÃ      sµ   ðõ ã å Hå 3ñ Ôß9ßhôWA¨5¯8©8¯?©?ô WAðx 15Ø59ñ2 Øð2 à §¡Ñ-ð2 ð % U§\¡\Ñ2ó2 ôjC u§x¡x§¡ô Cô*°e·h±h·o±oô *ôZs! §¡§¡õ s!r<   