Ë
    S^(h	ö  ã                   óú  — d Z ddlZddlmZmZmZ ddlZddlZddlZddlm	Z	 ddl
mZmZmZ ddlmZ ddlmZmZmZ dd	lmZ dd
lmZ ddlmZmZ ddlmZmZmZmZ ddl m!Z! ddl"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z( ddl)m*Z*m+Z+ ddl,m-Z-  e&«       rddl.m/Z/ ddl0m1Z1  e«       rddlm2Z2  e(jf                  e4«      Z5dZ6dZ7dZ8de9de9dejt                  fd„Z;ejx                  jz                  d„ «       Z>dejt                  dejt                  fd„Z?dejt                  d ejt                  d!ejt                  dejt                  fd"„Z@ G d#„ d$e	j‚                  «      ZB G d%„ d&eB«      ZCeBeCd'œZD G d(„ d)e	j‚                  «      ZE G d*„ d+e	j‚                  «      ZF G d,„ d-e!«      ZGd.ZHd/ZId0ZJd1ZK e$d2eH«       G d3„ d4eG«      «       ZL e$d5eH«       G d6„ d7eGe«      «       ZM e$d8eH«       G d9„ d:eG«      «       ZN e$d;eH«       G d<„ d=eG«      «       ZOg d>¢ZPy)?zPyTorch GPT-J model.é    N)ÚOptionalÚTupleÚUnion)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )ÚACT2FN)ÚCacheÚDynamicCacheÚStaticCache)ÚGenerationMixin)ÚAttentionMaskConverter)Ú!flash_attn_supports_top_left_maskÚis_flash_attn_available)ÚBaseModelOutputWithPastÚCausalLMOutputWithPastÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPast)ÚPreTrainedModel)Úadd_code_sample_docstringsÚadd_start_docstringsÚ%add_start_docstrings_to_model_forwardÚis_torch_flex_attn_availableÚis_torch_fx_proxyÚlogging)Úassert_device_mapÚget_device_mapé   )Ú
GPTJConfig)Ú	BlockMask)Úmake_flex_block_causal_mask)Ú_flash_attention_forwardz$hf-internal-testing/tiny-random-gptjzEleutherAI/gpt-j-6Br!   Únum_posÚdimÚreturnc                 óŒ  — ddt        j                  d|dt         j                  ¬«      |z  z  z  }t        j                  dt        j                  | t         j                  ¬«      j	                  «       |«      j	                  «       }t        j
                  t        j                  |«      t        j                  |«      fd¬«      S )	Nç      ð?i'  r   é   ©Údtypezi , j -> i jr    ©r&   )ÚtorchÚarangeÚint64ÚeinsumÚfloatÚcatÚsinÚcos)r%   r&   Úinv_freqÚsinusoid_inps       úd/var/www/skyplay_api_hub/venv/lib/python3.12/site-packages/transformers/models/gptj/modeling_gptj.pyÚcreate_sinusoidal_positionsr9   C   s…   € Ø�e¤§¡¨Q°°Q¼e¿k¹kÔ JÈSÑ PÑQÑR€HÜ—<‘< ´·±¸WÌEÏKÉKÔ0X×0^Ñ0^Ó0`ÐbjÓk×qÑqÓs€LÜ�9‰9”e—i‘i Ó-¬u¯y©y¸Ó/FÐGÈQÔOÐOó    c                 ót   — | j                  |j                  «      j                  |j                  d   dd«      S ©Nr   r    )ÚtoÚdeviceÚrepeatÚshape)Úembed_positionsÚposition_idss     r8   Úget_embed_positionsrC   I   s5   € à×Ñ˜l×1Ñ1Ó2×9Ñ9¸,×:LÑ:LÈQÑ:OÐQRÐTUÓVÐVr:   Úxc                 ó    — | d d …d d …d d …d d d…f   }| d d …d d …d d …dd d…f   }t        j                  | |fd¬«      } | j                  d«      S )Nr*   r    éÿÿÿÿr-   éþÿÿÿ)r.   ÚstackÚflatten)rD   Úx1Úx2s      r8   Úrotate_every_tworL   N   sS   € Ø	
Š1Ša’‘C�a�Cˆ<‰€BØ	
Š1Ša’�A�D�q�Dˆ=Ñ	€BÜ�‰�b�S˜"�I 2Ô&€AØ�9‰9�R‹=Ðr:   Útensorr4   r5   c                 óº   — t        j                  |d d …d d …d d d …f   dd«      }t        j                  |d d …d d …d d d …f   dd«      }| |z  t        | «      |z  z   S )Nr*   r
   )r.   Úrepeat_interleaverL   )rM   r4   r5   s      r8   Úapply_rotary_pos_embrP   U   s^   € Ü
×
!Ñ
! #¢aª¨D²! mÑ"4°a¸Ó
;€CÜ
×
!Ñ
! #¢aª¨D²! mÑ"4°a¸Ó
;€CØ�S‰LÔ-¨fÓ5¸Ñ;Ñ<Ð<r:   c                   ó¨  ‡ — e Zd Zdˆ fd„	Zd„ Zd„ Z	 	 dd„Zd„ Z	 	 	 	 	 	 	 ddej                  de
e   de
ej                     d	e
ej                     d
e
ej                     de
e   de
e   de
ej                     deeej                   eej                      f   e
eej                   eej                      eej                   df   f      f   fd„Zˆ xZS )ÚGPTJAttentionc                 ó:  •— t         ‰| �  «        || _        |j                  }t	        j
                  |j                  «      | _        t	        j
                  |j                  «      | _	        d| _
        || _        |€-t        j                  d| j                  j                  › d�«       |j                   | _        |j$                  | _        | j"                  | j$                  z  | _        | j&                  | j$                  z  | j"                  k7  r&t)        d| j"                  › d| j$                  › d�«      ‚t+        j,                  t+        j.                  | j&                  t*        j0                  ¬«      «      j3                  t+        j4                  «       «      | _        t	        j8                  | j"                  | j"                  d¬	«      | _        t	        j8                  | j"                  | j"                  d¬	«      | _        t	        j8                  | j"                  | j"                  d¬	«      | _        t	        j8                  | j"                  | j"                  d¬	«      | _         |jB                  | _!        | jB                  xs | j"                  }tE        ||«      | _#        y )
NTzInstantiating z¹ without passing a `layer_idx` is not recommended and will lead to errors during the forward call if caching is used. Please make sure to provide a `layer_idx` when creating this class.zEembed_dim must be divisible by num_attention_heads (got `embed_dim`: z and `num_attention_heads`: z).r+   F©Úbias)$ÚsuperÚ__init__ÚconfigÚmax_position_embeddingsr   ÚDropoutÚ
attn_pdropÚattn_dropoutÚresid_pdropÚresid_dropoutÚ	is_causalÚ	layer_idxÚloggerÚwarning_onceÚ	__class__Ú__name__Úhidden_sizeÚ	embed_dimÚnum_attention_headsÚhead_dimÚ
ValueErrorr.   ÚsqrtrM   Úfloat32r=   Úget_default_dtypeÚ
scale_attnÚLinearÚk_projÚv_projÚq_projÚout_projÚ
rotary_dimr9   rA   )ÚselfrX   r`   Úmax_positionsÚpos_embd_dimrc   s        €r8   rW   zGPTJAttention.__init__\   sè  ø€ Ü‰ÑÔØˆŒØ×6Ñ6ˆäŸJ™J v×'8Ñ'8Ó9ˆÔÜŸZ™Z¨×(:Ñ(:Ó;ˆÔàˆŒØ"ˆŒØÐÜ×ÑØ  §¡×!8Ñ!8Ð 9ð :,ð ,ôð  ×+Ñ+ˆŒØ#)×#=Ñ#=ˆÔ ØŸ™¨$×*BÑ*BÑBˆŒØ�=‰=˜4×3Ñ3Ñ3°t·~±~ÒEÜØWÐX\×XfÑXfÐWgð h+Ø+/×+CÑ+CÐ*DÀBðHóð ô  Ÿ*™*¤U§\¡\°$·-±-ÄuÇ}Á}Ô%UÓV×YÑYÔZ_×ZqÑZqÓZsÓtˆŒä—i‘i §¡°·±ÀUÔKˆŒÜ—i‘i §¡°·±ÀUÔKˆŒÜ—i‘i §¡°·±ÀUÔKˆŒÜŸ	™	 $§.¡.°$·.±.ÀuÔMˆŒØ ×+Ñ+ˆŒØ—‘Ò8¨$¯.©.ˆÜ:¸=È,ÓWˆÕr:   c                 óP  — |j                  «       dd ||fz   }|j                  |«      }|r|S t        |j                  «      dk(  r|j	                  ddddd«      S t        |j                  «      dk(  r|j	                  dddd«      S t        d	t        |j                  «      › �«      ‚)
zO
        Splits hidden dim into attn_head_size and num_attention_heads
        NrF   é   r   r    r
   r*   é   ú3Input tensor rank should be one of [4, 5], but is: )ÚsizeÚviewÚlenr@   Úpermuteri   )rt   rM   rg   Úattn_head_sizeÚrotaryÚ	new_shapes         r8   Ú_split_headszGPTJAttention._split_heads   s¦   € ð —K‘K“M # 2Ð&Ð*=¸~Ð)NÑNˆ	Ø—‘˜YÓ'ˆÙØˆMÜˆv�|‰|Ó Ò!Ø—>‘> ! Q¨¨1¨aÓ0Ð0Ü�—‘Ó !Ò#Ø—>‘> ! Q¨¨1Ó-Ð-äÐRÔSVÐW]×WcÑWcÓSdÐReÐfÓgÐgr:   c                 óˆ  — t        |j                  «      dk(  r$|j                  ddddd«      j                  «       }n\t        |j                  «      dk(  r#|j                  dddd«      j                  «       }n!t	        dt        |j                  «      › �«      ‚|j                  «       dd	 ||z  fz   }|j                  |«      S )
zR
        Merges attn_head_size dim and num_attn_heads dim into hidden dim
        rx   r   r    r
   r*   ry   rz   NrG   )r}   r@   r~   Ú
contiguousri   r{   r|   )rt   rM   rg   r   r�   s        r8   Ú_merge_headszGPTJAttention._merge_headsŽ   s²   € ô ˆv�|‰|Ó Ò!Ø—^‘^ A q¨!¨Q°Ó2×=Ñ=Ó?‰FÜ�—‘Ó !Ò#Ø—^‘^ A q¨!¨QÓ/×:Ñ:Ó<‰FäÐRÔSVÐW]×WcÑWcÓSdÐReÐfÓgÐgØ—K‘K“M # 2Ð&Ð*=ÀÑ*NÐ)PÑPˆ	Ø�{‰{˜9Ó%Ð%r:   c                 ó  — |j                  t        j                  «      }|j                  t        j                  «      }t        j                  ||j	                  dd«      «      }|| j
                  z  }|�#|d d …d d …d d …d |j                  d   …f   }||z   }t        j                  j                  |d¬«      }|j                  |j                  «      }| j                  |«      }|�||z  }t        j                  ||«      }||fS )NrF   rG   r-   )r=   r.   rk   ÚmatmulÚ	transposerm   r@   r   Ú
functionalÚsoftmaxr,   r\   )	rt   ÚqueryÚkeyÚvalueÚattention_maskÚ	head_maskÚattn_weightsÚcausal_maskÚattn_outputs	            r8   Ú_attnzGPTJAttention._attn›   sè   € ð —‘œŸ™Ó'ˆØ�f‰f”U—]‘]Ó#ˆä—|‘| E¨3¯=©=¸¸RÓ+@ÓAˆØ# d§o¡oÑ5ˆàÐ%Ø(ªªAªq°/°C·I±I¸b±M°/Ð)AÑBˆKØ'¨+Ñ5ˆLä—}‘}×,Ñ,¨\¸rÐ,ÓBˆØ#—‘ u§{¡{Ó3ˆØ×(Ñ(¨Ó6ˆð Ð Ø'¨)Ñ3ˆLä—l‘l <°Ó7ˆà˜LÐ(Ð(r:   c                 óÐ   — | j                   }|j                  |j                  k7  r"|j                  |j                  «      }|| _         |j                  |j                  d   dd«      S r<   )rA   r>   r=   r?   r@   )rt   rB   rA   s      r8   Ú_get_embed_positionsz"GPTJAttention._get_embed_positionsº   s_   € Ø×.Ñ.ˆØ×!Ñ! \×%8Ñ%8Ò8Ø-×0Ñ0°×1DÑ1DÓEˆOØ#2ˆDÔ Ø×%Ñ% l×&8Ñ&8¸Ñ&;¸QÀÓBÐBr:   Úhidden_statesÚ
layer_pastrŽ   rB   r�   Ú	use_cacheÚoutput_attentionsÚcache_positionr'   .c	                 ó0  — | j                  |«      }	| j                  |«      }
| j                  |«      }| j                  |	| j                  | j
                  d«      }	| j                  |
| j                  | j
                  d«      }
| j                  || j                  | j
                  d«      }t        |«      st        j                  j                  «       rt        | j                  |«      }n| j                  |«      }|j                  d«      j                  dd|j                  d   «      }t        j                   |d|«      }t        j"                  ||j                  d   dz  d¬«      \  }}| j$                  �¹|
d d …d d …d d …d | j$                  …f   }|
d d …d d …d d …| j$                  d …f   }|	d d …d d …d d …d | j$                  …f   }|	d d …d d …d d …| j$                  d …f   }t'        |||«      }t'        |||«      }t        j(                  ||gd¬«      }
t        j(                  ||gd¬«      }	nt'        |
||«      }
t'        |	||«      }	|
j+                  dddd«      }
|	j+                  dddd«      }	|�2||| j$                  |d	œ}|j-                  |
|| j.                  |«      \  }
}| j1                  |	|
|||«      \  }}| j3                  || j                  | j
                  «      }| j5                  |«      }| j7                  |«      }||f}|r||fz  }|S )
NTFrF   r    r*   r-   r   r
   ©r4   r5   Úpartial_rotation_sizerš   )rq   ro   rp   r‚   rg   rh   r   r.   ÚjitÚ
is_tracingrC   rA   r•   Ú	unsqueezer?   r@   ÚgatherÚsplitrs   rP   r3   r~   Úupdater`   r“   r…   rr   r^   )rt   r–   r—   rŽ   rB   r�   r˜   r™   rš   r‹   rŒ   r�   rA   Úrepeated_position_idsÚsincosr4   r5   Úk_rotÚk_passÚq_rotÚq_passÚcache_kwargsr’   r�   Úoutputss                            r8   ÚforwardzGPTJAttention.forwardÁ   sã  € ð —‘˜MÓ*ˆØ�k‰k˜-Ó(ˆØ—‘˜MÓ*ˆà×!Ñ! %¨×)AÑ)AÀ4Ç=Á=ÐRVÓWˆØ×Ñ  T×%=Ñ%=¸t¿}¹}ÈdÓSˆØ×!Ñ! %¨×)AÑ)AÀ4Ç=Á=ÐRWÓXˆä˜\Ô*¬e¯i©i×.BÑ.BÔ.Dô 2°$×2FÑ2FÈÓU‰Oà"×7Ñ7¸ÓEˆOà ,× 6Ñ 6°rÓ :× AÑ AÀ!ÀQÈ×H]ÑH]Ð^`ÑHaÓ bÐÜ—‘˜o¨qÐ2GÓHˆÜ—;‘;˜v v§|¡|°BÑ'7¸1Ñ'<À"ÔE‰ˆˆSà�?‰?Ð&Øšš1šaÐ!2 4§?¡?Ð!2Ð2Ñ3ˆEØššAšq $§/¡/Ñ"3Ð3Ñ4ˆFàš!šQ¢Ð#4 T§_¡_Ð#4Ð4Ñ5ˆEØš1ša¢ D§O¡OÑ$5Ð5Ñ6ˆFä(¨°°SÓ9ˆEÜ(¨°°SÓ9ˆEä—)‘)˜U F˜O°Ô4ˆCÜ—I‘I˜u f˜o°2Ô6‰Eä& s¨C°Ó5ˆCÜ(¨°°SÓ9ˆEà�k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆàÐ!àØØ)-¯©Ø"0ñ	ˆLð $×*Ñ*¨3°°t·~±~À|ÓT‰JˆC�ð %)§J¡J¨u°c¸5À.ÐR[Ó$\Ñ!ˆ�\à×'Ñ'¨°T×5MÑ5MÈtÏ}É}Ó]ˆØ—m‘m KÓ0ˆØ×(Ñ(¨Ó5ˆà 
Ð+ˆÙØ˜�Ñ&ˆGàˆr:   ©N)NN©NNNNFFN)rd   Ú
__module__Ú__qualname__rW   r‚   r…   r“   r•   r.   ÚFloatTensorr   r   Ú
LongTensorÚboolr   r   ÚTensorr¬   Ú__classcell__©rc   s   @r8   rR   rR   [   s8  ø„ õ!XòFhò&ð$ Øó)ò>Cð '+Ø6:Ø37Ø15Ø$)Ø,1Ø59ñHà×(Ñ(ðHð ˜U‘OðHð ! ×!2Ñ!2Ñ3ð	Hð
 ˜u×/Ñ/Ñ0ðHð ˜E×-Ñ-Ñ.ðHð ˜D‘>ðHð $ D™>ðHð ! ×!1Ñ!1Ñ2ðHð 
Øˆe�l‰l˜E %§,¡,Ñ/Ð/Ñ0Ø��u—|‘| U¨5¯<©<Ñ%8¸%ÀÇÁÈcÐ@QÑ:RÐRÑSÑTð	Vñ
÷Hr:   rR   c                   óŒ  ‡ — e Zd ZdZˆ fd„Z	 	 	 	 	 	 	 ddej                  dee   deej                     deej                     deej                     dee
   d	ee
   d
eej                     deeej                  eej                     f   eeej                  eej                     eej                  df   f      f   fd„Zˆ xZS )ÚGPTJFlashAttention2aD  
    GPTJ flash attention module. This module inherits from `GPTJAttention` as the weights of the module stays
    untouched. The only required change would be on the forward pass where it needs to correctly call the public API of
    flash attention and deal with padding tokens in case the input contains any of them.
    c                 óB   •— t        ‰| �  |i |¤Ž t        «       | _        y r­   )rV   rW   r   Ú_flash_attn_uses_top_left_mask)rt   ÚargsÚkwargsrc   s      €r8   rW   zGPTJFlashAttention2.__init__  s#   ø€ Ü‰Ñ˜$Ð) &Ò)ô
 /PÓ.QˆÕ+r:   r–   r—   rŽ   rB   r�   r˜   r™   rš   r'   .c	           
      óˆ	  — | j                  |«      }	| j                  |«      }
| j                  |«      }| j                  |	| j                  | j
                  d«      }	| j                  |
| j                  | j
                  d«      }
| j                  || j                  | j
                  d«      }t        |«      st        j                  j                  «       rt        | j                  |«      }n| j                  |«      }|j                  d«      j                  dd|j                  d   «      }t        j                   |d|«      }t        j"                  ||j                  d   dz  d¬«      \  }}| j$                  �¹|
d d …d d …d d …d | j$                  …f   }|
d d …d d …d d …| j$                  d …f   }|	d d …d d …d d …d | j$                  …f   }|	d d …d d …d d …| j$                  d …f   }t'        |||«      }t'        |||«      }t        j(                  ||gd¬«      }
t        j(                  ||gd¬«      }	nt'        |
||«      }
t'        |	||«      }	|
j+                  dddd«      }
|	j+                  dddd«      }	|�2||| j$                  |d	œ}|j-                  |
|| j.                  |«      \  }
}|
j+                  dddd«      j1                  «       }
|	j+                  dddd«      j1                  «       }	|j+                  dddd«      j1                  «       }|	j2                  }|t        j4                  k(  rÂt        j6                  «       rt        j8                  «       }nMt;        | j<                  d
«      r| j<                  j>                  }n | j                   j@                  j2                  }tB        jE                  d|› d�«       |	jG                  |«      }	|
jG                  |«      }
|jG                  |«      }| jH                  r| j<                  jJ                  nd}|	j                  d   }tM        |	|
||||| jN                  | jP                  ¬«      }|jS                  |j                  d   |j                  d   |j                  d   |j                  d   z  «      }| jU                  |«      }| jW                  |«      }||f}|r||fz  }|S )NTFrF   r    r*   r-   r   r
   rœ   Ú_pre_quantization_dtypez¾The input hidden states seems to be silently casted in float32, this might be related to the fact you have upcasted embedding or layer norm layers in float32. We will cast back the input in ú.ç        )Údropoutr_   Úuse_top_left_mask),rq   ro   rp   r‚   rg   rh   r   r.   rž   rŸ   rC   rA   r•   r    r?   r@   r¡   r¢   rs   rP   r3   r~   r£   r`   r„   r,   rk   Úis_autocast_enabledÚget_autocast_gpu_dtypeÚhasattrrX   r¾   Úweightra   rb   r=   Útrainingr[   r$   r_   rº   Úreshaperr   r^   )rt   r–   r—   rŽ   rB   r�   r˜   r™   rš   r‹   rŒ   r�   rA   r¤   r¥   r4   r5   r¦   r§   r¨   r©   rª   Úinput_dtypeÚtarget_dtypeÚattention_dropoutÚquery_lengthr�   r’   r«   s                                r8   r¬   zGPTJFlashAttention2.forward  sT  € ð —‘˜MÓ*ˆØ�k‰k˜-Ó(ˆØ—‘˜MÓ*ˆà×!Ñ! %¨×)AÑ)AÀ4Ç=Á=ÐRVÓWˆØ×Ñ  T×%=Ñ%=¸t¿}¹}ÈdÓSˆØ×!Ñ! %¨×)AÑ)AÀ4Ç=Á=ÐRWÓXˆä˜\Ô*¬e¯i©i×.BÑ.BÔ.Dô 2°$×2FÑ2FÈÓU‰Oà"×7Ñ7¸ÓEˆOà ,× 6Ñ 6°rÓ :× AÑ AÀ!ÀQÈ×H]ÑH]Ð^`ÑHaÓ bÐÜ—‘˜o¨qÐ2GÓHˆÜ—;‘;˜v v§|¡|°BÑ'7¸1Ñ'<À"ÔE‰ˆˆSà�?‰?Ð&Øšš1šaÐ!2 4§?¡?Ð!2Ð2Ñ3ˆEØššAšq $§/¡/Ñ"3Ð3Ñ4ˆFàš!šQ¢Ð#4 T§_¡_Ð#4Ð4Ñ5ˆEØš1ša¢ D§O¡OÑ$5Ð5Ñ6ˆFä(¨°°SÓ9ˆEÜ(¨°°SÓ9ˆEä—)‘)˜U F˜O°Ô4ˆCÜ—I‘I˜u f˜o°2Ô6‰Eä& s¨C°Ó5ˆCÜ(¨°°SÓ9ˆEð
 �k‰k˜!˜Q  1Ó%ˆØ—‘˜a  A qÓ)ˆð Ð!àØØ)-¯©Ø"0ñ	ˆLð $×*Ñ*¨3°°t·~±~À|ÓT‰JˆC�ð �k‰k˜!˜Q  1Ó%×0Ñ0Ó2ˆØ—‘˜a  A qÓ)×4Ñ4Ó6ˆØ—‘˜a  A qÓ)×4Ñ4Ó6ˆð —k‘kˆØœ%Ÿ-™-Ò'Ü×(Ñ(Ô*Ü$×;Ñ;Ó=‘ä˜Ÿ™Ð&?Ô@Ø#Ÿ{™{×BÑB‘à#Ÿ{™{×1Ñ1×7Ñ7�ä×Ñðà �> ð$ôð —H‘H˜\Ó*ˆEØ—&‘&˜Ó&ˆCØ—H‘H˜\Ó*ˆEà6:·m²m˜DŸK™K×2Ò2ÈÐà—{‘{ 1‘~ˆô 0ØØØØØØ%Ø—n‘nØ"×AÑAô	
ˆð #×*Ñ*Ø×Ñ˜qÑ! <×#5Ñ#5°aÑ#8¸,×:LÑ:LÈQÑ:OÐR^×RdÑRdÐefÑRgÑ:gó
ˆð —m‘m KÓ0ˆØ×(Ñ(¨Ó5ˆà 
Ð+ˆÙØ˜�Ñ&ˆGàˆr:   r®   )rd   r¯   r°   Ú__doc__rW   r.   r±   r   r   r²   r³   r   r   r´   r¬   rµ   r¶   s   @r8   r¸   r¸     s  ø„ ñôRð '+Ø6:Ø37Ø15Ø$)Ø,1Ø59ñ~à×(Ñ(ð~ð ˜U‘Oð~ð ! ×!2Ñ!2Ñ3ð	~ð
 ˜u×/Ñ/Ñ0ð~ð ˜E×-Ñ-Ñ.ð~ð ˜D‘>ð~ð $ D™>ð~ð ! ×!1Ñ!1Ñ2ð~ð 
Øˆe�l‰l˜E %§,¡,Ñ/Ð/Ñ0Ø��u—|‘| U¨5¯<©<Ñ%8¸%ÀÇÁÈcÐ@QÑ:RÐRÑSÑTð	Vñ
÷~r:   r¸   )ÚeagerÚflash_attention_2c                   ó\   ‡ — e Zd Zˆ fd„Zdeej                     dej                  fd„Zˆ xZS )ÚGPTJMLPc                 ó  •— t         ‰| �  «        |j                  }t        j                  ||«      | _        t        j                  ||«      | _        t        |j                     | _	        t        j                  |j                  «      | _        y r­   )rV   rW   Ún_embdr   rn   Úfc_inÚfc_outr   Úactivation_functionÚactrZ   r]   rÁ   )rt   Úintermediate_sizerX   rf   rc   s       €r8   rW   zGPTJMLP.__init__£  se   ø€ Ü‰ÑÔØ—M‘Mˆ	ä—Y‘Y˜yÐ*;Ó<ˆŒ
Ü—i‘iÐ 1°9Ó=ˆŒä˜&×4Ñ4Ñ5ˆŒÜ—z‘z &×"4Ñ"4Ó5ˆ�r:   r–   r'   c                 óŽ   — | j                  |«      }| j                  |«      }| j                  |«      }| j                  |«      }|S r­   )rÔ   r×   rÕ   rÁ   )rt   r–   s     r8   r¬   zGPTJMLP.forward­  s@   € ØŸ
™
 =Ó1ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆØŸ™ ]Ó3ˆØÐr:   )	rd   r¯   r°   rW   r   r.   r±   r¬   rµ   r¶   s   @r8   rÑ   rÑ   ¢  s,   ø„ ô6ð X¨e×.?Ñ.?Ñ%@ð ÀU×EVÑEV÷ r:   rÑ   c                   óV  ‡ — e Zd Zdˆ fd„	Z	 	 	 	 	 	 	 ddeej                     dee   deej                     deej                     deej                     dee	   dee	   d	eej                     d
e
eej                     eeej                  eej                  df   f      f   fd„Zˆ xZS )Ú	GPTJBlockc                 ó.  •— t         ‰| �  «        |j                  �|j                  nd|j                  z  }t	        j
                  |j                  |j                  ¬«      | _        t        |j                     ||«      | _
        t        ||«      | _        y )Nry   ©Úeps)rV   rW   Ún_innerrÓ   r   Ú	LayerNormÚlayer_norm_epsilonÚln_1ÚGPTJ_ATTENTION_CLASSESÚ_attn_implementationÚattnrÑ   Úmlp)rt   rX   r`   Ú	inner_dimrc   s       €r8   rW   zGPTJBlock.__init__¶  so   ø€ Ü‰ÑÔØ&,§n¡nÐ&@�F—N’NÀaÈ&Ï-É-ÑFWˆ	Ü—L‘L §¡°F×4MÑ4MÔNˆŒ	Ü*¨6×+FÑ+FÑGÈÐPYÓZˆŒ	Ü˜9 fÓ-ˆ�r:   r–   r—   rŽ   rB   r�   r˜   r™   rš   r'   .c	           
      óÊ   — |}	| j                  |«      }| j                  ||||||||¬«      }
|
d   }|
dd  }| j                  |«      }||z   |	z   }|r|f|z   }|S |f|dd  z   }|S )N©r–   r—   rŽ   rB   r�   r˜   r™   rš   r   r    )râ   rå   ræ   )rt   r–   r—   rŽ   rB   r�   r˜   r™   rš   ÚresidualÚattn_outputsr’   r«   Úfeed_forward_hidden_statess                 r8   r¬   zGPTJBlock.forward½  s©   € ð !ˆØŸ	™	 -Ó0ˆØ—y‘yØ'Ø!Ø)Ø%ØØØ/Ø)ð !ó 	
ˆð # 1‘oˆØ˜q˜rÐ"ˆà%)§X¡X¨mÓ%<Ð"Ø#Ð&@Ñ@À8ÑKˆáØ$Ð&¨Ñ0ˆGð ˆð %Ð&¨°°¨Ñ4ˆGàˆr:   r­   r®   )rd   r¯   r°   rW   r   r.   r±   r   r²   r³   r   r   r´   r¬   rµ   r¶   s   @r8   rÛ   rÛ   µ  sö   ø„ õ.ð '+Ø6:Ø37Ø15Ø$)Ø,1Ø59ñ"à × 1Ñ 1Ñ2ð"ð ˜U‘Oð"ð ! ×!2Ñ!2Ñ3ð	"ð
 ˜u×/Ñ/Ñ0ð"ð ˜E×-Ñ-Ñ.ð"ð ˜D‘>ð"ð $ D™>ð"ð ! ×!1Ñ!1Ñ2ð"ð 
ˆu�U—\‘\Ñ" H¨U°5·<±<ÀÀu×GXÑGXÐZ]ÐG]ÑA^Ð3^Ñ-_Ñ$`Ð`Ñ	a÷"r:   rÛ   c                   óV   ‡ — e Zd ZdZeZdZdZdZdgZ	dZ
dZdZdZdZdZˆ fd„Zd„ Zˆ xZS )	ÚGPTJPreTrainedModelz†
    An abstract class to handle weights initialization and a simple interface for downloading and loading pretrained
    models.
    ÚtransformerTrÛ   Úpast_key_valuesFc                 ó$   •— t        ‰| �  |i |¤Ž y r­   )rV   rW   )rt   Úinputsr¼   rc   s      €r8   rW   zGPTJPreTrainedModel.__init__ô  s   ø€ Ü‰Ñ˜&Ð+ FÓ+r:   c                 ó  — t        |t        j                  f«      rm|j                  j                  j                  d| j                  j                  ¬«       |j                  �%|j                  j                  j                  «        yyt        |t        j                  «      rz|j                  j                  j                  d| j                  j                  ¬«       |j                  �2|j                  j                  |j                     j                  «        yyt        |t        j                  «      rJ|j                  j                  j                  «        |j                  j                  j                  d«       yy)zInitialize the weights.rÀ   )ÚmeanÚstdNr)   )Ú
isinstancer   rn   rÆ   ÚdataÚnormal_rX   Úinitializer_rangerU   Úzero_Ú	EmbeddingÚpadding_idxrà   Úfill_)rt   Úmodules     r8   Ú_init_weightsz!GPTJPreTrainedModel._init_weights÷  s  € ä�fœrŸy™y˜lÔ+ð �M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ�{‰{Ð&Ø—‘× Ñ ×&Ñ&Õ(ð 'ä˜¤§¡Ô-Ø�M‰M×Ñ×&Ñ&¨C°T·[±[×5RÑ5RÐ&ÔSØ×!Ñ!Ð-Ø—‘×"Ñ" 6×#5Ñ#5Ñ6×<Ñ<Õ>ð .ä˜¤§¡Ô-Ø�K‰K×Ñ×"Ñ"Ô$Ø�M‰M×Ñ×$Ñ$ SÕ)ð .r:   )rd   r¯   r°   rÍ   r!   Úconfig_classÚbase_model_prefixÚis_parallelizableÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attn_2Ú_supports_cache_classÚ_supports_quantized_cacheÚ_supports_static_cacheÚ!_supports_param_buffer_assignmentrW   rÿ   rµ   r¶   s   @r8   rî   rî   â  sW   ø„ ñð
 €LØ%ÐØÐØ&*Ð#Ø$˜ÐØ"3ÐØ!ÐØ ÐØ $ÐØ!ÐØ(-Ð%ô,ö*r:   rî   aG  
    This model is a PyTorch [torch.nn.Module](https://pytorch.org/docs/stable/nn.html#torch.nn.Module) sub-class. Use
    it as a regular PyTorch Module and refer to the PyTorch documentation for all matter related to general usage and
    behavior.

    Parameters:
        config ([`GPTJConfig`]): Model configuration class with all the parameters of the model.
            Initializing with a config file does not load the weights associated with the model, only the
            configuration. Check out the [`~PreTrainedModel.from_pretrained`] method to load the model weights.
aÙ  
    Args:
        input_ids (`torch.LongTensor` of shape `({0})`):
            Indices of input sequence tokens in the vocabulary.

            Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
            [`PreTrainedTokenizer.__call__`] for details.

            [What are input IDs?](../glossary#input-ids)
        attention_mask (`torch.FloatTensor` of shape `({0})`, *optional*):
            Mask to avoid performing attention on padding token indices. Mask values selected in `[0, 1]`:

            - 1 for tokens that are **not masked**,
            - 0 for tokens that are **masked**.

            [What are attention masks?](../glossary#attention-mask)
        token_type_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
            1]`:

            - 0 corresponds to a *sentence A* token,
            - 1 corresponds to a *sentence B* token.

            [What are token type IDs?](../glossary#token-type-ids)
        position_ids (`torch.LongTensor` of shape `({0})`, *optional*):
            Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
            config.n_positions - 1]`.

            [What are position IDs?](../glossary#position-ids)
        head_mask (`torch.FloatTensor` of shape `(num_attention_heads,)` or `(n_layer, num_attention_heads)`, *optional*):
            Mask to nullify selected heads of the self-attention modules. Mask values selected in `[0, 1]`:

            - 1 indicates the head is **not masked**,
            - 0 indicates the head is **masked**.

        inputs_embeds (`torch.FloatTensor` of shape `({0}, hidden_dim)`, *optional*):
            Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
            is useful if you want more control over how to convert *input_ids* indices into associated vectors than the
            model's internal embedding lookup matrix.
        past_key_values (`Cache` or `tuple(tuple(torch.FloatTensor))`, *optional*):
            Pre-computed hidden-states (key and values in the self-attention blocks and in the cross-attention
            blocks) that can be used to speed up sequential decoding. This typically consists in the `past_key_values`
            returned by the model at a previous stage of decoding, when `use_cache=True` or `config.use_cache=True`.

            Two formats are allowed:
            - a [`~cache_utils.Cache`] instance, see our
            [kv cache guide](https://huggingface.co/docs/transformers/en/kv_cache);
            - Tuple of `tuple(torch.FloatTensor)` of length `config.n_layers`, with each tuple having 2 tensors of
            shape `(batch_size, num_heads, sequence_length, embed_size_per_head)`). This is also known as the legacy
            cache format.

            The model will output the same cache format that is fed as input. If no `past_key_values` are passed, the
            legacy cache format will be returned.

            If `past_key_values` are used, the user can optionally input only the last `input_ids` (those that don't
            have their past key value states given to this model) of shape `(batch_size, 1)` instead of all `input_ids`
            of shape `(batch_size, sequence_length)`.
        output_attentions (`bool`, *optional*):
            Whether or not to return the attentions tensors of all attention layers. See `attentions` under returned
            tensors for more detail.
        output_hidden_states (`bool`, *optional*):
            Whether or not to return the hidden states of all layers. See `hidden_states` under returned tensors for
            more detail.
        return_dict (`bool`, *optional*):
            Whether or not to return a [`~utils.ModelOutput`] instead of a plain tuple.
        cache_position (`torch.LongTensor` of shape `(sequence_length)`, *optional*):
            Indices depicting the position of the input sequence tokens in the sequence. Contrarily to `position_ids`,
            this tensor is not affected by padding. It is used to update the cache in the correct position and to infer
            the complete sequence length.
a¯  
    This is an experimental feature and is a subject to change at a moment's notice. Uses a device map to distribute
    attention modules of the model across several devices. If no device map is given, it will evenly distribute blocks
    across all devices.

    Args:
        device_map (`Dict[int, list]`, *optional*):
            A dictionary that maps attention modules to devices. Note that the embedding module and LMHead are always
            automatically mapped to the first device (for esoteric reasons). That means that the first device should
            have fewer attention modules mapped to it than other devices. For reference, the GPT-J models have the
            following number of attention modules:

                - gpt-j-6B: 28

    Example:

    ```python
    # Here is an example of a device map on a machine with 4 GPUs using gpt-j-6B, which has a total of 28 attention modules:
    model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
    device_map = {
        0: [0, 1, 2, 3, 4, 5, 6],
        1: [7, 8, 9, 10, 11, 12, 13],
        2: [14, 15, 16, 17, 18, 19, 20],
        3: [21, 22, 23, 24, 25, 26, 27],
    }
    model.parallelize(device_map)
    ```
a6  
    Moves the model to CPU from a model parallel state.

    Example:

    ```python
    # On a 4 GPU machine with gpt-j-6B:
    model = GPTJForCausalLM.from_pretrained("EleutherAI/gpt-j-6B")
    device_map = {
        0: [0, 1, 2, 3, 4, 5, 6],
        1: [7, 8, 9, 10, 11, 12, 13],
        2: [14, 15, 16, 17, 18, 19, 20],
        3: [21, 22, 23, 24, 25, 26, 27],
    }
    model.parallelize(device_map)  # Splits the model across several devices
    model.deparallelize()  # Put the model back on cpu and cleans memory by calling torch.cuda.empty_cache()
    ```
z_The bare GPT-J Model transformer outputting raw hidden-states without any specific head on top.c                   óî  ‡ — e Zd Zˆ fd„Z ee«      dd„«       Z ee«      d„ «       Zd„ Z	d„ Z
 eej                  d«      «       eeeee¬«      	 	 	 	 	 	 	 	 	 	 	 	 ddeej*                     d	eeeeeej2                        f      d
eej4                     deej*                     deej*                     deej4                     deej4                     dee   dee   dee   dee   deej*                     deeef   fd„«       «       Z	 d d
ej2                  dej2                  dej2                  d	edef
d„Zed
ej2                  dededej@                  dejB                  dej2                  defd„«       Z"ˆ xZ#S )!Ú	GPTJModelc           	      óv  •— t         ‰| �  |«       |j                  | _        |j                  | _        t        j                  |j                  | j                  «      | _        t        j                  |j                  «      | _
        t        j                  t        |j                  «      D �cg c]  }t        ||¬«      ‘Œ c}«      | _        t        j                   | j                  |j"                  ¬«      | _        d| _        d | _        d| _        | j-                  «        |j.                  dk(  | _        y c c}w )N)r`   rÝ   FrÏ   )rV   rW   rÓ   rf   Ú
vocab_sizer   rû   ÚwterZ   Ú
embd_pdropÚdropÚ
ModuleListÚrangeÚn_layerrÛ   Úhrà   rá   Úln_fÚmodel_parallelÚ
device_mapÚgradient_checkpointingÚ	post_initrä   Ú_use_flash_attention_2)rt   rX   Úirc   s      €r8   rW   zGPTJModel.__init__�  sà   ø€ Ü‰Ñ˜Ô àŸ™ˆŒØ ×+Ñ+ˆŒÜ—<‘< × 1Ñ 1°4·>±>ÓBˆŒÜ—J‘J˜v×0Ñ0Ó1ˆŒ	Ü—‘ÌÈfÏnÉnÓH]Ö^À1¤	¨&¸AÖ >Ò^Ó_ˆŒÜ—L‘L §¡°V×5NÑ5NÔOˆŒ	ð $ˆÔØˆŒØ&+ˆÔ#ð 	�‰Ôà&,×&AÑ&AÐEXÑ&XˆÕ#ùò  _s   Â,D6c                 óª  — t        j                  dt        «       |€Dt        t	        | j
                  «      t        t        j                  j                  «       «      «      n|| _
        t        | j                  t	        | j
                  «      «       d| _        d| j                  j                  «       v rdn.dt        t        | j                  j                  «       «      «      z   | _        dt        t#        | j                  j                  «       «      «      z   | _        | j&                  j)                  | j                   «      | _        | j                  j+                  «       D ]E  \  }}|D ];  }dt        |«      z   }| j
                  |   j)                  |«      | j
                  |<   Œ= ŒG | j,                  j)                  | j$                  «      | _        y )Na6  `GPTJModel.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own `device_map` but it needs to be a dictionary module_name to device, so for instance {'h.0': 0, 'h.1': 1, ...}TÚcpuúcuda:)ÚwarningsÚwarnÚFutureWarningr   r}   r  r  r.   ÚcudaÚdevice_countr  r   r  ÚkeysÚstrÚminÚfirst_deviceÚmaxÚlast_devicer  r=   Úitemsr  )rt   r  ÚkÚvÚblockÚcuda_devices         r8   ÚparallelizezGPTJModel.parallelize¤  se  € ä�‰ðô ô	
ð NXÐM_ŒNœ3˜tŸv™v›;¬¬e¯j©j×.EÑ.EÓ.GÓ(HÔIÐeoð 	Œô 	˜$Ÿ/™/¬3¨t¯v©v«;Ô7Ø"ˆÔØ%*¨d¯o©o×.BÑ.BÓ.DÑ%D™EÈ'ÔTWÔX[Ð\`×\kÑ\k×\pÑ\pÓ\rÓXsÓTtÑJtˆÔØ"¤S¬¨T¯_©_×-AÑ-AÓ-CÓ)DÓ%EÑEˆÔØ—8‘8—;‘;˜t×0Ñ0Ó1ˆŒà—O‘O×)Ñ)Ó+ò 	>‰DˆAˆqØò >�Ø%¬¨A«Ñ.�Ø $§¡ u¡× 0Ñ 0°Ó =�—‘�u’ñ>ð	>ð
 —I‘I—L‘L ×!1Ñ!1Ó2ˆ�	r:   c                 óÈ  — t        j                  dt        «       d| _        d | _        d| _        d| _        | j                  j                  d«      | _        t        t        | j                  «      «      D ]-  }| j                  |   j                  d«      | j                  |<   Œ/ | j                  j                  d«      | _        t        j                  j                  «        y )Nú\Like `parallelize`, `deparallelize` is deprecated and will be removed in v5 of Transformers.Fr  )r   r!  r"  r  r  r(  r*  r  r=   r  r}   r  r  r.   r#  Úempty_cache)rt   Úindexs     r8   ÚdeparallelizezGPTJModel.deparallelize¾  s©   € ä�‰ØjÜô	
ð $ˆÔØˆŒØ!ˆÔØ ˆÔØ—8‘8—;‘;˜uÓ%ˆŒÜœ3˜tŸv™v›;Ó'ò 	4ˆEØ ŸF™F 5™M×,Ñ,¨UÓ3ˆD�F‰F�5ŠMð	4à—I‘I—L‘L Ó'ˆŒ	Ü�
‰
×ÑÕ r:   c                 ó   — | j                   S r­   ©r  ©rt   s    r8   Úget_input_embeddingszGPTJModel.get_input_embeddingsÎ  s   € Ø�x‰xˆr:   c                 ó   — || _         y r­   r7  ©rt   Únew_embeddingss     r8   Úset_input_embeddingszGPTJModel.set_input_embeddingsÑ  s	   € Ø!ˆ�r:   úbatch_size, sequence_length©Ú
checkpointÚoutput_typer   Úreal_checkpointÚ	input_idsrð   rŽ   Útoken_type_idsrB   r�   Úinputs_embedsr˜   r™   Úoutput_hidden_statesÚreturn_dictrš   r'   c                 ó¬  — |	�|	n| j                   j                  }	|
�|
n| j                   j                  }
|�|n| j                   j                  }|�|n| j                   j                  }|d u |d uz  rt        d«      ‚| j                  r%| j                  r|rt        j                  d«       d}|€| j                  |«      }d}|rIt        |t        «      s9d}|€t        «       }n*t        j                  |«      }t        j                  d«       |j                  d   }|€9|�|j!                  «       nd}t#        j$                  |||z   |j&                  ¬«      }|€|j)                  d«      }| j+                  |||||	«      }| j-                  || j                   j.                  «      }|}|�(|j1                  d	|«      }| j                  |«      }||z   }| j3                  |«      }d	||j5                  d	«      f}d }|	rd
nd }|
rd
nd }t7        | j8                  «      D �]Ï  \  }}| j:                  rÑt"        j<                  j?                  |j&                  «       |�T|j@                  jC                  |j&                  «      |_         |jD                  jC                  |j&                  «      |_"        |�|jC                  |j&                  «      }t        |t"        jF                  «      r|jC                  |j&                  «      }|
r||fz   }| j                  r3| j                  r'| jI                  |jJ                  |d ||||   ||	|«	      }n |||||||   ||	|¬«      }|d   }|du r|d   }|	r|||rdnd   fz   }| j:                  s�Œi| jL                  jO                  «       D ]J  \  }}||d	   k(  sŒdtQ        |«      z   | jR                  k7  sŒ+|jC                  dtQ        |dz   «      z   «      }ŒL �ŒÒ | jU                  |«      }|j1                  |«      }|
r||fz   }|r|nd }|r|jW                  «       }|stY        d„ ||||fD «       «      S t[        ||||¬«      S )Nz:You must specify exactly one of input_ids or inputs_embedszZ`use_cache=True` is incompatible with gradient checkpointing. Setting `use_cache=False`...FTzÿWe detected that you are passing `past_key_values` as a tuple of tuples. This is deprecated and will be removed in v4.47. Please convert your cache or use an appropriate `Cache` class (https://huggingface.co/docs/transformers/kv_cache#legacy-cache-format)r    r   ©r>   rF   © ré   r*   r  c              3   ó&   K  — | ]	  }|€Œ|–— Œ y ­wr­   rJ  )Ú.0r-  s     r8   ú	<genexpr>z$GPTJModel.forward.<locals>.<genexpr>o  s   è ø€ ò ØÐbcÑbo”ñùs   ‚Š)Úlast_hidden_staterð   r–   Ú
attentions).rX   r™   rF  r˜   Úuse_return_dictri   r  rÇ   ra   rb   r  rö   r   r   Úfrom_legacy_cacher@   Úget_seq_lengthr.   r/   r>   r    Ú_update_causal_maskÚget_head_maskr  r|   r  r{   Ú	enumerater  r  r#  Ú
set_deviceÚ	key_cacher=   Úvalue_cacher´   Ú_gradient_checkpointing_funcÚ__call__r  r+  r&  r*  r  Úto_legacy_cacheÚtupler   )rt   rC  rð   rŽ   rD  rB   r�   rE  r˜   r™   rF  rG  rš   Úreturn_legacy_cacheÚ
seq_lengthÚpast_key_values_lengthr‘   r–   Útoken_type_embedsÚoutput_shapeÚnext_decoder_cacheÚall_self_attentionsÚall_hidden_statesr  r.  r«   r,  r-  Ú
next_caches                                r8   r¬   zGPTJModel.forwardÔ  s›  € ð, 2CÐ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ð$DÑ È$Ï+É+×JjÑJjð 	ð "+Ð!6‘I¸D¿K¹K×<QÑ<Qˆ	Ø%0Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà˜Ð -°tÐ";Ò<ÜÐYÓZÐZà×&Ò&¨4¯=ª=ÙÜ×#Ñ#Øpôð "�	àÐ Ø ŸH™H YÓ/ˆMð $ÐÙœZ¨¼Ô?Ø"&ÐØÐ&Ü".£.‘ä".×"@Ñ"@ÀÓ"Q�Ü×#Ñ#ð^ôð #×(Ñ(¨Ñ+ˆ
ØÐ!ØIXÐId _×%CÑ%CÔ%EÐjkÐ"Ü"Ÿ\™\Ø&Ð(>ÀÑ(KÐTa×ThÑThôˆNð ÐØ)×3Ñ3°AÓ6ˆLà×.Ñ.Ø˜M¨>¸?ÐL]ó
ˆð ×&Ñ& y°$·+±+×2EÑ2EÓFˆ	Ø%ˆàÐ%Ø+×0Ñ0°°ZÓ@ˆNØ $§¡¨Ó 8ÐØ)Ð,=Ñ=ˆMàŸ	™	 -Ó0ˆØ˜J¨×(:Ñ(:¸2Ó(>Ð?ˆà!ÐÙ$5™b¸4ÐÙ"6™B¸DÐÜ! $§&¡&Ó)ó 5	O‰HˆAˆuà×"Ò"Ü—
‘
×%Ñ% m×&:Ñ&:Ô;ð #Ð.Ø0?×0IÑ0I×0LÑ0LÈ]×MaÑMaÓ0b�OÔ-Ø2A×2MÑ2M×2PÑ2PÐQ^×QeÑQeÓ2f�OÔ/ð Ð*Ø"-§.¡.°×1EÑ1EÓ"F�KÜ˜i¬¯©Ô6Ø )§¡¨]×-AÑ-AÓ B�IÙ#Ø$5¸Ð8HÑ$HÐ!à×*Ò*¨t¯}ª}Ø×;Ñ;Ø—N‘NØ!ØØØ Ø˜a‘LØØ%Ø"ó
‘ñ  Ø"/Ø.Ø#.Ø!-Ø'¨™lØ'Ø&7Ø#1ô	�ð $ A™JˆMØ˜DÑ Ø%,¨Q¡ZÐ"á Ø&9¸WÉ)ÁQÐYZÑ=[Ð<]Ñ&]Ð#ð ×"Ô"Ø ŸO™O×1Ñ1Ó3ò O‘D�A�qØ˜A˜b™E“z g´°A³Ñ&6¸$×:JÑ:JÓ&JØ(5×(8Ñ(8¸Ä3ÀqÈ1ÁuÃ:Ñ9MÓ(N™òOðg5	Oðn Ÿ	™	 -Ó0ˆà%×*Ñ*¨<Ó8ˆáØ 1°]Ð4DÑ DÐá+4Ñ'¸$ˆ
ÙØ#×3Ñ3Ó5ˆJáÜñ Ø)¨:Ð7HÐJ]Ð^ôó ð ô 'Ø+Ø&Ø+Ø*ô	
ð 	
r:   Úinput_tensorc           
      óÆ  — | j                   j                  dk(  r|�|dk(  j                  «       r|S y | j                   j                  dk(  r7t        |t        j
                  «      rt        |«      }t        |t        «      r|S |�|j                  «       nd}t        |t        «      }| j                   j                  dk(  r(|s&|s$t        j                  |||| j                  ¬«      ry |j                  |j                  }	}|j                  d   }
|r|j!                  «       }n1t        |t        j
                  «      r|j                  d   n||
z   dz   }| j#                  ||
|||	||j                  d   ¬	«      }| j                   j                  dk(  rQ|�O|j                  j$                  d
v r7|s5t	        j&                  |«      j(                  }t        j*                  ||«      }|S )NrÏ   rÀ   Úflex_attentionr   Úsdpa)rE  r_  Úis_trainingr    rF   )Úsequence_lengthÚtarget_lengthr,   r>   rš   Ú
batch_size)r#  Úxpu)rX   rä   Úanyrö   r.   r´   r#   r"   rR  r   r   Ú_ignore_causal_mask_sdparÇ   r,   r>   r@   Úget_max_cache_shapeÚ5_prepare_4d_causal_attention_mask_with_cache_positionÚtypeÚfinfor'  Ú_unmask_unattended)rt   rŽ   rf  rš   rð   r™   Úpast_seen_tokensÚusing_static_cacher,   r>   rk  rl  r‘   Ú	min_dtypes                 r8   rS  zGPTJModel._update_causal_mask{  sÖ  € ð �;‰;×+Ñ+Ð/BÒBØÐ)¨~ÀÑ/D×.IÑ.IÔ.KØ%Ð%ØØ�;‰;×+Ñ+Ð/?Ò?Ü˜.¬%¯,©,Ô7Ü!<¸^Ó!L�Ü˜.¬)Ô4Ø%Ð%ð
 @OÐ?Z˜?×9Ñ9Ô;Ð`aÐÜ'¨¼ÓEÐð �;‰;×+Ñ+¨vÒ5Ñ>PÑYjÜ%×>Ñ>ØØ*Ø'7Ø ŸM™Mõ	ð à$×*Ñ*¨L×,?Ñ,?ˆvˆØ&×,Ñ,¨QÑ/ˆÙØ+×?Ñ?ÓA‰Mô ˜n¬e¯l©lÔ;ð ×$Ñ$ RÒ(à%¨Ñ7¸!Ñ;ð ð ×PÑPØØ+Ø'ØØØ)Ø#×)Ñ)¨!Ñ,ð Qó 
ˆð �K‰K×,Ñ,°Ò6ØÐ*Ø×%Ñ%×*Ñ*¨oÑ=Ù%ô
 Ÿ™ EÓ*×.Ñ.ˆIÜ0×CÑCÀKÐQZÓ[ˆKàÐr:   rk  rl  r,   r>   rm  c                 ó˜  — | �| j                  «       dk(  r| }|S t        j                  |«      j                  }	t        j                  ||f|	||¬«      }|dk7  rt        j
                  |d¬«      }|t        j                  ||¬«      |j                  dd«      kD  z  }|dddd…dd…f   j                  |ddd«      }| �Œ|j                  «       }| j                  d   }
|dd…dd…dd…d|
…f   | dd…dddd…f   j                  |j                  «      z   }|dk(  }|dd…dd…dd…d|
…f   j                  ||	«      |dd…dd…dd…d|
…f<   |S )	a°  
        Creates a causal 4D mask of shape `(batch_size, 1, query_length, key_value_length)` from a 2D mask of shape
        `(batch_size, key_value_length)`, or if the input `attention_mask` is already 4D, do nothing.

        Args:
            attention_mask (`torch.Tensor`):
                A 2D attention mask of shape `(batch_size, key_value_length)` or a 4D attention mask of shape
                `(batch_size, 1, query_length, key_value_length)`.
            sequence_length (`int`):
                The sequence length being processed.
            target_length (`int`):
                The target length: when generating with static cache, the mask should be as long as the static cache,
                to account for the 0 padding, the part of the cache that is not filled yet.
            dtype (`torch.dtype`):
                The dtype to use for the 4D attention mask.
            device (`torch.device`):
                The device to place the 4D attention mask on.
            cache_position (`torch.Tensor`):
                Indices depicting the position of the input sequence tokens in the sequence.
            batch_size (`torch.Tensor`):
                Batch size.
        Nry   )Ú
fill_valuer,   r>   r    )ÚdiagonalrI  rF   r   )r&   r.   rt  r'  ÚfullÚtriur/   rÈ   ÚexpandÚcloner@   r=   r>   Úmasked_fill)rŽ   rk  rl  r,   r>   rš   rm  r¼   r‘   rx  Úmask_lengthÚpadding_masks               r8   rr  z?GPTJModel._prepare_4d_causal_attention_mask_with_cache_positionÁ  sy  € ðD Ð%¨.×*<Ñ*<Ó*>À!Ò*Cà(ˆKð* Ðô' Ÿ™ EÓ*×.Ñ.ˆIÜŸ*™*Ø  -Ð0¸YÈeÐ\bôˆKð  !Ò#Ü#Ÿj™j¨¸qÔA�Øœ5Ÿ<™<¨¸fÔEÈ×H^ÑH^Ð_aÐcdÓHeÑeÑeˆKØ% d¨D²!²QÐ&6Ñ7×>Ñ>¸zÈ1ÈbÐRTÓUˆKØÐ)Ø)×/Ñ/Ó1�Ø,×2Ñ2°2Ñ6�Ø*ª1ªa²°L°[°LÐ+@ÑAÀNÒSTÐVZÐ\`ÒbcÐScÑDd×DgÑDgØ×&Ñ&óEñ  �ð  ,¨qÑ0�Ø5@ÂÂAÂqÈ,È;È,ÐAVÑ5W×5cÑ5cØ  )ó6�šAšq¢! \ k \Ð1Ñ2ð Ðr:   r­   ©NNNNNNNNNNNN)F)$rd   r¯   r°   rW   r   ÚPARALLELIZE_DOCSTRINGr0  ÚDEPARALLELIZE_DOCSTRINGr5  r9  r=  r   ÚGPTJ_INPUTS_DOCSTRINGÚformatr   Ú_CHECKPOINT_FOR_DOCr   Ú_CONFIG_FOR_DOCÚ_REAL_CHECKPOINT_FOR_DOCr   r.   r²   r   r   r   r´   r±   r³   r¬   rS  ÚstaticmethodÚintr,   r>   rr  rµ   r¶   s   @r8   r  r  ‹  su  ø„ ô
Yñ( Ð/Ó0ò3ó 1ð3ñ2 Ð1Ó2ñ!ó 3ð!òò"ñ +Ð+@×+GÑ+GÐHeÓ+fÓgÙØ&Ø+Ø$Ø0ô	ð 15ØNRØ6:Ø59Ø37Ø15Ø59Ø$(Ø,0Ø/3Ø&*Ø59ñ]
à˜E×,Ñ,Ñ-ð]
ð " %¨¨u°U¸5¿<¹<Ñ5HÑ/IÐ(IÑ"JÑKð]
ð ! ×!2Ñ!2Ñ3ð	]
ð
 ! ×!1Ñ!1Ñ2ð]
ð ˜u×/Ñ/Ñ0ð]
ð ˜E×-Ñ-Ñ.ð]
ð   × 1Ñ 1Ñ2ð]
ð ˜D‘>ð]
ð $ D™>ð]
ð ' t™nð]
ð ˜d‘^ð]
ð ! ×!1Ñ!1Ñ2ð]
ð 
ˆuÐ-Ð-Ñ	.ò]
óó hð]
ðL #(ñDàŸ™ðDð —l‘lðDð Ÿ™ð	Dð
 ðDð  óDðL ð7ØŸ™ð7àð7ð ð7ð �{‰{ð	7ð
 —‘ð7ð Ÿ™ð7ð ò7ó ô7r:   r  zK
    The GPT-J Model transformer with a language modeling head on top.
    c            !       ó¬  ‡ — e Zd ZdgZˆ fd„Z ee«      dd„«       Z ee«      d„ «       Z	d„ Z
d„ Z eej                  d«      «       eeeee¬«      	 	 	 	 	 	 	 	 	 	 	 	 	 dd	eej,                     d
eeeeeej4                        f      deej6                     deej,                     deej,                     deej6                     deej6                     deej,                     dee   dee   dee   dee   deej,                     deeef   fd„«       «       Zed
eeej4                        dej4                  deeej4                        fd„«       Zˆ xZ S )ÚGPTJForCausalLMzlm_head.weightc                 óÞ   •— t         ‰| �  |«       t        |«      | _        t	        j
                  |j                  |j                  «      | _        d| _	        d | _
        | j                  «        y ©NF)rV   rW   r  rï   r   rn   rÓ   r  Úlm_headr  r  r  ©rt   rX   rc   s     €r8   rW   zGPTJForCausalLM.__init__  sV   ø€ Ü‰Ñ˜Ô Ü$ VÓ,ˆÔÜ—y‘y §¡°×0AÑ0AÓBˆŒð $ˆÔØˆŒð 	�‰Õr:   c                 ó  — t        j                  dt        «       |€Nt        t	        | j
                  j                  «      t        t        j                  j                  «       «      «      n|| _        t        | j                  t	        | j
                  j                  «      «       | j
                  j                  | j                  «       | j                  j                  | j
                  j                   «      | _        d| _        y )NaT  `GPTJForCausalLM.parallelize` is deprecated and will be removed in v5 of Transformers, you should load your model with `device_map='balanced'` in the call to `from_pretrained`. You can also provide your own `device_map` but it needs to be a dictionary module_name to device, so for instance {'transformer.h.0': 0, 'transformer.h.1': 1, ...}T)r   r!  r"  r   r}   rï   r  r  r.   r#  r$  r  r   r0  r‘  r=   r(  r  )rt   r  s     r8   r0  zGPTJForCausalLM.parallelize  sº   € ä�‰ð-ô ô	
ð Ð!ô œ3˜t×/Ñ/×1Ñ1Ó2´E¼%¿*¹*×:QÑ:QÓ:SÓ4TÔUàð 	Œô
 	˜$Ÿ/™/¬3¨t×/?Ñ/?×/AÑ/AÓ+BÔCØ×Ñ×$Ñ$ T§_¡_Ô5Ø—|‘|—‘ t×'7Ñ'7×'DÑ'DÓEˆŒØ"ˆÕr:   c                 ó6  — t        j                  dt        «       | j                  j	                  «        | j                  j                  d«      | _        | j                  j                  d«      | _        d| _        t        j                  j                  «        y )Nr2  r  F)r   r!  r"  rï   r5  r=   r‘  r  r.   r#  r3  r8  s    r8   r5  zGPTJForCausalLM.deparallelize%  sm   € ä�‰ØjÜô	
ð 	×Ñ×&Ñ&Ô(Ø×+Ñ+×.Ñ.¨uÓ5ˆÔØ—|‘|—‘ uÓ-ˆŒØ#ˆÔÜ�
‰
×ÑÕ r:   c                 ó   — | j                   S r­   ©r‘  r8  s    r8   Úget_output_embeddingsz%GPTJForCausalLM.get_output_embeddings1  s   € Ø�|‰|Ðr:   c                 ó   — || _         y r­   r–  r;  s     r8   Úset_output_embeddingsz%GPTJForCausalLM.set_output_embeddings4  s	   € Ø%ˆ�r:   r>  r?  rC  rð   rŽ   rD  rB   r�   rE  Úlabelsr˜   r™   rF  rG  rš   r'   c                 ó   — |�|n| j                   j                  }| j                  ||||||||	|
|||¬«      }|d   }| j                  rbt        j
                  j                  | j                  j                  «       |j                  | j                  j                  j                  «      }| j                  |«      j                  t        j                  «      }d}|�`|j                  |j                  «      } | j                  ||fd| j                   j                  i|¤Ž}|j                  |j                  «      }|s|f|dd z   }|�|f|z   S |S t!        |||j"                  |j$                  |j&                  ¬«      S )a³  
        labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
            Labels for language modeling. Note that the labels **are shifted** inside the model, i.e. you can set
            `labels = input_ids` Indices are selected in `[-100, 0, ..., config.vocab_size]` All labels set to `-100`
            are ignored (masked), the loss is only computed for labels in `[0, ..., config.vocab_size]`
        N)rð   rŽ   rD  rB   r�   rE  r˜   r™   rF  rG  rš   r   r  r    ©ÚlossÚlogitsrð   r–   rO  )rX   rP  rï   r  r.   r#  rV  r(  r=   r‘  rÆ   r>   rk   Úloss_functionr  r,   r   rð   r–   rO  )rt   rC  rð   rŽ   rD  rB   r�   rE  rš  r˜   r™   rF  rG  rš   r¼   Útransformer_outputsr–   Ú	lm_logitsr�  Úoutputs                       r8   r¬   zGPTJForCausalLM.forward7  s�  € ð< &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà"×.Ñ.ØØ+Ø)Ø)Ø%ØØ'ØØ/Ø!5Ø#Ø)ð /ó 
Ðð ,¨AÑ.ˆð ×ÒÜ�J‰J×!Ñ! $×"2Ñ"2×"?Ñ"?Ô@Ø)×,Ñ,¨T¯\©\×-@Ñ-@×-GÑ-GÓHˆMð
 —L‘L Ó/×2Ñ2´5·=±=ÓAˆ	àˆØÐà—Y‘Y˜y×/Ñ/Ó0ˆFà%�4×%Ñ%ØØñð  Ÿ;™;×1Ñ1ðð ñ	ˆDð —7‘7˜=×.Ñ.Ó/ˆDáØ�\Ð$7¸¸Ð$;Ñ;ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä%ØØØ/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r:   Úbeam_idxc                 ó,   ‡— t        ˆfd„| D «       «      S )a  
        This function is used to re-order the `past_key_values` cache if [`~PretrainedModel.beam_search`] or
        [`~PretrainedModel.beam_sample`] is called. This is required to match `past_key_values` with the correct
        beam_idx at every generation step.
        c              3   óF   •K  — | ]  }t        ˆfd „|D «       «      –— Œ y­w)c              3   ót   •K  — | ]/  }|j                  d ‰j                  |j                  «      «      –— Œ1 y­w)r   N)Úindex_selectr=   r>   )rL  Ú
past_stater£  s     €r8   rM  z;GPTJForCausalLM._reorder_cache.<locals>.<genexpr>.<genexpr>•  s.   øè ø€ ÒjÐQ[�*×)Ñ)¨!¨X¯[©[¸×9JÑ9JÓ-K×LÑjùs   ƒ58N©r\  )rL  r—   r£  s     €r8   rM  z1GPTJForCausalLM._reorder_cache.<locals>.<genexpr>”  s%   øè ø€ ò 
àô ÓjÐ_iÔj×jñ
ùs   ƒ!r©  )rð   r£  s    `r8   Ú_reorder_cachezGPTJForCausalLM._reorder_cache‹  s   ø€ ô ó 
à-ô
ó 
ð 	
r:   r­   )NNNNNNNNNNNNN)!rd   r¯   r°   Ú_tied_weights_keysrW   r   r„  r0  r…  r5  r—  r™  r   r†  r‡  r   rˆ  r   r‰  rŠ  r   r.   r²   r   r   r   r´   r±   r³   r¬   r‹  rª  rµ   r¶   s   @r8   rŽ  rŽ  ý  s1  ø„ ð +Ð+Ðô
ñ Ð/Ó0ò#ó 1ð#ñ$ Ð1Ó2ñ	!ó 3ð	!òò&ñ +Ð+@×+GÑ+GÐHeÓ+fÓgÙØ&Ø*Ø$Ø0ô	ð 15ØNRØ6:Ø59Ø37Ø15Ø59Ø-1Ø$(Ø,0Ø/3Ø&*Ø59ñK
à˜E×,Ñ,Ñ-ðK
ð " %¨¨u°U¸5¿<¹<Ñ5HÑ/IÐ(IÑ"JÑKðK
ð ! ×!2Ñ!2Ñ3ð	K
ð
 ! ×!1Ñ!1Ñ2ðK
ð ˜u×/Ñ/Ñ0ðK
ð ˜E×-Ñ-Ñ.ðK
ð   × 1Ñ 1Ñ2ðK
ð ˜×)Ñ)Ñ*ðK
ð ˜D‘>ðK
ð $ D™>ðK
ð ' t™nðK
ð ˜d‘^ðK
ð ! ×!1Ñ!1Ñ2ðK
ð  
ˆuÐ,Ð,Ñ	-ò!K
óó hðK
ðZ ð
Ø˜u U§\¡\Ñ2Ñ3ð
Ø?D¿|¹|ð
à	ˆu�U—\‘\Ñ"Ñ	#ò
ó ô
r:   rŽ  aã  
    The GPT-J Model transformer with a sequence classification head on top (linear layer).

    [`GPTJForSequenceClassification`] uses the last token in order to do the classification, as other causal models
    (e.g. GPT, GPT-2, GPT-Neo) do.

    Since it does classification on the last token, it requires to know the position of the last token. If a
    `pad_token_id` is defined in the configuration, it finds the last token that is not a padding token in each row. If
    no `pad_token_id` is defined, it simply takes the last value in each row of the batch. Since it cannot guess the
    padding tokens when `inputs_embeds` are passed instead of `input_ids`, it does the same (take the last value in
    each row of the batch).
    c                   óÄ  ‡ — e Zd Zˆ fd„Z eej                  d«      «       edee	e
¬«      	 	 	 	 	 	 	 	 	 	 	 	 ddeej                     deeeej                           deej                      deej                     d	eej                     d
eej                      deej                      deej                     dee   dee   dee   dee   deeef   fd„«       «       Zˆ xZS )ÚGPTJForSequenceClassificationc                 ó  •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  | j                  d¬«      | _        d| _	        d | _
        | j                  «        y )NFrT   )rV   rW   Ú
num_labelsr  rï   r   rn   rÓ   Úscorer  r  r  r’  s     €r8   rW   z&GPTJForSequenceClassification.__init__ª  sc   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ$ VÓ,ˆÔÜ—Y‘Y˜vŸ}™}¨d¯o©oÀEÔJˆŒ
ð $ˆÔØˆŒð 	�‰Õr:   r>  z4ydshieh/tiny-random-gptj-for-sequence-classificationr?  rC  rð   rŽ   rD  rB   r�   rE  rš  r˜   r™   rF  rG  r'   c                 ó   — |�|n| j                   j                  }| j                  ||||||||	|
||¬«      }|d   }| j                  |«      }|�|j                  d   }n|j                  d   }| j                   j
                  €|dk7  rt        d«      ‚| j                   j
                  €d}nÃ|�“|| j                   j
                  k7  j                  |j                  t        j                  «      }t        j                  |j                  d   |j                  t        j                  ¬«      }||z  j                  d«      }n.d}t        j                  | j                  j                   › d�«       |t        j                  ||j                  ¬	«      |f   }d}|��¢|j                  |j                  «      }| j                   j"                  €�| j$                  dk(  rd
| j                   _        nl| j$                  dkD  rL|j&                  t        j(                  k(  s|j&                  t        j*                  k(  rd| j                   _        nd| j                   _        | j                   j"                  d
k(  rIt-        «       }| j$                  dk(  r& ||j/                  «       |j/                  «       «      }nŒ |||«      }n‚| j                   j"                  dk(  r=t1        «       } ||j3                  d| j$                  «      |j3                  d«      «      }n,| j                   j"                  dk(  rt5        «       } |||«      }|s|f|dd z   }|�|f|z   S |S t7        |||j8                  |j:                  |j<                  ¬«      S )a�  
        labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
            config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
            `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
        N)
rð   rŽ   rD  rB   r�   rE  r˜   r™   rF  rG  r   r    z=Cannot handle batch sizes > 1 if no padding token is defined.rF   )r>   r,   zŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`rI  Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrœ  )rX   rP  rï   r°  r@   Úpad_token_idri   r=   r>   r.   Úint32r/   Úargmaxra   rb   rc   rd   Úproblem_typer¯  r,   ÚlongrŒ  r	   Úsqueezer   r|   r   r   rð   r–   rO  )rt   rC  rð   rŽ   rD  rB   r�   rE  rš  r˜   r™   rF  rG  r   r–   rž  rm  Úlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesÚpooled_logitsr�  Úloss_fctr¢  s                           r8   r¬   z%GPTJForSequenceClassification.forward·  s$  € ð8 &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà"×.Ñ.ØØ+Ø)Ø)Ø%ØØ'ØØ/Ø!5Ø#ð /ó 
Ðð ,¨AÑ.ˆØ—‘˜MÓ*ˆàÐ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×#Ñ#Ð+°
¸a²ÜÐ\Ó]Ð]Ø�;‰;×#Ñ#Ð+Ø!#ÑØÐ"à%¨¯©×)AÑ)AÑA×EÑEÀfÇmÁmÔUZ×U`ÑU`ÓaˆLÜ!ŸL™L¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÔfˆMØ"/°,Ñ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|™|¨J¸v¿}¹}ÔMÐOaÐaÑbˆàˆØÑØ—Y‘Y˜}×3Ñ3Ó4ˆFØ�{‰{×'Ñ'Ð/Ø—?‘? aÒ'Ø/;�D—K‘KÕ,Ø—_‘_ qÒ(¨f¯l©l¼e¿j¹jÒ.HÈFÏLÉLÔ\a×\eÑ\eÒLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ò7Ü"›9�Ø—?‘? aÒ'Ù# M×$9Ñ$9Ó$;¸V¿^¹^Ó=MÓN‘Dá# M°6Ó:‘DØ—‘×)Ñ)Ð-JÒJÜ+Ó-�Ù × 2Ñ 2°2°t·±Ó GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÒIÜ,Ó.�Ù ¨vÓ6�ÙØ#Ð%Ð(;¸A¸BÐ(?Ñ?ˆFØ)-Ð)9�T�G˜fÑ$ÐE¸vÐEä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ô
ð 	
r:   rƒ  )rd   r¯   r°   rW   r   r†  r‡  r   r   r‰  rŠ  r   r.   r²   r   r´   r±   r³   r   r¬   rµ   r¶   s   @r8   r­  r­  š  sx  ø„ ô ñ +Ð+@×+GÑ+GÐHeÓ+fÓgÙØIØ4Ø$Ø0ô	ð 15Ø@DØ6:Ø59Ø37Ø15Ø59Ø-1Ø$(Ø,0Ø/3Ø&*ñ_
à˜E×,Ñ,Ñ-ð_
ð " %¨¨e¯l©lÑ(;Ñ"<Ñ=ð_
ð ! ×!2Ñ!2Ñ3ð	_
ð
 ! ×!1Ñ!1Ñ2ð_
ð ˜u×/Ñ/Ñ0ð_
ð ˜E×-Ñ-Ñ.ð_
ð   × 1Ñ 1Ñ2ð_
ð ˜×)Ñ)Ñ*ð_
ð ˜D‘>ð_
ð $ D™>ð_
ð ' t™nð_
ð ˜d‘^ð_
ð 
ˆuÐ6Ð6Ñ	7ò_
óó hô_
r:   r­  zî
    The GPT-J Model transformer with a span classification head on top for extractive question-answering tasks like
    SQuAD (a linear layers on top of the hidden-states output to compute `span start logits` and `span end logits`).
    c                   ó¬  ‡ — e Zd Zˆ fd„Z eej                  d«      «       eee	e
e¬«      	 	 	 	 	 	 	 	 	 	 	 ddeej                     deej                     deej                     deej                     deej                     d	eej                     d
eej                     deej                     dee   dee   dee   deee	f   fd„«       «       Zˆ xZS )ÚGPTJForQuestionAnsweringc                 ó   •— t         ‰| �  |«       |j                  | _        t        |«      | _        t        j                  |j                  |j                  «      | _        d| _	        d | _
        | j                  «        y r�  )rV   rW   r¯  r  rï   r   rn   re   Ú
qa_outputsr  r  r  r’  s     €r8   rW   z!GPTJForQuestionAnswering.__init__(  se   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒÜ$ VÓ,ˆÔÜŸ)™) F×$6Ñ$6¸×8IÑ8IÓJˆŒð $ˆÔØˆŒð 	�‰Õr:   r>  r?  rC  rŽ   rD  rB   r�   rE  Ústart_positionsÚend_positionsr™   rF  rG  r'   c                 óŒ  — |�|n| j                   j                  }| j                  |||||||	|
|¬«	      }|d   }| j                  |«      }|j	                  dd¬«      \  }}|j                  d«      j                  «       }|j                  d«      j                  «       }d}|�é|�çt        |j                  «       «      dkD  r*|j                  d«      j                  |j                  «      }t        |j                  «       «      dkD  r*|j                  d«      j                  |j                  «      }|j                  d«      }|j                  d|«      }|j                  d|«      }t        |¬«      } |||«      } |||«      }||z   dz  }|s||f|dd z   }|�|f|z   S |S t        ||||j                  |j                  ¬	«      S )
a  
        start_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the start of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        end_positions (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
            Labels for position (index) of the end of the labelled span for computing the token classification loss.
            Positions are clamped to the length of the sequence (`sequence_length`). Position outside of the sequence
            are not taken into account for computing the loss.
        N)rŽ   rD  rB   r�   rE  r™   rF  rG  r   r    rF   r-   )Úignore_indexr*   )r�  Ústart_logitsÚ
end_logitsr–   rO  )rX   rP  rï   rÃ  r¢   rº  r„   r}   r{   r=   r>   Úclampr   r   r–   rO  )rt   rC  rŽ   rD  rB   r�   rE  rÄ  rÅ  r™   rF  rG  r«   Úsequence_outputrž  rÈ  rÉ  Ú
total_lossÚignored_indexr¿  Ú
start_lossÚend_lossr¢  s                          r8   r¬   z GPTJForQuestionAnswering.forward5  sæ  € ð> &1Ð%<‘kÀ$Ç+Á+×B]ÑB]ˆà×"Ñ"ØØ)Ø)Ø%ØØ'Ø/Ø!5Ø#ð #ó 

ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ó#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÐ&¨=Ð+Dä�?×'Ñ'Ó)Ó*¨QÒ.Ø"1×"9Ñ"9¸"Ó"=×"@Ñ"@À×ATÑATÓ"U�Ü�=×%Ñ%Ó'Ó(¨1Ò,Ø -× 5Ñ 5°bÓ 9× <Ñ <¸Z×=NÑ=NÓ O�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÔCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJáØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ð/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ô
ð 	
r:   )NNNNNNNNNNN)rd   r¯   r°   rW   r   r†  r‡  r   rˆ  r   r‰  rŠ  r   r.   r²   r±   r³   r   r   r¬   rµ   r¶   s   @r8   rÁ  rÁ     s^  ø„ ôñ +Ð+@×+GÑ+GÐHeÓ+fÓgÙØ&Ø0Ø$Ø0ô	ð 15Ø6:Ø59Ø37Ø15Ø59Ø6:Ø48Ø,0Ø/3Ø&*ñH
à˜E×,Ñ,Ñ-ðH
ð ! ×!2Ñ!2Ñ3ðH
ð ! ×!1Ñ!1Ñ2ð	H
ð
 ˜u×/Ñ/Ñ0ðH
ð ˜E×-Ñ-Ñ.ðH
ð   × 1Ñ 1Ñ2ðH
ð " %×"2Ñ"2Ñ3ðH
ð   × 0Ñ 0Ñ1ðH
ð $ D™>ðH
ð ' t™nðH
ð ˜d‘^ðH
ð 
ˆuÐ2Ð2Ñ	3òH
óó hôH
r:   rÁ  )rŽ  rÁ  r­  r  rî   )QrÍ   r   Útypingr   r   r   r.   Útorch.fxÚtorch.utils.checkpointr   Útorch.nnr   r   r	   Úactivationsr   Úcache_utilsr   r   r   Ú
generationr   Úmodeling_attn_mask_utilsr   Úmodeling_flash_attention_utilsr   r   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úutilsr   r   r   r   r   r   Úutils.model_parallel_utilsr   r   Úconfiguration_gptjr!   Ú!torch.nn.attention.flex_attentionr"   Úintegrations.flex_attentionr#   r$   Ú
get_loggerrd   ra   rˆ  rŠ  r‰  rŒ  r´   r9   ÚfxÚwraprC   rL   rP   ÚModulerR   r¸   rã   rÑ   rÛ   rî   ÚGPTJ_START_DOCSTRINGr†  r„  r…  r  rŽ  r­  rÁ  Ú__all__rJ  r:   r8   ú<module>ræ     si  ðñ ã ß )Ñ )ã Û Û Ý ß AÑ Aå !ß ;Ñ ;Ý )Ý >ß h÷ó õ .÷÷ ÷ LÝ *ñ  Ô!Ý;åJñ ÔÝJð 
ˆ×	Ñ	˜HÓ	%€à<Ð Ø0Ð Ø€ðP¨ð P°3ð P¸5¿<¹<ó Pð ‡�‡�ñWó ðWð˜Ÿ™ð ¨¯©ó ð= §¡ð =°E·L±Lð =ÀuÇ|Á|ð =ÐX]×XdÑXdó =ôn�B—I‘Iô nôbM˜-ô Mðb Ø,ñÐ ôˆb�i‰iô ô&*�—	‘	ô *ôZ#*˜/ô #*ðL	Ð ðEÐ ðNÐ ð:Ð ñ( ØeØóôkÐ#ó kó	ðkñ\ ðð ó	ôT
Ð)¨?ó T
óðT
ñn ðð óôt
Ð$7ó t
óðt
ñn ðð óô]
Ð2ó ]
óð]
ò@�r:   