ó
    ˆŽ¡j©P  ã                   óH  • S SK Jr  S SKrS SKJr  SSKJr  SSKJrJ	r	  SSK
Jr  SSKJrJr  SSKJr  SS	KJrJrJr  SS
KJrJr  SSKJrJr  SSKJrJr  SSKJr  SSK J!r!J"r"  SSK#J$r$  SSK%J&r&J'r'J(r(  SSK)J*r*  SSK+J,r,   " S S\RZ                  5      r.S r/\" S5      S7S j5       r0S\Rb                  S\2S\Rb                  4S jr3 S8S\RZ                  S\Rb                  S\Rb                  S \Rb                  S!\Rb                  S-  S"\4S#\4S$\\!   4S% jjr5\" \05       " S& S'\RZ                  5      5       r6 " S( S)\RZ                  5      r7 " S* S+\5      r8\" " S, S-\5      5       r9\" " S. S/\95      5       r:\" " S0 S1\9\5      5       r; " S2 S3\\95      r< " S4 S5\\95      r=/ S6Qr>g)9é    )ÚCallableNé   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstring)Údeprecate_kwarg)Úcan_return_tupleÚmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	PhiConfigc            	       óÒ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\\	R                  \4   4S jj5       5       r\	R                  " 5       \S	 5       5       rS
rU =r$ )ÚPhiRotaryEmbeddingé!   Údevicez5.18)ÚversionÚconfigc                 óÄ  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        U R
                  R                  S   U l        U R                  nU R                  S:w  a  [        U R                     nU" U R
                  U5      u  o@l
        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚdefaultF)Ú
persistent)ÚsuperÚ__init__Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr$   Úrope_parametersr&   Úcompute_default_rope_parametersr   Úattention_scalingÚnnÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)Úselfr$   r"   Úrope_init_fnr3   Ú	__class__s        €Ú²C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/phi/modeling_phi.pyr*   ÚPhiRotaryEmbedding.__init__"   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÓ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕó    Úreturnc           	      óR  • U R                   S   nU R                   R                  SS5      n[        U SS5      =(       d    U R                  U R                  -  n[        XT-  5      nSnSU[        R                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )	a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   é   ©Údtype)
r.   ÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintÚtorchÚarangeÚfloatÚto)	r$   r"   ÚkwargsÚbaser?   r@   ÚdimÚattention_factorr3   s	            r9   r/   Ú2PhiRotaryEmbedding.compute_default_rope_parameters3   s£   € ð ×%Ñ% lÑ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×h¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ñ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r;   c                 ó  • U R                   S S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        USS9   X4-  R                  SS	5      n[        R                  " Xf4SS
9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR                  UR                   S9W	R                  UR                   S94$ ! , (       d  f       N@= f)Nr   éÿÿÿÿr   )rC   r"   ÚmpsÚcpuF)Údevice_typeÚenabledrA   ©rO   rB   )r3   ÚexpandÚshaperL   rI   rK   r"   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosr0   ÚsinrC   )
r6   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrV   ÚfreqsÚembr`   ra   s
             r9   ÚforwardÚPhiRotaryEmbedding.forwardI   s;  € ð �M‰M˜$¢ 4˜-Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÐtð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   ÃA%E7Å7
F)r0   r$   r3   r,   r5   r-   r&   ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r   r*   ÚstaticmethodÚtuplerI   ÚTensorrK   r/   Úno_gradr   rh   Ú__static_attributes__Ú__classcell__©r8   s   @r9   r    r    !   sŒ   ø† Ù�X vÑ.ñO˜y÷ Oó /ðOð  Ù�X vÑ.ñ5°	ð 5ÐUZÐ[`×[gÑ[gÐinÐ[nÑUoô 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ö<r;   r    c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..NrS   rA   rX   )rZ   rI   r_   )rb   Úx1Úx2s      r9   Úrotate_halfry   \   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r;   Úrotary_pos_embc                 ó˜   • UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXV4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezery   )ÚqÚkr`   ra   Úunsqueeze_dimÚq_embedÚk_embeds          r9   Úapply_rotary_pos_embr‚   c   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØÐÐr;   Úhidden_statesÚn_repr<   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)rZ   rY   Úreshape)rƒ   r„   ÚbatchÚnum_key_value_headsÚslenr@   s         r9   Ú	repeat_kvrŠ   }   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr;   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrM   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )NrA   r   rS   )rO   rC   )ÚpÚtrainingr   )rŠ   Únum_key_value_groupsrI   Úmatmulr^   r1   Ú
functionalÚsoftmaxÚfloat32rL   rC   r‘   r”   Ú
contiguous)r‹   rŒ   r�   rŽ   r�   r�   r‘   rM   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r9   Úeager_attention_forwardrŸ   ‰   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r;   c                   óú   ^ • \ rS rSrSrS\S\4U 4S jjr SS\R                  S\
\R                  \R                  4   S	\R                  S-  S
\S-  S\
\R                  \R                  S-  4   4
S jjrSrU =r$ )ÚPhiAttentioné¢   z=Multi-headed attention from 'Attention Is All You Need' paperr$   Ú	layer_idxc                 óŠ  >• [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  SS9U l        [        R                  " UR
                  UR                  U R                  -  SS9U l        [        R                  " UR
                  UR                  U R                  -  SS9U l        [        R                  " UR                  U R                  -  UR
                  SS9U l        ['        U R                  UR(                  S   -  5      U l        UR,                  U l        U R,                  (       ay  [        R.                  " UR
                  UR                  -  UR0                  SS9U l        [        R.                  " UR
                  UR                  -  UR0                  SS9U l        g g )Nr@   g      à¿T©Úbiasr?   )ÚepsÚelementwise_affine)r)   r*   r$   r£   rE   rF   rG   r@   rˆ   r•   r�   Úattention_dropoutÚ	is_causalr1   ÚLinearÚq_projÚk_projÚv_projÚdenserH   r.   Úrotary_ndimsÚqk_layernormÚ	LayerNormÚlayer_norm_epsÚq_layernormÚk_layernorm©r6   r$   r£   r8   s      €r9   r*   ÚPhiAttention.__init__¦   sÈ  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÑjˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÑjˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÑjˆŒÜ—Y’Y˜v×9Ñ9¸D¿M¹MÑIÈ6×K]ÑK]ÐdhÑiˆŒ
Ü §¡°×0FÑ0FÐG^Ñ0_Ñ _Ó`ˆÔØ"×/Ñ/ˆÔØ××Ü!Ÿ|š|Ø×"Ñ" f×&@Ñ&@Ñ@Àf×F[ÑF[Ðptñ ˆDÔô  "Ÿ|š|Ø×"Ñ" f×&@Ñ&@Ñ@Àf×F[ÑF[Ðptñ ˆDÕð	 r;   Nrƒ   Úposition_embeddingsr�   Úpast_key_valuesr<   c                 ób  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  (       a"  U R                  U5      nU R                  U	5      n	Uu  p¼USS U R                  24   USU R                  S 24   píU	SS U R                  24   U	SU R                  S 24   nn[        XßX¼5      u  pß[        R                  " XÞ4SS9n[        R                  " UU4SS9n	Ub  UR                  XšU R                  5      u  pš[         R"                  " U R$                  R&                  [(        5      nU" U UU	U
U4U R*                  (       d  SOU R,                  U R.                  S.UD6u  nnUR0                  " / UQSP76 R3                  5       nU R5                  U5      nUU4$ )NrS   r   rA   .rX   ç        )r‘   r�   )rZ   r@   r¬   Úviewr^   r­   r®   r±   r´   rµ   r°   r‚   rI   r_   Úupdater£   r   Úget_interfacer$   Ú_attn_implementationrŸ   r”   r©   r�   r†   rš   r¯   )r6   rƒ   r¸   r�   r¹   rM   Úinput_shapeÚhidden_shapeÚquery_statesr›   rœ   r`   ra   Ú	query_rotÚ
query_passÚkey_rotÚkey_passÚattention_interfacerž   r�   s                       r9   rh   ÚPhiAttention.forward½   sD  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà××Ø×+Ñ+¨LÓ9ˆLØ×)Ñ)¨*Ó5ˆJà&‰ˆð ˜Ð1 × 1Ñ 1Ð1Ð1Ñ2Ø˜˜d×/Ñ/Ñ1Ð1Ñ2ð ð
 �sÐ/˜d×/Ñ/Ð/Ð/Ñ0Ø�s˜D×-Ñ-Ñ/Ð/Ñ0ð ˆô
 2°)ÀcÓOÑˆ	ô —y’y )Ð!8¸bÑAˆÜ—Y’Y ¨Ð2¸Ñ;ˆ
àÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—j‘j Ó-ˆØ˜LÐ(Ð(r;   )r©   r$   r¯   r@   rª   rµ   r­   r£   r•   r´   r¬   r±   r°   r�   r®   rj   )rk   rl   rm   rn   Ú__doc__r   rH   r*   rI   rq   rp   r   rh   rs   rt   ru   s   @r9   r¡   r¡   ¢   s’   ø† áGð˜yð °S÷ ð8 )-ñ8)à—|‘|ð8)ð # 5§<¡<°·±Ð#=Ñ>ð8)ð Ÿ™ tÑ+ð	8)ð
  ™ð8)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷8)ó 8)r;   r¡   c                   ób   ^ • \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )ÚPhiMLPéø   c                 ó  >• [         TU ]  5         Xl        [        UR                     U l        [        R                  " UR                  UR                  5      U l
        [        R                  " UR                  UR                  5      U l        g rj   )r)   r*   r$   r   Ú
hidden_actÚactivation_fnr1   r«   rF   Úintermediate_sizeÚfc1Úfc2©r6   r$   r8   s     €r9   r*   ÚPhiMLP.__init__ù   sb   ø€ Ü‰ÑÔØŒÜ# F×$5Ñ$5Ñ6ˆÔÜ—9’9˜V×/Ñ/°×1IÑ1IÓJˆŒÜ—9’9˜V×5Ñ5°v×7IÑ7IÓJˆ�r;   rƒ   r<   c                 ól   • U R                  U5      nU R                  U5      nU R                  U5      nU$ rj   )rÑ   rÏ   rÒ   )r6   rƒ   s     r9   rh   ÚPhiMLP.forward   s4   € ØŸ™ Ó/ˆØ×*Ñ*¨=Ó9ˆØŸ™ Ó/ˆØÐr;   )rÏ   r$   rÑ   rÒ   )
rk   rl   rm   rn   r*   rI   rq   rh   rs   rt   ru   s   @r9   rË   rË   ø   s)   ø† õKð U§\¡\ð °e·l±l÷ ò r;   rË   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚPhiDecoderLayeri  r$   r£   c                 ó  >• [         TU ]  5         [        XS9U l        [	        U5      U l        [        R                  " UR                  UR                  S9U l
        [        R                  " UR                  5      U l        g )N)r£   ©r§   )r)   r*   r¡   Ú	self_attnrË   Úmlpr1   r²   rF   r³   Úinput_layernormÚDropoutÚresid_pdropÚresid_dropoutr¶   s      €r9   r*   ÚPhiDecoderLayer.__init__  s[   ø€ Ü‰ÑÔÜ% fÑBˆŒÜ˜&“>ˆŒÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÑZˆÔÜŸZšZ¨×(:Ñ(:Ó;ˆÕr;   Nrƒ   r�   rc   r¹   Ú	use_cacher¸   rM   r<   c           
      óÒ   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pšU R                  U	5      n	U R                  U R                  U5      5      nX›-   U-   nU$ )N)rƒ   r�   rc   r¹   râ   r¸   © )rÝ   rÛ   rà   rÜ   )r6   rƒ   r�   rc   r¹   râ   r¸   rM   ÚresidualÚattn_outputsÚ_Úfeed_forward_hidden_statess               r9   rh   ÚPhiDecoderLayer.forward  s‰   € ð !ˆà×,Ñ,¨]Ó;ˆàŸ.š.ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
‰ˆð ×)Ñ)¨,Ó7ˆà%)×%7Ñ%7¸¿¹ÀÓ8OÓ%PÐ"Ø$ÑAÀHÑLˆàÐr;   )rÝ   rÜ   rà   rÛ   )NNNFN)rk   rl   rm   rn   r   rH   r*   rI   rq   Ú
LongTensorr   Úboolrp   r   r   rh   rs   rt   ru   s   @r9   rØ   rØ     sÀ   ø† ð<˜yð <°S÷ <ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷ó r;   rØ   c                   óR   • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.rSrg	)
ÚPhiPreTrainedModeli.  r$   ÚmodelTrØ   r¹   )rƒ   Ú
attentionsrä   N)rk   rl   rm   rn   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrØ   r¡   Ú_can_record_outputsrs   rä   r;   r9   rí   rí   .  sQ   ‡ àÓØÐØ&*Ð#Ø*Ð+ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà(Ø"ñÓr;   rí   c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚPhiModeliA  r$   c           	      óh  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        US9U l        SU l        [
        R"                  " UR$                  5      U l        [
        R(                  " UR                  UR*                  S9U l        U R/                  5         g s  snf )N©r$   FrÚ   )r)   r*   Úpad_token_idÚpadding_idxÚ
vocab_sizer1   Ú	EmbeddingrF   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrØ   Úlayersr    Ú
rotary_embÚgradient_checkpointingrÞ   Ú
embd_pdropÚembed_dropoutr²   r³   Úfinal_layernormÚ	post_initr¶   s      €r9   r*   ÚPhiModel.__init__C  så   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÒA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô -°FÑ;ˆŒØ&+ˆÔ#ÜŸZšZ¨×(9Ñ(9Ó:ˆÔÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÑZˆÔð 	�‰Õùò bs   ÂD/NÚ	input_idsr�   rc   r¹   Úinputs_embedsrâ   rM   r<   c           
      ó`  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U R                  UUUUS9n	U R                  U5      nUn
U R                  X£S9nU R                  S U R                  R                    H  nU" U
4U	UUUUS.UD6n
M     U R                  U
5      n
[!        U
US	9$ )
Nz:You must specify exactly one of input_ids or inputs_embedsrþ   r   r   )r"   )r$   r  r�   r¹   rc   )rc   )r�   rc   r¹   râ   r¸   )Úlast_hidden_stater¹   )Ú
ValueErrorr  r   r$   Úget_seq_lengthrI   rJ   rZ   r"   r|   r   r  r  r  r  r  r   )r6   r  r�   rc   r¹   r  râ   rM   Úpast_seen_tokensÚcausal_maskrƒ   r¸   Údecoder_layers                r9   rh   ÚPhiModel.forwardT  sX  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð ×*Ñ*¨=Ó9ˆØ%ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñŠMñ Jð ×,Ñ,¨]Ó;ˆä&Ø+Ø+ñ
ð 	
r;   )r  r  r  r	  r  r   r  r  )NNNNNN)rk   rl   rm   rn   r   r*   r   r   r   rI   rê   rq   r   ÚFloatTensorrë   r   r   r   rh   rs   rt   ru   s   @r9   rü   rü   A  sÍ   ø† ð˜y÷ ð"  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ4
à×#Ñ# dÑ*ð4
ð Ÿ™ tÑ+ð4
ð ×&Ñ&¨Ñ-ð	4
ð
  ™ð4
ð ×(Ñ(¨4Ñ/ð4
ð ˜$‘;ð4
ð Ð+Ñ,ð4
ð 
!ô4
ó ó ó  ö4
r;   rü   c                   óX  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
        SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚPhiForCausalLMiŽ  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrƒ   ÚlogitsÚkeep_full_weightc                 óä   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g )NTr¥   )
r)   r*   rü   rî   r  r1   r«   rF   r  r  rÓ   s     €r9   r*   ÚPhiForCausalLM.__init__•  sU   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈTÑRˆŒð 	�‰Õr;   Nr  r�   rc   r¹   r  Úlabelsrâ   Úlogits_to_keeprM   r<   c	           
      ó|  • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nSnUb)  U R                  " SXÖU R                  R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9$ )ac  
Example:

```python
>>> from transformers import AutoTokenizer, PhiForCausalLM

>>> model = PhiForCausalLM.from_pretrained("meta-phi/Phi-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-phi/Phi-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```)r  r�   rc   r¹   r  râ   N)r  r"  r  )Úlossr  r¹   rƒ   rï   rä   )rî   r  r[   rH   Úslicer  Úloss_functionr$   r  r   r¹   rƒ   rï   )r6   r  r�   rc   r¹   r  r"  râ   r#  rM   Úoutputsrƒ   Úslice_indicesr  r%  s                  r9   rh   ÚPhiForCausalLM.forwardž  sÖ   € ð> ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò%Ðp¨VÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r;   )r  rî   r  )NNNNNNNr   )rk   rl   rm   rn   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr*   r   r   rI   rê   rq   r   r  rë   rH   r   r   r   rh   rs   rt   ru   s   @r9   r  r  Ž  s"  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ô6
ó ó ö6
r;   r  c                   ó   • \ rS rSrSrg)ÚPhiForSequenceClassificationiÙ  rä   N©rk   rl   rm   rn   rs   rä   r;   r9   r0  r0  Ù  ó   † Úr;   r0  c                   ó   • \ rS rSrSrg)ÚPhiForTokenClassificationiÝ  rä   Nr1  rä   r;   r9   r4  r4  Ý  r2  r;   r4  )rí   rü   r  r0  r4  )r   )r»   )?Úcollections.abcr   rI   Útorch.nnr1   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   r
   Úmasking_utilsr   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   Úconfiguration_phir   ÚModuler    ry   r‚   rq   rH   rŠ   rK   rŸ   r¡   rË   rØ   rí   rü   r  r0  r4  Ú__all__rä   r;   r9   Ú<module>rH     sà  ðõ %ã Ý å !ß .Ý )ß LÝ /÷ñ ÷
 Pß Kß FÝ &ß 7Ý 0ß YÑ YÝ 5Ý (ô8<˜Ÿ™ô 8<òv(ñ Ð-Ó.óó /ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ñ2 Ð)Ó*ôR)�2—9‘9ó R)ó +ðR)ôjˆR�Y‰Yô ô$Ð0ô $ðN ô˜ó ó ðð$ ôI
Ð!ó I
ó ðI
ðX ôG
Ð'¨ó G
ó ðG
ôT	Ð#CÐEWô 	ô	Ð =Ð?Qô 	ò�r;   