ó
    ˆŽ¡jC[  ã                   óh  • S SK Jr  S SKrS SKJr  SSKJr  SSKJrJr  SSK	J
r
  SSKJr  SS	KJrJr  SS
KJr  SSKJrJrJr  SSKJrJr  SSKJrJr  SSKJrJr  SSKJ r   SSK!J"r"J#r#J$r$  SSK%J&r&  SSK'J(r(J)r)  SSK*J+r+  SSK,J-r-   " S S\R\                  5      r/ " S S\R\                  5      r0S r1S\Rd                  S\3S\Rd                  4S jr4 S;S\R\                  S \Rd                  S!\Rd                  S"\Rd                  S#\Rd                  S-  S$\5S%\5S&\ \"   4S' jjr6S<S( jr7 " S) S*\R\                  5      r8\" S+5       " S, S-\R\                  5      5       r9 " S. S/\5      r:\# " S0 S1\5      5       r;\# " S2 S3\;5      5       r<\# " S4 S5\;\
5      5       r= " S6 S7\\;5      r> " S8 S9\\;5      r?/ S:Qr@g)=é    )ÚCallableN)Únné   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú
Phi3Configc                   ób   ^ • \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )ÚPhi3MLPé1   c                 ó   >• [         TU ]  5         Xl        [        R                  " UR
                  SUR                  -  SS9U l        [        R                  " UR                  UR
                  SS9U l        [        UR                     U l        g )Né   F©Úbias)ÚsuperÚ__init__Úconfigr   ÚLinearÚhidden_sizeÚintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn©Úselfr*   Ú	__class__s     €Ú´C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/phi3/modeling_phi3.pyr)   ÚPhi3MLP.__init__2   sn   ø€ Ü‰ÑÔàŒÜŸIšI f×&8Ñ&8¸!¸f×>VÑ>VÑ:VÐ]bÑcˆÔÜŸš 6×#;Ñ#;¸V×=OÑ=OÐV[Ñ\ˆŒÜ# F×$5Ñ$5Ñ6ˆÕó    Úhidden_statesÚreturnc                 ó�   • U R                  U5      nUR                  SSS9u  p2X R                  U5      -  nU R                  U5      $ )Nr%   éÿÿÿÿ©Údim)r.   Úchunkr1   r/   )r3   r8   Ú	up_statesÚgates       r5   ÚforwardÚPhi3MLP.forward:   sH   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ð4‰ˆØ× 2Ñ 2°4Ó 8Ñ8ˆ	à�~‰~˜iÓ(Ð(r7   )r1   r*   r/   r.   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r)   ÚtorchÚFloatTensorrA   Ú__static_attributes__Ú__classcell__©r4   s   @r5   r"   r"   1   s,   ø† õ7ð) U×%6Ñ%6ð )¸5×;LÑ;L÷ )ò )r7   r"   c            	       óÒ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\\	R                  \4   4S jj5       5       r\	R                  " 5       \S	 5       5       rS
rU =r$ )ÚPhi3RotaryEmbeddingéC   Údevicez5.18)Úversionr*   c                 óÄ  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        U R
                  R                  S   U l        U R                  nU R                  S:w  a  [        U R                     nU" U R
                  U5      u  o@l
        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚdefaultF)Ú
persistent)r(   r)   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr*   Úrope_parametersrR   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r3   r*   rO   Úrope_init_fnr\   r4   s        €r5   r)   ÚPhi3RotaryEmbedding.__init__D   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÓ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕr7   r9   c           	      óR  • U R                   S   nU R                   R                  SS5      n[        U SS5      =(       d    U R                  U R                  -  n[        XT-  5      nSnSU[        R                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )	a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   r%   ©Údtype)
rX   ÚgetÚgetattrr,   Únum_attention_headsÚintrG   ÚarangeÚfloatÚto)	r*   rO   ÚkwargsÚbaserc   rd   r=   Úattention_factorr\   s	            r5   rY   Ú3Phi3RotaryEmbedding.compute_default_rope_parametersU   s£   € ð ×%Ñ% lÑ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×h¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ñ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r7   c                 ó  • U R                   S S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        USS9   X4-  R                  SS	5      n[        R                  " Xf4SS
9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR                  UR                   S9W	R                  UR                   S94$ ! , (       d  f       N@= f)Nr   r;   r   )rf   rO   ÚmpsÚcpuF)Údevice_typeÚenabledr%   r<   re   )r\   ÚexpandÚshaperm   rG   rl   rO   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrZ   Úsinrf   )
r3   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedru   ÚfreqsÚembr~   r   s
             r5   rA   ÚPhi3RotaryEmbedding.forwardk   s;  € ð �M‰M˜$¢ 4˜-Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÐtð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   ÃA%E7Å7
F)rZ   r*   r\   rV   r^   rW   rR   ©N)rC   rD   rE   rF   r   r    r)   ÚstaticmethodÚtuplerG   ÚTensorrl   rY   Úno_gradr   rA   rI   rJ   rK   s   @r5   rM   rM   C   sŒ   ø† Ù�X vÑ.ñO˜z÷ Oó /ðOð  Ù�X vÑ.ñ5°
ð 5ÐV[Ð\a×\hÑ\hÐjoÐ\oÑVpô 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ö<r7   rM   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..Nr;   r%   r<   )rx   rG   r}   )r€   Úx1Úx2s      r5   Úrotate_halfr�   ~   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r7   r8   Ún_repr9   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)rx   rw   Úreshape)r8   r�   ÚbatchÚnum_key_value_headsÚslenrd   s         r5   Ú	repeat_kvr–   …   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr7   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrn   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )Nr%   r   r;   )r=   rf   )ÚpÚtrainingr   )r–   Únum_key_value_groupsrG   Úmatmulr|   r   Ú
functionalÚsoftmaxÚfloat32rm   rf   r�   r    Ú
contiguous)r—   r˜   r™   rš   r›   rœ   r�   rn   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r5   Úeager_attention_forwardr«   ‘   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r7   c                 óN  • UR                  U5      nUR                  U5      nUR                  S   nU SSU24   U SUS24   pvUSSU24   USUS24   p˜[        R                  " Xb-  [	        U5      U-  -   U/SS9n
[        R                  " X‚-  [	        U5      U-  -   U	/SS9nX«4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
r;   .Nr<   )Ú	unsqueezerx   rG   r}   r�   )ÚqÚkr~   r   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r5   Úapply_rotary_pos_embr¸   ª   sÆ   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cà—‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6Ø�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6ä�iŠi˜%™+¬+°eÓ*<¸sÑ*BÑCÀVÐLÐRTÑU€GÜ�iŠi˜%™+¬+°eÓ*<¸sÑ*BÑCÀVÐLÐRTÑU€GØÐÐr7   c                   ó0  ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr SS\R                  S\
\R                  \R                  4   S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  S-  \
\R                     S-  4   4S jjrSrU =r$ )ÚPhi3AttentionéÈ   z=Multi-headed attention from 'Attention Is All You Need' paperNr*   Ú	layer_idxc                 óp  >• [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        UR                  U l        U R                  S-  U l
        UR                  U l        SU l        UR                  U R                  -  SUR                  U R                  -  -  -   n[        R                  " UR                  U R                  -  UR
                  SS9U l        [        R                  " UR
                  USS9U l        g )Nrd   g      à¿Tr%   Fr&   )r(   r)   r*   r¼   rh   r,   ri   rd   r”   r¡   rœ   Úattention_dropoutÚ	is_causalr   r+   Úo_projÚqkv_proj)r3   r*   r¼   Úop_sizer4   s       €r5   r)   ÚPhi3Attention.__init__Ë   sù   ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø#)×#=Ñ#=ˆÔ Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒà×,Ñ,¨t¯}©}Ñ<¸qÀF×D^ÑD^Ðae×anÑanÑDnÑ?oÑoˆÜ—i’i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÑkˆŒÜŸ	š	 &×"4Ñ"4°gÀEÑJˆ�r7   r8   Úposition_embeddingsr›   Úpast_key_valuesrn   r9   c           
      óà  • UR                   S S n/ UQSPU R                  P7nU R                  U5      nU R                  R                  U R                  -  n	USS U	24   n
USX™U R
                  U R                  -  -   24   nUSX�R
                  U R                  -  -   S 24   nU
R                  U5      R                  SS5      n
UR                  U5      R                  SS5      nUR                  U5      R                  SS5      nUu  pÞ[        X«XÞ5      u  p«Ub  UR                  X¼U R                  5      u  p¼[        R                  " U R                  R                  [        5      nU" U U
UUU4U R                  (       d  SOU R                   U R"                  [%        U R                  SS 5      S.UD6u  nnUR&                  " / UQSP76 R)                  5       nU R+                  U5      nUU4$ )Nr;   .r   r%   ç        Úsliding_window)r�   rœ   rÈ   )rx   rd   rÁ   r*   ri   r”   Úviewr|   r¸   Úupdater¼   r   Úget_interfaceÚ_attn_implementationr«   r    r¾   rœ   rh   r’   r¦   rÀ   )r3   r8   rÄ   r›   rÅ   rn   Úinput_shapeÚhidden_shapeÚqkvÚ	query_posÚquery_statesr§   r¨   r~   r   Úattention_interfacerª   r©   s                     r5   rA   ÚPhi3Attention.forwardÚ   sø  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà�m‰m˜MÓ*ˆØ—K‘K×3Ñ3°d·m±mÑCˆ	Ø˜3 
  
˜?Ñ+ˆØ˜˜i°d×6NÑ6NÐQU×Q^ÑQ^Ñ6^Ñ*^Ð^Ð^Ñ_ˆ
Ø˜3 	×,DÑ,DÀtÇ}Á}Ñ,TÑ TÑ VÐVÑWˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r7   )
r¾   r*   rd   r¿   r¼   r¡   r”   rÀ   rÁ   rœ   r‡   )rC   rD   rE   rF   Ú__doc__r    rj   r)   rG   rŠ   r‰   r   r   r   rA   rI   rJ   rK   s   @r5   rº   rº   È   s¿   ø† ÙGñK˜zð K°c¸D±j÷ Kð Kð( )-ñ-)à—|‘|ð-)ð # 5§<¡<°·±Ð#=Ñ>ð-)ð Ÿ™ tÑ+ð	-)ð
  ™ð-)ð Ð-Ñ.ð-)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷-)ó -)r7   rº   ÚRMSNormc                   óx   ^ • \ rS rSrS
S\SS4U 4S jjjrS\R                  S\R                  4S jrS r	S	r
U =r$ )ÚPhi3RMSNormi
  Úepsr9   Nc                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z*
Phi3RMSNorm is equivalent to T5LayerNorm
N)r(   r)   r   Ú	ParameterrG   ÚonesÚweightÚvariance_epsilon)r3   r,   rØ   r4   s      €r5   r)   ÚPhi3RMSNorm.__init__  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Õr7   r8   c                 ó  • UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )Nr%   r;   T)Úkeepdim)	rf   rm   rG   r¥   ÚpowÚmeanÚrsqrtrÝ   rÜ   )r3   r8   Úinput_dtypeÚvariances       r5   rA   ÚPhi3RMSNorm.forward  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ð>ˆØ%¬¯ª°H×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r7   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)r‰   rÜ   rx   rÝ   )r3   s    r5   Ú
extra_reprÚPhi3RMSNorm.extra_repr  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr7   )rÝ   rÜ   )g�íµ ÷Æ°>)rC   rD   rE   rF   rl   r)   rG   rŠ   rA   rè   rI   rJ   rK   s   @r5   r×   r×   
  sB   ø† ñ$¨ð $¸$÷ $ð $ð; U§\¡\ð ;°e·l±lô ;÷Jð Jr7   r×   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚPhi3DecoderLayeri  r*   r¼   c                 ó¬  >• [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        [        UR                  UR                  S9U l
        Xl        [        R                  " UR                  5      U l        [        R                  " UR                  5      U l        g )N)r*   r¼   ©rØ   )r(   r)   r,   rº   Ú	self_attnr"   Úmlpr×   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr*   r   ÚDropoutÚresid_pdropÚresid_attn_dropoutÚresid_mlp_dropout©r3   r*   r¼   r4   s      €r5   r)   ÚPhi3DecoderLayer.__init__   sœ   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ&¨fÑJˆŒÜ˜6“?ˆŒÜ*¨6×+=Ñ+=À6×CVÑCVÑWˆÔÜ(3°F×4FÑ4FÈF×L_ÑL_Ñ(`ˆÔ%ØŒÜ"$§*¢*¨V×-?Ñ-?Ó"@ˆÔÜ!#§¢¨F×,>Ñ,>Ó!?ˆÕr7   Nr8   r›   r�   rÅ   Ú	use_cacherÄ   rn   r9   c           
      óö   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pX€R                  U5      -   nUnU R                  U5      nU R	                  U5      nX€R                  U5      -   nU$ )N)r8   r›   r�   rÅ   rù   rÄ   © )rñ   rî   rõ   rò   rï   rö   )
r3   r8   r›   r�   rÅ   rù   rÄ   rn   ÚresidualÚself_attn_weightss
             r5   rA   ÚPhi3DecoderLayer.forward+  sœ   € ð !ˆØ×,Ñ,¨]Ó;ˆà+/¯>ª>ð ,
Ø'Ø)Ø%Ø+ØØ 3ñ,
ð ñ,
Ñ(ˆð !×#:Ñ#:¸=Ó#IÑIˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ ×#9Ñ#9¸-Ó#HÑHˆØÐr7   )r*   r,   rñ   rï   rò   rõ   rö   rî   )NNNFN)rC   rD   rE   rF   r    rj   r)   rG   rŠ   Ú
LongTensorr   Úboolr‰   r   r   rA   rI   rJ   rK   s   @r5   rë   rë     sÃ   ø† ð	@˜zð 	@°c÷ 	@ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð-Ñ.ðð 
�‰÷ó r7   rë   c                   óV   • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.rSrS	rg
)ÚPhi3PreTrainedModeliJ  r*   ÚmodelTrë   rÅ   )r8   Ú
attentionsz0.0.5rû   N)rC   rD   rE   rF   r    Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrë   rº   Ú_can_record_outputsÚ_versionrI   rû   r7   r5   r  r  J  sX   ‡ àÓØÐØ&*Ð#Ø+Ð,ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà)Ø#ñÐð ƒHr7   r  c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )Ú	Phi3Modeli^  r*   c           	      ó  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nrí   ©r*   F)r(   r)   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr,   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrë   Úlayersr×   rð   ÚnormrM   Ú
rotary_embÚgradient_checkpointingÚ	post_initr÷   s      €r5   r)   ÚPhi3Model.__init__`  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜBGÈ×H`ÑH`ÔBaÓbÒBa°YÔ˜fÖ0ÑBaÑbó
ˆŒô   × 2Ñ 2¸×8KÑ8KÑLˆŒ	Ü-°VÑ<ˆŒØ&+ˆÔ#ð 	�‰Õùò cs   ÂC?NÚ	input_idsr›   r�   rÅ   Úinputs_embedsrù   rn   r9   c           
      ó”  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      nU R                  R                  c  [        O[        n	U	" U R                  UUUUS9n
UnU R                  X³S9nU R                  S U R                  R                    H  nU" U4U
UUUUS.UD6nM     U R!                  U5      n[#        UU(       a  US	9$ S S	9$ )
Nz:You must specify exactly one of input_ids or inputs_embedsr  r   r   )rO   )r*   r$  r›   rÅ   r�   )r�   )r›   r�   rÅ   rù   rÄ   )Úlast_hidden_staterÅ   )Ú
ValueErrorr  r   r*   Úget_seq_lengthrG   rk   rx   rO   r­   rÈ   r   r   r  r  r  r  r   )r3   r#  r›   r�   rÅ   r$  rù   rn   Úpast_seen_tokensÚmask_functionÚcausal_maskr8   rÄ   Údecoder_layers                 r5   rA   ÚPhi3Model.forwardp  sm  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÑ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð &ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Þ/8˜Oñ
ð 	
à>Bñ
ð 	
r7   )r  r   r  r  r  r  r  )NNNNNN)rC   rD   rE   rF   r    r)   r   r   r   rG   rÿ   rŠ   r   rH   r   r   r   r   rA   rI   rJ   rK   s   @r5   r  r  ^  sÍ   ø† ð˜z÷ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!ô2
ó ó ó  ö2
r7   r  c                   ót  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
        SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       r      SU 4S jjrSrU =r$ )ÚPhi3ForCausalLMi¨  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr8   ÚlogitsÚkeep_full_weightc                 óä   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g )NFr&   )
r(   r)   r  r  r  r   r+   r,   r0  r!  r2   s     €r5   r)   ÚPhi3ForCausalLM.__init__¯  sU   ø€ Ü‰Ñ˜Ô Ü˜vÓ&ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒð 	�‰Õr7   Nr#  r›   r�   rÅ   r$  Úlabelsrù   Úlogits_to_keeprn   r9   c	           
      ó|  • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nSnUb)  U R                  " SXÖU R                  R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9$ )ai  
Example:

```python
>>> from transformers import AutoTokenizer, Phi3ForCausalLM

>>> model = Phi3ForCausalLM.from_pretrained("meta-phi3/Phi3-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-phi3/Phi3-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```)r#  r›   r�   rÅ   r$  rù   N)r2  r6  r  )Úlossr2  rÅ   r8   r  rû   )r  r&  ry   rj   Úslicer0  Úloss_functionr*   r  r   rÅ   r8   r  )r3   r#  r›   r�   rÅ   r$  r6  rù   r7  rn   Úoutputsr8   Úslice_indicesr2  r9  s                  r5   rA   ÚPhi3ForCausalLM.forward¸  sÖ   € ð> ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò%Ðp¨VÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r7   c                 ó&  >• U(       ap  [        U R                  S5      (       aU  UR                  S   U R                  R                  S-   :¼  a+  UR	                  5       n	X�R                  R                  ::  a  S n[
        TU ]  " SUUUUUUUS.UD6n
U
$ )NÚ original_max_position_embeddingsr   )r#  rÅ   r›   r$  r�   rù   r7  rû   )Úhasattrr*   rx   r@  r(  r(   Úprepare_inputs_for_generation)r3   r#  rÅ   r›   r$  r�   rù   r7  rn   Úpast_lengthÚmodel_inputsr4   s              €r5   rB  Ú-Phi3ForCausalLM.prepare_inputs_for_generationò  s™   ø€ ö" Ü˜Ÿ™Ð%G×HÑHØ—‘ Ñ" d§k¡k×&RÑ&RÐUVÑ&VÓVà)×8Ñ8Ó:ˆKØŸk™k×JÑJÓJØ"&�ä‘wÒ<ð 	
ØØ+Ø)Ø'Ø%ØØ)ñ	
ð ñ	
ˆð Ðr7   )r0  r  r  )NNNNNNNr   )NNNNTr   )rC   rD   rE   rF   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr)   r   r   rG   rÿ   rŠ   r   rH   r   rj   r   r   r   rA   rB  rI   rJ   rK   s   @r5   r/  r/  ¨  sA  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ô6
ó ó ð6
ðv ØØØØØ÷#õ #r7   r/  c                   ó   • \ rS rSrSrg)ÚPhi3ForSequenceClassificationi  rû   N©rC   rD   rE   rF   rI   rû   r7   r5   rK  rK    ó   † Úr7   rK  c                   ó   • \ rS rSrSrg)ÚPhi3ForTokenClassificationi  rû   NrL  rû   r7   r5   rO  rO    rM  r7   rO  )r  r  r/  rK  rO  )rÇ   )r   )AÚcollections.abcr   rG   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úintegrationsr
   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_phi3r    ÚModuler"   rM   r�   rŠ   rj   r–   rl   r«   r¸   rº   r×   rë   r  r  r/  rK  rO  Ú__all__rû   r7   r5   Ú<module>rc     sÝ  ðõ, %ã Ý å !ß .Ý )Ý 7ß RÝ B÷ñ ÷
 Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý *ô)ˆb�i‰iô )ô$8<˜"Ÿ)™)ô 8<òv(ð	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ô2ô<?)�B—I‘Iô ?)ñD ˜YÓ'ôJ�"—)‘)ó Jó (ðJô((Ð1ô (ðV ô˜/ó ó ðð& ôF
Ð#ó F
ó ðF
ðR ôlÐ)¨?ó ló ðlô^	Ð$DÐFYô 	ô	Ð!>Ð@Sô 	ò�r7   