ó
    ˆŽ¡j\(  ã                   óÎ  • S r SSKJr  SSKrSSKJr  SSKJr  SSKJr  SSK	J
r
  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJrJrJrJrJrJrJr  SSKJr  SSKJr  \R>                  " \ 5      r!Sr"Sr# " S S\RH                  5      r% " S S\5      r&S&S jr' " S S\RH                  5      r( " S S\5      r) " S S\5      r* " S S \5      r+ " S! S"\5      r, " S# S$\5      r-/ S%Qr.g)'zPyTorch Phi-3 model.é    )ÚCallableN)Únné   )ÚACT2FN)ÚCache)ÚGenerationMixin)ÚFlashAttentionKwargs)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)Úloggingé   )ÚMistralDecoderLayerÚMistralForCausalLMÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralPreTrainedModelÚeager_attention_forwardÚrotate_half)ÚPhiRotaryEmbeddingé   )Ú
Phi3Configz microsoft/Phi-3-mini-4k-instructr   c                   ób   ^ • \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )ÚPhi3MLPé0   c                 ó   >• [         TU ]  5         Xl        [        R                  " UR
                  SUR                  -  SS9U l        [        R                  " UR                  UR
                  SS9U l        [        UR                     U l        g )Nr   F©Úbias)ÚsuperÚ__init__Úconfigr   ÚLinearÚhidden_sizeÚintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn)Úselfr    Ú	__class__s     €Ú³C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/phi3/modular_phi3.pyr   ÚPhi3MLP.__init__1   sn   ø€ Ü‰ÑÔàŒÜŸIšI f×&8Ñ&8¸!¸f×>VÑ>VÑ:VÐ]bÑcˆÔÜŸš 6×#;Ñ#;¸V×=OÑ=OÐV[Ñ\ˆŒÜ# F×$5Ñ$5Ñ6ˆÕó    Úhidden_statesÚreturnc                 ó�   • U R                  U5      nUR                  SSS9u  p2X R                  U5      -  nU R                  U5      $ )Nr   éÿÿÿÿ©Údim)r$   Úchunkr'   r%   )r(   r-   Ú	up_statesÚgates       r*   ÚforwardÚPhi3MLP.forward9   sH   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ð4‰ˆØ× 2Ñ 2°4Ó 8Ñ8ˆ	à�~‰~˜iÓ(Ð(r,   )r'   r    r%   r$   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   ÚtorchÚFloatTensorr6   Ú__static_attributes__Ú__classcell__©r)   s   @r*   r   r   0   s,   ø† õ7ð) U×%6Ñ%6ð )¸5×;LÑ;L÷ )ò )r,   r   c                   ó   • \ rS rSrSrg)ÚPhi3RotaryEmbeddingéB   © N©r8   r9   r:   r;   r>   rD   r,   r*   rB   rB   B   ó   † Úr,   rB   c                 óN  • UR                  U5      nUR                  U5      nUR                  S   nU SSU24   U SUS24   pvUSSU24   USUS24   p˜[        R                  " Xb-  [	        U5      U-  -   U/SS9n
[        R                  " X‚-  [	        U5      U-  -   U	/SS9nX«4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
r0   .Nr1   )Ú	unsqueezeÚshaper<   Úcatr   )ÚqÚkÚcosÚsinÚunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r*   Úapply_rotary_pos_embrW   F   sÆ   € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cà—‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6Ø�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6ä�iŠi˜%™+¬+°eÓ*<¸sÑ*BÑCÀVÐLÐRTÑU€GÜ�iŠi˜%™+¬+°eÓ*<¸sÑ*BÑCÀVÐLÐRTÑU€GØÐÐr,   c                   ó0  ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr SS\R                  S\
\R                  \R                  4   S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  S-  \
\R                     S-  4   4S jjrSrU =r$ )ÚPhi3Attentionéd   z=Multi-headed attention from 'Attention Is All You Need' paperNr    Ú	layer_idxc                 óp  >• [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        UR                  U l        U R                  S-  U l
        UR                  U l        SU l        UR                  U R                  -  SUR                  U R                  -  -  -   n[        R                  " UR                  U R                  -  UR
                  SS9U l        [        R                  " UR
                  USS9U l        g )NÚhead_dimg      à¿Tr   Fr   )r   r   r    r[   Úgetattrr"   Únum_attention_headsr]   Únum_key_value_headsÚnum_key_value_groupsÚscalingÚattention_dropoutÚ	is_causalr   r!   Úo_projÚqkv_proj)r(   r    r[   Úop_sizer)   s       €r*   r   ÚPhi3Attention.__init__g   sù   ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø#)×#=Ñ#=ˆÔ Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒà×,Ñ,¨t¯}©}Ñ<¸qÀF×D^ÑD^Ðae×anÑanÑDnÑ?oÑoˆÜ—i’i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÑkˆŒÜŸ	š	 &×"4Ñ"4°gÀEÑJˆ�r,   r-   Úposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsr.   c           
      óà  • UR                   S S n/ UQSPU R                  P7nU R                  U5      nU R                  R                  U R                  -  n	USS U	24   n
USX™U R
                  U R                  -  -   24   nUSX�R
                  U R                  -  -   S 24   nU
R                  U5      R                  SS5      n
UR                  U5      R                  SS5      nUR                  U5      R                  SS5      nUu  pÞ[        X«XÞ5      u  p«Ub  UR                  X¼U R                  5      u  p¼[        R                  " U R                  R                  [        5      nU" U U
UUU4U R                  (       d  SOU R                   U R"                  [%        U R                  SS 5      S.UD6u  nnUR&                  " / UQSP76 R)                  5       nU R+                  U5      nUU4$ )Nr0   .r   r   g        Úsliding_window)Údropoutrb   rn   )rI   r]   rf   r    r_   r`   ÚviewÚ	transposerW   Úupdater[   r
   Úget_interfaceÚ_attn_implementationr   Útrainingrc   rb   r^   ÚreshapeÚ
contiguousre   )r(   r-   ri   rj   rk   rl   Úinput_shapeÚhidden_shapeÚqkvÚ	query_posÚquery_statesÚ
key_statesÚvalue_statesrM   rN   Úattention_interfaceÚattn_outputÚattn_weightss                     r*   r6   ÚPhi3Attention.forwardv   sø  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà�m‰m˜MÓ*ˆØ—K‘K×3Ñ3°d·m±mÑCˆ	Ø˜3 
  
˜?Ñ+ˆØ˜˜i°d×6NÑ6NÐQU×Q^ÑQ^Ñ6^Ñ*^Ð^Ð^Ñ_ˆ
Ø˜3 	×,DÑ,DÀtÇ}Á}Ñ,TÑ TÑ VÐVÑWˆà#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆØ—_‘_ \Ó2×<Ñ<¸QÀÓBˆ
Ø#×(Ñ(¨Ó6×@Ñ@ÀÀAÓFˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r,   )
rc   r    r]   rd   r[   ra   r`   re   rf   rb   )N)r8   r9   r:   r;   Ú__doc__r   Úintr   r<   ÚTensorÚtupler   r   r	   r6   r>   r?   r@   s   @r*   rY   rY   d   s¿   ø† ÙGñK˜zð K°c¸D±j÷ Kð Kð( )-ñ-)à—|‘|ð-)ð # 5§<¡<°·±Ð#=Ñ>ð-)ð Ÿ™ tÑ+ð	-)ð
  ™ð-)ð Ð-Ñ.ð-)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷-)ó -)r,   rY   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚPhi3DecoderLayeré¦   r    r[   c                 ó   >• [         TU ]  X5        Xl        [        XS9U l        [        U5      U l        [        R                  " UR                  5      U l
        [        R                  " UR                  5      U l        g )N)r    r[   )r   r   r    rY   Ú	self_attnr   Úmlpr   ÚDropoutÚresid_pdropÚresid_attn_dropoutÚresid_mlp_dropout)r(   r    r[   r)   s      €r*   r   ÚPhi3DecoderLayer.__init__§   sZ   ø€ Ü‰Ñ˜Ô+ØŒÜ&¨fÑJˆŒÜ˜6“?ˆŒÜ"$§*¢*¨V×-?Ñ-?Ó"@ˆÔÜ!#§¢¨F×,>Ñ,>Ó!?ˆÕr,   Nr-   rj   Úposition_idsrk   Ú	use_cacheri   rl   r.   c           
      óö   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pX€R                  U5      -   nUnU R                  U5      nU R	                  U5      nX€R                  U5      -   nU$ )N)r-   rj   r’   rk   r“   ri   rD   )Úinput_layernormr‹   r�   Úpost_attention_layernormrŒ   r�   )
r(   r-   rj   r’   rk   r“   ri   rl   ÚresidualÚself_attn_weightss
             r*   r6   ÚPhi3DecoderLayer.forward¯   sœ   € ð !ˆØ×,Ñ,¨]Ó;ˆà+/¯>ª>ð ,
Ø'Ø)Ø%Ø+ØØ 3ñ,
ð ñ,
Ñ(ˆð !×#:Ñ#:¸=Ó#IÑIˆà ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ ×#9Ñ#9¸-Ó#HÑHˆØÐr,   )r    rŒ   r�   r�   r‹   )NNNFN)r8   r9   r:   r;   r   r„   r   r<   r…   Ú
LongTensorr   Úboolr†   r   r	   r6   r>   r?   r@   s   @r*   rˆ   rˆ   ¦   sÃ   ø† ð@˜zð @°c÷ @ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð-Ñ.ðð 
�‰÷ó r,   rˆ   c                   ó   • \ rS rSrSrSrg)ÚPhi3PreTrainedModeléÎ   z0.0.5rD   N)r8   r9   r:   r;   Ú_versionr>   rD   r,   r*   r�   r�   Î   s   † ØƒHr,   r�   c                   ó*   • \ rS rSr      SS jrSrg)ÚPhi3ForCausalLMéÒ   Nc                 ó2  • U(       ap  [        U R                  S5      (       aU  UR                  S   U R                  R                  S-   :¼  a+  UR	                  5       n	X�R                  R                  ::  a  S n[
        R                  " U 4UUUUUUUS.UD6n
U
$ )NÚ original_max_position_embeddingsr   )Ú	input_idsrk   rj   Úinputs_embedsr’   r“   Úlogits_to_keep)Úhasattrr    rI   r¤   Úget_seq_lengthr   Úprepare_inputs_for_generation)r(   r¥   rk   rj   r¦   r’   r“   r§   rl   Úpast_lengthÚmodel_inputss              r*   rª   Ú-Phi3ForCausalLM.prepare_inputs_for_generationÓ   sœ   € ö" Ü˜Ÿ™Ð%G×HÑHØ—‘ Ñ" d§k¡k×&RÑ&RÐUVÑ&VÓVà)×8Ñ8Ó:ˆKØŸk™k×JÑJÓJØ"&�ä&×DÒDØð

àØ+Ø)Ø'Ø%ØØ)ñ

ð ñ

ˆð Ðr,   rD   )NNNNTr   )r8   r9   r:   r;   rª   r>   rD   r,   r*   r¡   r¡   Ò   s   † ð ØØØØØ÷$r,   r¡   c                   ó   • \ rS rSrSrg)ÚPhi3ForSequenceClassificationéú   rD   NrE   rD   r,   r*   r¯   r¯   ú   rF   r,   r¯   c                   ó   • \ rS rSrSrg)ÚPhi3ForTokenClassificationéþ   rD   NrE   rD   r,   r*   r²   r²   þ   rF   r,   r²   )r�   Ú	Phi3Modelr¡   r¯   r²   )r   )/rƒ   Úcollections.abcr   r<   r   Úactivationsr   Úcache_utilsr   Ú
generationr   Úmodeling_flash_attention_utilsr	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   Úmistral.modeling_mistralr   r   r   r   r   r   r   Úphi.modeling_phir   Úconfiguration_phi3r   Ú
get_loggerr8   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCÚModuler   rB   rW   rY   rˆ   r�   r¡   r¯   r²   Ú__all__rD   r,   r*   Ú<module>rÆ      sÙ   ðñ å $ã Ý å !Ý  Ý )Ý BÝ 5Ý &Ý ÷÷ ñ õ 2Ý *ð 
×	Ò	˜HÓ	%€à8Ð Ø€ô)ˆb�i‰iô )ô$	Ð,ô 	ôô<?)�B—I‘Iô ?)ôD%Ð*ô %ôPÐ0ô ô%Ð(ô %ôP	Ð$Dô 	ô	Ð!>ô 	ò�r,   