ó
    ˆŽ¡j‘(  ã                   óÚ  • S SK Jr  S SKrS SKJr  SSKJrJr  SSKJ	r	  SSK
Jr  SSKJr  SSKJr  SS	KJr  SS
KJrJrJr  SSKJr  SSKJr  SSKJr  SSKJrJrJrJ r J!r!J"r"J#r#J$r$J%r%  SSK&J'r'  \RP                  " \)5      r*Sr+Sr, " S S\#5      r- " S S\5      r. " S S\5      r/ " S S\5      r0 " S S\"5      r1 " S S\!5      r2 " S  S!\5      r3 " S" S#\5      r4 " S$ S%\ 5      r5/ S&Qr6g)'é    )ÚCallableNé   )ÚCacheÚDynamicCache)Úcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚCLIPMLP)	ÚLlamaAttentionÚLlamaForCausalLMÚLlamaForSequenceClassificationÚLlamaForTokenClassificationÚ
LlamaModelÚLlamaPreTrainedModelÚLlamaRotaryEmbeddingÚapply_rotary_pos_embÚeager_attention_forwardé   )Ú	PhiConfigzmicrosoft/phi-1r   c                   óH   • \ rS rSrSS\S\\R                  \4   4S jjr	Sr
g)ÚPhiRotaryEmbeddingé&   NÚconfigÚreturnc           	      óR  • U R                   S   nU R                   R                  SS5      n[        U SS5      =(       d    U R                  U R                  -  n[        XT-  5      nSnSU[        R                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )	a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   r   )Údtype)
Úrope_parametersÚgetÚgetattrÚhidden_sizeÚnum_attention_headsÚintÚtorchÚarangeÚfloatÚto)	r!   ÚdeviceÚkwargsÚbaser%   r&   ÚdimÚattention_factorÚinv_freqs	            Ú±C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/phi/modular_phi.pyÚcompute_default_rope_parametersÚ2PhiRotaryEmbedding.compute_default_rope_parameters'   s£   € ð ×%Ñ% lÑ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×h¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ñ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4ó    © ©N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   Útupler.   ÚTensorr0   r9   Ú__static_attributes__r<   r;   r8   r   r   &   s-   † ñ5°	ð 5ÐUZÐ[`×[gÑ[gÐinÐ[nÑUo÷ 5ð 5r;   r   c                   óö   ^ • \ rS rSrS\S\4U 4S jjr SS\R                  S\	\R                  \R                  4   S\R                  S-  S	\
S-  S
\	\R                  \R                  S-  4   4
S jjrSrU =r$ )ÚPhiAttentioné<   r!   Ú	layer_idxc                 ó’  >• [         TU ]  X5        [        R                  " UR                  UR
                  U R                  -  SS9U l        [        R                  " UR                  UR                  U R                  -  SS9U l	        [        R                  " UR                  UR                  U R                  -  SS9U l
        [        R                  " UR
                  U R                  -  UR                  SS9U l        U ?[        U R                  UR                  S   -  5      U l        UR                   U l        U R                   (       ay  [        R"                  " UR                  UR
                  -  UR$                  SS9U l        [        R"                  " UR                  UR
                  -  UR$                  SS9U l        g g )NT©Úbiasr%   )ÚepsÚelementwise_affine)ÚsuperÚ__init__ÚnnÚLinearr+   r,   r&   Úq_projÚnum_key_value_headsÚk_projÚv_projÚdenseÚo_projr-   r(   Úrotary_ndimsÚqk_layernormÚ	LayerNormÚlayer_norm_epsÚq_layernormÚk_layernorm©Úselfr!   rH   Ú	__class__s      €r8   rO   ÚPhiAttention.__init__=   sf  ø€ Ü‰Ñ˜Ô+Ü—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÑjˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÑjˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐeiÑjˆŒÜ—Y’Y˜v×9Ñ9¸D¿M¹MÑIÈ6×K]ÑK]ÐdhÑiˆŒ
ØˆKÜ §¡°×0FÑ0FÐG^Ñ0_Ñ _Ó`ˆÔØ"×/Ñ/ˆÔØ××Ü!Ÿ|š|Ø×"Ñ" f×&@Ñ&@Ñ@Àf×F[ÑF[Ðptñ ˆDÔô  "Ÿ|š|Ø×"Ñ" f×&@Ñ&@Ñ@Àf×F[ÑF[Ðptñ ˆDÕð	 r;   NÚhidden_statesÚposition_embeddingsÚattention_maskÚpast_key_valuesr"   c                 ób  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
U R                  (       a"  U R                  U5      nU R                  U	5      n	Uu  p¼USS U R                  24   USU R                  S 24   píU	SS U R                  24   U	SU R                  S 24   nn[        XßX¼5      u  pß[        R                  " XÞ4SS9n[        R                  " UU4SS9n	Ub  UR                  XšU R                  5      u  pš[         R"                  " U R$                  R&                  [(        5      nU" U UU	U
U4U R*                  (       d  SOU R,                  U R.                  S.UD6u  nnUR0                  " / UQSP76 R3                  5       nU R5                  U5      nUU4$ )Néÿÿÿÿr   r   .)r5   g        )ÚdropoutÚscaling)Úshaper&   rR   ÚviewÚ	transposerT   rU   rY   r\   r]   rX   r   r.   ÚcatÚupdaterH   r
   Úget_interfacer!   Ú_attn_implementationr   ÚtrainingÚattention_dropoutri   ÚreshapeÚ
contiguousrV   )r_   rb   rc   rd   re   r3   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚ	query_rotÚ
query_passÚkey_rotÚkey_passÚattention_interfaceÚattn_outputÚattn_weightss                       r8   ÚforwardÚPhiAttention.forwardN   sD  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà××Ø×+Ñ+¨LÓ9ˆLØ×)Ñ)¨*Ó5ˆJà&‰ˆð ˜Ð1 × 1Ñ 1Ð1Ð1Ñ2Ø˜˜d×/Ñ/Ñ1Ð1Ñ2ð ð
 �sÐ/˜d×/Ñ/Ð/Ð/Ñ0Ø�s˜D×-Ñ-Ñ/Ð/Ñ0ð ˆô
 2°)ÀcÓOÑˆ	ô —y’y )Ð!8¸bÑAˆÜ—Y’Y ¨Ð2¸Ñ;ˆ
àÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—j‘j Ó-ˆØ˜LÐ(Ð(r;   )rV   r]   rT   r\   rR   rY   rX   rU   r=   )r>   r?   r@   rA   r   r-   rO   r.   rC   rB   r   rƒ   rD   Ú__classcell__©r`   s   @r8   rF   rF   <   s�   ø† ð˜yð °S÷ ð, )-ñ8)à—|‘|ð8)ð # 5§<¡<°·±Ð#=Ñ>ð8)ð Ÿ™ tÑ+ð	8)ð
  ™ð8)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷8)ó 8)r;   rF   c                   ó   • \ rS rSrSrg)ÚPhiMLPé‰   r<   N©r>   r?   r@   rA   rD   r<   r;   r8   rˆ   rˆ   ‰   ó   † Úr;   rˆ   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚPhiDecoderLayeré�   r!   rH   c                 ó  >• [         TU ]  5         [        XS9U l        [	        U5      U l        [        R                  " UR                  UR                  S9U l
        [        R                  " UR                  5      U l        g )N)rH   ©rL   )rN   rO   rF   Ú	self_attnrˆ   ÚmlprP   rZ   r+   r[   Úinput_layernormÚDropoutÚresid_pdropÚresid_dropoutr^   s      €r8   rO   ÚPhiDecoderLayer.__init__Ž   s[   ø€ Ü‰ÑÔÜ% fÑBˆŒÜ˜&“>ˆŒÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÑZˆÔÜŸZšZ¨×(:Ñ(:Ó;ˆÕr;   Nrb   rd   Úposition_idsre   Ú	use_cacherc   r3   r"   c           
      óÒ   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pšU R                  U	5      n	U R                  U R                  U5      5      nX›-   U-   nU$ )N)rb   rd   r˜   re   r™   rc   r<   )r“   r‘   r–   r’   )r_   rb   rd   r˜   re   r™   rc   r3   ÚresidualÚattn_outputsÚ_Úfeed_forward_hidden_statess               r8   rƒ   ÚPhiDecoderLayer.forward•   s‰   € ð !ˆà×,Ñ,¨]Ó;ˆàŸ.š.ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
‰ˆð ×)Ñ)¨,Ó7ˆà%)×%7Ñ%7¸¿¹ÀÓ8OÓ%PÐ"Ø$ÑAÀHÑLˆàÐr;   )r“   r’   r–   r‘   )NNNFN)r>   r?   r@   rA   r   r-   rO   r.   rC   Ú
LongTensorr   ÚboolrB   r   r   rƒ   rD   r…   r†   s   @r8   r�   r�   �   sÀ   ø† ð<˜yð <°S÷ <ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷ó r;   r�   c                   ó   • \ rS rSr\\S.rSrg)ÚPhiPreTrainedModelé´   )rb   Ú
attentionsr<   N)r>   r?   r@   rA   r�   rF   Ú_can_record_outputsrD   r<   r;   r8   r£   r£   ´   s   † à(Ø"ñÓr;   r£   c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚPhiModelé»   r!   c           	      óh  >• [         TU ]  U5        [        R                  " [	        UR
                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        R                  " UR                  5      U l
        [        R                  " UR                  UR                  S9U l        U ?g s  snf )Nr�   )rN   rO   rP   Ú
ModuleListÚrangeÚnum_hidden_layersr�   Úlayersr”   Ú
embd_pdropÚembed_dropoutrZ   r+   r[   Úfinal_layernormÚnormr^   s      €r8   rO   ÚPhiModel.__init__¼   s‡   ø€ Ü‰Ñ˜Ô Ü—m’mÜAFÀv×G_ÑG_ÔA`ÓaÒA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô  ŸZšZ¨×(9Ñ(9Ó:ˆÔÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DYÑDYÑZˆÔØ‰Iùò	 bs   ¸B/NÚ	input_idsrd   r˜   re   Úinputs_embedsr™   r3   r"   c           
      ó`  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U R                  UUUUS9n	U R                  U5      nUn
U R                  X£S9nU R                  S U R                  R                    H  nU" U
4U	UUUUS.UD6n
M     U R                  U
5      n
[!        U
US	9$ )
Nz:You must specify exactly one of input_ids or inputs_embeds)r!   r   r   )r2   )r!   rµ   rd   re   r˜   )r˜   )rd   r˜   re   r™   rc   )Úlast_hidden_statere   )Ú
ValueErrorÚembed_tokensr   r!   Úget_seq_lengthr.   r/   rj   r2   Ú	unsqueezer   r°   Ú
rotary_embr®   r­   r±   r	   )r_   r´   rd   r˜   re   rµ   r™   r3   Úpast_seen_tokensÚcausal_maskrb   rc   Údecoder_layers                r8   rƒ   ÚPhiModel.forwardÅ   sX  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð ×*Ñ*¨=Ó9ˆØ%ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñŠMñ Jð ×,Ñ,¨]Ó;ˆä&Ø+Ø+ñ
ð 	
r;   )r°   r±   r®   )NNNNNN)r>   r?   r@   rA   r   rO   r   r   r   r.   r    rC   r   ÚFloatTensorr¡   r   r   r	   rƒ   rD   r…   r†   s   @r8   r¨   r¨   »   sÍ   ø† ð˜y÷ ð  ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ4
à×#Ñ# dÑ*ð4
ð Ÿ™ tÑ+ð4
ð ×&Ñ&¨Ñ-ð	4
ð
  ™ð4
ð ×(Ñ(¨4Ñ/ð4
ð ˜$‘;ð4
ð Ð+Ñ,ð4
ð 
!ô4
ó ó ó  ö4
r;   r¨   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚPhiForCausalLMéÿ   c                 ó‚   >• [         TU ]  U5        [        R                  " UR                  UR
                  SS9U l        g )NTrJ   )rN   rO   rP   rQ   r+   Ú
vocab_sizeÚlm_head)r_   r!   r`   s     €r8   rO   ÚPhiForCausalLM.__init__   s0   ø€ Ü‰Ñ˜Ô Ü—y’y ×!3Ñ!3°V×5FÑ5FÈTÑRˆ�r;   )rÇ   )r>   r?   r@   rA   rO   rD   r…   r†   s   @r8   rÃ   rÃ   ÿ   s   ø† ÷Só Sr;   rÃ   c                   ó   • \ rS rSrSrg)ÚPhiForSequenceClassificationi  r<   NrŠ   r<   r;   r8   rÊ   rÊ     r‹   r;   rÊ   c                   ó   • \ rS rSrSrg)ÚPhiForTokenClassificationi	  r<   NrŠ   r<   r;   r8   rÌ   rÌ   	  r‹   r;   rÌ   )r£   r¨   rÃ   rÊ   rÌ   )7Úcollections.abcr   r.   Útorch.nnrP   Úcache_utilsr   r   Úmasking_utilsr   Úmodeling_layersr   Úmodeling_outputsr	   Úmodeling_utilsr
   Úprocessing_utilsr   Úutilsr   r   r   Úutils.genericr   Úutils.output_capturingr   Úclip.modeling_clipr   Úllama.modeling_llamar   r   r   r   r   r   r   r   r   Úconfiguration_phir   Ú
get_loggerr>   ÚloggerÚ_CHECKPOINT_FOR_DOCÚ_CONFIG_FOR_DOCr   rF   rˆ   r�   r£   r¨   rÃ   rÊ   rÌ   Ú__all__r<   r;   r8   Ú<module>rà      sæ   ðÝ $ã Ý ç .Ý /Ý 9õõ 6Ý &ß @Ñ @Ý 7Ý 5Ý (÷
÷ 
õ 
õ )ð 
×	Ò	˜HÓ	%€à'Ð Ø€ô5Ð-ô 5ô,J)�>ô J)ôZ	ˆWô 	ô$Ð0ô $ôNÐ-ô ôA
ˆzô A
ôHSÐ%ô Sô	Ð#Aô 	ô	Ð ;ô 	ò�r;   