ó
    ˆŽ¡jK#  ã                   óº  • S r SSKJr  SSKrSSKJr  SSKJr  SSKJrJ	r	  SSK
JrJr  SS	KJr  SS
KJr  SSKJr  SSKJr  SSKJrJr  SSKJr  SSKJr  SSKJrJrJrJ r J!r!J"r"J#r#J$r$  SSK%J&r&  \RN                  " \(5      r) " S S\RT                  5      r+ " S S\5      r, " S S\5      r- " S S\"5      r. " S S\5      r/ " S S\ 5      r0 " S  S!\!5      r1/ S"Qr2g)#zPyTorch Starcoder2 model.é    )ÚCallableN)Únné   )ÚACT2FN)ÚCacheÚDynamicCache)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚBaseModelOutputWithPast)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚlogging)Úmerge_with_config_defaults)Úcapture_outputsé   )ÚMistralAttentionÚMistralDecoderLayerÚMistralForCausalLMÚ MistralForSequenceClassificationÚMistralForTokenClassificationÚMistralModelÚapply_rotary_pos_embÚeager_attention_forwardé   )ÚStarcoder2Configc                   óv   ^ • \ rS rSrS\4U 4S jjrS\\R                     S-  S\R                  4S jr	Sr
U =r$ )	ÚStarcoder2MLPé4   Úconfigc                 óD  >• [         TU ]  5         UR                  n[        R                  " X!R
                  UR                  S9U l        [        R                  " UR
                  X!R                  S9U l        [        UR                     U l        UR                  U l        g )N©Úbias)ÚsuperÚ__init__Úhidden_sizer   ÚLinearÚintermediate_sizeÚuse_biasÚc_fcÚc_projr   Ú
hidden_actÚactÚresidual_dropout)Úselfr!   Ú	embed_dimÚ	__class__s      €Ú¿C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/starcoder2/modular_starcoder2.pyr&   ÚStarcoder2MLP.__init__5   sq   ø€ Ü‰ÑÔØ×&Ñ&ˆ	Ü—I’I˜i×)AÑ)AÈÏÉÑXˆŒ	Ü—i’i × 8Ñ 8¸)Ï/É/ÑZˆŒÜ˜&×+Ñ+Ñ,ˆŒØ &× 7Ñ 7ˆÕó    Úhidden_statesNÚreturnc                 óÐ   • U R                  U5      nU R                  U5      nU R                  U5      n[        R                  R                  XR                  U R                  S9nU$ )N©ÚpÚtraining)r+   r.   r,   r   Ú
functionalÚdropoutr/   r;   )r0   r6   s     r3   ÚforwardÚStarcoder2MLP.forward=   sX   € ØŸ	™	 -Ó0ˆØŸ™ Ó/ˆØŸ™ MÓ2ˆÜŸ™×-Ñ-¨m×?TÑ?TÐ_c×_lÑ_lÐ-ÐmˆØÐr5   )r.   r+   r,   r/   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r   r&   ÚtupleÚtorchÚFloatTensorr>   Ú__static_attributes__Ú__classcell__©r2   s   @r3   r   r   4   s>   ø† ð8Ð/÷ 8ð U¨5×+<Ñ+<Ñ%=ÀÑ%Dð È×IZÑIZ÷ ò r5   r   c                   ó,  ^ • \ rS rSrSS\S\S-  4U 4S jjjr SS\R                  S\	\R                  \R                  4   S\R                  S-  S	\
S-  S
\\   S\	\R                  \R                  S-  \	\R                     S-  4   4S jjrSrU =r$ )ÚStarcoder2AttentionéE   Nr!   Ú	layer_idxc                 ór  >• [         TU ]  XS9  UR                  U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l	        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        g )N©r!   rM   r#   )r%   r&   r/   r   r(   r'   Únum_attention_headsÚhead_dimr*   Úq_projÚnum_key_value_headsÚk_projÚv_projÚo_proj©r0   r!   rM   r2   s      €r3   r&   ÚStarcoder2Attention.__init__F   sê   ø€ Ü‰Ñ ÐÑ<Ø &× 7Ñ 7ˆÔÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÑuˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÑuˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^Ðek×etÑetÑuˆŒÜ—i’i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^Ðek×etÑetÑuˆ�r5   r6   Úposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsr7   c           
      ó˜  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [        5      nU" U UU	U
U4U R                  (       d  SOU R                   U R"                  [%        U R                  SS 5      S.UD6u  pïUR&                  " / UQSP76 R)                  5       nU R+                  U5      n[,        R.                  R1                  XàR2                  U R                  S9nXï4$ )Néÿÿÿÿr   r   g        Úsliding_window)r=   Úscalingr_   r9   )ÚshaperQ   rR   ÚviewÚ	transposerT   rU   r   ÚupdaterM   r   Úget_interfacer!   Ú_attn_implementationr   r;   Úattention_dropoutr`   ÚgetattrÚreshapeÚ
contiguousrV   r   r<   r=   r/   )r0   r6   rY   rZ   r[   r\   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesÚcosÚsinÚattention_interfaceÚattn_outputÚattn_weightss                   r3   r>   ÚStarcoder2Attention.forwardN   s¹  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆÜ—m‘m×+Ñ+Ø×0Ñ0¸4¿=¹=ð ,ð 
ˆð Ð(Ð(r5   )rT   rV   rR   r/   rU   )N)r@   rA   rB   rC   r   Úintr&   rE   ÚTensorrD   r   r   r   r>   rG   rH   rI   s   @r3   rK   rK   E   s½   ø† ñvÐ/ð v¸CÀ$¹J÷ vð vð )-ñ+)à—|‘|ð+)ð # 5§<¡<°·±Ð#=Ñ>ð+)ð Ÿ™ tÑ+ð	+)ð
  ™ð+)ð Ð-Ñ.ð+)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0°%¸¿¹Ñ2EÈÑ2LÐLÑ	M÷+)ó +)r5   rK   c                   ó4   ^ • \ rS rSrS\S\4U 4S jjrSrU =r$ )ÚStarcoder2DecoderLayeré|   r!   rM   c                 ó  >• [         TU ]  X5        [        XS9U l        [	        U5      U l        [        R                  " UR                  UR                  S9U l
        [        R                  " UR                  UR                  S9U l        g )NrO   ©Úeps)r%   r&   rK   Ú	self_attnr   Úmlpr   Ú	LayerNormr'   Únorm_epsilonÚinput_layernormÚpost_attention_layernormrW   s      €r3   r&   ÚStarcoder2DecoderLayer.__init__}   sf   ø€ Ü‰Ñ˜Ô+Ü,°FÑPˆŒÜ  Ó(ˆŒÜ!Ÿ|š|¨F×,>Ñ,>ÀF×DWÑDWÑXˆÔÜ(*¯ª°V×5GÑ5GÈV×M`ÑM`Ñ(aˆÕ%r5   )r‚   r   rƒ   r~   )	r@   rA   rB   rC   r   rv   r&   rG   rH   rI   s   @r3   ry   ry   |   s    ø† ðbÐ/ð b¸C÷ bõ br5   ry   c                   óþ   ^ • \ rS rSrS\4U 4S jjr\\      SS\R                  S-  S\R                  S-  S\R                  S-  S\S-  S	\R                  S-  S
\S-  S\\   S\\-  4S jj5       5       rSrU =r$ )ÚStarcoder2Modelé…   r!   c           	      ó<  >• [         TU ]  U5        [        R                  " [	        UR
                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        R                  " UR                  UR                  S9U l        UR                  U l        g s  snf )Nr|   )r%   r&   r   Ú
ModuleListÚrangeÚnum_hidden_layersry   Úlayersr€   r'   r�   ÚnormÚembedding_dropoutrW   s      €r3   r&   ÚStarcoder2Model.__init__†   sy   ø€ Ü‰Ñ˜Ô Ü—m’mÜHMÈf×NfÑNfÔHgÓhÒHg¸9Ô# FÖ6ÑHgÑhó
ˆŒô —L’L ×!3Ñ!3¸×9LÑ9LÑMˆŒ	Ø!'×!9Ñ!9ˆÕùò is   ¸BNÚ	input_idsrZ   Úposition_idsr[   Úinputs_embedsÚ	use_cacher\   r7   c           
      óø  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      nU R                  R                  c  [        O[        n	U	" U R                  UUUUS9n
Un[        R                  R                  X°R                   U R"                  S9nU R%                  X³S9nU R&                  S U R                  R(                    H  nU" U4U
UUUUS	.UD6nM     U R+                  U5      n[-        UU(       a  US
9$ S S
9$ )Nz:You must specify exactly one of input_ids or inputs_embeds)r!   r   r   )Údevice)r!   r’   rZ   r[   r‘   r9   )r‘   )rZ   r‘   r[   r“   rY   )Úlast_hidden_stater[   )Ú
ValueErrorÚembed_tokensr   r!   Úget_seq_lengthrE   Úarangera   r•   Ú	unsqueezer_   r	   r
   r   r<   r=   rŽ   r;   Ú
rotary_embrŒ   r‹   r�   r   )r0   r�   rZ   r‘   r[   r’   r“   r\   Úpast_seen_tokensÚmask_functionÚcausal_maskr6   rY   Údecoder_layers                 r3   r>   ÚStarcoder2Model.forwardŽ   s—  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÑ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð &ˆÜŸ™×-Ñ-Ø×3Ñ3¸d¿m¹mð .ð 
ˆð #Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñŠMñ Jð Ÿ	™	 -Ó0ˆä&Ø+Þ/8˜Oñ
ð 	
à>Bñ
ð 	
r5   )rŽ   rŒ   r�   )NNNNNN)r@   rA   rB   rC   r   r&   r   r   rE   Ú
LongTensorrw   r   rF   Úboolr   r   rD   r   r>   rG   rH   rI   s   @r3   r†   r†   …   sÌ   ø† ð:Ð/÷ :ð  Øð .2Ø.2Ø04Ø(,Ø26Ø!%ñ7
à×#Ñ# dÑ*ð7
ð Ÿ™ tÑ+ð7
ð ×&Ñ&¨Ñ-ð	7
ð
  ™ð7
ð ×(Ñ(¨4Ñ/ð7
ð ˜$‘;ð7
ð Ð+Ñ,ð7
ð 
Ð(Ñ	(ô7
ó ó  ö7
r5   r†   c                   ó   • \ rS rSrSrg)ÚStarcoder2ForCausalLMéÊ   © N©r@   rA   rB   rC   rG   r§   r5   r3   r¥   r¥   Ê   ó   † Úr5   r¥   c                   ó   • \ rS rSrSrg)Ú#Starcoder2ForSequenceClassificationéÎ   r§   Nr¨   r§   r5   r3   r«   r«   Î   r©   r5   r«   c                   ó   • \ rS rSrSrg)Ú Starcoder2ForTokenClassificationéÒ   r§   Nr¨   r§   r5   r3   r®   r®   Ò   r©   r5   r®   )r¥   r†   ÚStarcoder2PreTrainedModelr«   r®   )3Ú__doc__Úcollections.abcr   rE   r   Úactivationsr   Úcache_utilsr   r   Úmasking_utilsr	   r
   Úmodeling_flash_attention_utilsr   Úmodeling_outputsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   Úutils.genericr   Úutils.output_capturingr   Úmistral.modeling_mistralr   r   r   r   r   r   r   r   Úconfiguration_starcoder2r   Ú
get_loggerr@   ÚloggerÚModuler   rK   ry   r†   r¥   r«   r®   Ú__all__r§   r5   r3   Ú<module>rÃ      sÁ   ðñ&  å $ã Ý å !ß .ß RÝ BÝ 7Ý 5Ý &ß 0Ý 7Ý 5÷	÷ 	ó 	õ 7ð 
×	Ò	˜HÓ	%€ô�B—I‘Iô ô"4)Ð*ô 4)ônbÐ0ô bôB
�lô B
ôJ	Ð.ô 	ô	Ð*Jô 	ô	Ð'Dô 	ò�r5   