ó
    ‡Ž¡jþx  ã                  ó  • S SK Jr  S SKrS SKrS SKrS SKJr  S SKJr  S SK	r	S SK
Jr  S SKJr  SSKJr  SSKJr  SS	KJrJr  SS
KJrJr  SSKJrJrJrJr  SSKJrJrJ r   SSK!J"r"  SSK#J$r$  SSK%J&r&J'r'J(r(J)r)J*r*  SSK+J,r,  \(       a  SSKJ-r-  SSK.J/r/  SSK0J1r1  \*Rd                  " \35      r4 " S S\Rj                  5      r6\( " S S5      5       r7\( " S S5      5       r8\( " S S5      5       r9 " S S\Rj                  5      r: " S S\5      r;g) é    )ÚannotationsN)Úpartial)ÚTYPE_CHECKING)Ú	safe_opené   )ÚCache)Úget_model_conversion_mapping)ÚWeightRenamingÚ$convert_and_load_state_dict_in_model)Ú&LAYER_PATTERN_TO_MASK_FUNCTION_MAPPINGÚcreate_causal_mask)ÚBaseModelOutputWithPastÚQuestionAnsweringModelOutputÚ SequenceClassifierOutputWithPastÚTokenClassifierOutput)ÚLoadStateDictConfigÚPreTrainedModelÚ_get_resolved_checkpoint_files)Ú	AutoModel)ÚUnpack)ÚContextManagersÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úlog_state_dict_report)ÚMtpCache)ÚPreTrainedConfig)ÚLogitsProcessorListc                  ó4   ^ • \ rS rSrSrSrSrU 4S jrSrU =r	$ )ÚGradientCheckpointingLayeré4   aÜ  Base class for layers with gradient checkpointing.

This class enables gradient checkpointing functionality for a layer. By default, gradient checkpointing is disabled
(`gradient_checkpointing = False`). When `model.set_gradient_checkpointing()` is called, gradient checkpointing is
enabled by setting `gradient_checkpointing = True` and assigning a checkpointing function to `_gradient_checkpointing_func`.

Important:

    When using gradient checkpointing with `use_reentrant=True`, inputs that require gradients (e.g. hidden states)
    must be passed as positional arguments (`*args`) rather than keyword arguments to properly propagate gradients.

    Example:

        ```python
        >>> # Correct - hidden_states passed as positional arg
        >>> out = self.layer(hidden_states, attention_mask=attention_mask)

        >>> # Incorrect - hidden_states passed as keyword arg
        >>> out = self.layer(hidden_states=hidden_states, attention_mask=attention_mask)
        ```
Fc                ó  >• U R                   (       aá  U R                  (       aÐ  SnU R                  R                  nSU S3nSU;   a  US   (       a  SUS'   US-  nSnU R                  (       d0  SU;   a  US   b  S US'   US-  nSnS	U;   a  US	   b  S US	'   US
-  nSnU(       a)  UR                  S5      S-   n[        R                  U5        U R                  " [        [        TU ],  40 UD6/UQ76 $ [        TU ],  " U0 UD6$ )NFz7Caching is incompatible with gradient checkpointing in z	. SettingÚ	use_cachez `use_cache=False`,TÚpast_key_valuesz `past_key_values=None`,Ú
layer_pastz `layer_past=None`,Ú,Ú.)Úgradient_checkpointingÚtrainingÚ	__class__Ú__name__Ú_can_checkpoint_with_cacheÚrstripÚloggerÚwarning_onceÚ_gradient_checkpointing_funcr   ÚsuperÚ__call__)ÚselfÚargsÚkwargsÚdo_warnÚ
layer_nameÚmessager+   s         €ÚªC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/modeling_layers.pyr3   Ú#GradientCheckpointingLayer.__call__P   s$  ø€ Ø×&×&¨4¯=¯=ØˆGØŸ™×0Ñ0ˆJØOÐPZÈ|Ð[dÐeˆGà˜fÓ$¨°×)<Ø&+��{Ñ#ØÐ0Ñ0�Ø�à×2×2à$¨Ó.°6Ð:KÑ3LÑ3XØ04�FÐ,Ñ-ØÐ9Ñ9�GØ"�Gà 6Ó)¨f°\Ñ.BÑ.NØ+/�F˜<Ñ(ØÐ4Ñ4�GØ"�Gö Ø!Ÿ.™.¨Ó-°Ñ3�Ü×#Ñ# GÔ,à×4Ò4´W¼U¹WÑ=MÑ5XÐQWÑ5XÐ`Ð[_Ò`Ð`Ü‰wÒ Ð0¨Ñ0Ð0ó    © )
r,   Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r)   r-   r3   Ú__static_attributes__Ú__classcell__©r+   s   @r:   r!   r!   4   s"   ø† ñð, #Ðð "'Ð÷1ó 1r<   r!   c                  ó~   ^ • \ rS rSrSrU 4S jr\\       S                 SS jj5       5       rSr	U =r
$ )Ú GenericForSequenceClassificationép   Úmodelc                ó4  >• [         TU ]  U5        UR                  U l        [        X R                  [
        R                  " U5      5        [        R                  " UR                  5       R                  U R                  SS9U l        U R                  5         g )NF©Úbias)r2   Ú__init__Ú
num_labelsÚsetattrÚbase_model_prefixr   Úfrom_configÚnnÚLinearÚget_text_configÚhidden_sizeÚscoreÚ	post_init©r4   Úconfigr+   s     €r:   rL   Ú)GenericForSequenceClassification.__init__t   sl   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä�×,Ñ,¬i×.CÒ.CÀFÓ.KÔLÜ—Y’Y˜v×5Ñ5Ó7×CÑCÀTÇ_Á_Ð[`ÑaˆŒ
ð 	�‰Õr<   c           	     óø  • [        X R                  5      " U4UUUUUS.UD6n	U	R                  n
U R                  U
5      nUb  UR                  S   nOUR                  S   nU R
                  R                  5       R                  c  US:w  a  [        S5      eU R
                  R                  5       R                  c  SnOÎUb�  XR
                  R                  5       R                  :g  R                  UR                  [        R                  5      n[        R                  " UR                  S   UR                  [        R                  S9nXþ-  R                  S5      nO.Sn[        R!                  U R"                  R$                   S35        U[        R                  " XËR                  S9U4   nS nUb  U R'                  X¶UU R
                  S	9n[)        UUU	R*                  U	R,                  U	R.                  S
9$ )N©Úattention_maskÚposition_idsr%   Úinputs_embedsr$   r   r   z=Cannot handle batch sizes > 1 if no padding token is defined.éÿÿÿÿ)ÚdeviceÚdtypezŠ will not detect padding tokens in `inputs_embeds`. Results may be unexpected if using padding tokens in conjunction with `inputs_embeds.`©r`   )ÚlogitsÚlabelsÚpooled_logitsrX   )Úlossrc   r%   Úhidden_statesÚ
attentions)ÚgetattrrO   Úlast_hidden_staterU   ÚshaperX   rS   Úpad_token_idÚ
ValueErrorÚtor`   ÚtorchÚint32ÚarangeÚargmaxr/   r0   r+   r,   Úloss_functionr   r%   rg   rh   )r4   Ú	input_idsr\   r]   r%   r^   rd   r$   r6   Útransformer_outputsrg   rc   Ú
batch_sizeÚlast_non_pad_tokenÚnon_pad_maskÚtoken_indicesre   rf   s                     r:   ÚforwardÚ(GenericForSequenceClassification.forward~   sâ  € ô 8?¸t×E[ÑE[Ô7\Øð8
à)Ø%Ø+Ø'Øñ8
ð ñ8
Ðð ,×=Ñ=ˆØ—‘˜MÓ*ˆàÑ Ø"Ÿ™¨Ñ+‰Jà&×,Ñ,¨QÑ/ˆJà�;‰;×&Ñ&Ó(×5Ñ5Ñ=À*ÐPQÃ/ÜÐ\Ó]Ð]Ø�;‰;×&Ñ&Ó(×5Ñ5Ñ=Ø!#ÑØÑ"à%¯©×)DÑ)DÓ)F×)SÑ)SÑS×WÑWÐX^×XeÑXeÔgl×grÑgrÓsˆLÜ!ŸLšL¨¯©¸Ñ)<ÀVÇ]Á]ÔZ_×ZeÑZeÑfˆMØ"/Ñ">×!FÑ!FÀrÓ!JÑà!#ÐÜ×ÑØ—>‘>×*Ñ*Ð+ð ,Zð Zôð
 œuŸ|š|¨J¿}¹}ÑMÐOaÐaÑbˆàˆØÑØ×%Ñ%¨VÐR_Ðhl×hsÑhsÐ%ÐtˆDä/ØØ Ø/×?Ñ?Ø-×;Ñ;Ø*×5Ñ5ñ
ð 	
r<   )rM   rU   ©NNNNNNN)rt   útorch.LongTensor | Noner\   útorch.Tensor | Noner]   r}   r%   úCache | Noner^   útorch.FloatTensor | Nonerd   r}   r$   úbool | Noner6   úUnpack[TransformersKwargs]Úreturnr   ©r,   r>   r?   r@   rO   rL   r   r   rz   rB   rC   rD   s   @r:   rF   rF   p   s™   ø† àÐõð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%ð8
à*ð8
ð ,ð8
ð .ð	8
ð
 &ð8
ð 0ð8
ð (ð8
ð ð8
ð -ð8
ð 
*ô8
ó ó ö8
r<   rF   c                  óŠ   ^ • \ rS rSrSrU 4S jrS rS r\\	       S                 S	S jj5       5       r
SrU =r$ )
ÚGenericForQuestionAnsweringé»   rH   c                óä   >• [         TU ]  U5        [        X R                  [        R
                  " U5      5        [        R                  " UR                  S5      U l	        U R                  5         g )Né   )r2   rL   rN   rO   r   rP   rQ   rR   rT   Ú
qa_outputsrV   rW   s     €r:   rL   Ú$GenericForQuestionAnswering.__init__¿   sO   ø€ Ü‰Ñ˜Ô ä�×,Ñ,¬i×.CÒ.CÀFÓ.KÔLÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰Õr<   c                ó@   • [        X R                  5      R                  $ ©N©ri   rO   Úembed_tokens)r4   s    r:   Úget_input_embeddingsÚ0GenericForQuestionAnswering.get_input_embeddingsÈ   s   € Ü�t×3Ñ3Ó4×AÑAÐAr<   c                ó8   • U[        X R                  5      l        g r�   rŽ   )r4   Úvalues     r:   Úset_input_embeddingsÚ0GenericForQuestionAnswering.set_input_embeddingsË   s   € Ø=BŒ�×,Ñ,Ó-Õ:r<   c                ó˜  • [        X R                  5      " U4UUUUS.UD6n	U	R                  n
U R                  U
5      nUR	                  SSS9u  pÍUR                  S5      R                  5       nUR                  S5      R                  5       nS nUb  Ub  U R                  " XÍXg40 UD6n[        UUUU	R                  U	R                  S9$ )N)r\   r]   r%   r^   r   r_   ©Údim)rf   Ústart_logitsÚ
end_logitsrg   rh   )ri   rO   rj   rŠ   ÚsplitÚsqueezeÚ
contiguousrs   r   rg   rh   )r4   rt   r\   r]   r%   r^   Ústart_positionsÚend_positionsr6   ÚoutputsÚsequence_outputrc   r™   rš   rf   s                  r:   rz   Ú#GenericForQuestionAnswering.forwardÎ   sé   € ô ,3°4×9OÑ9OÔ+PØð,
à)Ø%Ø+Ø'ñ,
ð ñ,
ˆð "×3Ñ3ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ð#:Ñ ˆØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆØÑ&¨=Ñ+DØ×%Ò% lÀÑiÐbhÑiˆDä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r<   )rŠ   r|   )rt   r}   r\   r~   r]   r}   r%   r   r^   r€   rž   r}   rŸ   r}   r6   r‚   rƒ   r   )r,   r>   r?   r@   rO   rL   r�   r”   r   r   rz   rB   rC   rD   s   @r:   r†   r†   »   s¥   ø† àÐõòBòCð Øð .2Ø.2Ø04Ø(,Ø26Ø37Ø15ð%
à*ð%
ð ,ð%
ð .ð	%
ð
 &ð%
ð 0ð%
ð 1ð%
ð /ð%
ð -ð%
ð 
&ô%
ó ó ö%
r<   r†   c                  ó~   ^ • \ rS rSrSrU 4S jr\\       S                 SS jj5       5       rSr	U =r
$ )ÚGenericForTokenClassificationéø   rH   c           	     óð  >• [         TU ]  U5        UR                  U l        [        X R                  [
        R                  " U5      5        [        USS 5      b  UR                  nO[        USS 5      b  UR                  nOSn[        R                  " U5      U l        [        R                  " UR                  5       R                  UR                  [        USS5      S9U l        U R#                  5         g )NÚclassifier_dropoutÚhidden_dropoutgš™™™™™¹?Útoken_classification_biasTrJ   )r2   rL   rM   rN   rO   r   rP   ri   r§   r¨   rQ   ÚDropoutÚdropoutrR   rS   rT   rU   rV   )r4   rX   r§   r+   s      €r:   rL   Ú&GenericForTokenClassification.__init__ü   sË   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä�×,Ñ,¬i×.CÒ.CÀFÓ.KÔLÜ�6Ð/°Ó6ÑBØ!'×!:Ñ!:ÑÜ�VÐ-¨tÓ4Ñ@Ø!'×!6Ñ!6Ñà!$ÐÜ—z’zÐ"4Ó5ˆŒÜ—Y’YØ×"Ñ"Ó$×0Ñ0Ø×ÑÜ˜Ð!<¸dÓCñ
ˆŒ
ð 	�‰Õr<   c           	     ó$  • [        X R                  5      " U4UUUUUS.UD6n	U	R                  n
U R                  U
5      n
U R	                  U
5      nS nUb  U R                  X¶U R                  5      n[        UUU	R                  U	R                  S9$ )Nr[   )rf   rc   rg   rh   )
ri   rO   rj   r«   rU   rs   rX   r   rg   rh   )r4   rt   r\   r]   r%   r^   rd   r$   r6   r    r¡   rc   rf   s                r:   rz   Ú%GenericForTokenClassification.forward  s©   € ô ,3°4×9OÑ9OÔ+PØð,
à)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð "×3Ñ3ˆØŸ,™, Ó7ˆØ—‘˜OÓ,ˆàˆØÑØ×%Ñ% f°d·k±kÓBˆDä$ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
r<   )r«   rM   rU   r|   )rt   r}   r\   r~   r]   r}   r%   r   r^   r€   rd   r}   r$   r�   r6   r‚   rƒ   r   r„   rD   s   @r:   r¤   r¤   ø   s™   ø† àÐõð* Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%ð!
à*ð!
ð ,ð!
ð .ð	!
ð
 &ð!
ð 0ð!
ð (ð!
ð ð!
ð -ð!
ð 
ô!
ó ó ö!
r<   r¤   c                  ój   ^ • \ rS rSr S         SU 4S jjjr              SS jrSrU =r$ )ÚMtpLayeri7  c                ó–  >• [         TU ]  5         Xl        XPl        U" UR                  UR
                  S9U l        U" UR                  UR
                  S9U l        [        R                  " UR                  S-  UR                  SS9U l
        U" X5      U l        U(       a!  U" UR                  UR
                  S9U l        g S U l        g )N©Úepsr‰   FrJ   )r2   rL   rX   Úuse_post_normrT   Úrms_norm_epsÚenormÚhnormrQ   rR   Úeh_projÚ	mtp_blockÚ	post_norm)r4   rX   Údecoder_layer_clsÚnorm_clsÚ	layer_idxr´   r+   s         €r:   rL   ÚMtpLayer.__init__8  s¤   ø€ ô 	‰ÑÔØŒØ*ÔÙ˜f×0Ñ0°f×6IÑ6IÑJˆŒ
Ù˜f×0Ñ0°f×6IÑ6IÑJˆŒ
Ü—y’y ×!3Ñ!3°aÑ!7¸×9KÑ9KÐRWÑXˆŒÙ*¨6Ó=ˆŒÞR_™ &×"4Ñ"4¸&×:MÑ:MÑNˆ�Ðeiˆ�r<   c                óª  • [        U R                  SS5      (       a6  [        R                  " U R	                  U5      U R                  U5      /SS9nO5[        R                  " U R                  U5      U R	                  U5      /SS9nU R                  U5      n	U R                  " U	4UUUUS.UD6n	U R                  (       a  U R                  U	5      n	U	$ )NÚmtp_hidden_states_firstFr_   r—   )r\   Úposition_embeddingsr]   r%   )
ri   rX   ro   Úcatr·   r¶   r¸   r¹   r´   rº   )
r4   r^   Úprevious_hidden_staterÁ   r\   r]   r%   r6   Úprojection_inputrg   s
             r:   rz   ÚMtpLayer.forwardI  sÇ   € ô �4—;‘;Ð 9¸5×AÑAÜ$Ÿyšy¨$¯*©*Ð5JÓ*KÈTÏZÉZÐXeÓMfÐ)gÐmoÑpÑä$Ÿyšy¨$¯*©*°]Ó*CÀTÇZÁZÐPeÓEfÐ)gÐmoÑpÐØŸ™Ð%5Ó6ˆØŸšØð
à)Ø 3Ø%Ø+ñ
ð ñ
ˆð ××Ø ŸN™N¨=Ó9ˆMàÐr<   )rX   r¸   r¶   r·   r¹   rº   r´   )T)
rX   r   r»   útype[nn.Module]r¼   rÆ   r½   Úintr´   Úbool)r^   útorch.TensorrÃ   rÉ   rÁ   ú!tuple[torch.Tensor, torch.Tensor]r\   r~   r]   r~   r%   r   rƒ   rÊ   )r,   r>   r?   r@   rL   rz   rB   rC   rD   s   @r:   r°   r°   7  s—   ø† ð #ðjà ðjð +ðjð "ð	jð
 ðjð ÷jð jð"à#ðð  ,ðð ?ð	ð
 ,ðð *ðð &ðð 
+÷ò r<   r°   c                  óò   ^ • \ rS rSrSrSrSrSS/rSS/rSU 4S jjr	SS jr
SS jr        SS	 jr    S                   SS
 jjr\SSS jj5       r\SS j5       r\SS j5       rSrU =r$ )ÚMtpModelig  Tzshared_head.head.weightzembed_tokens.weightzshared_head.weightc                ó,  >• [         TU ]  UR                  R                  5       5        SU l        X l        UR                  5       n[        UR                  S   5      n[        S UR                  S   R                  5        5       5      nSU l        SU l        [        U R                  S5      (       a"  SU l        U R                  R                  U l        [        R                   " [#        U5       Vs/ s H$  n[%        U R                  XEX`R                  5      PM&     sn5      U l        U R                  (       a4  U" U R                  R&                  U R                  R(                  S9U l        U R-                  U5        U R/                  5         g s  snf )NÚForCausalLMr_   c              3  óL   #   • U  H  u  pS U;   d  M  [        U5      v •  M     g7f)ÚnormN)Útype)Ú.0ÚnameÚmodules      r:   Ú	<genexpr>Ú$MtpModel.__init__.<locals>.<genexpr>z  s(   é € ð 
â E‘�Ø˜‰~ó ŒD��LˆLÚ Eùs   ‚$’$TFÚchain_hidden_post_normr²   )r2   rL   rX   Úget_mtp_configÚ	loss_typeÚnum_mtp_layersÚget_decoderrÑ   ÚlayersÚnextÚnamed_modulesr´   Úuse_shared_post_normÚhasattrr×   rQ   Ú
ModuleListÚranger°   rT   rµ   Úshared_post_normÚtie_with_main_modelrV   )r4   Ú
main_modelrÚ   Ú
base_modelÚ	layer_clsr¼   Úkr+   s          €r:   rL   ÚMtpModel.__init__r  sF  ø€ Ü‰Ñ˜×*Ñ*×9Ñ9Ó;Ô<à&ˆŒØ,Ôà×+Ñ+Ó-ˆ
Ü˜×*Ñ*¨2Ñ.Ó/ˆ	Üñ 
à *× 1Ñ 1°"Ñ 5× CÑ CÔ Eó
ó 
ˆð "ˆÔØ$)ˆÔ!Ü�4—;‘;Ð 8×9Ñ9Ø!&ˆDÔØ(,¯©×(JÑ(JˆDÔ%ô —m’mÜX]Ð^lÔXmÓnÒXmÐSTŒX�d—k‘k 9¸×;MÑ;MÖNÑXmÑnó
ˆŒð ×$×$Ù$,¨T¯[©[×-DÑ-DÈ$Ï+É+×JbÑJbÑ$cˆDÔ!ð 	× Ñ  Ô,à�‰Õùò os   Ã3+Fc                ó”   • UR                  5       U l        UR                  U l        UR	                  5       n[        USS5      U l        g)z8Tie the embedding/head/rotary layer with the main model.Ú
rotary_embN)r�   r�   Úlm_headÚshared_headrÛ   ri   rë   )r4   rå   ræ   s      r:   rä   ÚMtpModel.tie_with_main_model’  sC   € ð '×;Ñ;Ó=ˆÔØ%×-Ñ-ˆÔð  ×+Ñ+Ó-ˆ
Ü! *¨l¸DÓAˆ�r<   c                óÎ   • [        U R                  SS5      nUb  X-  nU R                  U5      n[        U R                  SS5      nUb  XCR                  S   :  a	  USSU24   nU$ )z[Apply the shared head the same way the main model does (muP scaling, unpadded vocab slice).Úlogits_mup_width_multiplierNÚunpadded_vocab_sizer_   .)ri   rX   rí   rk   )r4   rg   Ú
multiplierrc   rñ   s        r:   Ú_project_to_logitsÚMtpModel._project_to_logitsœ  sx   € ä˜TŸ[™[Ð*GÈÓNˆ
ØÑ!Ø)Ñ6ˆMØ×!Ñ! -Ó0ˆÜ% d§k¡kÐ3HÈ$ÓOÐØÑ*Ð/BÇ\Á\ÐRTÑEUÓ/UØ˜CÐ!5Ð"5Ð!5Ð5Ñ6ˆFØˆr<   c                óF  • U R                   R                  (       a  U R                   R                  U   OU R                   nUUSUUUS.n[        U R                   SS5      nUb  Xq   OSn0 n	UbW  U[        ;   aM  [        U   n
[        U
[        5      (       a$  U
R                  5        H  u  p¼U" S
0 UD6X›'   M     OU
" S
0 UD6X˜'   O[        S
0 UD6U	S'   [        U	5      S:”  a  [        S5      eSSSS	.nU	R                  5        VVs0 s H
  u  pïXÞ   U_M     n	nnU	$ s  snnf )z¬
Create the (potentially several) masks required for layer `layer_idx`. This relies on the `layer_types` from the MTP config
if defined, and otherwise uses full attention.
N)rX   r^   r\   r%   r]   r½   Úlayer_typesÚfull_attentionr‰   zLYou should have at most 2 masks, 1 for attention, and 1 for linear attentionr\   Ú	conv_mask)r÷   Úsliding_attentionÚlinear_attentionr=   )rX   Úis_heterogeneousÚper_layer_configri   r   Ú
isinstanceÚdictÚitemsr   Úlenrm   )r4   r½   r^   Ú	mtp_cacher]   Úlayer_configÚmask_kwargsÚmtp_layer_typesÚmtp_layer_typeÚmasksÚmask_functionÚactual_patternÚactual_functionÚ%internal_layer_expected_kwarg_mappingrè   Úvs                   r:   Úcreate_masks_for_mtp_layerÚ#MtpModel.create_masks_for_mtp_layer§  s@  € ð CGÇ+Á+×B^×B^�t—{‘{×3Ñ3°IÒ>Ðdh×doÑdoˆà"Ø*Ø"Ø(Ø(à"ñ
ˆô " $§+¡+¨}¸dÓCˆØ7FÑ7R˜Ò3ÐX\ˆØˆØÑ%¨.Ô<bÓ*bÜBÀ>ÑRˆMä˜-¬×.Ñ.Ø7D×7JÑ7JÖ7LÑ3�NÙ,;Ñ,J¸kÑ,J�EÓ)ò 8Mñ )6Ñ(D¸Ñ(D�Ò%ä&8Ñ&G¸;Ñ&GˆEÐ"Ñ#äˆu‹:˜‹>ÜÐkÓlÐlð /Ø!1Ø +ñ1
Ð-ð JOÏÉÌÔWÊÁÀÐ6Ñ9¸1Ò<ÉˆÑWàˆùó Xs   ÄDc
                ó  • UR                   S   n/ n/ nSn[        U R                  5       GH£  u  nnU R                  U5      R	                  UR
                  5      nU R                  b  U R                  UUS9OSnU R                  UUXT5      nU" UU4UUUS.UDU
D6nU R                  (       a  U R                  U5      nUc  [        SS5      O[        SS5      nU R                  USS2USS24   5      nUbc  [        R                  R                  USU4SS9SUS24   R                  5       nXàR                   " UU4U R"                  R$                  US	.U
D6-  nUSS2SSS24   R	                  UR
                  S
9nUb'  U	b$  U" U	UR	                  [&        R(                  S95      nUR+                  USS2SSS24   5        U(       aC  [        R                  R-                  US[&        R(                  S9n[&        R.                  " USS9nO[&        R0                  " USSS9nUR+                  U5        [&        R2                  " USS2SS24   U/SS9n[&        R2                  " USS2SS24   UR5                  US5      /SS9n[&        R2                  " USS2SS24   USS2SS24   S-   /SS9nU	c  GMŒ  [&        R2                  " U	U/SS9n	GM¦     [&        R2                  " USS9n[&        R2                  " USS9nUUU4$ )aG  
Sample 1 new token for each mtp layers present in this model. Note that the inputs are assumed to be already sliced and correct
here, i.e. if the main model just processed inputs corresponding to tokens at positions [N-1, N] in the sequence, then from it
you draft a new token for position N+1, and the `input_ids`/`position_ids`/`attention_mask` here are assumed to correspond to
data for tokens at positions [N, N+1], i.e. shifted by 1 from the main model, by the newly drafted token. The `last_hidden_states`
though will correspond to the same as the main model, i.e. positions [N-1, N] in the sequence length dimension.

`full_input_ids` correspond to the full sequence of `input_ids`, which is used in case we have any `logits_processor` as some
processors may require to check the length/value of the full previous sequence of ids.
r   N)r]   )rÁ   r]   r%   r_   iœÿÿÿ)r“   .)Ú
vocab_sizeÚshift_labelsrb   )ra   )r˜   ra   r   )Únum_samplesT)r˜   Úkeepdimr—   )rk   Ú	enumeraterÜ   r�   rn   r`   rë   r  rß   rã   Úsliceró   rQ   Ú
functionalÚpadr�   rs   rX   r  ro   Úfloat32ÚappendÚsoftmaxÚmultinomialrr   rÂ   Únew_ones)r4   rt   Úlast_hidden_statesr\   r]   r  rd   Ú	do_sampleÚlogits_processorÚfull_input_idsr6   rv   Údrafted_logitsÚdrafted_tokensrf   ÚiÚ	mtp_layerr^   rÁ   r  Úslice_indicesrc   r  Únext_token_logitsÚprobsÚnext_mtp_tokenÚnew_candidate_idsÚcandidate_logitss                               r:   rz   ÚMtpModel.forwardÖ  s+  € ð0 —_‘_ QÑ'ˆ
àˆØˆØˆÜ% d§k¡k×2‰LˆAˆyà ×-Ñ-¨iÓ8×;Ñ;Ð<N×<UÑ<UÓVˆMàMQÏ_É_ÑMh�—‘ ¸L�ÑIÐnrð  ð
 ×3Ñ3°A°}ÀiÓ^ˆEá!*ØØ"ð"ð %8Ø)Ø )ñ"ð ð"ð ñ"Ðð ×(×(Ø%)×%:Ñ%:Ð;MÓ%NÐ"ð 06©~œE " dœOÄ5ÈÈtÓCTˆMØ×,Ñ,Ð-?ÂÀ=ÒRSÐ@SÑ-TÓUˆFð Ñ!ä!Ÿ}™}×0Ñ0°¸!¸Q¸ÀtÐ0ÐLÈSÐRSÑRTÈWÑU×`Ñ`Ób�Ø×*Ò*Ø˜FðØ/3¯{©{×/EÑ/EÐT`ñØdjññ �ð
 !'¢q¨"ªa xÑ 0× 3Ñ 3¸9×;KÑ;KÐ 3Ð LÐØÑ+°Ñ0JÙ$4°^ÐEV×EYÑEYÔ`e×`mÑ`mÐEYÐEnÓ$oÐ!à×!Ñ!Ð"3²A°tºQ°JÑ"?Ô@ÞÜŸ™×-Ñ-Ð.?ÀRÌuÏ}É}Ð-Ð]�Ü!&×!2Ò!2°5ÀaÑ!H‘ä!&§¢Ð.?ÀRÐQUÑ!V�Ø×!Ñ! .Ô1ô Ÿ	š	 9ªQ°±¨UÑ#3°^Ð"DÈ"ÑMˆIÜ"ŸYšY¨²q¸!¹"°uÑ(=¸~×?VÑ?VÐWaÐcdÓ?eÐ'fÐlnÑoˆNÜ Ÿ9š9 l²1°a±b°5Ñ&9¸<ÊÈ2É3ÈÑ;OÐRSÑ;SÐ%TÐZ\Ñ]ˆLð Ô)Ü!&§¢¨N¸NÐ+KÐQSÑ!T“ñm 3ôp "ŸIšI n¸!Ñ<ÐÜ Ÿ9š9 ^¸Ñ;ÐØ Ð"2°DÐ8Ð8r<   c           
     óH  • UR                   R                  nUR                   R                  5       R                  nUR                  R                  5       n/ nU HN  n[        R                  " SU5      n	U	b   [        U	R                  S5      5      U:  a  M=  UR                  U5        MP     [        U5      S:X  a"  [        UR                  R                   S35      e[        R                  " SR!                  S U 5       5      5      n
UR                   R                  5       R"                  nU R%                  UR                   R&                  SSS 5      n[)        U5         U " X5      nS S S 5        [+        US S SS SS	9u  pïUnS nUb‚  US
   R-                  5        VVs0 s H  u  nnU
R                  U5      c  M  UU_M     nnnU Vs/ s H8  n[.        R0                  R3                  U5      UR5                  5       ;   d  M6  UPM:     nn0 n[7        5       nUS L=(       a     [9        S UR5                  5        5       5      nU(       d  [:        R<                  S:X  a  SOSnU H{  n[?        USSUS9nURA                  U5        URC                  5        HG  nUb  UURC                  5       ;   d  Ub  M  U
R                  U5      c  M3  URE                  U5      UU'   MI     M}     [G        XUU-   5       Vs/ s H  n[I        SU S3SUU-
   S3S9PM     nnURK                  [M        WSS95        URK                  URN                  5        [Q        UU[S        UX!R                   R&                  S9S9u  nnU H  nURU                  S S S 5        M     URW                  U5        URX                  (       a1  [[        SU R                   SU S[]        URX                  5       35      eUR_                  U5        [a        UUSU[b        S9  U$ ! , (       d  f       GN¡= fs  snnf s  snf s  snf )Nz\.(\d+)r   r   z7 does not seem to register any known MTP layer patternsÚ|c              3  ó.   #   • U  H  nS U S3v •  M     g7f)Ú(Ú)Nr=   )rÒ   Úpatterns     r:   rÕ   Ú+MtpModel.from_pretrained.<locals>.<genexpr>@  s   é € Ð'ZÒGY¸G¨1¨W¨I°Q­ÒGYùs   ‚FT)Úpretrained_model_name_or_pathÚvariantÚ	gguf_fileÚuse_safetensorsÚ
user_agentÚis_remote_codeÚ
weight_mapc              3  ó‚   #   • U  H5  n[        U[        R                  5      (       a  UR                  OUS :H  v •  M7     g7f)ÚmpsN)rý   ro   r`   rÑ   )rÒ   Úds     r:   rÕ   r1  ]  s/   é € ð 0
ÚM`È”z !¤U§\¡\×2Ñ2ˆQ�VŠV¸¸eÖCÒM`ùs   ‚=?Úwin32ÚpreadÚmmapÚptÚcpu)Ú	frameworkr`   Úbackendzlayers.r(   z.mtp_block.)Úsource_patternsÚtarget_patterns)Ú
add_legacy)Úweight_mappingÚ
device_mapra   )rH   Ú
state_dictÚload_configzThe following z weights are missing from z9 (checkpoint keys not matching the conversion mapping?): )rH   r2  Úignore_mismatched_sizesÚloading_infor/   )2rX   Úname_or_pathrS   Únum_hidden_layersÚ"_keys_to_ignore_on_load_unexpectedÚcopyÚreÚsearchrÇ   Úgroupr  r   rm   r+   r,   ÚcompileÚjoinrÚ   Úget_init_contextra   r   r   rÿ   ÚosÚpathÚbasenameÚvaluesÚsetÚanyÚsysÚplatformr   ÚaddÚkeysÚ	get_slicerâ   r
   Úextendr	   Ú_weight_conversionsr   r   Ú__exit__Ú#_adjust_missing_and_unexpected_keysÚmissing_keysÚRuntimeErrorÚsortedrä   r   r/   )Úclsrå   rG  r6   r2  rM  Úmtp_patternsÚfinal_mtp_patternsr0  Úmatch_objectÚ	mtp_regexrÚ   ÚcontextsÚ	mtp_modelÚcheckpoint_filesÚsharded_metadataÚ	mtp_filesÚmtp_weight_maprè   r  ÚfileÚmtp_state_dictÚall_pointerÚis_mpsrB  Úfile_pointerÚNÚweight_conversionsrK  Ú_s                                 r:   Úfrom_pretrainedÚMtpModel.from_pretrained/  s%  € à(2×(9Ñ(9×(FÑ(FÐ%Ø&×-Ñ-×=Ñ=Ó?×QÑQÐð "×DÑD×IÑIÓKˆð  ÐÛ#ˆGÜŸ9š9 Z°Ó9ˆLØÑ'¬C°×0BÑ0BÀ1Ó0EÓ,FÐIZÓ,ZÙØ×%Ñ% gÖ.ñ	 $ô
 Ð!Ó" aÓ'Ü 
× 4Ñ 4× =Ñ =Ð>Ð>uÐvÓwÐwÜ—J’J˜sŸx™xÑ'ZÑGYÓ'ZÓZÓ[ˆ	ð $×*Ñ*×:Ñ:Ó<×KÑKˆØ×'Ñ'¨
×(9Ñ(9×(?Ñ(?ÀÈÈtÓTˆÜ˜XÕ&Ù˜JÓ7ˆI÷ 'ô .LØ*GØØØ ØØ ñ.
Ñ*Ðð %ˆ	ØˆàÑ'à!1°,Ñ!?×!EÑ!EÔ!GôÚ!G™˜˜AÈ9×K[ÑK[Ð\]ÓK^“��1’Ñ!Gð ñ ñ +;ÓpÒ*: $¼b¿g¹g×>NÑ>NÈtÓ>TÐXf×XmÑXmÓXoÑ>oŸÑ*:ˆIÐpð ˆÜ“eˆØ 4Ð'÷ 
¬Cñ 0
ØMW×M^ÑM^ÔM`ó0
ó -
ˆö $¤s§|¡|°wÓ'>‘'ÀFˆÛˆDÜ$ T°TÀ%ÐQXÑYˆLØ�O‰O˜LÔ)Ø!×&Ñ&Ö(�à"Ñ.°1¸×8KÑ8KÓ8MÓ3MØ"Ó*¨y×/?Ñ/?ÀÓ/BÓ/Nà(4×(>Ñ(>¸qÓ(A�N 1Ó%ó )ñ ô$ Ð,À.Ñ.PÔQó	
ò R�ô Ø")¨!¨¨A À'È!ÐN_ÑJ_ÐI`Ð`kÐ@lôñ Rð	 	ð 
ð 	×!Ñ!Ô">¸yÐUZÑ"[Ô\Ø×!Ñ! *×"@Ñ"@ÔAô ?ØØ%Ü+Ø1¸j×PaÑPa×PgÑPgññ
‰ˆ�aó ˆAØ�J‰J�t˜T 4Ö(ñ ð 	×5Ñ5°lÔCð ×$×$ÜØ  §¡ Ð.HÐIfÐHgð hKÜKQÐR^×RkÑRkÓKlÐJmðoóð ð 	×%Ñ% jÔ1äØØ*GØ$)Ø%Üò	
ð Ð÷i 'Ö&üó ùò qùò.
s*   Å	PÆPÆ:PÇ5PÈPË0PÐ
Pc                ó   • g©NTr=   ©rh  s    r:   Ú_can_set_attn_implementationÚ%MtpModel._can_set_attn_implementation›  ó   € ð r<   c                ó   • gr~  r=   r  s    r:   Ú_can_set_experts_implementationÚ(MtpModel._can_set_experts_implementation   r‚  r<   )	r�   rÜ   rÙ   rÚ   rë   rí   rã   r´   rß   )rå   r   rÚ   rÇ   )rå   r   )rg   rÉ   rƒ   rÉ   )r½   rÇ   r^   rÉ   r  r   r]   rÉ   )NFNN)rt   rÉ   r  rÉ   r\   r~   r]   r~   r  zMtpCache | Nonerd   r}   r  rÈ   r  zLogitsProcessorList | Noner  r~   rƒ   rÊ   r�   )rå   r   rƒ   rÌ   )rƒ   rÈ   )r,   r>   r?   r@   Ú_supports_sdpaÚ_supports_flex_attnÚ_supports_flash_attnrN  Ú_keys_to_ignore_on_load_missingrL   rä   ró   r  rz   Úclassmethodr{  r€  r„  rB   rC   rD   s   @r:   rÌ   rÌ   g  s%  ø† ð €NØÐØÐà*CÐEZÐ)[Ð&à';Ð=RÐ&SÐ#÷ô@Bô	ð-Øð-Ø-9ð-ØFNð-Ø^jô-ðl +/àØ7;Ø.2ðW9àðW9ð )ðW9ð ,ð	W9ð
 *ðW9ð #ðW9ð (ðW9ð ðW9ð 5ðW9ð ,ðW9ð 
+õW9ðr õió ðiðV óó ðð óó ör<   rÌ   )<Ú
__future__r   rV  rP  r\  Ú	functoolsr   Útypingr   ro   Útorch.nnrQ   Úsafetensorsr   Úcache_utilsr   Úconversion_mappingr	   Úcore_model_loadingr
   r   Úmasking_utilsr   r   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   r   r   Úmodels.autor   Úprocessing_utilsr   Úutilsr   r   r   r   r   Úutils.loading_reportr   r   Úconfiguration_utilsr   Úgeneration.logits_processr   Ú
get_loggerr,   r/   ÚModuler!   rF   r†   r¤   r°   rÌ   r=   r<   r:   Ú<module>rž     sñ   ðõ #ã 	Û 	Û 
Ý Ý  ã Ý Ý !å Ý <ß Tß U÷ó ÷ aÑ `Ý "Ý $ß aÕ aÝ 7ö Ý%Ý5Ý>ð 
×	Ò	˜HÓ	%€ô91 §¡ô 91ðx ÷G
ð G
ó ðG
ðT ÷9
ð 9
ó ð9
ðx ÷;
ð ;
ó ð;
ô|-ˆr�y‰yô -ô`|ˆõ |r<   