ó
    ‡Ž¡je  ã                   ó¾  • S SK r S SKJr  S SKrS SKJr  SSKJr  SSKJ	r	  SSK
JrJr  SSKJr  SS	KJrJr  SS
KJr  SSKJrJrJrJr  SSKJrJr  SSKJrJr  SSKJ r J!r!  SSK"J#r#  SSK$J%r%J&r&J'r'  SSK(J)r)  SSK*J+r+J,r,  SSK-J.r.  SSK/J0r0   " S S\Rb                  5      r2 " S S\Rb                  5      r3S r4\" S5      S?S j5       r5S\Rl                  S\7S\Rl                  4S  jr8 S@S!\Rb                  S"\Rl                  S#\Rl                  S$\Rl                  S%\Rl                  S-  S&\9S'\9S(\#\%   4S) jjr:S* r;\" \55       " S+ S,\Rb                  5      5       r<\" S-5       " S. S/\Rb                  5      5       r= " S0 S1\5      r>\& " S2 S3\!5      5       r?\& " S4 S5\?5      5       r@\& " S6 S7\?\5      5       rA " S8 S9\\?5      rB " S: S;\\?5      rC " S< S=\\?5      rD/ S>QrEg)Aé    N)ÚCallable)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚDiffLlamaConfigc                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚDiffLlamaMLPé4   c                 ó¼  >• [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l	        [        UR                     U l        g ©NF©Úbias)ÚsuperÚ__init__ÚconfigÚhidden_sizeÚintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©Úselfr+   Ú	__class__s     €Ú¾C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/diffllama/modeling_diffllama.pyr*   ÚDiffLlamaMLP.__init__5   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÑXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÑVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÑXˆŒÜ˜V×.Ñ.Ñ/ˆ�ó    c                 óˆ   • U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ ©N)r1   r3   r/   r0   )r5   Úxr1   s      r7   ÚforwardÚDiffLlamaMLP.forward?   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr9   )r3   r+   r1   r/   r,   r-   r0   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r*   r=   Ú__static_attributes__Ú__classcell__©r6   s   @r7   r#   r#   4   s   ø† õ0÷ð r9   r#   c            	       óÒ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\\	R                  \4   4S jj5       5       r\	R                  " 5       \S	 5       5       rS
rU =r$ )ÚDiffLlamaRotaryEmbeddingéD   Údevicez5.18)Úversionr+   c                 óÄ  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        U R
                  R                  S   U l        U R                  nU R                  S:w  a  [        U R                     nU" U R
                  U5      u  o@l
        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚdefaultF)Ú
persistent)r)   r*   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr+   Úrope_parametersrL   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r5   r+   rI   Úrope_init_fnrV   r6   s        €r7   r*   Ú!DiffLlamaRotaryEmbedding.__init__E   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÓ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕr9   Úreturnc           	      ó   • U R                   S   n[        U SS5      =(       d    U R                  U R                  -  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?r   é   ©Údtype)rR   Úgetattrr,   Únum_attention_headsÚtorchÚarangeÚfloatÚto)r+   rI   ÚkwargsÚbaseÚdimÚattention_factorrV   s          r7   rS   Ú8DiffLlamaRotaryEmbedding.compute_default_rope_parametersV   s{   € ð ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r9   c                 ó  • U R                   S S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        USS9   X4-  R                  SS	5      n[        R                  " Xf4SS
9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR                  UR                   S9W	R                  UR                   S94$ ! , (       d  f       N@= f)Nr   éÿÿÿÿr    )ra   rI   ÚmpsÚcpuF)Údevice_typeÚenabledr_   ©rj   r`   )rV   ÚexpandÚshaperg   rd   rf   rI   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrT   Úsinra   )
r5   r<   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrq   ÚfreqsÚembr{   r|   s
             r7   r=   Ú DiffLlamaRotaryEmbedding.forwardj   s;  € ð �M‰M˜$¢ 4˜-Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÐtð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   ÃA%E7Å7
F)rT   r+   rV   rP   rX   rQ   rL   r;   )r?   r@   rA   rB   r   r!   r*   ÚstaticmethodÚtuplerd   ÚTensorrf   rS   Úno_gradr   r=   rC   rD   rE   s   @r7   rG   rG   D   sŒ   ø† Ù�X vÑ.ñO˜÷ Oó /ðOð  Ù�X vÑ.ñ5°ð 5Ð[`Ðaf×amÑamÐotÐatÑ[uô 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ö<r9   rG   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..Nrn   r_   rs   )ru   rd   rz   )r<   Úx1Úx2s      r7   Úrotate_halfrŠ   }   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r9   Úrotary_pos_embc                 ó˜   • UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXV4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerŠ   )ÚqÚkr{   r|   Úunsqueeze_dimÚq_embedÚk_embeds          r7   Úapply_rotary_pos_embr“   „   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØÐÐr9   Úhidden_statesÚn_repr[   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r    N)ru   rt   Úreshape)r”   r•   ÚbatchÚnum_key_value_headsÚslenr^   s         r7   Ú	repeat_kvr›   ž   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr9   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrh   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )Nr_   r   rn   ©rj   ra   )ÚpÚtrainingr    )r›   Únum_key_value_groupsrd   Úmatmulry   r   Ú
functionalÚsoftmaxÚfloat32rg   ra   r¢   r¦   Ú
contiguous)rœ   r�   rž   rŸ   r    r¡   r¢   rh   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r7   Úeager_attention_forwardr±   ª   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r9   c                 ó@   • SS[         R                  " SU -  5      -  -
  $ )Ngš™™™™™é?g333333ã?g333333Ó¿)ÚmathÚexp)Ú	layer_idxs    r7   Úlambda_init_fnr¶   Ã   s    € Ø�”t—x’x  yÑ 0Ó1Ñ1Ñ1Ð1r9   c                   ó   ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr  SS\R                  S\
\R                  \R                  4   S	\R                  S-  S
\S-  S\
\R                  \R                  4   4
S jjrSrU =r$ )ÚDiffLlamaAttentionéÇ   uç  Multi-headed differential attention (https://huggingface.co/papers/2410.05258).

Computes ``(softmax(Q1 K1áµ€) - Î» Â· softmax(Q2 K2áµ€)) Â· V`` as two standard attention calls
sharing Q and K over the two halves of V. The two-call structure is ~30% faster than the
V-doubling shortcut at production shapes, since asymmetric V (``head_dim_v != head_dim_q``)
forces SDPA off Flash/cuDNN onto the memory-efficient/math kernel; Flash Attention 2 also
requires ``head_dim_v == head_dim_q``.
Nr+   rµ   c                 óœ  >• [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        UR                  S:”  a  [)        S5      eUR                  b  UR                  S-  S:w  a  [)        S	UR                   S
35      e[+        U5      U l        [        R.                  " [0        R2                  " SUR4                  U R                  4S95      U l        [        R.                  " [0        R2                  " SUR4                  U R                  4S95      U l        [        R.                  " [0        R2                  " SUR4                  U R                  4S95      U l        [        R.                  " [0        R2                  " SUR4                  U R                  4S95      U l        [        R>                  " SU R                  -  UR@                  SS9U l!        g )Nr^   g      à¿Tr'   ç        z€DiffLlama does not support `attention_dropout > 0`: the differential attention mechanism has no paper-defined dropout semantics.r_   r   z–DiffLlama requires `num_key_value_heads` to be even (and at least 2): the two-call differential attention splits the value tensor along KV heads, got Ú.)ÚsizeF)ÚepsÚelementwise_affine)"r)   r*   r+   rµ   rb   r,   rc   r^   r™   r§   r¡   Úattention_dropoutÚ	is_causalr   r.   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚ
ValueErrorr¶   Úlambda_initÚ	Parameterrd   ÚnormalÚlambda_std_devÚ	lambda_q1Ú	lambda_k1Ú	lambda_q2Ú	lambda_k2ÚRMSNormÚrms_norm_epsÚ	groupnorm©r5   r+   rµ   r6   s      €r7   r*   ÚDiffLlamaAttention.__init__Ò   sœ  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfñ
ˆŒð ×#Ñ# cÓ)ÜðDóð ð ×%Ñ%Ñ-°×1KÑ1KÈaÑ1OÐSTÓ1TÜðVØV\×VpÑVpÐUqÐqrðtóð ô *¨)Ó4ˆÔÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÑ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÑ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÑ&cÓdˆŒÜŸš¤e§l¢l°1°f×6KÑ6KÐSW×S`ÑS`ÐRbÑ&cÓdˆŒÜŸš A¨¯©Ñ$5¸6×;NÑ;NÐchÑiˆ�r9   r”   Úposition_embeddingsr    Úpast_key_valuesr[   c                 ó¤  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pšS [        R                  " U
SSS9 5       u  pÞ[        R                  " U R                  R                  [         5      nU" U UU	UU4SU R"                  S.UD6u  nnU" U UU	UU4SU R"                  S.UD6u  nn[        R$                  " UU/SS9n[        R                  " USSS9u  nn[        R&                  " [        R(                  " U R*                  U R,                  -  S[        R.                  S95      R1                  UR2                  5      n[        R&                  " [        R(                  " U R4                  U R6                  -  S[        R.                  S95      R1                  UR2                  5      nUU-
  U R8                  -   nUUU-  -
  nSU R8                  -
  U R;                  U5      -  nUR<                  " / UQSP76 nU R?                  U5      nUU4$ )	Nrn   r    r_   c              3   óH   #   • U  H  oR                  S SS S 5      v •  M     g7f)r    r_   N)Úrepeat)Ú.0Úvs     r7   Ú	<genexpr>Ú-DiffLlamaAttention.forward.<locals>.<genexpr>  s"   é € Ð'jÒFiÀ¯©°°A°q¸!×(<Ð(<ÒFiùs   ‚ "rs   r»   )r¢   r¡   r¤   ) ru   r^   rÃ   Úviewry   rÄ   rÅ   r“   Úupdaterµ   rd   Úchunkr   Úget_interfacer+   Ú_attn_implementationr±   r¡   rz   r´   ÚsumrÌ   rÍ   r«   rg   ra   rÎ   rÏ   rÈ   rÒ   r—   rÆ   )r5   r”   rÕ   r    rÖ   rh   Úinput_shapeÚhidden_shapeÚquery_statesr­   r®   r{   r|   Úvalue_states1Úvalue_states2Úattention_interfaceÚattn_output1r¯   Úattn_output2Ú_r°   Úlambda_1Úlambda_2Úlambda_fulls                           r7   r=   ÚDiffLlamaAttention.forwardÿ   s¿  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJñ (kÄeÇkÂkÐR^Ð`aÐghÒFiÓ'jÑ$ˆä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ &9ØØØØØð	&
ð Ø—L‘Lñ	&
ð ñ	&
Ñ"ˆ�lñ .ØØØØØð	
ð Ø—L‘Lñ	
ð ñ	
‰ˆ�aô —i’i ¨|Ð <À"ÑEˆô &+§[¢[°¸aÀQÑ%GÑ"ˆ�lÜ—9’9œUŸYšY t§~¡~¸¿¹Ñ'FÈBÔV[×VcÑVcÑdÓe×hÑhØ×Ñó
ˆô —9’9œUŸYšY t§~¡~¸¿¹Ñ'FÈBÔV[×VcÑVcÑdÓe×hÑhØ×Ñó
ˆð  Ñ)¨D×,<Ñ,<Ñ<ˆØ" [°<Ñ%?Ñ?ˆØ˜4×+Ñ+Ñ+¨t¯~©~¸kÓ/JÑJˆØ!×)Ò)Ð;¨;Ð;¸Ò;ˆØ—k‘k +Ó.ˆØ˜LÐ(Ð(r9   )rÀ   r+   rÒ   r^   rÁ   rÄ   rÈ   rÍ   rÏ   rÌ   rÎ   rµ   r§   rÆ   rÃ   r¡   rÅ   r;   )NN)r?   r@   rA   rB   Ú__doc__r!   Úintr*   rd   r…   r„   r   r=   rC   rD   rE   s   @r7   r¸   r¸   Ç   s§   ø† ññ+j˜ð +j¸3À¹:÷ +jð +jðb /3Ø(,ñC)à—|‘|ðC)ð # 5§<¡<°·±Ð#=Ñ>ðC)ð Ÿ™ tÑ+ð	C)ð
  ™ðC)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷C)ó C)r9   r¸   rÐ   c                   óx   ^ • \ rS rSrS
S\SS4U 4S jjjrS\R                  S\R                  4S jrS r	S	r
U =r$ )ÚDiffLlamaRMSNormiE  r¾   r[   Nc                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z/
DiffLlamaRMSNorm is equivalent to T5LayerNorm
N)r)   r*   r   rÉ   rd   ÚonesÚweightÚvariance_epsilon)r5   r,   r¾   r6   s      €r7   r*   ÚDiffLlamaRMSNorm.__init__G  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Õr9   r”   c                 ó  • UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )Nr_   rn   T)Úkeepdim)	ra   rg   rd   r«   ÚpowÚmeanÚrsqrtrø   r÷   )r5   r”   Úinput_dtypeÚvariances       r7   r=   ÚDiffLlamaRMSNorm.forwardO  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ð>ˆØ%¬¯ª°H×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r9   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)r„   r÷   ru   rø   )r5   s    r7   Ú
extra_reprÚDiffLlamaRMSNorm.extra_reprV  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr9   )rø   r÷   )g�íµ ÷Æ°>)r?   r@   rA   rB   rf   r*   rd   r…   r=   r  rC   rD   rE   s   @r7   rô   rô   E  sB   ø† ñ$¨ð $¸$÷ $ð $ð; U§\¡\ð ;°e·l±lô ;÷Jð Jr9   rô   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚDiffLlamaDecoderLayeriZ  r+   rµ   c                 ó  >• [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        [        UR                  UR                  S9U l
        g )N)r+   rµ   ©r¾   )r)   r*   r,   r¸   Ú	self_attnr#   Úmlprô   rÑ   Úinput_layernormÚpost_attention_layernormrÓ   s      €r7   r*   ÚDiffLlamaDecoderLayer.__init__[  sk   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä+°6ÑOˆŒä Ó'ˆŒÜ/°×0BÑ0BÈ×H[ÑH[Ñ\ˆÔÜ(8¸×9KÑ9KÐQW×QdÑQdÑ(eˆÕ%r9   Nr”   r    r}   rÖ   Ú	use_cacherÕ   rh   r[   c           
      óº   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pX�-   nUnU R                  U5      nU R                  U5      nX�-   nU$ )N)r”   r    r}   rÖ   r  rÕ   © )r  r	  r  r
  )
r5   r”   r    r}   rÖ   r  rÕ   rh   Úresidualrì   s
             r7   r=   ÚDiffLlamaDecoderLayer.forwarde  sˆ   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆð !Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Ñ0ˆØÐr9   )r,   r  r
  r  r	  )NNNFN)r?   r@   rA   rB   r!   rò   r*   rd   r…   Ú
LongTensorr   Úboolr„   r   r   r=   rC   rD   rE   s   @r7   r  r  Z  sÃ   ø† ðf˜ð f¸3÷ fð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷ó r9   r  c                   óŽ   ^ • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.r\R&                  " 5       U 4S j5       rS	rU =r$ )
ÚDiffLlamaPreTrainedModeli…  r+   ÚmodelTr  rÖ   )r”   Ú
attentionsc                 ó   >• [         TU ]  U5        [        U[        5      (       aÙ  [        R
                  " UR                  SU R                  R                  5        [        R
                  " UR                  SU R                  R                  5        [        R
                  " UR                  SU R                  R                  5        [        R
                  " UR                  SU R                  R                  5        g g )Nr   )r)   Ú_init_weightsrv   r¸   ÚinitÚnormal_rÌ   r+   rË   rÍ   rÎ   rÏ   )r5   rœ   r6   s     €r7   r  Ú&DiffLlamaPreTrainedModel._init_weights—  s£   ø€ ä‰Ñ˜fÔ%Ü�fÔ0×1Ñ1Ü�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÔIÜ�LŠL˜×)Ñ)¨1¨d¯k©k×.HÑ.HÕIð	 2r9   r  )r?   r@   rA   rB   r!   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr  r¸   Ú_can_record_outputsrd   r†   r  rC   rD   rE   s   @r7   r  r  …  sn   ø‡ àÓØÐØ&*Ð#Ø0Ð1ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà.Ø(ñÐð
 ‡]‚]ƒ_ôJó öJr9   r  c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚDiffLlamaModeli¡  r+   c           	      ó  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nr  ©r+   F)r)   r*   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr,   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr  Úlayersrô   rÑ   ÚnormrG   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÓ   s      €r7   r*   ÚDiffLlamaModel.__init__£  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜGLÈV×MeÑMeÔGfÓgÒGf¸)Ô" 6Ö5ÑGfÑgó
ˆŒô % V×%7Ñ%7¸V×=PÑ=PÑQˆŒ	Ü2¸&ÑAˆŒØ&+ˆÔ#ð 	�‰Õùò hs   ÂC?NÚ	input_idsr    r}   rÖ   Úinputs_embedsr  rh   r[   c           
      ó>  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U R                  UUUUS9n	Un
U R                  X£S9nU R                  S U R                  R                    H  nU" U
4U	UUUUS.UD6n
M     U R                  U
5      n
[        U
US	9$ )
Nz:You must specify exactly one of input_ids or inputs_embedsr,  r   r    )rI   )r+   r<  r    rÖ   r}   )r}   )r    rÕ   r}   rÖ   r  )Úlast_hidden_staterÖ   )rÇ   r1  r	   r+   Úget_seq_lengthrd   re   ru   rI   r�   r   r7  r5  r4  r6  r   )r5   r;  r    r}   rÖ   r<  r  rh   Úpast_seen_tokensÚcausal_maskr”   rÕ   Údecoder_layers                r7   r=   ÚDiffLlamaModel.forward³  sF  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø*.×*;Ñ*;¸IÓ*FˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð &ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø$7Ø)Ø /Ø#ñð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ñ
ð 	
r9   )r1  r8  r5  r6  r.  r7  r/  )NNNNNN)r?   r@   rA   rB   r!   r*   r   r   r   rd   r  r…   r   ÚFloatTensorr  r   r   r   r=   rC   rD   rE   s   @r7   r*  r*  ¡  sÍ   ø† ð˜÷ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!ô2
ó ó ó  ö2
r9   r*  c                   óX  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
        SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚDiffLlamaForCausalLMië  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr”   ÚlogitsÚkeep_full_weightc                 óä   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g r&   )
r)   r*   r*  r  r/  r   r.   r,   rG  r9  r4   s     €r7   r*   ÚDiffLlamaForCausalLM.__init__ò  sU   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒð 	�‰Õr9   Nr;  r    r}   rÖ   r<  Úlabelsr  Úlogits_to_keeprh   r[   c	           
      ó|  • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nSnUb)  U R                  " SXÖU R                  R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9$ )a,  
Example:

```python
>>> from transformers import AutoTokenizer, DiffLlamaForCausalLM

>>> model = DiffLlamaForCausalLM.from_pretrained("google/diffllama-7b")
>>> tokenizer = AutoTokenizer.from_pretrained("google/diffllama-7b")

>>> prompt = "What is your favorite condiment?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"What is your favorite condiment?"
```)r;  r    r}   rÖ   r<  r  N)rI  rM  r/  )ÚlossrI  rÖ   r”   r  r  )r  r>  rv   rò   ÚslicerG  Úloss_functionr+   r/  r   rÖ   r”   r  )r5   r;  r    r}   rÖ   r<  rM  r  rN  rh   Úoutputsr”   Úslice_indicesrI  rP  s                  r7   r=   ÚDiffLlamaForCausalLM.forwardû  sÖ   € ð> ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò%Ðp¨VÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r9   )rG  r  r/  )NNNNNNNr   )r?   r@   rA   rB   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr*   r   r   rd   r  r…   r   rD  r  rò   r   r   r   r=   rC   rD   rE   s   @r7   rF  rF  ë  s"  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ô6
ó ó ö6
r9   rF  c                   ó   • \ rS rSrSrg)Ú"DiffLlamaForSequenceClassificationi6  r  N©r?   r@   rA   rB   rC   r  r9   r7   r[  r[  6  ó   † Úr9   r[  c                   ó   • \ rS rSrSrSrg)ÚDiffLlamaForQuestionAnsweringi:  Útransformerr  N)r?   r@   rA   rB   r  rC   r  r9   r7   r_  r_  :  s   † Ø%Ór9   r_  c                   ó   • \ rS rSrSrg)ÚDiffLlamaForTokenClassificationi>  r  Nr\  r  r9   r7   rb  rb  >  r]  r9   rb  )r  r*  rF  r[  r_  rb  )r    )r»   )Fr³   Úcollections.abcr   rd   r   Ú r   r  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   Úmasking_utilsr   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_diffllamar!   ÚModuler#   rG   rŠ   r“   r…   rò   r›   rf   r±   r¶   r¸   rô   r  r  r*  rF  r[  r_  rb  Ú__all__r  r9   r7   Ú<module>rv     s%  ðó. Ý $ã Ý å &Ý !ß .Ý )ß LÝ /÷ó ÷ Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý 4ô�2—9‘9ô ô 6<˜rŸy™yô 6<òr(ñ Ð-Ó.óó /ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ò22ñ Ð)Ó*ôz)˜Ÿ™ó z)ó +ðz)ñz ˜YÓ'ôJ�r—y‘yó Jó (ðJô((Ð6ô (ðV ôJ˜ó Jó ðJð6 ôF
Ð-ó F
ó ðF
ðR ôG
Ð3°_ó G
ó ðG
ôT	Ð)IÐKcô 	ô&Ð$?ÐAYô &ô	Ð&CÐE]ô 	ò�r9   