ó
    ˆŽ¡jÙu  ã                   óÐ  • S SK Jr  S SKrS SKJs  Jr  S SKJr  SSKJr	  SSK
Jr  SSKJrJr  SSKJr  SS	KJrJrJr  SS
KJrJr  SSKJr  SSKJrJrJrJr  SSKJ r J!r!  SSK"J#r#J$r$  SSK%J&r&J'r'  SSK(J)r)  SSK*J+r+J,r,J-r-  SSK.J/r/  SSK0J1r1J2r2  SSK3J4r4J5r5  SSK6J7r7  \ " S S\Rp                  5      5       r9 " S S\Rp                  5      r: " S S\Rp                  5      r;\" S5       " S S\Rp                  5      5       r< " S  S!\Rp                  5      r=S" r>\" S#5      SFS$ j5       r?S%\R€                  S&\AS'\R€                  4S( jrB SGS)\Rp                  S*\R€                  S+\R€                  S,\R€                  S-\R€                  S-  S.\CS/\CS0\)\+   4S1 jjrD\" \?5       " S2 S3\Rp                  5      5       rE " S4 S5\5      rF\, " S6 S7\'5      5       rG\, " S8 S9\G5      5       rH   SHS:\R€                  \I\R€                     -  S-  S;\AS-  S-\R€                  S-  S'\R€                  \A-  4S< jjrJ\, " S= S>\G\5      5       rK " S? S@\\G5      rL " SA SB\\G5      rM " SC SD\\G5      rN/ SEQrOg)Ié    )ÚCallableN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚMixtralConfigc                   óž   ^ • \ rS rSrSrS\4U 4S jjrS\R                  S\R                  S\R                  S\R                  4S	 jr	S
r
U =r$ )ÚMixtralExpertsé8   z2Collection of expert weights stored as 3D tensors.Úconfigc                 óú  >• [         TU ]  5         UR                  U l        UR                  U l        UR                  U l        [        R                  " [        R                  " U R                  SU R                  -  U R
                  5      5      U l        [        R                  " [        R                  " U R                  U R
                  U R                  5      5      U l        [        UR                     U l        g )Né   )ÚsuperÚ__init__Únum_local_expertsÚnum_expertsÚhidden_sizeÚ
hidden_dimÚintermediate_sizeÚintermediate_dimr   Ú	ParameterÚtorchÚemptyÚgate_up_projÚ	down_projr   Ú
hidden_actÚact_fn©Úselfr)   Ú	__class__s     €ÚºC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/mixtral/modeling_mixtral.pyr-   ÚMixtralExperts.__init__<   s­   ø€ Ü‰ÑÔØ!×3Ñ3ˆÔØ ×,Ñ,ˆŒØ &× 8Ñ 8ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�ó    Úhidden_statesÚtop_k_indexÚtop_k_weightsÚreturnc                 óX  • [         R                  " U5      n[         R                  " 5          [         R                  R                  R                  X R                  S9nUR                  SSS5      n[         R                  " UR                  SS9S5      R                  5       nS S S 5        W Hè  nUS   nXpR                  :X  a  M  [         R                  " WU   5      u  p‰X   n
[        R                  R                  X R                  U   5      R                  SSS9u  p¼U R                  U5      U-  n[        R                  R                  XÐR                   U   5      nXÓX˜S 4   -  nUR#                  SX�R%                  UR&                  5      5        Mê     U$ ! , (       d  f       Nþ= f)N)Únum_classesr+   r$   r   )éÿÿÿÿéþÿÿÿ©ÚdimrG   )r5   Ú
zeros_likeÚno_gradr   Ú
functionalÚone_hotr/   ÚpermuteÚgreaterÚsumÚnonzeroÚwhereÚlinearr7   Úchunkr:   r8   Ú
index_add_ÚtoÚdtype)r<   rA   rB   rC   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 r>   ÚforwardÚMixtralExperts.forwardE   so  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ð`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ð'DÀaÓH×PÑPÓRˆJ÷ ó
 %ˆJØ# A™ˆJØ×-Ñ-Ó-ÙÜ#(§;¢;¨{¸:Ñ/FÓ#GÑ ˆIØ)Ñ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ðj‰HˆDØ$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9È)Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9lÖmñ %ð #Ð"÷# �_ús   ¬A7FÆ
F))r:   r8   r7   r1   r3   r/   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r%   r-   r5   ÚTensorrc   Ú__static_attributes__Ú__classcell__©r=   s   @r>   r'   r'   8   sR   ø† á<ð0˜}÷ 0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#ò #r@   r'   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚMixtralTopKRouteré`   c                 ó  >• [         TU ]  5         UR                  U l        UR                  U l        UR                  U l        [        R                  " [        R                  " U R
                  U R                  5      5      U l        g ©N)r,   r-   Únum_experts_per_tokÚtop_kr.   r/   r0   r1   r   r4   r5   r6   Úweightr;   s     €r>   r-   ÚMixtralTopKRouter.__init__a   s[   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×3Ñ3ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆ�r@   c                 óX  • UR                  SU R                  5      n[        R                  " XR                  5      n[
        R                  R                  R                  UR                  5       SS9n[
        R                  " X0R                  SS9u  pEXDR                  SSS9-  nUnX&U4$ )NrG   rI   T)rJ   Úkeepdim)Úreshaper1   ÚFrT   ru   r5   r   rM   ÚsoftmaxÚfloatÚtopkrt   rQ   )r<   rA   Úrouter_logitsÚrouter_probsÚrouter_top_valueÚrouter_indicesÚrouter_scoress          r>   rc   ÚMixtralTopKRouter.forwardh   s”   € Ø%×-Ñ-¨b°$·/±/ÓBˆÜŸš ·±Ó<ˆÜ—x‘x×*Ñ*×2Ñ2°=×3FÑ3FÓ3HÈbÐ2ÐQˆÜ+0¯:ª:°lÇJÁJÐTVÑ+WÑ(ÐØ×0Ñ0°RÀÐ0ÐFÑFÐØ(ˆØ¨^Ð;Ð;r@   )r1   r/   rt   ru   )re   rf   rg   rh   r-   rc   rk   rl   rm   s   @r>   ro   ro   `   s   ø† õS÷<ð <r@   ro   c                   ób   ^ • \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )ÚMixtralSparseMoeBlockér   c                 ó¦   >• [         TU ]  5         UR                  U l        UR                  U l        [        U5      U l        [        U5      U l	        g rr   )
r,   r-   rs   rt   Úrouter_jitter_noiseÚjitter_noisero   r`   r'   Úexpertsr;   s     €r>   r-   ÚMixtralSparseMoeBlock.__init__s   sA   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø"×6Ñ6ˆÔÜ% fÓ-ˆŒ	Ü% fÓ-ˆ�r@   rA   rD   c                 óš  • UR                   u  p#nU R                  (       aS  U R                  S:”  aC  U[        R                  " U5      R                  SU R                  -
  SU R                  -   5      -  nUR                  SUR                   S   5      nU R                  U5      u  pVnU R                  XU5      nUR                  X#U5      nU$ )Nr   ç      ð?rG   )
ÚshapeÚtrainingr‰   r5   Ú
empty_likeÚuniform_Úviewr`   rŠ   ry   )r<   rA   Ú
batch_sizeÚsequence_lengthr1   Ú_rC   rB   s           r>   rc   ÚMixtralSparseMoeBlock.forwardz   s¸   € Ø2?×2EÑ2EÑ/ˆ
 ZØ�=�=˜T×.Ñ.°Ó2ØœU×-Ò-¨mÓ<×EÑEÀcÈD×L]ÑL]ÑF]Ð_bÐei×evÑevÑ_vÓwÑwˆMØ%×*Ñ*¨2¨}×/BÑ/BÀ2Ñ/FÓGˆØ(,¯	©	°-Ó(@Ñ%ˆ˜+ØŸ™ ]ÀÓOˆØ%×-Ñ-¨jÈ:ÓVˆØÐr@   )rŠ   r`   r‰   rt   )
re   rf   rg   rh   r-   r5   rj   rc   rk   rl   rm   s   @r>   r…   r…   r   s(   ø† õ.ð U§\¡\ð °e·l±l÷ ò r@   r…   ÚRMSNormc                   óx   ^ • \ rS rSrS
S\SS4U 4S jjjrS\R                  S\R                  4S jrS r	S	r
U =r$ )ÚMixtralRMSNormé…   ÚepsrD   Nc                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z-
MixtralRMSNorm is equivalent to T5LayerNorm
N)r,   r-   r   r4   r5   Úonesru   Úvariance_epsilon)r<   r0   r›   r=   s      €r>   r-   ÚMixtralRMSNorm.__init__‡   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Õr@   rA   c                 ó  • UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )Nr+   rG   T)rx   )	rX   rW   r5   Úfloat32ÚpowÚmeanÚrsqrtrž   ru   )r<   rA   Úinput_dtypeÚvariances       r>   rc   ÚMixtralRMSNorm.forward�   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ð>ˆØ%¬¯ª°H×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r@   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)Útupleru   rŽ   rž   )r<   s    r>   Ú
extra_reprÚMixtralRMSNorm.extra_repr–   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr@   )rž   ru   )g�íµ ÷Æ°>)re   rf   rg   rh   r|   r-   r5   rj   rc   rª   rk   rl   rm   s   @r>   r™   r™   …   sB   ø† ñ$¨ð $¸$÷ $ð $ð; U§\¡\ð ;°e·l±lô ;÷Jð Jr@   r™   c            	       óÒ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\\	R                  \4   4S jj5       5       r\	R                  " 5       \S	 5       5       rS
rU =r$ )ÚMixtralRotaryEmbeddingéš   Údevicez5.18)Úversionr)   c                 óÄ  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        U R
                  R                  S   U l        U R                  nU R                  S:w  a  [        U R                     nU" U R
                  U5      u  o@l
        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚdefaultF)Ú
persistent)r,   r-   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr)   Úrope_parametersr²   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r<   r)   r¯   Úrope_init_fnr¼   r=   s        €r>   r-   ÚMixtralRotaryEmbedding.__init__›   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÓ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕr@   rD   c           	      ó   • U R                   S   n[        U SS5      =(       d    U R                  U R                  -  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNr�   r   r+   ©rX   )r¸   Úgetattrr0   Únum_attention_headsr5   Úaranger|   rW   )r)   r¯   ÚkwargsÚbaserJ   Úattention_factorr¼   s          r>   r¹   Ú6MixtralRotaryEmbedding.compute_default_rope_parameters¬   s{   € ð ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r@   c                 ó  • U R                   S S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        USS9   X4-  R                  SS	5      n[        R                  " Xf4SS
9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR                  UR                   S9W	R                  UR                   S94$ ! , (       d  f       N@= f)Nr   rG   r$   ©rX   r¯   ÚmpsÚcpuF)Údevice_typeÚenabledr+   rI   rÄ   )r¼   ÚexpandrŽ   rW   r5   r|   r¯   Ú
isinstanceÚtypeÚstrr    Ú	transposeÚcatÚcosrº   ÚsinrX   )
r<   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrÐ   ÚfreqsÚembrØ   rÙ   s
             r>   rc   ÚMixtralRotaryEmbedding.forwardÀ   s;  € ð �M‰M˜$¢ 4˜-Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÐtð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   ÃA%E7Å7
F)rº   r)   r¼   r¶   r¾   r·   r²   rr   )re   rf   rg   rh   r   r%   r-   Ústaticmethodr©   r5   rj   r|   r¹   rL   r   rc   rk   rl   rm   s   @r>   r­   r­   š   sŒ   ø† Ù�X vÑ.ñO˜}÷ Oó /ðOð  Ù�X vÑ.ñ5°ð 5ÐY^Ð_d×_kÑ_kÐmrÐ_rÑYsô 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ö<r@   r­   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..NrG   r+   rI   )rŽ   r5   r×   )rÚ   Úx1Úx2s      r>   Úrotate_halfrå   Ó   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r@   Úrotary_pos_embc                 ó˜   • UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXV4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezerå   )ÚqÚkrØ   rÙ   Úunsqueeze_dimÚq_embedÚk_embeds          r>   Úapply_rotary_pos_embrî   Ú   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØÐÐr@   rA   Ún_reprD   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r$   N)rŽ   rÒ   ry   )rA   rï   ÚbatchÚnum_key_value_headsÚslenrÃ   s         r>   Ú	repeat_kvrô   ô   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr@   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrÈ   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )Nr+   r   rG   )rJ   rX   )Úpr�   r$   )rô   Únum_key_value_groupsr5   ÚmatmulrÖ   r   rM   r{   r¡   rW   rX   rû   r�   Ú
contiguous)rõ   rö   r÷   rø   rù   rú   rû   rÈ   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r>   Úeager_attention_forwardr     sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r@   c                   ó  ^ • \ rS rSrSrS\S\4U 4S jjr SS\R                  S\
\R                  \R                  4   S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  S-  4   4S jjrSrU =r$ )ÚMixtralAttentioni  z=Multi-headed attention from 'Attention Is All You Need' paperr)   Ú	layer_idxc                 ó  >• [         TU ]  5         Xl        X l        [	        USS 5      =(       d    UR
                  UR                  -  U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  SS9U l        [        R                  " UR
                  UR                  U R                  -  SS9U l        [        R                  " UR
                  UR                  U R                  -  SS9U l        [        R                  " UR                  U R                  -  UR
                  SS9U l        g )NrÃ   g      à¿TF©Úbias)r,   r-   r)   r  rÅ   r0   rÆ   rÃ   rò   rþ   rú   Úattention_dropoutÚ	is_causalr   ÚLinearÚq_projÚk_projÚv_projÚo_proj©r<   r)   r  r=   s      €r>   r-   ÚMixtralAttention.__init__  s.  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°DÓ9×m¸V×=OÑ=OÐSY×SmÑSmÑ=mˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÑkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÑkˆŒÜ—i’i × 2Ñ 2°F×4NÑ4NÐQU×Q^ÑQ^Ñ4^ÐejÑkˆŒÜ—i’i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÑkˆ�r@   NrA   Úposition_embeddingsrù   Úpast_key_valuesrÈ   rD   c           
      ó4  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [        5      nU" U UU	U
U4U R                  (       d  SOU R                   U R"                  [%        U R                  SS 5      S.UD6u  pïUR&                  " / UQSP76 R)                  5       nU R+                  U5      nXï4$ )NrG   r$   r+   ç        Úsliding_window)rû   rú   r  )rŽ   rÃ   r  r’   rÖ   r  r  rî   Úupdater  r   Úget_interfacer)   Ú_attn_implementationr  r�   r  rú   rÅ   ry   r   r  )r<   rA   r  rù   r  rÈ   Úinput_shapeÚhidden_shapeÚquery_statesr  r  rØ   rÙ   Úattention_interfacer  r  s                   r>   rc   ÚMixtralAttention.forward+  s�  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘LÜ" 4§;¡;Ð0@À$ÓGñ
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r@   )r  r)   rÃ   r  r  r  rþ   r  r  rú   r  rr   )re   rf   rg   rh   ri   r%   Úintr-   r5   rj   r©   r   r   r   rc   rk   rl   rm   s   @r>   r  r    s¥   ø† áGðl˜}ð l¸÷ lð& )-ñ')à—|‘|ð')ð # 5§<¡<°·±Ð#=Ñ>ð')ð Ÿ™ tÑ+ð	')ð
  ™ð')ð Ð-Ñ.ð')ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷')ó ')r@   r  c                   ó  ^ • \ rS rSrS\S\4U 4S jjr    SS\R                  S\	\R                  \R                  4   S-  S\R                  S-  S	\R                  S-  S
\S-  S\\   S\R                  4S jjrSrU =r$ )ÚMixtralDecoderLayeriU  r)   r  c                 ó  >• [         TU ]  5         UR                  U l        [        X5      U l        [        U5      U l        [        UR                  UR                  S9U l	        [        UR                  UR                  S9U l
        g )N©r›   )r,   r-   r0   r  Ú	self_attnr…   Úmlpr™   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormr  s      €r>   r-   ÚMixtralDecoderLayer.__init__V  sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä)¨&Ó<ˆŒä(¨Ó0ˆŒÜ-¨f×.@Ñ.@Àf×FYÑFYÑZˆÔÜ(6°v×7IÑ7IÈv×ObÑObÑ(cˆÕ%r@   NrA   r  rù   rÛ   r  rÈ   rD   c           	      ó¸   • UnU R                  U5      nU R                  " SUUUUUS.UD6u  pXq-   nUnU R                  U5      nU R                  U5      nXq-   nU$ )N)rA   r  rù   rÛ   r  © )r*  r'  r+  r(  )	r<   rA   r  rù   rÛ   r  rÈ   Úresidualr•   s	            r>   rc   ÚMixtralDecoderLayer.forward`  sƒ   € ð !ˆØ×,Ñ,¨]Ó;ˆØŸ>š>ð 
Ø'Ø 3Ø)Ø%Ø+ñ
ð ñ
Ñˆð !Ñ0ˆØ ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Ñ0ˆØÐr@   )r0   r*  r(  r+  r'  )NNNN)re   rf   rg   rh   r%   r"  r-   r5   rj   r©   Ú
LongTensorr   r   r   rc   rk   rl   rm   s   @r>   r$  r$  U  s´   ø† ðd˜}ð d¸÷ dð IMØ.2Ø04Ø(,ñà—|‘|ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ÿ™ tÑ+ð	ð
 ×&Ñ&¨Ñ-ðð  ™ðð Ð+Ñ,ðð 
�‰÷ó r@   r$  c                   óš   ^ • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\" \SS9\\S	.r\R*                  " 5       U 4S
 j5       rSrU =r$ )ÚMixtralPreTrainedModeli{  r)   ÚmodelTr$  r  r   )Úindex)r~   rA   Ú
attentionsc                 óh  >• [         TU ]  U5        U R                  R                  n[	        U[
        5      (       aA  [        R                  " UR                  SUS9  [        R                  " UR                  SUS9  g [	        U[        5      (       a!  [        R                  " UR                  SUS9  g g )Nr  )r£   Ústd)r,   Ú_init_weightsr)   Úinitializer_rangerÓ   r'   ÚinitÚnormal_r7   r8   ro   ru   )r<   rõ   r8  r=   s      €r>   r9  Ú$MixtralPreTrainedModel._init_weightsŽ  s   ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœn×-Ñ-Ü�LŠL˜×,Ñ,°3¸CÒ@Ü�LŠL˜×)Ñ)°¸Ó=Ü˜Ô 1×2Ñ2Ü�LŠL˜Ÿ™¨S°cÓ:ð 3r@   r.  )re   rf   rg   rh   r%   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr"   ro   r$  r  Ú_can_record_outputsr5   rL   r9  rk   rl   rm   s   @r>   r3  r3  {  sw   ø‡ àÓØÐØ&*Ð#Ø.Ð/ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðá'Ð(9ÀÑCØ,Ø&ñÐð ‡]‚]ƒ_ô;ó ö;r@   r3  c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚMixtralModeli™  r)   c           	      ó  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nr&  ©r)   F)r,   r-   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr0   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersr$  Úlayersr™   r)  Únormr­   Ú
rotary_embÚgradient_checkpointingÚ	post_initr  s      €r>   r-   ÚMixtralModel.__init__›  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜEJÈ6×KcÑKcÔEdÓeÒEd¸	Ô  Ö3ÑEdÑeó
ˆŒô # 6×#5Ñ#5¸6×;NÑ;NÑOˆŒ	Ü0¸Ñ?ˆŒØ&+ˆÔ#ð 	�‰Õùò fs   ÂC?NÚ	input_idsrù   rÛ   r  Úinputs_embedsÚ	use_cacherÈ   rD   c           
      ó~  • US L US L-  (       a  [        S5      eU(       a  Uc  [        U R                  S9nUc  U R                  U5      nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      nU R                  R                  c  [        O[        n	U	" U R                  UUUUS9n
UnU R                  X³S9nU R                  S U R                  R                    H  nU" U4U
UUUUS.UD6nM     U R!                  U5      n[#        UUS	9$ )
Nz:You must specify exactly one of input_ids or inputs_embedsrL  r   r$   )r¯   )r)   r\  rù   r  rÛ   )rÛ   )rù   rÛ   r  r]  r  )Úlast_hidden_stater  )Ú
ValueErrorr	   r)   rQ  Úget_seq_lengthr5   rÇ   rŽ   r¯   rè   r  r   r   rW  rU  rT  rV  r   )r<   r[  rù   rÛ   r  r\  r]  rÈ   Úpast_seen_tokensÚmask_functionÚcausal_maskrA   r  Údecoder_layers                 r>   rc   ÚMixtralModel.forward«  s^  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑ Ø ×-Ñ-¨iÓ8ˆMàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLà.2¯k©k×.HÑ.HÑ.PÕ*ÔVwˆÙ#Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð &ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø)Ø /Ø#Ø$7ñð ñŠMñ Jð Ÿ	™	 -Ó0ˆä%Ø+Ø+ñ
ð 	
r@   )rQ  rX  rU  rV  rN  rW  rO  )NNNNNN)re   rf   rg   rh   r%   r-   r!   r#   r   r5   r1  rj   r   ÚFloatTensorÚboolr   r   r   rc   rk   rl   rm   s   @r>   rJ  rJ  ™  sÍ   ø† ð˜}÷ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ4
à×#Ñ# dÑ*ð4
ð Ÿ™ tÑ+ð4
ð ×&Ñ&¨Ñ-ð	4
ð
  ™ð4
ð ×(Ñ(¨4Ñ/ð4
ð ˜$‘;ð4
ð Ð+Ñ,ð4
ð 
 ô4
ó ó ó  ö4
r@   rJ  Úgate_logitsr/   c           	      ód  • U b  [        U [        5      (       d  gU S   R                  n[        R                  " U[        R
                  US9n[        R                  " U[        R
                  US9nSnUb-  UR                  S5      R                  U[        R
                  S9nU  GHU  n	[        R                  R                  R                  U	R                  U5      SS9n
[        R                  " X¢SS9u  p¼Ucg  U[        R                  " UR                  S5      US9R                  5       -   nXjR                  5       R                  SS9-   nXzR                  S   -   nM¼  U[        R                  " U[        R
                  US9R!                  SUR                  S5      WR#                  U5      5      -   nXjR                  5       UR%                  S5      -  R                  SS9-   nXxR                  5       -   nGMX     XW-  nXg-  n[        R                  " XÞR%                  S5      -  5      nXñ-  $ )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
r   rÍ   r  rG   )r¯   rX   rI   )Ú	minlength)rÓ   r©   r¯   r5   Úzerosr¡   ry   rW   r   rM   r{   r}   Úbincountr|   rQ   rŽ   Úscatter_add_Úrepeat_interleaverè   )ri  r/   rt   rù   Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gateÚrouting_weightsr•   Úselected_expertsÚtokens_per_expertÚrouter_prob_per_expertÚoverall_losss                   r>   Úload_balancing_loss_funcr{  å  sÿ  € ð: Ñ¤*¨[¼%×"@Ñ"@Øð ! ‘^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ñ`ÐÜ—k’k +´U·]±]È>ÑZ€OØ€JàÑ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ð]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ð\ˆÜ#Ÿjšj¨ÀRÑHÑˆØÑ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÑ(k×(qÑ(qÓ(sÑsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÐ0RÑRˆOØ#×&;Ñ&;¸AÑ&>Ñ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ñ=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_ñ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÑ1b×0gÑ0gÐlmÐ0gÐ0nÑnˆOØ#§m¡m£oÑ5‹Jñ# "ð& .Ñ:ÐØ,Ñ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÑTÓU€LØÑ%Ð%r@   c                   ód  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
         SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚMixtralForCausalLMi*  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrA   ÚlogitsÚkeep_full_weightc                 óJ  >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        UR                  U l	        UR                  U l        UR                  U l        U R                  5         g )NFr
  )r,   r-   rJ  r4  rO  r   r  r0   r~  Úrouter_aux_loss_coefr.   r/   rs   rY  r;   s     €r>   r-   ÚMixtralForCausalLM.__init__1  s   ø€ Ü‰Ñ˜Ô Ü! &Ó)ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×3Ñ3ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Õr@   Nr[  rù   rÛ   r  r\  Úlabelsr]  Úoutput_router_logitsÚlogits_to_keeprÈ   rD   c
                 óz  • Ub  UOU R                   R                  nU R                  " SUUUUUUUS.U
D6nUR                  n[	        U	[
        5      (       a  [        U	* S5      OU	nU R                  USS2USS24   5      nSnUb  U R                  " XæU R                  40 U
D6nSnU(       aY  [        UR                  U R                  U R                  U5      nUb*  XðR                  UR                  UR                   5      -  -  n[#        UUUUR$                  UR&                  UR(                  UR                  S9$ )aþ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, MixtralForCausalLM

>>> model = MixtralForCausalLM.from_pretrained("mistralai/Mixtral-8x7B-v0.1")
>>> tokenizer = AutoTokenizer.from_pretrained("mistralai/Mixtral-8x7B-v0.1")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```N)r[  rù   rÛ   r  r\  r]  r†  )ÚlossÚaux_lossr€  r  rA   r6  r~   r.  )r)   r†  r4  r_  rÓ   r"  Úslicer~  Úloss_functionrO  r{  r~   r/   rs   rƒ  rW   r¯   r   r  rA   r6  )r<   r[  rù   rÛ   r  r\  r…  r]  r†  r‡  rÈ   ÚoutputsrA   Úslice_indicesr€  r‰  rŠ  s                    r>   rc   ÚMixtralForCausalLM.forward=  sP  € ðN %9Ñ$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò% f°d·o±oÑPÈÑPˆDàˆÞÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ñ!Ø×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ñ
ð 	
r@   )r~  r4  r/   rs   rƒ  rO  )	NNNNNNNNr   )re   rf   rg   rh   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr-   r   r   r5   r1  rj   r   rg  rh  r"  r   r   r   rc   rk   rl   rm   s   @r>   r}  r}  *  s@  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõ
ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñP
à×#Ñ# dÑ*ðP
ð Ÿ™ tÑ+ðP
ð ×&Ñ&¨Ñ-ð	P
ð
  ™ðP
ð ×(Ñ(¨4Ñ/ðP
ð × Ñ  4Ñ'ðP
ð ˜$‘;ðP
ð # T™kðP
ð ˜eŸl™lÑ*ðP
ð Ð+Ñ,ðP
ð 
#ôP
ó ó öP
r@   r}  c                   ó   • \ rS rSrSrg)Ú MixtralForSequenceClassificationi’  r.  N©re   rf   rg   rh   rk   r.  r@   r>   r•  r•  ’  ó   † Úr@   r•  c                   ó   • \ rS rSrSrg)ÚMixtralForTokenClassificationi–  r.  Nr–  r.  r@   r>   r™  r™  –  r—  r@   r™  c                   ó   • \ rS rSrSrg)ÚMixtralForQuestionAnsweringiš  r.  Nr–  r.  r@   r>   r›  r›  š  r—  r@   r›  )r}  r›  rJ  r3  r•  r™  )r$   )r  )Nr+   N)PÚcollections.abcr   r5   Útorch.nn.functionalr   rM   rz   Ú r   r;  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr    r!   Úutils.output_capturingr"   r#   Úconfiguration_mixtralr%   ÚModuler'   ro   r…   r™   r­   rå   rî   rj   r"  rô   r|   r  r  r$  r3  rJ  r©   r{  r}  r•  r™  r›  Ú__all__r.  r@   r>   Ú<module>r±     sº  ðõ4 %ã ß Ð Ý å &Ý !ß .Ý )ß hÑ hß RÝ B÷ó ÷ Rß Kß FÝ &ß IÑ IÝ 0ß Gß EÝ 0ð ô$#�R—Y‘Yó $#ó ð$#ôN<˜Ÿ	™	ô <ô$˜BŸI™Iô ñ& ˜YÓ'ôJ�R—Y‘Yó Jó (ðJô(6<˜RŸY™Yô 6<òr(ñ Ð-Ó.óó /ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ñ2 Ð)Ó*ô8)�r—y‘yó 8)ó +ð8)ôv#Ð4ô #ðL ô;˜_ó ;ó ð;ð: ôH
Ð)ó H
ó ðH
ðZ #Ø
Ø*.ñ	B&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðB&à�t‘ðB&ð —L‘L 4Ñ'ð	B&ð
 ‡\�\�CÑõB&ðJ ôd
Ð/°ó d
ó ðd
ôN	Ð'GÐI_ô 	ô	Ð$AÐCYô 	ô	Ð"=Ð?Uô 	ò�r@   