ó
    ˆŽ¡jà†  ã                   óv  • S SK Jr  S SKrS SKJs  Jr  S SKJr  SSKJr	  SSK
Jr  SSKJrJr  SSKJr  SS	KJrJr  SS
KJrJr  SSKJr  SSKJr  SSKJrJr  SSKJrJ r   SSK!J"r"J#r#  SSK$J%r%  SSK&J'r'J(r(  SSK)J*r*  SSK+J,r,J-r-J.r.  SSK/J0r0J1r1  SSK2J3r3  \" S5       " S S\Rh                  5      5       r5 " S S\Rh                  5      r6 " S S\Rh                  5      r7 " S S\Rh                  5      r8\ " S  S!\Rh                  5      5       r9 " S" S#\Rh                  5      r:S$ r;SAS% jr<S&\Rz                  S'\>S(\Rz                  4S) jr? SBS*\Rh                  S+\Rz                  S,\Rz                  S-\Rz                  S.\Rz                  S-  S/\@S0\@S1\%\,   4S2 jjrA " S3 S4\Rh                  5      rB " S5 S6\5      rC\' " S7 S8\#5      5       rD\' " S9 S:\D5      5       rE   SCS;\Rz                  \F\Rz                     -  S-  S<\>S-  S.\Rz                  S-  S(\Rz                  \>-  4S= jjrG\' " S> S?\D\5      5       rH/ S@QrIg)Dé    )ÚCallableN)Únné   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_experts_implementationÚuse_kernel_forward_from_hub)Úcreate_causal_maskÚ!create_sliding_window_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚMoeCausalLMOutputWithPastÚMoeModelOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)Úauto_docstringÚcan_return_tuple)Údeprecate_kwarg)ÚTransformersKwargsÚmaybe_autocastÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )ÚLagunaConfigÚRMSNormc                   óx   ^ • \ rS rSrS
S\SS4U 4S jjjrS\R                  S\R                  4S jrS r	S	r
U =r$ )ÚLagunaRMSNormé.   ÚepsÚreturnNc                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z,
LagunaRMSNorm is equivalent to T5LayerNorm
N)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizer&   Ú	__class__s      €Ú¸C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/laguna/modeling_laguna.pyr*   ÚLagunaRMSNorm.__init__0   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Õó    Úhidden_statesc                 ó  • UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtor,   Úfloat32ÚpowÚmeanÚrsqrtr/   r.   )r0   r6   Úinput_dtypeÚvariances       r3   ÚforwardÚLagunaRMSNorm.forward8   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ð>ˆØ%¬¯ª°H×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r5   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)Útupler.   Úshaper/   )r0   s    r3   Ú
extra_reprÚLagunaRMSNorm.extra_repr?   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr5   )r/   r.   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Úfloatr*   r,   ÚTensorrC   rH   Ú__static_attributes__Ú__classcell__©r2   s   @r3   r$   r$   .   sB   ø† ñ$¨ð $¸$÷ $ð $ð; U§\¡\ð ;°e·l±lô ;÷Jð Jr5   r$   c                   óÞ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9 SS\S\S-  S	\	\
R                  \4   4S
 jj5       5       r\
R                  " 5       \S 5       5       rSrU =r$ )ÚLagunaRotaryEmbeddingéC   Údevicez5.18)ÚversionNÚconfigc           
      óž  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        [        [        UR                  5      5      U l        0 U l	        U R                   HÜ  nU R
                  R                  U   nUc  M!  US   U R                  U'   U R                  nU R                  U   S:w  a  [        U R                  U      nU" U R
                  X#S9u  pg[        X S3[        R                  " USS95        [        X S3[        R                  " UR!                  5       SS95        [        X S3U5        MÞ     g )	NÚ	rope_typeÚdefault©Ú
layer_typeÚ	_inv_freqF)Ú
persistentÚ_original_inv_freqÚ_attention_scaling)r)   r*   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrX   ÚsortedÚsetÚlayer_typesrZ   Úrope_parametersÚcompute_default_rope_parametersr   Úsetattrr   ÚBufferÚclone)	r0   rX   rV   r]   Úrope_paramsÚrope_init_fnÚcurr_inv_freqÚcurr_attention_scalingr2   s	           €r3   r*   ÚLagunaRotaryEmbedding.__init__D   s'  ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!ØŒÜ!¤# f×&8Ñ&8Ó"9Ó:ˆÔØˆŒØ×*Ô*ˆJØŸ+™+×5Ñ5°jÑAˆKØÑ"Ùà)4°[Ñ)AˆD�N‰N˜:Ñ&Ø%)×%IÑ%IˆLØ�~‰~˜jÑ)¨YÓ6Ü2°4·>±>À*Ñ3MÑN�Ù4@ÀÇÁÈfÑ4lÑ1ˆMÜ�D˜L¨	Ð2´B·I²I¸mÐX]Ñ4^Ô_Ü�D˜LÐ(:Ð;¼R¿YºYÀ}×GZÑGZÓG\ÐinÑ=oÔpÜ�D˜LÐ(:Ð;Ð=SÖTò +r5   r]   r'   c           	      ó^  • U R                   U   S   nU R                   U   R                  SS5      n[        U SS5      =(       d    U R                  U R                  -  n[        Xe-  5      nSnSU[        R                  " SUS[        R                  S9U-  -  -  n	U	R                  U5      U4$ )	a9  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
    layer_type (`str`, *optional*):
        The current layer type if the model has different RoPE parameters per type.
        Should not be used unless `config.layer_types is not None`
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   r8   ©r;   )
rh   ÚgetÚgetattrr1   Únum_attention_headsÚintr,   ÚarangerN   r<   )
rX   rV   r]   ÚkwargsÚbasert   ru   ÚdimÚattention_factorÚinv_freqs
             r3   ri   Ú5LagunaRotaryEmbedding.compute_default_rope_parametersZ   s¯   € ð" ×%Ñ% jÑ1°,Ñ?ˆà &× 6Ñ 6°zÑ B× FÑ FÐG^Ð`cÓ dÐÜ˜6 :¨tÓ4×h¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ñ2Ó3ˆàÐð ˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r5   c                 ó  • [        X S35      n[        X S35      nUS S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        US	S
9   Xg-  R                  SS5      n	[        R                  " X™4SS9n
U
R                  5       U-  nU
R                  5       U-  nS S S 5        WR                  UR                  S9WR                  UR                  S94$ ! , (       d  f       N@= f)Nr^   ra   r   r9   r    ©r;   rV   ÚmpsÚcpuF)Údevice_typeÚenabledr8   ©r~   rv   )rx   ÚexpandrG   r<   r,   rN   rV   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosÚsinr;   )r0   ÚxÚposition_idsr]   r€   Úattention_scalingÚinv_freq_expandedÚposition_ids_expandedr†   ÚfreqsÚembr�   r�   s                r3   rC   ÚLagunaRotaryEmbedding.forwardw   sS  € ô ˜4 <¨yÐ!9Ó:ˆÜ# D¨LÐ8JÐ*KÓLÐð �Tš1˜d�]Ñ#×*Ñ*¨<×+=Ñ+=¸aÑ+@À"ÀaÓH×KÑKÔRW×R]ÑR]Ðfg×fnÑfnÐKÐoð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)Ð/Ñ/ˆCØ—'‘'“)Ð/Ñ/ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   Ã*AE5Å5
F)rX   rg   rc   rd   rZ   ©N)NN)rJ   rK   rL   rM   r   r!   r*   ÚstaticmethodrŒ   rF   r,   rO   rN   ri   Úno_gradr   rC   rP   rQ   rR   s   @r3   rT   rT   C   s˜   ø† Ù�X vÑ.ñU˜|÷ Uó /ðUð* Ù�X vÑ.àDHñ5Øð5Ø7:¸T±zð5à	ˆu�|‰|˜UÐ"Ñ	#ô5ó /ó ð5ð6 ‡]‚]ƒ_Øñ<ó ó ö<r5   rT   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )Ú	LagunaMLPé�   c                 óÆ  >• [         TU ]  5         Xl        UR                  U l        Uc  UR                  OUU l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l	        [        UR                     U l        g ©NF©Úbias)r)   r*   rX   r1   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn)r0   rX   r£   r2   s      €r3   r*   ÚLagunaMLP.__init__Ž   s¬   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ=NÑ=V ×!9Ò!9Ð\mˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÑXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÑVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÑXˆŒÜ˜V×.Ñ.Ñ/ˆ�r5   c                 óˆ   • U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ r™   )r§   r©   r¥   r¦   )r0   r‘   r§   s      r3   rC   ÚLagunaMLP.forward˜   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr5   )r©   rX   r§   r¥   r1   r£   r¦   r™   )rJ   rK   rL   rM   r*   rC   rP   rQ   rR   s   @r3   r�   r�   �   s   ø† ÷0÷ð r5   r�   c                   ó–   ^ • \ rS rSrU 4S jrS\R                  S\\R                  \R                  \R                  4   4S jrSr	U =r
$ )ÚLagunaTopKRouteré�   c                 ó¢  >• [         TU ]  5         UR                  U l        UR                  U l        UR
                  U l        [        R                  " [        R                  " U R                  U R                  5      5      U l        [        R                  " [        R                  " UR                  5      SS9U l        UR                  U l        g )NF)Úrequires_grad)r)   r*   Únum_experts_per_tokÚtop_kÚnum_expertsr1   Ú
hidden_dimr   r+   r,   Úzerosr.   Úe_score_correction_biasÚmoe_router_logit_softcappingÚrouter_logit_softcapping©r0   rX   r2   s     €r3   r*   ÚLagunaTopKRouter.__init__ž   sŒ   ø€ Ü‰ÑÔØ×/Ñ/ˆŒ
Ø!×-Ñ-ˆÔØ ×,Ñ,ˆŒÜ—l’l¤5§;¢;¨t×/?Ñ/?ÀÇÁÓ#QÓRˆŒÜ')§|¢|´E·K²KÀ×@RÑ@RÓ4SÐchÑ'iˆÔ$Ø(.×(KÑ(KˆÕ%r5   r6   r'   c                 óV  • UR                  SU R                  5      n[        R                  " XR                  5      R                  5       nU R                  S:”  a/  [        R                  " X R                  -  5      U R                  -  n[        R                  " U5      nX0R                  R                  UR                  5      -   n[        R                  " X@R                  SS9u  pVUR                  SU5      nXwR!                  SSS9-  nUR                  UR                  5      nX'U4$ )Nr9   ç        rˆ   T)r~   r:   )Úreshaperµ   ÚFÚlinearr.   rN   r¹   r,   ÚtanhÚsigmoidr·   r<   r;   Útopkr³   ÚgatherÚsum)r0   r6   Úrouter_logitsÚrouting_scoresÚscores_for_selectionÚ_Úselected_expertsÚrouting_weightss           r3   rC   ÚLagunaTopKRouter.forward§   sû   € ð &×-Ñ-¨b°$·/±/ÓBˆÜŸš ·±Ó<×BÑBÓDˆà×(Ñ(¨3Ó.Ü!ŸJšJ }×7TÑ7TÑ'TÓUÐX\×XuÑXuÑuˆMäŸš }Ó5ˆà-×0LÑ0L×0OÑ0OÐP^×PdÑPdÓ0eÑeÐÜ#ŸjšjÐ)=¿z¹zÈrÑRÑˆØ(×/Ñ/°Ð4DÓEˆØ)×,?Ñ,?ÀBÐPTÐ,?Ð,UÑUˆØ)×,Ñ,¨]×-@Ñ-@ÓAˆàÐ/?Ð?Ð?r5   )r·   rµ   r´   r¹   r³   r.   )rJ   rK   rL   rM   r*   r,   rO   rF   rC   rP   rQ   rR   s   @r3   r®   r®   �   sE   ø† õLð@à—|‘|ð@ð 
ˆu�|‰|˜UŸ\™\¨5¯<©<Ð7Ñ	8÷@ò @r5   r®   c                   ó–   ^ • \ rS rSrSrU 4S jrS\R                  S\R                  S\R                  S\R                  4S jrS	r	U =r
$ )
ÚLagunaExpertsé¼   z2Collection of expert weights stored as 3D tensors.c                 óú  >• [         TU ]  5         UR                  U l        UR                  U l        UR
                  U l        [        R                  " [        R                  " U R                  SU R                  -  U R                  5      5      U l        [        R                  " [        R                  " U R                  U R                  U R                  5      5      U l        [        UR                     U l        g )Nr8   )r)   r*   r´   r1   rµ   Úmoe_intermediate_sizeÚintermediate_dimr   r+   r,   ÚemptyÚgate_up_projr§   r   r¨   r©   rº   s     €r3   r*   ÚLagunaExperts.__init__À   s­   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔØ ×,Ñ,ˆŒØ &× <Ñ <ˆÔÜŸLšL¬¯ª°T×5EÑ5EÀqÈ4×K`ÑK`ÑG`Ðbf×bqÑbqÓ)rÓsˆÔÜŸš¤e§k¢k°$×2BÑ2BÀDÇOÁOÐUY×UjÑUjÓ&kÓlˆŒÜ˜V×.Ñ.Ñ/ˆ�r5   r6   Útop_k_indexÚtop_k_weightsr'   c                 óX  • [         R                  " U5      n[         R                  " 5          [         R                  R                  R                  X R                  S9nUR                  SSS5      n[         R                  " UR                  SS9S5      R                  5       nS S S 5        W Hè  nUS   nXpR                  :X  a  M  [         R                  " WU   5      u  p‰X   n
[        R                  R                  X R                  U   5      R                  SSS9u  p¼U R                  U5      U-  n[        R                  R                  XÐR                   U   5      nXÓX˜S 4   -  nUR#                  SX�R%                  UR&                  5      5        Mê     U$ ! , (       d  f       Nþ= f)N)Únum_classesr8   r    r   )r9   éþÿÿÿrˆ   r9   )r,   Ú
zeros_liker›   r   Ú
functionalÚone_hotr´   ÚpermuteÚgreaterrÅ   ÚnonzeroÚwhererÀ   rÔ   Úchunkr©   r§   Ú
index_add_r<   r;   )r0   r6   rÖ   r×   Úfinal_hidden_statesÚexpert_maskÚ
expert_hitÚ
expert_idxÚ	top_k_posÚ	token_idxÚcurrent_stateÚgateÚupÚcurrent_hidden_statess                 r3   rC   ÚLagunaExperts.forwardÉ   so  € ô $×.Ò.¨}Ó=ÐÜ�]Š]�_ÜŸ(™(×-Ñ-×5Ñ5°k×O_ÑO_Ð5Ð`ˆKØ%×-Ñ-¨a°°AÓ6ˆKÜŸš {§¡¸8 Ð'DÀaÓH×PÑPÓRˆJ÷ ó
 %ˆJØ# A™ˆJØ×-Ñ-Ó-ÙÜ#(§;¢;¨{¸:Ñ/FÓ#GÑ ˆIØ)Ñ4ˆMÜ—}‘}×+Ñ+¨M×;LÑ;LÈZÑ;XÓY×_Ñ_Ð`aÐgiÐ_Ðj‰HˆDØ$(§K¡K°Ó$5¸Ñ$:Ð!Ü$&§M¡M×$8Ñ$8Ð9N×P^ÑP^Ð_iÑPjÓ$kÐ!Ø$9È)Ð`dÐJdÑ<eÑ$eÐ!Ø×*Ñ*¨1¨i×9QÑ9QÐRe×RkÑRkÓ9lÖmñ %ð #Ð"÷# �_ús   ¬A7FÆ
F))r©   r§   rÔ   rµ   rÒ   r´   )rJ   rK   rL   rM   Ú__doc__r*   r,   rO   rC   rP   rQ   rR   s   @r3   rÎ   rÎ   ¼   sK   ø† á<õ0ð#à—|‘|ð#ð —\‘\ð#ð —|‘|ð	#ð
 
�‰÷#ò #r5   rÎ   c                   ój   ^ • \ rS rSrS\4U 4S jjrS\R                  S\R                  4S jrSr	U =r
$ )ÚLagunaSparseMoeBlockéä   rX   c                 ó´   >• [         TU ]  5         [        U5      U l        [	        U5      U l        [        XR                  S9U l        UR                  U l
        g )N©r£   )r)   r*   rÎ   Úexpertsr®   rë   r�   Úshared_expert_intermediate_sizeÚshared_expertsÚmoe_routed_scaling_factorÚrouted_scaling_factorrº   s     €r3   r*   ÚLagunaSparseMoeBlock.__init__å   sG   ø€ Ü‰ÑÔÜ$ VÓ,ˆŒÜ$ VÓ,ˆŒ	Ü'¨×BhÑBhÑiˆÔØ%+×%EÑ%EˆÕ"r5   r6   r'   c                 óþ   • UR                   u  p#nUR                  SU5      nU R                  U5      nU R                  U5      u  pgnU R	                  XU5      nXR
                  -  nX-   nUR                  X#U5      nU$ )Nr9   )rG   Úviewr÷   rë   rõ   rù   r¾   )	r0   r6   Ú
batch_sizeÚsequence_lengthrµ   Úshared_outputrÉ   rË   rÊ   s	            r3   rC   ÚLagunaSparseMoeBlock.forwardì   s†   € Ø2?×2EÑ2EÑ/ˆ
 ZØ%×*Ñ*¨2¨zÓ:ˆØ×+Ñ+¨MÓ:ˆà/3¯y©y¸Ó/GÑ,ˆÐ,ØŸ™ ]ÀoÓVˆà%×(BÑ(BÑBˆØ%Ñ5ˆà%×-Ñ-¨jÈ:ÓVˆØÐr5   )rõ   rë   rù   r÷   )rJ   rK   rL   rM   r!   r*   r,   rO   rC   rP   rQ   rR   s   @r3   rñ   rñ   ä   s1   ø† ðF˜|÷ Fð U§\¡\ð °e·l±l÷ ò r5   rñ   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..Nr9   r8   rˆ   )rG   r,   rŽ   )r‘   Úx1Úx2s      r3   Úrotate_halfr  û   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r5   c                 óR  • UR                  U5      nUR                  U5      nUR                  S   nU SSU24   U SUS24   pvUSSU24   USUS24   p˜Xb-  [        U5      U-  -   n
X‚-  [        U5      U-  -   n[        R                  " X§/SS9n
[        R                  " X¹/SS9nX«4$ )a{  Applies Rotary Position Embedding to the query and key tensors.

Removes the interleaving of cos and sin from GLM

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
r9   .Nrˆ   )Ú	unsqueezerG   r  r,   rŽ   )ÚqÚkr�   r�   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r3   Úapply_rotary_pos_embr    sÐ   € ð( �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð —‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6Ø�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6ð ‰{œ{¨5Ó1°CÑ7Ñ8€GØ‰{œ{¨5Ó1°CÑ7Ñ8€Gô �iŠi˜Ð)¨rÑ2€GÜ�iŠi˜Ð)¨rÑ2€GØÐÐr5   r6   Ún_repr'   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r    N)rG   r‰   r¾   )r6   r  ÚbatchÚnum_key_value_headsÚslenru   s         r3   Ú	repeat_kvr  )  s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr5   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutr|   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )Nr8   r   r9   )r~   r;   )ÚpÚtrainingr    )r  Únum_key_value_groupsr,   Úmatmulr�   r   rÜ   Úsoftmaxr=   r<   r;   r  r!  Ú
contiguous)r  r  r  r  r  r  r  r|   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r3   Úeager_attention_forwardr*  5  sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r5   c                   ó  ^ • \ rS rSrSrS\S\S\4U 4S jjr SS\R                  S	\
\R                  \R                  4   S
\R                  S-  S\S-  S\\   S\
\R                  \R                  S-  4   4S jjrSrU =r$ )ÚLagunaAttentioniN  zSAfmoe-style SWA/GQA attention with Laguna-specific gating and per-layer head count.rX   Ú	layer_idxÚ	num_headsc                 óf  >• [         TU ]  5         X0l        Xl        X l        [        USUR                  UR                  -  5      U l        U R                  UR                  -  U l
        U R                  S-  U l        UR                  U l        SU l        [        R                  " UR                  U R                  U R                  -  UR                   S9U l        [        R                  " UR                  UR                  U R                  -  UR                   S9U l        [        R                  " UR                  UR                  U R                  -  UR                   S9U l        [        R                  " U R                  U R                  -  UR                  UR                   S9U l        UR*                  U   S:H  U l        U R,                  (       a  UR.                  OS U l        [1        U R                  UR2                  S9U l        [1        U R                  UR2                  S9U l        UR8                  SL =(       d    UR8                  S:H  U l        U R:                  (       a  U R                  OU R                  U R                  -  n[        R                  " UR                  USS9U l        g )	Nru   g      à¿Tr¡   Úsliding_attention©r&   zper-headF)r)   r*   r.  rX   r-  rx   r1   ry   ru   r  r"  r  Úattention_dropoutÚ	is_causalr   r¤   Úattention_biasÚq_projÚk_projÚv_projÚo_projrg   Úis_local_attentionÚsliding_windowr$   Úrms_norm_epsÚq_normÚk_normÚgatingÚgate_per_headÚg_proj)r0   rX   r-  r.  Ú
g_proj_dimr2   s        €r3   r*   ÚLagunaAttention.__init__Q  s   ø€ Ü‰ÑÔà"ŒØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$(§N¡N°f×6PÑ6PÑ$PˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’i × 2Ñ 2°D·N±NÀTÇ]Á]Ñ4RÐY_×YnÑYnÑoˆŒÜ—i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’i §¡°·±Ñ >À×@RÑ@RÐY_×YnÑYnÑoˆŒð #)×"4Ñ"4°YÑ"?ÐCVÑ"VˆÔØ7;×7N×7N˜f×3Ò3ÐTXˆÔä# D§M¡M°v×7JÑ7JÑKˆŒÜ# D§M¡M°v×7JÑ7JÑKˆŒØ#Ÿ]™]¨dÐ2×Q°f·m±mÀzÑ6QˆÔØ'+×'9×'9�T—^’^¸t¿~¹~ÐPT×P]ÑP]Ñ?]ˆ
Ü—i’i × 2Ñ 2°JÀUÑKˆ�r5   Nr6   Úposition_embeddingsr  Úpast_key_valuesr|   r'   c                 ó´  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      nU R	                  U5      R                  U5      n	U R                  U5      R                  U5      n
U R                  U5      R                  SS5      nU R                  U	5      R                  SS5      n	U
R                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [         5      nU" U UU	U
U4U R"                  (       d  SOU R$                  U R&                  U R(                  S.UD6u  pïUR*                  " / UQSP76 R-                  5       n[.        R0                  " U R3                  U5      R5                  5       5      R7                  UR8                  5      nU R:                  (       aE  UR                  " / UQSPU R                  P76 UR=                  S5      -  R                  " / UQSP76 nOUU-  nU R?                  U5      nXï4$ )Nr9   r    r8   r½   )r  r  r:  ) rG   ru   r5  rü   r6  r7  r<  r�   r=  r  Úupdater-  r   Úget_interfacerX   Ú_attn_implementationr*  r!  r2  r  r:  r¾   r%  r¿   Úsoftplusr@  rN   r<   r;   r?  r  r8  )r0   r6   rC  r  rD  r|   Úinput_shapeÚhidden_shapeÚquery_statesr&  r'  r�   r�   Úattention_interfacer)  r(  rë   s                    r3   rC   ÚLagunaAttention.forwardp  s6  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆà—{‘{ <Ó0×:Ñ:¸1¸aÓ@ˆØ—[‘[ Ó,×6Ñ6°q¸!Ó<ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘LØ×.Ñ.ñ
%
ð ñ
%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆä�zŠz˜$Ÿ+™+ mÓ4×:Ñ:Ó<Ó=×@Ñ@À×ARÑARÓSˆØ××Ø&×+Ò+ÐL¨[ÐL¸"ÐL¸d¿m¹mÒLÈtÏ~É~Ð^`ÓOaÑa×gÒgð ØðØ ò‰Kð &¨Ñ,ˆKà—k‘k +Ó.ˆØÐ(Ð(r5   )r2  rX   r@  r?  ru   r3  r9  r=  r6  r-  r.  r"  r8  r<  r5  r  r:  r7  r™   )rJ   rK   rL   rM   rï   r!   rz   r*   r,   rO   rF   r   r   r   rC   rP   rQ   rR   s   @r3   r,  r,  N  s®   ø† Ù]ðL˜|ð L¸ð LÈ÷ LðH )-ñ3)à—|‘|ð3)ð # 5§<¡<°·±Ð#=Ñ>ð3)ð Ÿ™ tÑ+ð	3)ð
  ™ð3)ð Ð-Ñ.ð3)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷3)ó 3)r5   r,  c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚLagunaDecoderLayeri¦  rX   r-  c                 ó„  >• [         TU ]  5         UR                  U l        [        XUR                  U   5      U l        UR                  U   S:X  a  [        U5      U l        O[        XR                  S9U l        [        UR                  UR                  S9U l        [        UR                  UR                  S9U l        g )NÚsparserô   r1  )r)   r*   r1   r,  Únum_attention_heads_per_layerÚ	self_attnÚmlp_layer_typesrñ   Úmlpr�   r£   r$   r;  Úinput_layernormÚpost_attention_layernorm©r0   rX   r-  r2   s      €r3   r*   ÚLagunaDecoderLayer.__init__§  s�   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ(¨¸F×<`Ñ<`ÐajÑ<kÓlˆŒØ×!Ñ! )Ñ,°Ó8Ü+¨FÓ3ˆD�Hä  ×;SÑ;SÑTˆDŒHÜ,¨V×-?Ñ-?ÀV×EXÑEXÑYˆÔÜ(5°f×6HÑ6HÈf×NaÑNaÑ(bˆÕ%r5   Nr6   r  r’   rD  Ú	use_cacherC  r|   r'   c           
      óº   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pX�-   nUnU R                  U5      nU R                  U5      nX�-   nU$ )N)r6   r  r’   rD  r[  rC  © )rW  rT  rX  rV  )
r0   r6   r  r’   rD  r[  rC  r|   ÚresidualrÉ   s
             r3   rC   ÚLagunaDecoderLayer.forward²  sˆ   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆð !Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Ñ0ˆØÐr5   )r1   rW  rV  rX  rT  )NNNFN)rJ   rK   rL   rM   r!   rz   r*   r,   rO   Ú
LongTensorr   ÚboolrF   r   r   rC   rP   rQ   rR   s   @r3   rP  rP  ¦  sÃ   ø† ð	c˜|ð 	c¸÷ 	cð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷ó r5   rP  c                   óš   ^ • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\" \SS9\\S	.r\R*                  " 5       U 4S
 j5       rSrU =r$ )ÚLagunaPreTrainedModeliÒ  rX   ÚmodelTrP  rD  r   )Úindex)rÆ   r6   Ú
attentionsc                 óf  >• [         TU ]  U5        U R                  R                  n[	        U[
        5      (       aA  [        R                  " UR                  SUS9  [        R                  " UR                  SUS9  O5[	        U[        5      (       a   [        R                  " UR                  SUS9  [	        U[        5      (       a4  [        R                  R                  R                  UR                  5        g [	        U[         5      (       a¢  UR"                   H‘  nUR$                  nUR&                  U   S:w  a  [(        UR&                  U      nU" UR                  US9u  pV[        R*                  " [-        X S35      U5        [        R*                  " [-        X S35      U5        M“     g g )Nr½   )r?   Ústdr[   r\   r^   r`   )r)   Ú_init_weightsrX   Úinitializer_rangerŠ   rÎ   ÚinitÚnormal_rÔ   r§   r®   r.   r,   r   Úzeros_r·   rT   rg   ri   rZ   r   Úcopy_rx   )r0   r  rh  r]   rn   ro   rÉ   r2   s          €r3   ri  Ú#LagunaPreTrainedModel._init_weightså  sB  ø€ ä‰Ñ˜fÔ%Ø�k‰k×+Ñ+ˆÜ�fœm×,Ñ,Ü�LŠL˜×,Ñ,°3¸CÒ@Ü�LŠL˜×)Ñ)°¸Ó=Ü˜Ô 0×1Ñ1Ü�LŠL˜Ÿ™¨S°cÒ:Ü�fÔ.×/Ñ/Ü�H‰H�M‰M× Ñ  ×!?Ñ!?Õ@Ü˜Ô 5×6Ñ6Ø$×0Ô0�
Ø%×EÑE�Ø×#Ñ# JÑ/°9Ó<Ü#6°v×7GÑ7GÈ
Ñ7SÑ#T�LÙ#/°·±È*Ñ#UÑ �Ü—
’
œ7 6¨\¸Ð+CÓDÀmÔTÜ—
’
œ7 6¨\Ð9KÐ+LÓMÈ}Ö]ò 1ð 7r5   r]  )rJ   rK   rL   rM   r!   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendr   r®   rP  r,  Ú_can_record_outputsr,   r›   ri  rP   rQ   rR   s   @r3   rc  rc  Ò  sy   ø‡ àÓØÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðá'Ð(8ÀÑBØ+Ø%ñÐð ‡]‚]ƒ_ô^ó ö^r5   rc  c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚLagunaModeliú  rX   c           	      ó  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nr1  ©rX   F)r)   r*   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr1   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrP  Úlayersr$   r;  ÚnormrT   Ú
rotary_embÚgradient_checkpointingÚ	post_initrY  s      €r3   r*   ÚLagunaModel.__init__ü  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÒDc°yÔ Ö2ÑDcÑdó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÑNˆŒ	Ü/°vÑ>ˆŒØ&+ˆÔ#ð 	�‰Õùò es   ÂC?NÚ	input_idsr  r’   rD  Úinputs_embedsr[  r|   r'   c           	      óÌ  ^• US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U=n	[        5      (       dR  U R                  UUUUS.mU4S jU4S jS	.n
0 n	[        U R                  R                  5       H  nX«   " 5       X›'   M     Un0 n[        U R                  R                  5       H  nU R                  XÃU5      XÛ'   M     [        U R                   S U R                  R"                   5       HE  u  pïU" U4X�R                  R                  U      XÐR                  R                  U      UUS
.UD6nMG     U R%                  U5      n['        UU(       a  US9$ S S9$ )Nz:You must specify exactly one of input_ids or inputs_embedsr~  r   r    )rV   )rX   rŽ  r  rD  r’   c                  ó   >• [        S0 T D6$ ©Nr]  )r   ©Úmask_kwargss   €r3   Ú<lambda>Ú%LagunaModel.forward.<locals>.<lambda>0  s   ø€ Ô*<Ñ*K¸{Ò*Kr5   c                  ó   >• [        S0 T D6$ r‘  )r   r’  s   €r3   r”  r•  1  s   ø€ Ô-NÑ-]ÐQ\Ò-]r5   )Úfull_attentionr0  )r  rC  r’   rD  )Úlast_hidden_staterD  )Ú
ValueErrorrƒ  r	   rX   Úget_seq_lengthr,   r{   rG   rV   r  rŠ   Údictrf   rg   r‰  Ú	enumerater‡  r†  rˆ  r   )r0   r�  r  r’   rD  rŽ  r[  r|   Úpast_seen_tokensÚcausal_mask_mappingÚmask_creation_functionsr]   r6   rC  ÚiÚdecoder_layerr“  s                   @r3   rC   ÚLagunaModel.forward  sö  ø€ ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä°Ð?Ð-Ä×FÑFàŸ+™+Ø!.Ø"0Ø#2Ø ,ñˆKô #LÜ%]ñ'Ð#ð #%ÐÜ! $§+¡+×"9Ñ"9Ö:�
Ø2IÒ2UÓ2WÐ#Ó/ñ ;ð &ˆØ ÐÜ˜dŸk™k×5Ñ5Ö6ˆJØ.2¯o©o¸mÐ[eÓ.fÐÓ+ñ 7ô !*¨$¯+©+Ð6U¸¿¹×8UÑ8UÐ*VÖ WÑˆAÙ)Øðà2·;±;×3JÑ3JÈ1Ñ3MÑNØ$7¿¹×8OÑ8OÐPQÑ8RÑ$SØ)Ø /ñð ñŠMñ !Xð Ÿ	™	 -Ó0ˆä%Ø+Þ/8˜Oñ
ð 	
à>Bñ
ð 	
r5   )rƒ  rŠ  r‡  rˆ  r€  r‰  r�  )NNNNNN)rJ   rK   rL   rM   r!   r*   r   r   r   r,   r`  rO   r   ÚFloatTensorra  r   r   r   rC   rP   rQ   rR   s   @r3   r|  r|  ú  sÍ   ø† ð˜|÷ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ<
à×#Ñ# dÑ*ð<
ð Ÿ™ tÑ+ð<
ð ×&Ñ&¨Ñ-ð	<
ð
  ™ð<
ð ×(Ñ(¨4Ñ/ð<
ð ˜$‘;ð<
ð Ð+Ñ,ð<
ð 
 ô<
ó ó ó  ö<
r5   r|  Úgate_logitsr´   c           	      ód  • U b  [        U [        5      (       d  gU S   R                  n[        R                  " U[        R
                  US9n[        R                  " U[        R
                  US9nSnUb-  UR                  S5      R                  U[        R
                  S9nU  GHU  n	[        R                  R                  R                  U	R                  U5      SS9n
[        R                  " X¢SS9u  p¼Ucg  U[        R                  " UR                  S5      US9R                  5       -   nXjR                  5       R                  SS9-   nXzR                  S   -   nM¼  U[        R                  " U[        R
                  US9R!                  SUR                  S5      WR#                  U5      5      -   nXjR                  5       UR%                  S5      -  R                  SS9-   nXxR                  5       -   nGMX     XW-  nXg-  n[        R                  " XÞR%                  S5      -  5      nXñ-  $ )ax  
Computes auxiliary load balancing loss as in Switch Transformer - implemented in Pytorch.

See Switch Transformer (https://huggingface.co/papers/2101.03961) for more details. This function implements the loss
function presented in equations (4) - (6) of the paper. It aims at penalizing cases where the routing between
experts is too unbalanced.

Args:
    gate_logits:
        Logits from the `gate`, should be a tuple of model.config.num_hidden_layers tensors of
        shape [batch_size X sequence_length, num_experts].
    num_experts:
        Number of experts
    top_k:
        The number of experts to route per-token, can be also interpreted as the `top-k` routing
        parameter.
    attention_mask (`torch.Tensor`, *optional*):
        The attention_mask used in forward function
        shape [batch_size X sequence_length] if not None.

Returns:
    The auxiliary loss.
r   rƒ   r½   r9   )rV   r;   rˆ   )Ú	minlength)rŠ   rF   rV   r,   r¶   r=   r¾   r<   r   rÜ   r$  rÃ   ÚbincountrN   rÅ   rG   Úscatter_add_Úrepeat_interleaver  )r¤  r´   r³   r  Úcompute_deviceÚtokens_per_expert_sumÚrouter_prob_sumÚ
total_rowsÚ	flat_maskÚ
layer_gaterË   rÉ   rÊ   Útokens_per_expertÚrouter_prob_per_expertÚoverall_losss                   r3   Úload_balancing_loss_funcr³  N  sÿ  € ð: Ñ¤*¨[¼%×"@Ñ"@Øð ! ‘^×*Ñ*€NÜ!ŸKšK¨¼5¿=¹=ÐQ_Ñ`ÐÜ—k’k +´U·]±]È>ÑZ€OØ€JàÑ!à"×*Ñ*¨2Ó.×1Ñ1¸ÌuÏ}É}Ð1Ð]ˆ	ä!ˆ
ÜŸ(™(×-Ñ-×5Ñ5°j·m±mÀNÓ6SÐY[Ð5Ð\ˆÜ#Ÿjšj¨ÀRÑHÑˆØÑ!ð &¬¯ªÐ7G×7OÑ7OÐPRÓ7SÐ_jÑ(k×(qÑ(qÓ(sÑsð "ð .×0EÑ0EÓ0G×0KÑ0KÐPQÐ0KÐ0RÑRˆOØ#×&;Ñ&;¸AÑ&>Ñ>ŠJð %:¼E¿KºKØ¤5§=¡=¸ñ=ç‰l˜1Ð.×6Ñ6°rÓ:¸I×<WÑ<WÐX]Ó<^Ó_ñ%`Ð!ð .×1FÑ1FÓ1HÈ9×K^ÑK^Ð_aÓKbÑ1b×0gÑ0gÐlmÐ0gÐ0nÑnˆOØ#§m¡m£oÑ5‹Jñ# "ð& .Ñ:ÐØ,Ñ9Ðä—9’9Ð.×1QÑ1QÐRSÓ1TÑTÓU€LØÑ%Ð%r5   c                   ód  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
         SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚLagunaForCausalLMi“  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr6   ÚlogitsÚkeep_full_weightc                 óJ  >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        UR                  U l	        UR                  U l
        UR                  U l        U R                  5         g r    )r)   r*   r|  rd  r�  r   r¤   r1   r¶  Úrouter_aux_loss_coefr´   r²   r‹  rº   s     €r3   r*   ÚLagunaForCausalLM.__init__›  s   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒØ$*×$?Ñ$?ˆÔ!Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Õr5   Nr�  r  r’   rD  rŽ  Úlabelsr[  Úoutput_router_logitsÚlogits_to_keepr|   r'   c
                 óz  • Ub  UOU R                   R                  nU R                  " SUUUUUUUS.U
D6nUR                  n[	        U	[
        5      (       a  [        U	* S5      OU	nU R                  USS2USS24   5      nSnUb  U R                  " XæU R                  40 U
D6nSnU(       aY  [        UR                  U R                  U R                  U5      nUb*  XðR                  UR                  UR                   5      -  -  n[#        UUUUR$                  UR&                  UR(                  UR                  S9$ )a…  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.
N)r�  r  r’   rD  rŽ  r[  r¾  )ÚlossÚaux_lossr¸  rD  r6   rf  rÆ   r]  )rX   r¾  rd  r˜  rŠ   rz   Úslicer¶  Úloss_functionr�  r³  rÆ   r´   r²   r»  r<   rV   r   rD  r6   rf  )r0   r�  r  r’   rD  rŽ  r½  r[  r¾  r¿  r|   Úoutputsr6   Úslice_indicesr¸  rÁ  rÂ  s                    r3   rC   ÚLagunaForCausalLM.forward§  sO  € ð. %9Ñ$DÑ È$Ï+É+×JjÑJjð 	ð
 +/¯*ª*ð 	+
ØØ)Ø%Ø+Ø'ØØ!5ñ	+
ð ñ	+
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò% f°d·o±oÑPÈÑPˆDàˆÞÜ/Ø×%Ñ%Ø× Ñ Ø×(Ñ(Øó	ˆHð Ñ!Ø×1Ñ1°H·K±KÀÇÁÓ4LÑLÑL�ä(ØØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)Ø!×/Ñ/ñ
ð 	
r5   )r¶  rd  r´   r²   r»  r�  )	NNNNNNNNr   )rJ   rK   rL   rM   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr*   r   r   r,   r`  rO   r   r£  ra  rz   r   r   r   rC   rP   rQ   rR   s   @r3   rµ  rµ  “  s@  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HàÐ/Ð0€Jõ
ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø,0Ø-.ñ@
à×#Ñ# dÑ*ð@
ð Ÿ™ tÑ+ð@
ð ×&Ñ&¨Ñ-ð	@
ð
  ™ð@
ð ×(Ñ(¨4Ñ/ð@
ð × Ñ  4Ñ'ð@
ð ˜$‘;ð@
ð # T™kð@
ð ˜eŸl™lÑ*ð@
ð Ð+Ñ,ð@
ð 
#ô@
ó ó ö@
r5   rµ  )rµ  r|  rc  )r    )r½   )Nr8   N)JÚcollections.abcr   r,   Útorch.nn.functionalr   rÜ   r¿   Ú r   rk  Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   Úmasking_utilsr   r   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   r   Úutils.output_capturingr   r   Úconfiguration_lagunar!   ÚModuler$   rT   r�   r®   rÎ   rñ   r  r  rO   rz   r  rN   r*  r,  rP  rc  r|  rF   r³  rµ  Ú__all__r]  r5   r3   Ú<module>rá     so  ðõ* %ã ß Ð Ý å &Ý !ß .Ý )ß Sß RÝ BÝ 9ß Qß Kß FÝ &ß 5Ý 0ß [Ñ [ß EÝ .ñ ˜YÓ'ôJ�B—I‘Ió Jó (ðJô(G<˜BŸI™Iô G<ôT�—	‘	ô ô @�r—y‘yô @ð> ô$#�B—I‘Ió $#ó ð$#ôN˜2Ÿ9™9ô ò.(ô#ðL	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ô2U)�b—i‘iô U)ôp)Ð3ô )ðX ô$^˜Oó $^ó ð$^ðN ôP
Ð'ó P
ó ðP
ðj #Ø
Ø*.ñ	B&Ø—‘  e§l¡lÑ 3Ñ3°dÑ:ðB&à�t‘ðB&ð —L‘L 4Ñ'ð	B&ð
 ‡\�\�CÑõB&ðJ ôU
Ð-¨ó U
ó ðU
òp H�r5   