ó
    ‡Ž¡jyT  ã                   óX  • S SK Jr  S SKrS SKJr  SSKJr  SSKJrJ	r	  SSK
Jr  SSKJr  SSKJr  SS	KJrJrJr  SS
KJrJr  SSKJrJr  SSKJrJr  SSKJr  SSKJ r J!r!J"r"  SSK#J$r$  SSK%J&r&J'r'  SSK(J)r)  SSK*J+r+   " S S\RX                  5      r- " S S\RX                  5      r.S\R^                  S\0S\R^                  4S jr1 S9S\RX                  S\R^                  S\R^                  S\R^                  S \R^                  S-  S!\2S"\2S#\\    4S$ jjr3S% r4S:S& jr5 " S' S(\RX                  5      r6\" S)5       " S* S+\RX                  5      5       r7 " S, S-\5      r8\! " S. S/\5      5       r9\! " S0 S1\95      5       r:\! " S2 S3\9\5      5       r; " S4 S5\\95      r< " S6 S7\\95      r=/ S8Qr>g);é    )ÚCallableNé   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hub)Úcreate_causal_mask)Ú GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )Ú	GlmConfigc                   ób   ^ • \ rS rSrU 4S jrS\R                  S\R                  4S jrSrU =r	$ )ÚGlmMLPé0   c                 ó   >• [         TU ]  5         Xl        [        R                  " UR
                  SUR                  -  SS9U l        [        R                  " UR                  UR
                  SS9U l        [        UR                     U l        g )Né   F©Úbias)ÚsuperÚ__init__ÚconfigÚnnÚLinearÚhidden_sizeÚintermediate_sizeÚgate_up_projÚ	down_projr   Ú
hidden_actÚactivation_fn©Úselfr'   Ú	__class__s     €Ú²C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/glm/modeling_glm.pyr&   ÚGlmMLP.__init__1   sn   ø€ Ü‰ÑÔàŒÜŸIšI f×&8Ñ&8¸!¸f×>VÑ>VÑ:VÐ]bÑcˆÔÜŸš 6×#;Ñ#;¸V×=OÑ=OÐV[Ñ\ˆŒÜ# F×$5Ñ$5Ñ6ˆÕó    Úhidden_statesÚreturnc                 ó�   • U R                  U5      nUR                  SSS9u  p2X R                  U5      -  nU R                  U5      $ )Nr"   éÿÿÿÿ©Údim)r,   Úchunkr/   r-   )r1   r6   Ú	up_statesÚgates       r3   ÚforwardÚGlmMLP.forward9   sH   € Ø×%Ñ% mÓ4ˆ	à#Ÿ/™/¨!°˜/Ð4‰ˆØ× 2Ñ 2°4Ó 8Ñ8ˆ	à�~‰~˜iÓ(Ð(r5   )r/   r'   r-   r,   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r&   ÚtorchÚFloatTensorr?   Ú__static_attributes__Ú__classcell__©r2   s   @r3   r   r   0   s,   ø† õ7ð) U×%6Ñ%6ð )¸5×;LÑ;L÷ )ò )r5   r   c            	       óÒ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\\	R                  \4   4S jj5       5       r\	R                  " 5       \S	 5       5       rS
rU =r$ )ÚGlmRotaryEmbeddingéB   Údevicez5.18)Úversionr'   c                 óÄ  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        U R
                  R                  S   U l        U R                  nU R                  S:w  a  [        U R                     nU" U R
                  U5      u  o@l
        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚdefaultF)Ú
persistent)r%   r&   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr'   Úrope_parametersrP   Úcompute_default_rope_parametersr   Úattention_scalingr(   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r1   r'   rM   Úrope_init_fnrZ   r2   s        €r3   r&   ÚGlmRotaryEmbedding.__init__C   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÓ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕr5   r7   c           	      óR  • U R                   S   nU R                   R                  SS5      n[        U SS5      =(       d    U R                  U R                  -  n[        XT-  5      nSnSU[        R                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )	a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚpartial_rotary_factorg      ð?Úhead_dimNr   r"   ©Údtype)
rV   ÚgetÚgetattrr*   Únum_attention_headsÚintrE   ÚarangeÚfloatÚto)	r'   rM   ÚkwargsÚbasera   rb   r;   Úattention_factorrZ   s	            r3   rW   Ú2GlmRotaryEmbedding.compute_default_rope_parametersT   s£   € ð ×%Ñ% lÑ3ˆØ &× 6Ñ 6× :Ñ :Ð;RÐTWÓ XÐÜ˜6 :¨tÓ4×h¸×8JÑ8JÈf×NhÑNhÑ8hˆÜ�(Ñ2Ó3ˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r5   c                 ó  • U R                   S S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        USS9   X4-  R                  SS	5      n[        R                  " Xf4SS
9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR                  UR                   S9W	R                  UR                   S94$ ! , (       d  f       N@= f)Nr   r9   r   )rd   rM   ÚmpsÚcpuF)Údevice_typeÚenabledr"   r:   rc   )rZ   ÚexpandÚshaperk   rE   rj   rM   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrX   Úsinrd   )
r1   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrs   ÚfreqsÚembr|   r}   s
             r3   r?   ÚGlmRotaryEmbedding.forwardj   s;  € ð �M‰M˜$¢ 4˜-Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÐtð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   ÃA%E7Å7
F)rX   r'   rZ   rT   r\   rU   rP   ©N)rA   rB   rC   rD   r   r   r&   ÚstaticmethodÚtuplerE   ÚTensorrj   rW   Úno_gradr   r?   rG   rH   rI   s   @r3   rK   rK   B   sŒ   ø† Ù�X vÑ.ñO˜y÷ Oó /ðOð  Ù�X vÑ.ñ5°	ð 5ÐUZÐ[`×[gÑ[gÐinÐ[nÑUoô 5ó /ó ð5ð( ‡]‚]ƒ_Øñ<ó ó ö<r5   rK   r6   Ún_repr7   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)rv   ru   Úreshape)r6   rŠ   ÚbatchÚnum_key_value_headsÚslenrb   s         r3   Ú	repeat_kvr�   }   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr5   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrl   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )Nr"   r   r9   )r;   rd   )ÚpÚtrainingr   )r�   Únum_key_value_groupsrE   Úmatmulrz   r(   Ú
functionalÚsoftmaxÚfloat32rk   rd   r—   rš   Ú
contiguous)r‘   r’   r“   r”   r•   r–   r—   rl   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r3   Úeager_attention_forwardr¥   ‰   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r5   c                 óx   • U SSSS24   nU SSSS24   n[         R                  " U* U4SS9R                  S5      $ )	z*Rotates half the hidden dims of the input..r   Nr"   r   r9   r:   éþÿÿÿ)rE   ÚstackÚflatten)r~   Úx1Úx2s      r3   Úrotate_halfr¬   ¢   sJ   € à	
ˆ3���1�ˆ9‰€BØ	
ˆ3���1�ˆ9‰€BÜ�;Š;˜˜˜R�y bÑ)×1Ñ1°"Ó5Ð5r5   c                 óî  • UR                  U5      nUR                  U5      nUSSUR                  S   S-  24   R                  SSS9nUSSUR                  S   S-  24   R                  SSS9nUR                  S   nU SSU24   U SUS24   pvUSSU24   USUS24   p˜Xb-  [        U5      U-  -   n
X‚-  [        U5      U-  -   n[        R
                  " X§/SS9n
[        R
                  " X¹/SS9nX«4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
.Nr9   r"   r:   )Ú	unsqueezerv   Úrepeat_interleaver¬   rE   r{   )ÚqÚkr|   r}   Úunsqueeze_dimÚ
rotary_dimÚq_rotÚq_passÚk_rotÚk_passÚq_embedÚk_embeds               r3   Úapply_rotary_pos_embrº   ©   s6  € ð$ �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€Cð ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ð
E€CØ
ˆcÐ'�S—Y‘Y˜r‘] aÑ'Ð'Ð'Ñ
(×
:Ñ
:¸1À"Ð
:Ð
E€Cð —‘˜2‘€JØ�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6Ø�c˜;˜J˜;Ð&Ñ'¨¨3°
±Ð+;Ñ)<ˆ6ð ‰{œ{¨5Ó1°CÑ7Ñ8€GØ‰{œ{¨5Ó1°CÑ7Ñ8€Gô �iŠi˜Ð)¨rÑ2€GÜ�iŠi˜Ð)¨rÑ2€GØÐÐr5   c                   ó  ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr   SS\R                  S\
\R                  \R                  4   S-  S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  4   4S jjrSrU =r$ )ÚGlmAttentionéÑ   z=Multi-headed attention from 'Attention Is All You Need' paperNr'   Ú	layer_idxc                 ó<  >• [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  SS9U l        g )Nrb   g      à¿Tr#   F)r%   r&   r'   r¾   rf   r*   rg   rb   rŽ   r›   r–   Úattention_dropoutÚ	is_causalr(   r)   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©r1   r'   r¾   r2   s      €r3   r&   ÚGlmAttention.__init__Ô   s@  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’i × :Ñ :¸T¿]¹]Ñ JÈF×L^ÑL^ÐejÑkˆ�r5   r6   Úposition_embeddingsr•   Úpast_key_valuesrl   r7   c                 ó  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [        5      nU" U UU	U
U4U R                  (       d  SOU R                   U R"                  S.UD6u  pïUR$                  " / UQSP76 R'                  5       nU R)                  U5      nXï4$ )Nr9   r   r"   ç        )r—   r–   )rv   rb   rÃ   Úviewrz   rÄ   rÅ   rº   Úupdater¾   r   Úget_interfacer'   Ú_attn_implementationr¥   rš   rÀ   r–   rŒ   r    rÆ   )r1   r6   rÉ   r•   rÊ   rl   Úinput_shapeÚhidden_shapeÚquery_statesr¡   r¢   r|   r}   Úattention_interfacer¤   r£   s                   r3   r?   ÚGlmAttention.forwardé   s~  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r5   )rÀ   r'   rb   rÁ   rÄ   r¾   r›   rÆ   rÃ   r–   rÅ   r…   )NNN)rA   rB   rC   rD   Ú__doc__r   rh   r&   rE   rˆ   r‡   r   r   r   r?   rG   rH   rI   s   @r3   r¼   r¼   Ñ   s·   ø† ÙGñl˜yð l°S¸4±Z÷ lð lð0 IMØ.2Ø(,ñ&)à—|‘|ð&)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð&)ð Ÿ™ tÑ+ð	&)ð
  ™ð&)ð Ð+Ñ,ð&)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&)ó &)r5   r¼   ÚRMSNormc                   óx   ^ • \ rS rSrS
S\SS4U 4S jjjrS\R                  S\R                  4S jrS r	S	r
U =r$ )Ú
GlmRMSNormi  Úepsr7   Nc                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z)
GlmRMSNorm is equivalent to T5LayerNorm
N)r%   r&   r(   Ú	ParameterrE   ÚonesÚweightÚvariance_epsilon)r1   r*   rÚ   r2   s      €r3   r&   ÚGlmRMSNorm.__init__  s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Õr5   r6   c                 ó  • UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )Nr"   r9   T)Úkeepdim)	rd   rk   rE   rŸ   ÚpowÚmeanÚrsqrtrß   rÞ   )r1   r6   Úinput_dtypeÚvariances       r3   r?   ÚGlmRMSNorm.forward  sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ð>ˆØ%¬¯ª°H×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r5   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)r‡   rÞ   rv   rß   )r1   s    r3   Ú
extra_reprÚGlmRMSNorm.extra_repr#  s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr5   )rß   rÞ   )g�íµ ÷Æ°>)rA   rB   rC   rD   rj   r&   rE   rˆ   r?   rê   rG   rH   rI   s   @r3   rÙ   rÙ     sB   ø† ñ$¨ð $¸$÷ $ð $ð; U§\¡\ð ;°e·l±lô ;÷Jð Jr5   rÙ   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚGlmDecoderLayeri'  r'   r¾   c                 ó  >• [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        [        UR                  UR                  S9U l
        g )N)r'   r¾   ©rÚ   )r%   r&   r*   r¼   Ú	self_attnr   ÚmlprÙ   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrÇ   s      €r3   r&   ÚGlmDecoderLayer.__init__(  si   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä%¨VÑIˆŒä˜&“>ˆŒÜ)¨&×*<Ñ*<À&×BUÑBUÑVˆÔÜ(2°6×3EÑ3EÈ6×K^ÑK^Ñ(_ˆÕ%r5   Nr6   r•   r   rÊ   Ú	use_cacherÉ   rl   r7   c           
      óº   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pX�-   nUnU R                  U5      nU R                  U5      nX�-   nU$ )N)r6   r•   r   rÊ   rö   rÉ   © )ró   rð   rô   rñ   )
r1   r6   r•   r   rÊ   rö   rÉ   rl   ÚresidualÚ_s
             r3   r?   ÚGlmDecoderLayer.forward2  sˆ   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆð !Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Ñ0ˆØÐr5   )r*   ró   rñ   rô   rð   )NNNFN)rA   rB   rC   rD   r   rh   r&   rE   rˆ   Ú
LongTensorr   Úboolr‡   r   r   r?   rG   rH   rI   s   @r3   rí   rí   '  sÃ   ø† ð`˜yð `°S÷ `ð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷ó r5   rí   c                   óR   • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.rSrg	)
ÚGlmPreTrainedModeliR  r'   ÚmodelTrí   rÊ   )r6   Ú
attentionsrø   N)rA   rB   rC   rD   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrí   r¼   Ú_can_record_outputsrG   rø   r5   r3   rÿ   rÿ   R  sQ   ‡ àÓØÐØ&*Ð#Ø*Ð+ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà(Ø"ñÓr5   rÿ   c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚGlmModelie  r'   c           	      ó  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nrï   ©r'   F)r%   r&   Úpad_token_idÚpadding_idxÚ
vocab_sizer(   Ú	Embeddingr*   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrí   ÚlayersrÙ   rò   ÚnormrK   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÇ   s      €r3   r&   ÚGlmModel.__init__g  sÉ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜAFÀv×G_ÑG_ÔA`ÓaÒA`°IŒ_˜VÖ/ÑA`Ñaó
ˆŒô ˜v×1Ñ1°v×7JÑ7JÑKˆŒ	Ü,°FÑ;ˆŒØ&+ˆÔ#ð 	�‰Õùò bs   ÂC?NÚ	input_idsr•   r   rÊ   Úinputs_embedsrö   rl   r7   c           
      ó>  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U R                  UUUUS9n	Un
U R                  X£S9nU R                  S U R                  R                    H  nU" U
4U	UUUUS.UD6n
M     U R                  U
5      n
[        U
US	9$ )
Nz:You must specify exactly one of input_ids or inputs_embedsr  r   r   )rM   )r'   r   r•   rÊ   r   )r   )r•   rÉ   r   rÊ   rö   )Úlast_hidden_staterÊ   )Ú
ValueErrorr  r   r'   Úget_seq_lengthrE   ri   rv   rM   r®   r
   r  r  r  r  r   )r1   r  r•   r   rÊ   r   rö   rl   Úpast_seen_tokensÚcausal_maskr6   rÉ   Údecoder_layers                r3   r?   ÚGlmModel.forwardw  sF  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø*.×*;Ñ*;¸IÓ*FˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð &ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø$7Ø)Ø /Ø#ñð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ñ
ð 	
r5   )r  r  r  r  r  r  r  )NNNNNN)rA   rB   rC   rD   r   r&   r   r   r   rE   rü   rˆ   r   rF   rý   r   r   r   r?   rG   rH   rI   s   @r3   r  r  e  sÍ   ø† ð˜y÷ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!ô2
ó ó ó  ö2
r5   r  c                   óX  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
        SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚGlmForCausalLMi¯  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr6   ÚlogitsÚkeep_full_weightc                 óä   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g )NFr#   )
r%   r&   r  r   r  r(   r)   r*   r+  r  r0   s     €r3   r&   ÚGlmForCausalLM.__init__¶  sU   ø€ Ü‰Ñ˜Ô Ü˜fÓ%ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒð 	�‰Õr5   Nr  r•   r   rÊ   r   Úlabelsrö   Úlogits_to_keeprl   r7   c	           
      ó|  • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nSnUb)  U R                  " SXÖU R                  R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9$ )ac  
Example:

```python
>>> from transformers import AutoTokenizer, GlmForCausalLM

>>> model = GlmForCausalLM.from_pretrained("meta-glm/Glm-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-glm/Glm-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```)r  r•   r   rÊ   r   rö   N)r-  r1  r  )Úlossr-  rÊ   r6   r  rø   )r   r"  rw   rh   Úslicer+  Úloss_functionr'   r  r   rÊ   r6   r  )r1   r  r•   r   rÊ   r   r1  rö   r2  rl   Úoutputsr6   Úslice_indicesr-  r4  s                  r3   r?   ÚGlmForCausalLM.forward¿  sÖ   € ð> ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò%Ðp¨VÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r5   )r+  r   r  )NNNNNNNr   )rA   rB   rC   rD   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr&   r   r   rE   rü   rˆ   r   rF   rý   rh   r   r   r   r?   rG   rH   rI   s   @r3   r*  r*  ¯  s"  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ô6
ó ó ö6
r5   r*  c                   ó   • \ rS rSrSrg)ÚGlmForSequenceClassificationiú  rø   N©rA   rB   rC   rD   rG   rø   r5   r3   r?  r?  ú  ó   † Úr5   r?  c                   ó   • \ rS rSrSrg)ÚGlmForTokenClassificationiþ  rø   Nr@  rø   r5   r3   rC  rC  þ  rA  r5   rC  )rÿ   r  r*  r?  rC  )rÌ   )r   )?Úcollections.abcr   rE   Útorch.nnr(   Úactivationsr   Úcache_utilsr   r   Ú
generationr   Úintegrationsr	   Úmasking_utilsr
   Úmodeling_layersr   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_glmr   ÚModuler   rK   rˆ   rh   r�   rj   r¥   r¬   rº   r¼   rÙ   rí   rÿ   r  r*  r?  rC  Ú__all__rø   r5   r3   Ú<module>rW     sÛ  ðõ, %ã Ý å !ß .Ý )Ý 7Ý /÷ñ ÷
 Pß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý (ô)ˆR�Y‰Yô )ô$8<˜Ÿ™ô 8<ðv	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ò26ô%ôP>)�2—9‘9ô >)ñB ˜YÓ'ôJ�—‘ó Jó (ðJô((Ð0ô (ðV ô˜ó ó ðð$ ôF
Ð!ó F
ó ðF
ðR ôG
Ð'¨ó G
ó ðG
ôT	Ð#CÐEWô 	ô	Ð =Ð?Qô 	ò�r5   