ó
    ‡Ž¡jŽY  ã                   ó
  • S SK Jr  S SKrS SKJr  SSKJr  SSKJrJr  SSK	J
r
  SSKJr  SS	KJr  SS
KJr  SSKJrJr  SSKJrJr  SSKJrJr  SSKJr  SSKJrJrJr  SSK J!r!  SSK"J#r#J$r$  SSK%J&r&  SSK'J(r(   " S S\RR                  5      r* " S S\RR                  5      r+ " S S\RR                  5      r,S\RZ                  S\.S\RZ                  4S jr/ S5S\RR                  S \RZ                  S!\RZ                  S"\RZ                  S#\RZ                  S-  S$\0S%\0S&\\   4S' jjr1S( r2S6S) jr3 " S* S+\RR                  5      r4 " S, S-\5      r5\ " S. S/\5      5       r6\ " S0 S1\65      5       r7\ " S2 S3\6\
5      5       r8/ S4Qr9g)7é    )ÚCallableN)Únné   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_causal_mask)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚCohereConfigc                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )ÚCohereLayerNormé3   c                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)zcThe hidden size can be a tuple or an int. The tuple is used for QKNorm to normalize across head_dimN)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚbiasÚ	__class__s       €Ú¸C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/cohere/modeling_cohere.pyr"   ÚCohereLayerNorm.__init__4   s-   ø€ ä‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Õó    c                 ó‚  • UR                   nUR                  [        R                  5      nUR	                  SSS9nX-
  R                  S5      R	                  SSS9nX-
  [        R                  " X@R                  -   5      -  nU R                  R                  [        R                  5      U-  nUR                  U5      $ )NéÿÿÿÿT)Úkeepdimé   )	ÚdtypeÚtor$   Úfloat32ÚmeanÚpowÚrsqrtr'   r&   )r(   Úhidden_statesÚinput_dtyper7   Úvariances        r-   ÚforwardÚCohereLayerNorm.forward:   s¤   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ×!Ñ! "¨dÐ!Ð3ˆØ!Ñ(×-Ñ-¨aÓ0×5Ñ5°bÀ$Ð5ÐGˆØ&Ñ-´·²¸X×H]ÑH]Ñ=]Ó1^Ñ^ˆØŸ™Ÿ™¤u§}¡}Ó5¸ÑEˆØ×Ñ Ó,Ð,r/   )r'   r&   )Ngñhãˆµøä>F©Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r"   r=   Ú__static_attributes__Ú__classcell__©r,   s   @r-   r   r   3   s   ø† ÷$÷-ð -r/   r   c            	       óÒ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\\	R                  \4   4S jj5       5       r\	R                  " 5       \S	 5       5       rS
rU =r$ )ÚCohereRotaryEmbeddingéD   Údevicez5.18)ÚversionÚconfigc                 óÄ  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        U R
                  R                  S   U l        U R                  nU R                  S:w  a  [        U R                     nU" U R
                  U5      u  o@l
        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚdefaultF)Ú
persistent)r!   r"   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenrL   Úrope_parametersrN   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r(   rL   rJ   Úrope_init_fnrX   r,   s        €r-   r"   ÚCohereRotaryEmbedding.__init__E   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÓ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕr/   Úreturnc           	      ó   • U R                   S   n[        U SS5      =(       d    U R                  U R                  -  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?r   r3   ©r4   )rT   Úgetattrr)   Únum_attention_headsr$   ÚarangeÚfloatr5   )rL   rJ   ÚkwargsÚbaseÚdimÚattention_factorrX   s          r-   rU   Ú5CohereRotaryEmbedding.compute_default_rope_parametersV   s{   € ð ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r/   c                 ó  • U R                   S S S 2S 4   R                  5       R                  UR                  S   SS5      nUS S 2S S S 24   R                  5       n[	        UR
                  R                  [        5      (       a0  UR
                  R                  S:w  a  UR
                  R                  OSn[        USS9   UR                  5       UR                  5       -  R                  SS5      n[        R                  " USSS	9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR                  UR                   S
9W	R                  UR                   S
94$ ! , (       d  f       N@= f)Nr   r1   r   ÚmpsÚcpuF)Údevice_typeÚenabledr3   ©rh   ra   )rX   re   ÚexpandÚshapeÚ
isinstancerJ   ÚtypeÚstrr   Ú	transposer$   Úrepeat_interleaveÚcosrV   Úsinr5   r4   )
r(   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrn   ÚfreqsÚembrx   ry   s
             r-   r=   ÚCohereRotaryEmbedding.forwardj   s>  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓeÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆÜ¨¸UÓCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ×)Ò)¨%°¸Ñ;ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   Ã BE<Å<
F
)rV   rL   rX   rR   rZ   rS   rN   ©N)r@   rA   rB   rC   r   r   r"   ÚstaticmethodÚtupler$   ÚTensorre   rU   Úno_gradr   r=   rD   rE   rF   s   @r-   rH   rH   D   sŒ   ø† Ù�X vÑ.ñO˜|÷ Oó /ðOð  Ù�X vÑ.ñ5°ð 5ÐX]Ð^c×^jÑ^jÐlqÐ^qÑXrô 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ö<r/   rH   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )Ú	CohereMLPéz   c                 ó¼  >• [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l        [
        R                  " U R                  U R                  SS9U l	        [        UR                     U l        g ©NF©r+   )r!   r"   rL   r)   Úintermediate_sizer   ÚLinearÚ	gate_projÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©r(   rL   r,   s     €r-   r"   ÚCohereMLP.__init__{   s¤   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÑXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÑVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÑXˆŒÜ˜V×.Ñ.Ñ/ˆ�r/   c                 óˆ   • U R                  U R                  U R                  U5      5      U R                  U5      -  5      nU$ r�   )r�   r’   rŽ   r�   )r(   rz   r�   s      r-   r=   ÚCohereMLP.forward…   s6   € Ø—N‘N 4§;¡;¨t¯~©~¸aÓ/@Ó#AÀDÇLÁLÐQRÃOÑ#SÓTˆ	ØÐr/   )r’   rL   r�   rŽ   r)   rŒ   r�   r?   rF   s   @r-   r‡   r‡   z   s   ø† õ0÷ð r/   r‡   r:   Ún_repr]   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)rr   rq   Úreshape)r:   r—   ÚbatchÚnum_key_value_headsÚslenr`   s         r-   Ú	repeat_kvr�   Š   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr/   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutrf   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )Nr3   r   r1   )rh   r4   )ÚpÚtrainingr   )r�   Únum_key_value_groupsr$   Úmatmulrv   r   Ú
functionalÚsoftmaxr6   r5   r4   r¤   r§   Ú
contiguous)rž   rŸ   r    r¡   r¢   r£   r¤   rf   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r-   Úeager_attention_forwardr±   –   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r/   c                 ó|   • U SS S S24   nU SSS S24   n[         R                  " U* U/SS9R                  S5      nU$ )N.r3   r   r1   rp   éþÿÿÿ)r$   ÚstackÚflatten)rz   Úx1Úx2Úrot_xs       r-   Úrotate_halfr¹   ¯   sL   € à	
ˆ3‘�!�ˆ8‰€BØ	
ˆ3���1�ˆ9‰€BÜ�KŠK˜"˜˜b˜	 rÑ*×2Ñ2°2Ó6€EØ€Lr/   c                 ó&  • U R                   nU R                  5       n UR                  5       nUR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nUR	                  US9UR	                  US94$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
ra   )r4   re   Ú	unsqueezer¹   r5   )ÚqÚkrx   ry   Úunsqueeze_dimr4   Úq_embedÚk_embeds           r-   Úapply_rotary_pos_embrÁ   ·   sŠ   € ð$ �G‰G€EØ	�‰‹	€AØ	�‰‹	€AØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØ�:‰:˜Eˆ:Ð" G§J¡J°U JÐ$;Ð;Ð;r/   c                   ó  ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr SS\R                  S\
\R                  \R                  4   S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  S-  4   4S jjrSrU =r$ )ÚCohereAttentionéÓ   z=Multi-headed attention from 'Attention Is All You Need' paperNrL   Ú	layer_idxc                 óR  >• [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        UR(                  U l        U R(                  (       a_  [+        UR                  U R                  4UR,                  S9U l        [+        UR                  U R                  4UR,                  S9U l        g g )Nr`   g      à¿Tr‹   ©r)   r*   )r!   r"   rL   rÅ   rb   r)   rc   r`   r›   r¨   r£   Úattention_dropoutÚ	is_causalr   r�   Úattention_biasÚq_projÚk_projÚv_projÚo_projÚuse_qk_normr   Úlayer_norm_epsÚq_normÚk_norm©r(   rL   rÅ   r,   s      €r-   r"   ÚCohereAttention.__init__Ö   sµ  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfñ
ˆŒð "×-Ñ-ˆÔØ××ä)Ø#×7Ñ7¸¿¹ÐGÈV×MbÑMbñˆDŒKô *Ø#×7Ñ7¸¿¹ÐGÈV×MbÑMbñˆD�Kð r/   r:   Úposition_embeddingsr¢   Úpast_key_valuesrf   r]   c                 óz  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      nU R	                  U5      R                  U5      n	U R                  U5      R                  U5      n
U R                  (       a"  U R                  U5      nU R                  U	5      n	UR                  SS5      nU	R                  SS5      n	U
R                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                   ["        5      nU" U UU	U
U4U R$                  (       d  SOU R&                  U R(                  S.UD6u  pïUR*                  " / UQSP76 R-                  5       nU R/                  U5      nXï4$ )Nr1   r   r3   ç        )r¤   r£   )rr   r`   rË   ÚviewrÌ   rÍ   rÏ   rÑ   rÒ   rv   rÁ   ÚupdaterÅ   r   Úget_interfacerL   Ú_attn_implementationr±   r§   rÈ   r£   r™   r¬   rÎ   )r(   r:   rÕ   r¢   rÖ   rf   Úinput_shapeÚhidden_shapeÚquery_statesr­   r®   rx   ry   Úattention_interfacer°   r¯   s                   r-   r=   ÚCohereAttention.forwardö   s¯  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆà××ØŸ;™; |Ó4ˆLØŸ™ ZÓ0ˆJà#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r/   )rÈ   rL   r`   rÉ   rÒ   rÌ   rÅ   r¨   rÎ   rÑ   rË   r£   rÏ   rÍ   r�   )r@   rA   rB   rC   Ú__doc__r   Úintr"   r$   r„   rƒ   r   r   r   r=   rD   rE   rF   s   @r-   rÃ   rÃ   Ó   s¬   ø† ÙGñ˜|ð ¸¸d¹
÷ ð ðJ )-ñ.)à—|‘|ð.)ð # 5§<¡<°·±Ð#=Ñ>ð.)ð Ÿ™ tÑ+ð	.)ð
  ™ð.)ð Ð-Ñ.ð.)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷.)ó .)r/   rÃ   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚCohereDecoderLayeri'  rL   rÅ   c                 óÆ   >• [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        g )N)rL   rÅ   rÇ   )
r!   r"   r)   rÃ   Ú	self_attnr‡   Úmlpr   rÐ   Úinput_layernormrÓ   s      €r-   r"   ÚCohereDecoderLayer.__init__(  sP   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ(°ÑLˆŒÜ˜VÓ$ˆŒÜ.¸F×<NÑ<NÐU[×UjÑUjÑkˆÕr/   Nr:   r¢   r{   rÖ   Ú	use_cacherÕ   rf   r]   c           
      ó’   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pšU R                  U5      nX‰-   U-   nU$ )a<  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`, *optional*):
        attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
        query_sequence_length, key_sequence_length)` if default attention is used.
    past_key_values (`Cache`, *optional*): cached past key and value projection states
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
)r:   r¢   r{   rÖ   rë   rÕ   © )ré   rç   rè   )r(   r:   r¢   r{   rÖ   rë   rÕ   rf   ÚresidualÚhidden_states_attentionÚ_Úhidden_states_mlps               r-   r=   ÚCohereDecoderLayer.forward/  sq   € ð6 !ˆØ×,Ñ,¨]Ó;ˆà%)§^¢^ð &
Ø'Ø)Ø%Ø+ØØ 3ñ&
ð ñ&
Ñ"Ðð !ŸH™H ]Ó3ÐØ Ñ:Ð=NÑNˆØÐr/   )r)   ré   rè   rç   )NNNFN)r@   rA   rB   rC   r   rã   r"   r$   r„   Ú
LongTensorr   Úboolrƒ   r   r   r=   rD   rE   rF   s   @r-   rå   rå   '  sÃ   ø† ðl˜|ð l¸÷ lð /3Ø04Ø(,Ø!&ØHLñ*à—|‘|ð*ð Ÿ™ tÑ+ð*ð ×&Ñ&¨Ñ-ð	*ð
  ™ð*ð ˜$‘;ð*ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð*ð Ð-Ñ.ð*ð 
�‰÷*ó *r/   rå   c                   óR   • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.rSrg	)
ÚCoherePreTrainedModeli\  rL   ÚmodelTrå   rÖ   )r:   Ú
attentionsrí   N)r@   rA   rB   rC   r   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrå   rÃ   Ú_can_record_outputsrD   rí   r/   r-   rö   rö   \  sQ   ‡ àÓØÐØ&*Ð#Ø-Ð.ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà+Ø%ñÓr/   rö   c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )ÚCohereModelio  rL   c           	      ó  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )NrÇ   ©rL   F)r!   r"   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr)   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrå   Úlayersr   rÐ   ÚnormrH   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÓ   s      €r-   r"   ÚCohereModel.__init__q  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜDIÈ&×JbÑJbÔDcÓdÒDc°yÔ Ö2ÑDcÑdó
ˆŒô $°×1CÑ1CÈ&×J_ÑJ_Ñ`ˆŒ	Ü/°vÑ>ˆŒØ&+ˆÔ#ð 	�‰Õùò es   ÂC?NÚ	input_idsr¢   r{   rÖ   Úinputs_embedsrë   rf   r]   c           
      ó>  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U R                  UUUUS9n	Un
U R                  X£S9nU R                  S U R                  R                    H  nU" U
4U	UUUUS.UD6n
M     U R                  U
5      n
[        U
US	9$ )
Nz:You must specify exactly one of input_ids or inputs_embedsr  r   r   )rJ   )rL   r  r¢   rÖ   r{   )r{   )r¢   rÕ   r{   rÖ   rë   )Úlast_hidden_staterÖ   )Ú
ValueErrorr  r   rL   Úget_seq_lengthr$   rd   rr   rJ   r»   r
   r  r  r  r  r   )r(   r  r¢   r{   rÖ   r  rë   rf   Úpast_seen_tokensÚcausal_maskr:   rÕ   Údecoder_layers                r-   r=   ÚCohereModel.forward�  sF  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø*.×*;Ñ*;¸IÓ*FˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð &ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø$7Ø)Ø /Ø#ñð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ñ
ð 	
r/   )r  r  r  r  r	  r  r
  )NNNNNN)r@   rA   rB   rC   r   r"   r   r   r   r$   ró   r„   r   ÚFloatTensorrô   r   r   r   r=   rD   rE   rF   s   @r-   r  r  o  sÍ   ø† ð˜|÷ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!ô2
ó ó ó  ö2
r/   r  c                   óX  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
        SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚCohereForCausalLMi¹  zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputr:   ÚlogitsÚkeep_full_weightc                 ó(  >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        UR                  U l	        UR                  U l
        U R                  5         g rŠ   )r!   r"   r  r÷   r
  r   r�   r)   r#  Úlogit_scaleÚtie_word_embeddingsr  r“   s     €r-   r"   ÚCohereForCausalLM.__init__À  sq   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒØ!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ ð 	�‰Õr/   Nr  r¢   r{   rÖ   r  Úlabelsrë   Úlogits_to_keeprf   r]   c	           
      ó˜  • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nXÐR                  -  nSnUb)  U R                  " SXÖU R                  R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9$ )au  
Example:

```python
>> from transformers import AutoTokenizer, CohereForCausalLM

>> model = CohereForCausalLM.from_pretrained("CohereForAI/c4ai-command-r-v01")
>> tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01")

>> prompt = "Hey, are you conscious? Can you talk to me?"
>> inputs = tokenizer(prompt, return_tensors="pt")

>> # Generate
>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```)r  r¢   r{   rÖ   r  rë   N)r%  r+  r
  )Úlossr%  rÖ   r:   rø   rí   )r÷   r  rs   rã   Úslicer#  r(  Úloss_functionrL   r
  r   rÖ   r:   rø   )r(   r  r¢   r{   rÖ   r  r+  rë   r,  rf   Úoutputsr:   Úslice_indicesr%  r.  s                  r-   r=   ÚCohereForCausalLM.forwardË  sä   € ð> ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ×*Ñ*Ñ*ˆàˆØÑØ×%Ò%Ðp¨VÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r/   )r#  r(  r÷   r)  r
  )NNNNNNNr   )r@   rA   rB   rC   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr"   r   r   r$   ró   r„   r   r   rô   rã   r   r   r   r=   rD   rE   rF   s   @r-   r"  r"  ¹  s"  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõ	ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ô6
ó ó ö6
r/   r"  )r"  r  rö   )rØ   )r   ):Úcollections.abcr   r$   r   Úactivationsr   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_coherer   ÚModuler   rH   r‡   r„   rã   r�   re   r±   r¹   rÁ   rÃ   rå   rö   r  r"  Ú__all__rí   r/   r-   Ú<module>rJ     sœ  ðõ: %ã Ý å !ß .Ý )Ý /Ý BÝ 9ß Oß Kß FÝ &ß IÑ IÝ 0ß GÝ 5Ý .ô-�b—i‘iô -ô"3<˜BŸI™Iô 3<ôl�—	‘	ô ð 	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ò2ô<ô8Q)�b—i‘iô Q)ôh2Ð3ô 2ðj ô˜Oó ó ðð$ ôF
Ð'ó F
ó ðF
ðR ôI
Ð-¨ó I
ó ðI
òX H�r/   