ó
    ‡Ž¡jš6  ã                   óÈ  • S r SSKJr  SSKrSSKJr  SSKJr  SSKJr  SSK	J
r
  SS	KJrJr  SS
KJr  SSKJr  SSKJr  SSKJrJrJrJr  SSKJrJr  SSKJrJrJrJ r J!r!J"r"  SSK#J$r$  \RJ                  " \&5      r' " S S\RP                  5      r) " S S\!5      r*S r+S$S jr, " S S\5      r-\ " S S\5      5       r. " S S\
5      r/ " S S \ 5      r0 " S! S"\5      r1/ S#Qr2g)%zPyTorch Cohere model.é    )ÚCallableN)Únné   )ÚCache)ÚFlashAttentionKwargs)ÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)Údynamic_rope_update)ÚALL_ATTENTION_FUNCTIONS)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmaybe_autocastÚno_inherit_decoratoré   )ÚLlamaAttentionÚLlamaForCausalLMÚLlamaMLPÚ
LlamaModelÚLlamaRotaryEmbeddingÚeager_attention_forwardé   )ÚCohereConfigc                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )ÚCohereLayerNormé4   c                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)zcThe hidden size can be a tuple or an int. The tuple is used for QKNorm to normalize across head_dimN)ÚsuperÚ__init__r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚbiasÚ	__class__s       €Ú·C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/cohere/modular_cohere.pyr"   ÚCohereLayerNorm.__init__5   s-   ø€ ä‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Õó    c                 ó‚  • UR                   nUR                  [        R                  5      nUR	                  SSS9nX-
  R                  S5      R	                  SSS9nX-
  [        R                  " X@R                  -   5      -  nU R                  R                  [        R                  5      U-  nUR                  U5      $ )NéÿÿÿÿT)Úkeepdimr   )	ÚdtypeÚtor$   Úfloat32ÚmeanÚpowÚrsqrtr'   r&   )r(   Úhidden_statesÚinput_dtyper6   Úvariances        r-   ÚforwardÚCohereLayerNorm.forward;   s¤   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ×!Ñ! "¨dÐ!Ð3ˆØ!Ñ(×-Ñ-¨aÓ0×5Ñ5°bÀ$Ð5ÐGˆØ&Ñ-´·²¸X×H]ÑH]Ñ=]Ó1^Ñ^ˆØŸ™Ÿ™¤u§}¡}Ó5¸ÑEˆØ×Ñ Ó,Ð,r/   )r'   r&   )Ngñhãˆµøä>F)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r"   r<   Ú__static_attributes__Ú__classcell__©r,   s   @r-   r   r   4   s   ø† ÷$÷-ð -r/   r   c                   óL   • \ rS rSr\R
                  " 5       \S 5       5       rSrg)ÚCohereRotaryEmbeddingéE   c                 ó  • U R                   S S S 2S 4   R                  5       R                  UR                  S   SS5      nUS S 2S S S 24   R                  5       n[	        UR
                  R                  [        5      (       a0  UR
                  R                  S:w  a  UR
                  R                  OSn[        USS9   UR                  5       UR                  5       -  R                  SS5      n[        R                  " USSS	9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR                  UR                   S
9W	R                  UR                   S
94$ ! , (       d  f       N@= f)Nr   r1   r   ÚmpsÚcpuF)Údevice_typeÚenabledr   ©Údim©r3   )Úinv_freqÚfloatÚexpandÚshapeÚ
isinstanceÚdeviceÚtypeÚstrr   Ú	transposer$   Úrepeat_interleaveÚcosÚattention_scalingÚsinr4   r3   )
r(   ÚxÚposition_idsÚinv_freq_expandedÚposition_ids_expandedrK   ÚfreqsÚembrZ   r\   s
             r-   r<   ÚCohereRotaryEmbedding.forwardF   s>  € ð !ŸM™M¨$²°4¨-Ñ8×>Ñ>Ó@×GÑGÈ×HZÑHZÐ[\ÑH]Ð_aÐcdÓeÐØ ,ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆÜ¨¸UÓCØ&×,Ñ,Ó.Ð1F×1LÑ1LÓ1NÑN×YÑYÐZ[Ð]^Ó_ˆEÜ×)Ò)¨%°¸Ñ;ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   Ã BE<Å<
F
© N)	r>   r?   r@   rA   r$   Úno_gradr   r<   rB   rd   r/   r-   rF   rF   E   s"   † Ø
‡]‚]ƒ_Øñ<ó ó ó<r/   rF   c                 ó|   • U SS S S24   nU SSS S24   n[         R                  " U* U/SS9R                  S5      nU$ )N.r   r   r1   rM   éþÿÿÿ)r$   ÚstackÚflatten)r]   Úx1Úx2Úrot_xs       r-   Úrotate_halfrm   V   sL   € à	
ˆ3‘�!�ˆ8‰€BØ	
ˆ3���1�ˆ9‰€BÜ�KŠK˜"˜˜b˜	 rÑ*×2Ñ2°2Ó6€EØ€Lr/   c                 ó&  • U R                   nU R                  5       n UR                  5       nUR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nUR	                  US9UR	                  US94$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
rO   )r3   rQ   Ú	unsqueezerm   r4   )ÚqÚkrZ   r\   Úunsqueeze_dimr3   Úq_embedÚk_embeds           r-   Úapply_rotary_pos_embru   ^   sŠ   € ð$ �G‰G€EØ	�‰‹	€AØ	�‰‹	€AØ
�-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØ�:‰:˜Eˆ:Ð" G§J¡J°U JÐ$;Ð;Ð;r/   c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )Ú	CohereMLPéz   c                 ó>  >• [         TU ]  U5        [        R                  " U R                  U R
                  SS9U l        [        R                  " U R                  U R
                  SS9U l        [        R                  " U R
                  U R                  SS9U l        g )NF)r+   )	r!   r"   r   ÚLinearr)   Úintermediate_sizeÚ	gate_projÚup_projÚ	down_proj©r(   Úconfigr,   s     €r-   r"   ÚCohereMLP.__init__{   ss   ø€ Ü‰Ñ˜Ô ÜŸš 4×#3Ñ#3°T×5KÑ5KÐRWÑXˆŒÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPUÑVˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRWÑXˆ�r/   )r~   r|   r}   )r>   r?   r@   rA   r"   rB   rC   rD   s   @r-   rw   rw   z   s   ø† ÷Yó Yr/   rw   c                   ó  ^ • \ rS rSrSrSS\S\S-  4U 4S jjjr SS\R                  S\
\R                  \R                  4   S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  S-  4   4S jjrSrU =r$ )ÚCohereAttentioné‚   z=Multi-headed attention from 'Attention Is All You Need' paperNr€   Ú	layer_idxc                 ó&  >• [         TU ]  X5        UR                  U l        U R                  (       a_  [        UR                  U R
                  4UR                  S9U l        [        UR                  U R
                  4UR                  S9U l	        g g ©N©r)   r*   )
r!   r"   Úuse_qk_normr   Únum_attention_headsÚhead_dimÚlayer_norm_epsÚq_normÚnum_key_value_headsÚk_norm©r(   r€   r…   r,   s      €r-   r"   ÚCohereAttention.__init__†   sz   ø€ Ü‰Ñ˜Ô+Ø!×-Ñ-ˆÔØ××ä)Ø#×7Ñ7¸¿¹ÐGÈV×MbÑMbñˆDŒKô *Ø#×7Ñ7¸¿¹ÐGÈV×MbÑMbñˆD�Kð r/   r9   Úposition_embeddingsÚattention_maskÚpast_key_valuesÚkwargsÚreturnc                 óz  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      nU R	                  U5      R                  U5      n	U R                  U5      R                  U5      n
U R                  (       a"  U R                  U5      nU R                  U	5      n	UR                  SS5      nU	R                  SS5      n	U
R                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                   ["        5      nU" U UU	U
U4U R$                  (       d  SOU R&                  U R(                  S.UD6u  pïUR*                  " / UQSP76 R-                  5       nU R/                  U5      nXï4$ )Nr1   r   r   g        )ÚdropoutÚscaling)rS   r‹   Úq_projÚviewÚk_projÚv_projr‰   r�   r�   rX   ru   Úupdater…   r   Úget_interfacer€   Ú_attn_implementationr   ÚtrainingÚattention_dropoutr™   ÚreshapeÚ
contiguousÚo_proj)r(   r9   r’   r“   r”   r•   Úinput_shapeÚhidden_shapeÚquery_statesÚ
key_statesÚvalue_statesrZ   r\   Úattention_interfaceÚattn_outputÚattn_weightss                   r-   r<   ÚCohereAttention.forward’   s¯  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓDˆØ—[‘[ Ó/×4Ñ4°\ÓBˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓDˆà××ØŸ;™; |Ó4ˆLØŸ™ ZÓ0ˆJà#×-Ñ-¨a°Ó3ˆØ×)Ñ)¨!¨QÓ/ˆ
Ø#×-Ñ-¨a°Ó3ˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r/   )r�   r�   r‰   ©N)r>   r?   r@   rA   Ú__doc__r   Úintr"   r$   ÚTensorÚtupler   r   r   r<   rB   rC   rD   s   @r-   rƒ   rƒ   ‚   s«   ø† áGñ
˜|ð 
¸¸d¹
÷ 
ð 
ð" )-ñ.)à—|‘|ð.)ð # 5§<¡<°·±Ð#=Ñ>ð.)ð Ÿ™ tÑ+ð	.)ð
  ™ð.)ð Ð-Ñ.ð.)ð 
ˆu�|‰|˜UŸ\™\¨DÑ0Ð0Ñ	1÷.)ó .)r/   rƒ   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚCohereDecoderLayeréÃ   r€   r…   c                 óÆ   >• [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        g )N)r€   r…   rˆ   )
r!   r"   r)   rƒ   Ú	self_attnrw   Úmlpr   rŒ   Úinput_layernormr�   s      €r-   r"   ÚCohereDecoderLayer.__init__Ä   sP   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔÜ(°ÑLˆŒÜ˜VÓ$ˆŒÜ.¸F×<NÑ<NÐU[×UjÑUjÑkˆÕr/   Nr9   r“   r^   r”   Ú	use_cacher’   r•   r–   c           
      ó’   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pšU R                  U5      nX‰-   U-   nU$ )a<  
Args:
    hidden_states (`torch.FloatTensor`): input to the layer of shape `(batch, seq_len, embed_dim)`
    attention_mask (`torch.FloatTensor`, *optional*):
        attention mask of size `(batch_size, sequence_length)` if flash attention is used or `(batch_size, 1,
        query_sequence_length, key_sequence_length)` if default attention is used.
    past_key_values (`Cache`, *optional*): cached past key and value projection states
    output_attentions (`bool`, *optional*):
        Whether or not to return the attentions tensors of all attention layers. See `attentions` under
        returned tensors for more detail.
    use_cache (`bool`, *optional*):
        If set to `True`, `past_key_values` key value states are returned and can be used to speed up decoding
        (see `past_key_values`).
    position_embeddings (`tuple[torch.FloatTensor, torch.FloatTensor]`, *optional*):
        Tuple containing the cosine and sine positional embeddings of shape `(batch_size, seq_len, head_dim)`,
        with `head_dim` being the embedding dimension of each attention head.
)r9   r“   r^   r”   r¼   r’   rd   )rº   r¸   r¹   )r(   r9   r“   r^   r”   r¼   r’   r•   ÚresidualÚhidden_states_attentionÚ_Úhidden_states_mlps               r-   r<   ÚCohereDecoderLayer.forwardË   sq   € ð6 !ˆØ×,Ñ,¨]Ó;ˆà%)§^¢^ð &
Ø'Ø)Ø%Ø+ØØ 3ñ&
ð ñ&
Ñ"Ðð !ŸH™H ]Ó3ÐØ Ñ:Ð=NÑNˆØÐr/   )r)   rº   r¹   r¸   )NNNFN)r>   r?   r@   rA   r   r±   r"   r$   r²   Ú
LongTensorr   Úboolr³   r   r   r<   rB   rC   rD   s   @r-   rµ   rµ   Ã   sÃ   ø† ðl˜|ð l¸÷ lð /3Ø04Ø(,Ø!&ØHLñ*à—|‘|ð*ð Ÿ™ tÑ+ð*ð ×&Ñ&¨Ñ-ð	*ð
  ™ð*ð ˜$‘;ð*ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð*ð Ð-Ñ.ð*ð 
�‰÷*ó *r/   rµ   c                   ó0   ^ • \ rS rSrS\4U 4S jjrSrU =r$ )ÚCohereModeléø   r€   c           	      ó  >• [         TU ]  U5        [        R                  " [	        UR
                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        g s  snf r‡   )r!   r"   r   Ú
ModuleListÚrangeÚnum_hidden_layersrµ   Úlayersr   r)   rŒ   Únormr�   s      €r-   r"   ÚCohereModel.__init__ù   sg   ø€ Ü‰Ñ˜Ô Ü—m’mÜDIÈ&×JbÑJbÔDcÓdÒDc°yÔ Ö2ÑDcÑdó
ˆŒô $°×1CÑ1CÈ&×J_ÑJ_Ñ`ˆ�	ùò es   ¸A=)rÌ   rÍ   )r>   r?   r@   rA   r   r"   rB   rC   rD   s   @r-   rÆ   rÆ   ø   s   ø† ða˜|÷ aõ ar/   rÆ   c                   ó0  ^ • \ rS rSrU 4S jr\\        SS\R                  S-  S\R                  S-  S\R                  S-  S\
S-  S\R                  S-  S	\R                  S-  S
\S-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚCohereForCausalLMi  c                 óˆ   >• [         TU ]  U5        [        U5      U l        UR                  U l        UR
                  U l        g r¯   )r!   r"   rÆ   ÚmodelÚlogit_scaleÚtie_word_embeddingsr   s     €r-   r"   ÚCohereForCausalLM.__init__  s8   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÕ r/   NÚ	input_idsr“   r^   r”   Úinputs_embedsÚlabelsr¼   Úlogits_to_keepr•   r–   c	           
      ó˜  • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nXÐR                  -  nSnUb)  U R                  " SXÖU R                  R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9$ )au  
Example:

```python
>> from transformers import AutoTokenizer, CohereForCausalLM

>> model = CohereForCausalLM.from_pretrained("CohereForAI/c4ai-command-r-v01")
>> tokenizer = AutoTokenizer.from_pretrained("CohereForAI/c4ai-command-r-v01")

>> prompt = "Hey, are you conscious? Can you talk to me?"
>> inputs = tokenizer(prompt, return_tensors="pt")

>> # Generate
>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```)rÖ   r“   r^   r”   r×   r¼   N)ÚlogitsrØ   Ú
vocab_size)ÚlossrÛ   r”   r9   Ú
attentionsrd   )rÒ   Úlast_hidden_staterT   r±   ÚsliceÚlm_headrÓ   Úloss_functionr€   rÜ   r
   r”   r9   rÞ   )r(   rÖ   r“   r^   r”   r×   rØ   r¼   rÙ   r•   Úoutputsr9   Úslice_indicesrÛ   rÝ   s                  r-   r<   ÚCohereForCausalLM.forward  sä   € ð> ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆÜ8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆØ×*Ñ*Ñ*ˆàˆØÑØ×%Ò%Ðp¨VÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r/   )rÓ   rÒ   rÔ   )NNNNNNNr   )r>   r?   r@   rA   r"   r   r   r$   rÃ   r²   r   ÚFloatTensorrÄ   r±   r   r   r
   r<   rB   rC   rD   s   @r-   rÐ   rÐ     sì   ø† õ>ð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ô6
ó ó ö6
r/   rÐ   )rÐ   rÆ   ÚCoherePreTrainedModel)r   )3r°   Úcollections.abcr   r$   r   Úcache_utilsr   Úmodeling_flash_attention_utilsr   Úmodeling_layersr   Úmodeling_outputsr	   r
   Úmodeling_rope_utilsr   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   r   Úllama.modeling_llamar   r   r   r   r   r   Úconfiguration_coherer   Ú
get_loggerr>   ÚloggerÚModuler   rF   rm   ru   rw   rƒ   rµ   rÆ   rÐ   Ú__all__rd   r/   r-   Ú<module>rø      sÔ   ðñ, å $ã Ý å  Ý BÝ 9ß OÝ 6Ý 5Ý &ß RÓ Rß A÷÷ õ /ð 
×	Ò	˜HÓ	%€ô-�b—i‘iô -ô"<Ð0ô <ò"ô<ô8Y�ô Yð ô=)�nó =)ó ð=)ô@2Ð3ô 2ôja�*ô aô?
Ð(ô ?
òD�r/   