ó
    ‡Ž¡jÍS  ã                   óê  • S SK Jr  S SKrS SKJr  S SKJr  SSKJr  SSKJ	r	J
r
  SSKJr  SS	KJrJr  SS
KJr  SSKJrJrJrJr  SSKJrJr  SSKJrJr  SSKJrJr  SSK J!r!  SSK"J#r#J$r$  SSK%J&r&  SSK'J(r(J)r)  SSK*J+r+  SSK,J-r-   " S S\R\                  5      r/\" S5       " S S\R\                  5      5       r0 " S S\R\                  5      r1S r2\" S5      S@S j5       r3S \Rh                  S!\5S"\Rh                  4S# jr6 SAS$\R\                  S%\Rh                  S&\Rh                  S'\Rh                  S(\Rh                  S-  S)\7S*\7S+\!\#   4S, jjr8\" \35       " S- S.\R\                  5      5       r9 " S/ S0\5      r:\ " S1 S2\5      5       r;\ " S3 S4\;5      5       r<\" S5S69 " S7 S8\;\5      5       r=\" S5S69 " S9 S:\\;5      5       r>\" S5S69 " S; S<\\;5      5       r?\" S5S69 " S= S>\\;5      5       r@/ S?QrAg)Bé    )ÚCallableN)Únn)Úauto_docstringé   )ÚACT2FN)ÚCacheÚDynamicCache)ÚGenerationMixin)Úuse_kernel_forward_from_hubÚuse_kernelized_func)Úcreate_causal_mask)ÚGenericForQuestionAnsweringÚ GenericForSequenceClassificationÚGenericForTokenClassificationÚGradientCheckpointingLayer)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚROPE_INIT_FUNCTIONSÚdynamic_rope_update)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚcan_return_tuple)Údeprecate_kwarg)Úmaybe_autocastÚmerge_with_config_defaults)Úcapture_outputsé   )ÚArceeConfigc                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚArceeMLPé2   c                 ó†  >• [         TU ]  5         Xl        UR                  U l        UR                  U l        [
        R                  " U R                  U R                  UR                  S9U l        [
        R                  " U R                  U R                  UR                  S9U l	        [        UR                     U l        g )N©Úbias)ÚsuperÚ__init__ÚconfigÚhidden_sizeÚintermediate_sizer   ÚLinearÚmlp_biasÚup_projÚ	down_projr   Ú
hidden_actÚact_fn©Úselfr)   Ú	__class__s     €Ú¶C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/arcee/modeling_arcee.pyr(   ÚArceeMLP.__init__3   s�   ø€ Ü‰ÑÔØŒØ!×-Ñ-ˆÔØ!'×!9Ñ!9ˆÔÜ—y’y ×!1Ñ!1°4×3IÑ3IÐPV×P_ÑP_Ñ`ˆŒÜŸš 4×#9Ñ#9¸4×;KÑ;KÐRX×RaÑRaÑbˆŒÜ˜V×.Ñ.Ñ/ˆ�ó    c                 ó`   • U R                  U R                  U R                  U5      5      5      $ ©N)r/   r1   r.   )r3   Úxs     r5   ÚforwardÚArceeMLP.forward<   s"   € Ø�~‰~˜dŸk™k¨$¯,©,°q«/Ó:Ó;Ð;r7   )r1   r)   r/   r*   r+   r.   )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r(   r;   Ú__static_attributes__Ú__classcell__©r4   s   @r5   r"   r"   2   s   ø† õ0÷<ð <r7   r"   ÚRMSNormc                   óx   ^ • \ rS rSrS
S\SS4U 4S jjjrS\R                  S\R                  4S jrS r	S	r
U =r$ )ÚArceeRMSNormé@   ÚepsÚreturnNc                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)z+
ArceeRMSNorm is equivalent to T5LayerNorm
N)r'   r(   r   Ú	ParameterÚtorchÚonesÚweightÚvariance_epsilon)r3   r*   rH   r4   s      €r5   r(   ÚArceeRMSNorm.__init__B   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #Õr7   Úhidden_statesc                 ó  • UR                   nUR                  [        R                  5      nUR	                  S5      R                  SSS9nU[        R                  " X0R                  -   5      -  nU R                  UR                  U5      -  $ )Né   éÿÿÿÿT)Úkeepdim)	ÚdtypeÚtorL   Úfloat32ÚpowÚmeanÚrsqrtrO   rN   )r3   rQ   Úinput_dtypeÚvariances       r5   r;   ÚArceeRMSNorm.forwardJ   sw   € Ø#×)Ñ)ˆØ%×(Ñ(¬¯©Ó7ˆØ ×$Ñ$ QÓ'×,Ñ,¨R¸Ð,Ð>ˆØ%¬¯ª°H×?TÑ?TÑ4TÓ(UÑUˆØ�{‰{˜]×-Ñ-¨kÓ:Ñ:Ð:r7   c                 ó^   • [        U R                  R                  5       SU R                   3$ )Nz, eps=)ÚtuplerN   ÚshaperO   )r3   s    r5   Ú
extra_reprÚArceeRMSNorm.extra_reprQ   s*   € Ü˜Ÿ™×)Ñ)Ó*Ð+¨6°$×2GÑ2GÐ1HÐIÐIr7   )rO   rN   )g�íµ ÷Æ°>)r=   r>   r?   r@   Úfloatr(   rL   ÚTensorr;   rb   rA   rB   rC   s   @r5   rF   rF   @   sB   ø† ñ$¨ð $¸$÷ $ð $ð; U§\¡\ð ;°e·l±lô ;÷Jð Jr7   rF   c            	       óÒ   ^ • \ rS rSr\" SSS9SS\4U 4S jjj5       r\\" SSS9SS\S\\	R                  \4   4S jj5       5       r\	R                  " 5       \S	 5       5       rS
rU =r$ )ÚArceeRotaryEmbeddingéU   Údevicez5.18)Úversionr)   c                 óÄ  >• [         TU ]  5         UR                  U l        UR                  U l        Xl        U R
                  R                  S   U l        U R                  nU R                  S:w  a  [        U R                     nU" U R
                  U5      u  o@l
        [        R                  " USS9U l        [        R                  " UR                  5       SS9U l        g )NÚ	rope_typeÚdefaultF)Ú
persistent)r'   r(   Úmax_position_embeddingsÚmax_seq_len_cachedÚoriginal_max_seq_lenr)   Úrope_parametersrl   Úcompute_default_rope_parametersr   Úattention_scalingr   ÚBufferÚinv_freqÚcloneÚoriginal_inv_freq)r3   r)   ri   Úrope_init_fnrv   r4   s        €r5   r(   ÚArceeRotaryEmbedding.__init__V   s¬   ø€ ä‰ÑÔØ"(×"@Ñ"@ˆÔØ$*×$BÑ$BˆÔ!àŒàŸ™×4Ñ4°[ÑAˆŒØ!%×!EÑ!EˆØ�>‰>˜YÓ&Ü.¨t¯~©~Ñ>ˆLÙ+7¸¿¹ÀVÓ+LÑ(ˆÔ(äŸ	š	 (°uÑ=ˆŒÜ!#§¢¨8¯>©>Ó+;ÈÑ!NˆÕr7   rI   c           	      ó   • U R                   S   n[        U SS5      =(       d    U R                  U R                  -  nSnSU[        R
                  " SUS[        R                  S9U-  -  -  nUR                  U5      U4$ )a~  
Computes the inverse frequencies according to the original RoPE implementation
Args:
    config ([`~transformers.PreTrainedConfig`]):
        The model configuration.
Returns:
    Tuple of (`torch.Tensor`, `float`), containing the inverse frequencies for the RoPE embeddings and the
    post-processing scaling factor applied to the computed cos/sin (unused in this type of RoPE).
Ú
rope_thetaÚhead_dimNg      ð?r   rS   ©rV   )rr   Úgetattrr*   Únum_attention_headsrL   Úarangerd   rW   )r)   ri   ÚkwargsÚbaseÚdimÚattention_factorrv   s          r5   rs   Ú4ArceeRotaryEmbedding.compute_default_rope_parametersg   s{   € ð ×%Ñ% lÑ3ˆÜ�f˜j¨$Ó/×c°6×3EÑ3EÈ×IcÑIcÑ3cˆàÐà˜$¤5§<¢<°°3¸ÄÇÁÑ#MÐPSÑ#SÑTÑUˆØ�{‰{˜6Ó"Ð$4Ð4Ð4r7   c                 ó  • U R                   S S S 2S 4   R                  UR                  S   SS5      R                  [        R
                  UR                  S9nUS S 2S S S 24   R                  5       n[        UR                  R                  [        5      (       a0  UR                  R                  S:w  a  UR                  R                  OSn[        USS9   X4-  R                  SS	5      n[        R                  " Xf4SS
9nUR                  5       U R                  -  nUR                  5       U R                  -  n	S S S 5        WR                  UR                   S9W	R                  UR                   S94$ ! , (       d  f       N@= f)Nr   rT   r   )rV   ri   ÚmpsÚcpuF)Údevice_typeÚenabledrS   ©r„   r~   )rv   Úexpandra   rW   rL   rd   ri   Ú
isinstanceÚtypeÚstrr   Ú	transposeÚcatÚcosrt   ÚsinrV   )
r3   r:   Úposition_idsÚinv_freq_expandedÚposition_ids_expandedrŠ   ÚfreqsÚembr“   r”   s
             r5   r;   ÚArceeRotaryEmbedding.forward{   s;  € ð �M‰M˜$¢ 4˜-Ñ(×/Ñ/°×0BÑ0BÀ1Ñ0EÀrÈ1ÓM×PÑPÔW\×WbÑWbÐkl×ksÑksÐPÐtð 	ð !-ªQ°²a¨ZÑ 8× >Ñ >Ó @Ðä'1°!·(±(·-±-Ä×'EÑ'EÈ!Ï(É(Ï-É-Ð[`ÓJ`�a—h‘h—m’mÐfkˆä¨¸UÓCØ&Ñ>×IÑIÈ!ÈQÓOˆEÜ—)’)˜U˜N°Ñ3ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆCØ—'‘'“)˜d×4Ñ4Ñ4ˆC÷	 Dð �v‰v˜AŸG™GˆvÐ$ c§f¡f°1·7±7 fÐ&;Ð;Ð;÷ DÕCús   ÃA%E7Å7
F)rt   r)   rv   rp   rx   rq   rl   r9   )r=   r>   r?   r@   r   r    r(   Ústaticmethodr`   rL   re   rd   rs   Úno_gradr   r;   rA   rB   rC   s   @r5   rg   rg   U   sŒ   ø† Ù�X vÑ.ñO˜{÷ Oó /ðOð  Ù�X vÑ.ñ5°ð 5ÐW\Ð]b×]iÑ]iÐkpÐ]pÑWqô 5ó /ó ð5ð$ ‡]‚]ƒ_Øñ<ó ó ö<r7   rg   c                 ó–   • U SSU R                   S   S-  24   nU SU R                   S   S-  S24   n[        R                  " U* U4SS9$ )z*Rotates half the hidden dims of the input..NrT   rS   rŒ   )ra   rL   r’   )r:   Úx1Úx2s      r5   Úrotate_halfr    Ž   sZ   € à	
ˆ3Ð"�!—'‘'˜"‘+ Ñ"Ð"Ð"Ñ	#€BØ	
ˆ3�—‘˜‘˜qÑ Ñ"Ð"Ñ	#€BÜ�9Š9�r�c˜2�Y BÑ'Ð'r7   Úrotary_pos_embc                 ó˜   • UR                  U5      nUR                  U5      nX-  [        U 5      U-  -   nX-  [        U5      U-  -   nXV4$ )aI  Applies Rotary Position Embedding to the query and key tensors.

Args:
    q (`torch.Tensor`): The query tensor.
    k (`torch.Tensor`): The key tensor.
    cos (`torch.Tensor`): The cosine part of the rotary embedding.
    sin (`torch.Tensor`): The sine part of the rotary embedding.
    unsqueeze_dim (`int`, *optional*, defaults to 1):
        The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
        sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
        that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
        k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
        cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
        the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
    `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
)Ú	unsqueezer    )ÚqÚkr“   r”   Úunsqueeze_dimÚq_embedÚk_embeds          r5   Úapply_rotary_pos_embr©   •   sS   € ð& �-‰-˜Ó
&€CØ
�-‰-˜Ó
&€CØ‰wœ; q›>¨CÑ/Ñ0€GØ‰wœ; q›>¨CÑ/Ñ0€GØÐÐr7   rQ   Ún_reprI   c                 ó    • U R                   u  p#pEUS:X  a  U $ U SS2SS2SSS2SS24   R                  X#XU5      n U R                  X#U-  XE5      $ )zÈ
This is the equivalent of torch.repeat_interleave(x, dim=1, repeats=n_rep). The hidden states go from (batch,
num_key_value_heads, seqlen, head_dim) to (batch, num_attention_heads, seqlen, head_dim)
r   N)ra   r�   Úreshape)rQ   rª   ÚbatchÚnum_key_value_headsÚslenr}   s         r5   Ú	repeat_kvr°   ¯   s_   € ð
 2?×1DÑ1DÑ.€E Ø�ƒzØÐØ!¢!¢Q¨ªa²Ð"2Ñ3×:Ñ:¸5ÐW\ÐdlÓm€MØ× Ñ  ¸eÑ(CÀTÓTÐTr7   ÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutr‚   c                 ó  • [        X R                  5      n[        X0R                  5      n	[        R                  " XR	                  SS5      5      U-  n
Ub  X¤-   n
[
        R                  R                  U
S[        R                  S9R                  UR                  5      n
[
        R                  R                  X¦U R                  S9n
[        R                  " X©5      nUR	                  SS5      R                  5       nXº4$ )NrS   r   rT   )r„   rV   )ÚpÚtrainingr   )r°   Únum_key_value_groupsrL   Úmatmulr‘   r   Ú
functionalÚsoftmaxrX   rW   rV   r·   rº   Ú
contiguous)r±   r²   r³   r´   rµ   r¶   r·   r‚   Ú
key_statesÚvalue_statesÚattn_weightsÚattn_outputs               r5   Úeager_attention_forwardrÄ   »   sÒ   € ô ˜3× ;Ñ ;Ó<€JÜ˜U×$?Ñ$?Ó@€Lä—<’< ×';Ñ';¸A¸qÓ'AÓBÀWÑL€LØÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2ÄUÇ]Á]Ð(ÐS×VÑVÐW\×WbÑWbÓc€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€LÜ—,’,˜|Ó:€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$r7   c                   ó  ^ • \ rS rSrSrS\S\4U 4S jjr   SS\R                  S\
\R                  \R                  4   S-  S	\R                  S-  S
\S-  S\\   S\
\R                  \R                  4   4S jjrSrU =r$ )ÚArceeAttentionéÔ   z=Multi-headed attention from 'Attention Is All You Need' paperr)   Ú	layer_idxc                 óP  >• [         TU ]  5         Xl        X l        [	        USUR
                  UR                  -  5      U l        UR                  UR                  -  U l	        U R                  S-  U l
        UR                  U l        SU l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR
                  UR                  U R                  -  UR                  S9U l        [        R                  " UR                  U R                  -  UR
                  UR                  S9U l        g )Nr}   g      à¿Tr%   )r'   r(   r)   rÈ   r   r*   r€   r}   r®   r»   r¶   Úattention_dropoutÚ	is_causalr   r,   Úattention_biasÚq_projÚk_projÚv_projÚo_proj©r3   r)   rÈ   r4   s      €r5   r(   ÚArceeAttention.__init__Ø   sI  ø€ Ü‰ÑÔØŒØ"ŒÜ ¨
°F×4FÑ4FÈ&×JdÑJdÑ4dÓeˆŒØ$*×$>Ñ$>À&×B\ÑB\Ñ$\ˆÔ!Ø—}‘} dÑ*ˆŒØ!'×!9Ñ!9ˆÔØˆŒä—i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×Ñ × :Ñ :¸T¿]¹]Ñ JÐQW×QfÑQfñ
ˆŒô —i’iØ×&Ñ&¨¯©Ñ6¸×8JÑ8JÐQW×QfÑQfñ
ˆ�r7   NrQ   Úposition_embeddingsrµ   Úpast_key_valuesr‚   rI   c                 ó  • UR                   S S n/ UQSPU R                  P7nU R                  U5      R                  U5      R	                  SS5      nU R                  U5      R                  U5      R	                  SS5      n	U R                  U5      R                  U5      R	                  SS5      n
Uu  p¼[        X‰X¼5      u  p‰Ub  UR                  XšU R                  5      u  pš[        R                  " U R                  R                  [        5      nU" U UU	U
U4U R                  (       d  SOU R                   U R"                  S.UD6u  pïUR$                  " / UQSP76 R'                  5       nU R)                  U5      nXï4$ )NrT   r   rS   ç        )r·   r¶   )ra   r}   rÍ   Úviewr‘   rÎ   rÏ   r©   ÚupdaterÈ   r   Úget_interfacer)   Ú_attn_implementationrÄ   rº   rÊ   r¶   r¬   r¿   rÐ   )r3   rQ   rÓ   rµ   rÔ   r‚   Úinput_shapeÚhidden_shapeÚquery_statesrÀ   rÁ   r“   r”   Úattention_interfacerÃ   rÂ   s                   r5   r;   ÚArceeAttention.forwardï   s~  € ð $×)Ñ)¨#¨2Ð.ˆØ8˜Ð8 bÐ8¨$¯-©-Ñ8ˆà—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆØ—[‘[ Ó/×4Ñ4°\ÓB×LÑLÈQÐPQÓRˆ
Ø—{‘{ =Ó1×6Ñ6°|ÓD×NÑNÈqÐRSÓTˆà&‰ˆÜ#7¸ÐRUÓ#[Ñ ˆàÑ&Ø'6×'=Ñ'=¸jÐX\×XfÑXfÓ'gÑ$ˆJä(?×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð	%
ð  $Ÿ}Ÿ}‘C°$×2HÑ2HØ—L‘Lñ	%
ð ñ	%
Ñ!ˆð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—k‘k +Ó.ˆØÐ(Ð(r7   )rÊ   r)   r}   rË   rÎ   rÈ   r»   rÐ   rÍ   r¶   rÏ   )NNN)r=   r>   r?   r@   Ú__doc__r    Úintr(   rL   re   r`   r   r   r   r;   rA   rB   rC   s   @r5   rÆ   rÆ   Ô   sª   ø† áGð
˜{ð 
°s÷ 
ð4 IMØ.2Ø(,ñ&)à—|‘|ð&)ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð&)ð Ÿ™ tÑ+ð	&)ð
  ™ð&)ð Ð+Ñ,ð&)ð 
ˆu�|‰|˜UŸ\™\Ð)Ñ	*÷&)ó &)r7   rÆ   c                   ó  ^ • \ rS rSrS\S\4U 4S jjr     SS\R                  S\R                  S-  S\R                  S-  S	\
S-  S
\S-  S\\R                  \R                  4   S-  S\\   S\R                  4S jjrSrU =r$ )ÚArceeDecoderLayeri  r)   rÈ   c                 ó  >• [         TU ]  5         UR                  U l        [        XS9U l        [        U5      U l        [        UR                  UR                  S9U l	        [        UR                  UR                  S9U l
        g )N)r)   rÈ   ©rH   )r'   r(   r*   rÆ   Ú	self_attnr"   ÚmlprF   Úrms_norm_epsÚinput_layernormÚpost_attention_layernormrÑ   s      €r5   r(   ÚArceeDecoderLayer.__init__  sj   ø€ Ü‰ÑÔØ!×-Ñ-ˆÔä'¨vÑKˆŒä˜FÓ#ˆŒÜ+¨F×,>Ñ,>ÀF×DWÑDWÑXˆÔÜ(4°V×5GÑ5GÈV×M`ÑM`Ñ(aˆÕ%r7   NrQ   rµ   r•   rÔ   Ú	use_cacherÓ   r‚   rI   c           
      óº   • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  pX�-   nUnU R                  U5      nU R                  U5      nX�-   nU$ )N)rQ   rµ   r•   rÔ   rì   rÓ   © )ré   ræ   rê   rç   )
r3   rQ   rµ   r•   rÔ   rì   rÓ   r‚   ÚresidualÚ_s
             r5   r;   ÚArceeDecoderLayer.forward#  sˆ   € ð !ˆØ×,Ñ,¨]Ó;ˆàŸ>š>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆð !Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Ñ0ˆØÐr7   )r*   ré   rç   rê   ræ   )NNNFN)r=   r>   r?   r@   r    rá   r(   rL   re   Ú
LongTensorr   Úboolr`   r   r   r;   rA   rB   rC   s   @r5   rã   rã     sÃ   ø† ðb˜{ð b°s÷ bð /3Ø04Ø(,Ø!&ØHLñà—|‘|ðð Ÿ™ tÑ+ðð ×&Ñ&¨Ñ-ð	ð
  ™ðð ˜$‘;ðð # 5§<¡<°·±Ð#=Ñ>ÀÑEðð Ð+Ñ,ðð 
�‰÷ó r7   rã   c                   óR   • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSrSr\\S.rSrg	)
ÚArceePreTrainedModeliC  r)   ÚmodelTrã   rÔ   )rQ   Ú
attentionsrî   N)r=   r>   r?   r@   r    Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_skip_keys_device_placementÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_can_compile_fullgraphÚ_supports_attention_backendrã   rÆ   Ú_can_record_outputsrA   rî   r7   r5   rõ   rõ   C  sQ   ‡ àÓØÐØ&*Ð#Ø,Ð-ÐØ#4Ð"5ÐØÐØ€NØÐà!ÐØ"&Ðà*Ø$ñÓr7   rõ   c                   ó  ^ • \ rS rSrS\4U 4S jjr\\\      SS\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S	\	R                  S-  S
\S-  S\\   S\4S jj5       5       5       rSrU =r$ )Ú
ArceeModeliV  r)   c           	      ó  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        [        UR                  UR                  S9U l        [#        US9U l        SU l        U R)                  5         g s  snf )Nrå   ©r)   F)r'   r(   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr*   Úembed_tokensÚ
ModuleListÚrangeÚnum_hidden_layersrã   ÚlayersrF   rè   Únormrg   Ú
rotary_embÚgradient_checkpointingÚ	post_initrÑ   s      €r5   r(   ÚArceeModel.__init__X  sÊ   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸLšL¨×):Ñ):¸F×<NÑ<NÐPT×P`ÑP`ÓaˆÔÜ—m’mÜCHÈ×IaÑIaÔCbÓcÒCb°iÔ˜vÖ1ÑCbÑcó
ˆŒô ! ×!3Ñ!3¸×9LÑ9LÑMˆŒ	Ü.°fÑ=ˆŒØ&+ˆÔ#ð 	�‰Õùò ds   ÂC?NÚ	input_idsrµ   r•   rÔ   Úinputs_embedsrì   r‚   rI   c           
      ó>  • US L US L-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      n[        U R                  UUUUS9n	Un
U R                  X£S9nU R                  S U R                  R                    H  nU" U
4U	UUUUS.UD6n
M     U R                  U
5      n
[        U
US	9$ )
Nz:You must specify exactly one of input_ids or inputs_embedsr  r   r   )ri   )r)   r  rµ   rÔ   r•   )r•   )rµ   rÓ   r•   rÔ   rì   )Úlast_hidden_staterÔ   )Ú
ValueErrorr  r	   r)   Úget_seq_lengthrL   r�   ra   ri   r£   r   r  r  r  r  r   )r3   r  rµ   r•   rÔ   r  rì   r‚   Úpast_seen_tokensÚcausal_maskrQ   rÓ   Údecoder_layers                r5   r;   ÚArceeModel.forwardh  sF  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø*.×*;Ñ*;¸IÓ*FˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLä(Ø—;‘;Ø'Ø)Ø+Ø%ñ
ˆð &ˆØ"Ÿo™o¨m˜oÐWÐà!Ÿ[™[Ð)H¨4¯;©;×+HÑ+HÓIˆMÙ)Øðà*Ø$7Ø)Ø /Ø#ñð ñŠMñ Jð Ÿ	™	 -Ó0ˆÜ&Ø+Ø+ñ
ð 	
r7   )r  r  r  r  r  r  r	  )NNNNNN)r=   r>   r?   r@   r    r(   r   r   r   rL   rò   re   r   ÚFloatTensorró   r   r   r   r;   rA   rB   rC   s   @r5   r  r  V  sÍ   ø† ð˜{÷ ð   ØØð .2Ø.2Ø04Ø(,Ø26Ø!%ñ2
à×#Ñ# dÑ*ð2
ð Ÿ™ tÑ+ð2
ð ×&Ñ&¨Ñ-ð	2
ð
  ™ð2
ð ×(Ñ(¨4Ñ/ð2
ð ˜$‘;ð2
ð Ð+Ñ,ð2
ð 
!ô2
ó ó ó  ö2
r7   r  zarcee-ai/AFM-4.5B)Ú
checkpointc                   óX  ^ • \ rS rSrSS0rSS0rSS/S/40rSS0rU 4S	 jr\	\
        SS\R                  S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\R                  S
-  S\R                  S
-  S\S
-  S\\R                  -  S\\   S\4S jj5       5       rSrU =r$ )ÚArceeForCausalLMi   zlm_head.weightzmodel.embed_tokens.weightÚlm_headÚcolwise_gather_outputrQ   ÚlogitsÚkeep_full_weightc                 óä   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        U R                  5         g )NFr%   )
r'   r(   r  rö   r	  r   r,   r*   r#  r  r2   s     €r5   r(   ÚArceeForCausalLM.__init__§  sU   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒð 	�‰Õr7   Nr  rµ   r•   rÔ   r  Úlabelsrì   Úlogits_to_keepr‚   rI   c	           
      ó|  • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nSnUb)  U R                  " SXÖU R                  R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9$ )ao  
Example:

```python
>>> from transformers import AutoTokenizer, ArceeForCausalLM

>>> model = ArceeForCausalLM.from_pretrained("meta-arcee/Arcee-2-7b-hf")
>>> tokenizer = AutoTokenizer.from_pretrained("meta-arcee/Arcee-2-7b-hf")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```)r  rµ   r•   rÔ   r  rì   N)r%  r)  r	  )Úlossr%  rÔ   rQ   r÷   rî   )rö   r  rŽ   rá   Úslicer#  Úloss_functionr)   r	  r   rÔ   rQ   r÷   )r3   r  rµ   r•   rÔ   r  r)  rì   r*  r‚   ÚoutputsrQ   Úslice_indicesr%  r,  s                  r5   r;   ÚArceeForCausalLM.forward°  sÖ   € ð> ,0¯:ª:ð ,
ØØ)Ø%Ø+Ø'Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò%Ðp¨VÈtÏ{É{×OeÑOeÑpÐioÑpˆDä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
r7   )r#  rö   r	  )NNNNNNNr   )r=   r>   r?   r@   Ú_tied_weights_keysÚ_tp_planÚ_pp_planÚ
_fsdp_planr(   r   r   rL   rò   re   r   r  ró   rá   r   r   r   r;   rA   rB   rC   s   @r5   r"  r"     s"  ø† à*Ð,GÐHÐØÐ2Ð3€HØ˜_Ð-°¨zÐ:Ð;€HØÐ/Ð0€Jõð Øð .2Ø.2Ø04Ø(,Ø26Ø*.Ø!%Ø-.ñ6
à×#Ñ# dÑ*ð6
ð Ÿ™ tÑ+ð6
ð ×&Ñ&¨Ñ-ð	6
ð
  ™ð6
ð ×(Ñ(¨4Ñ/ð6
ð × Ñ  4Ñ'ð6
ð ˜$‘;ð6
ð ˜eŸl™lÑ*ð6
ð Ð+Ñ,ð6
ð 
 ô6
ó ó ö6
r7   r"  c                   ó   • \ rS rSrSrg)ÚArceeForSequenceClassificationië  rî   N©r=   r>   r?   r@   rA   rî   r7   r5   r7  r7  ë  ó   † âr7   r7  c                   ó   • \ rS rSrSrSrg)ÚArceeForQuestionAnsweringið  Útransformerrî   N)r=   r>   r?   r@   rù   rA   rî   r7   r5   r;  r;  ð  s   † à%Ór7   r;  c                   ó   • \ rS rSrSrg)ÚArceeForTokenClassificationiõ  rî   Nr8  rî   r7   r5   r>  r>  õ  r9  r7   r>  )r"  r;  r7  r>  r  rõ   )r   )rÖ   )BÚcollections.abcr   rL   r   Útransformers.utilsr   Úactivationsr   Úcache_utilsr   r	   Ú
generationr
   Úintegrationsr   r   Úmasking_utilsr   Úmodeling_layersr   r   r   r   Úmodeling_outputsr   r   Úmodeling_rope_utilsr   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   Úutils.deprecationr   Úutils.genericr   r   Úutils.output_capturingr   Úconfiguration_arceer    ÚModuler"   rF   rg   r    r©   re   rá   r°   rd   rÄ   rÆ   rã   rõ   r  r"  r7  r;  r>  Ú__all__rî   r7   r5   Ú<module>rR     s\  ðõ* %ã Ý å -å !ß .Ý )ß LÝ /÷ó ÷ Pß Kß FÝ &ß 9Ý 0ß GÝ 5Ý ,ô<ˆr�y‰yô <ñ ˜YÓ'ôJ�2—9‘9ó Jó (ðJô(6<˜2Ÿ9™9ô 6<òr(ñ Ð-Ó.óó /ðð2	U˜UŸ\™\ð 	U°#ð 	U¸%¿,¹,ô 	Uð& ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð ð%ð ð%ð Ð'Ñ(õ%ñ2 Ð)Ó*ô@)�R—Y‘Yó @)ó +ð@)ôF(Ð2ô (ðV ô˜?ó ó ðð$ ôF
Ð%ó F
ó ðF
ñR Ð.Ñ/ôG
Ð+¨_ó G
ó 0ðG
ñT Ð.Ñ/ô	Ð%EÐG[ó 	ó 0ð	ñ Ð.Ñ/ô&Ð ;Ð=Qó &ó 0ð&ñ Ð.Ñ/ô	Ð"?ÐAUó 	ó 0ð	ò�r7   