ó
    ‡Ž¡jQ  ã                   óÆ  • S SK Jr  SSKJr  SSKJr  SSKJr  SSKJ	r	  SSK
JrJr  SS	KJrJrJrJrJr  SS
KJrJrJrJrJrJr  \R4                  " \5      r\" SS9\ " S S\5      5       5       r " S S\5      r " S S\5      r " S S\5      r  " S S\5      r! " S S\5      r" " S S\5      r# " S S\5      r$ " S S\5      r% " S S \5      r& " S! S"\5      r'/ S#Qr(g$)%é    )Ústricté   )ÚPreTrainedConfig)ÚCausalLMOutputWithPast)ÚRopeParameters)ÚUnpack)Úauto_docstringÚloggingé   )ÚDeepseekV3DecoderLayerÚDeepseekV3MLPÚDeepseekV3MoEÚDeepseekV3PreTrainedModelÚDeepseekV3TopkRouter)ÚQwen3AttentionÚQwen3ForCausalLMÚ
Qwen3ModelÚQwen3RMSNormÚQwen3RotaryEmbeddingÚTransformersKwargszrednote-hilab/dots.llm1.base)Ú
checkpointc                   óº  ^ • \ rS rSr% SrSrS/rSSSSSSSSS	SSSSSSS
.rS/S/4SS/S/4S/S/4S.rSSSS	S.r	SS0r
Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S'   Sr\\S '   Sr\S!-  \S"'   S!r\S!-  \S#'   S!r\S!-  \S'   S$r\S!-  \S%'   S$r\S!-  \S&'   S!r\S!-  \S''   S(r\S!-  \S)'   S*r\S!-  \S+'   S,r\\S-'   S.r\\S/'   S0r\ \S1'   S2r!\ \S3'   S4r"\\S5'   S*r#\\S6'   S!r$\%\&-  S!-  \S7'   S*r'\\S8'   S9r(\ \-  S!-  \S:'   S;r)\ \S<'   S=r*\S!-  \S>'   Sr+\S!-  \S?'   S!r,\-\   S!-  \S@'   S!r.\S!-  \SA'   S!r/\S!-  \SB'   S!r0\\-\   -  S!-  \SC'   U 4SD jr1SEr2U =r3$ )FÚDots1Configé)   aÎ  
n_group (`int`, *optional*, defaults to 1):
    Number of groups for routed experts.
first_k_dense_replace (`int`, *optional*, defaults to 0):
    Number of dense layers at the beginning of the model before the first MoE layer.

Examples:

```python
>>> from transformers import Dots1Model, Dots1Config
>>> # Initializing a Dots1 style configuration
>>> configuration = Dots1Config()
>>> # Accessing the model configuration
>>> configuration = model.config
```
Údots1Úpast_key_valuesÚcolwiseÚrowwiseÚreplicated_with_grad_allreduceÚpacked_colwiseÚmoe_tp_experts)zlayers.*.self_attn.q_projzlayers.*.self_attn.k_projzlayers.*.self_attn.v_projzlayers.*.self_attn.o_projzlayers.*.self_attn.q_normzlayers.*.self_attn.k_normú!layers.*.mlp.experts.gate_up_projúlayers.*.mlp.experts.down_projúlayers.*.mlp.expertsz%layers.*.mlp.shared_experts.gate_projz#layers.*.mlp.shared_experts.up_projz%layers.*.mlp.shared_experts.down_projzlayers.*.mlp.gate_projzlayers.*.mlp.up_projzlayers.*.mlp.down_projÚ	input_idsÚinputs_embedsÚhidden_statesÚattention_mask)Úembed_tokensÚlayersÚnormÚ	ep_routerÚgrouped_gemm)zlayers.*.mlp.gater"   r#   r$   Únum_local_expertsÚn_routed_expertsi R Ú
vocab_sizei   Úhidden_sizeiÀ*  Úintermediate_sizei€  Úmoe_intermediate_sizeé>   Únum_hidden_layersé    Únum_attention_headsNÚnum_key_value_headsÚn_shared_expertsé   Ún_groupÚ
topk_groupÚnum_experts_per_tokr   Úfirst_k_dense_replaceFÚnorm_topk_probÚsiluÚ
hidden_acti   Úmax_position_embeddingsg{®Gáz”?Úinitializer_rangeg�íµ ÷Æ°>Úrms_norm_epsTÚ	use_cacheÚtie_word_embeddingsÚrope_parametersÚattention_biasg        Úattention_dropoutg      ð?Úrouted_scaling_factori   Úsliding_windowÚmax_window_layersÚlayer_typesÚpad_token_idÚbos_token_idÚeos_token_idc                 ó  >• U R                   c  U R                  U l         U R                  cI  [        U R                  5       Vs/ s H#  nU R
                  b  X R                  :¼  a  SOSPM%     snU l        [        TU ]   " S0 UD6  g s  snf )NÚsliding_attentionÚfull_attention© )	r8   r7   rM   Úranger5   rK   rL   ÚsuperÚ__post_init__)ÚselfÚkwargsÚiÚ	__class__s      €ÚµC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/dots1/modular_dots1.pyrW   ÚDots1Config.__post_init__�   s’   ø€ Ø×#Ñ#Ñ+Ø'+×'?Ñ'?ˆDÔ$à×ÑÑ#ô
 ˜t×5Ñ5Ô6ó	 ò 7�Að ×&Ñ&Ñ2°q×<RÑ<RÓ7Rñ $à%ò&ñ 7ñ	 ˆDÔô 	‰ÒÑ' Ó'ùò s   Á*B)rM   r8   )4Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú
model_typeÚkeys_to_ignore_at_inferenceÚbase_model_tp_planÚbase_model_pp_planÚbase_model_ep_planÚattribute_mapr0   ÚintÚ__annotations__r1   r2   r3   r5   r7   r8   r9   r/   r;   r<   r=   r>   r?   ÚboolrA   ÚstrrB   rC   ÚfloatrD   rE   rF   rG   r   ÚdictrH   rI   rJ   rK   rL   rM   ÚlistrN   rO   rP   rW   Ú__static_attributes__Ú__classcell__©r[   s   @r\   r   r   )   sJ  ø‡ ñð" €JØ#4Ð"5Ðð &/Ø%.Ø%.Ø%.Ø%EØ%EØ-=Ø*3Ø 0Ø1:Ø/8Ø1:Ø"+Ø )Ø"+ñÐð& &˜¨Ð(9Ð:Ø#Ð%5Ð6¸Ð8IÐJØ!Ð" _Ð$5Ð6ñÐð )Ø-;Ø*8Ø 0ñ	Ðð 	Ð/ð€Mð €J�ÓØ€K�ÓØ"Ð�sÓ"Ø!%Ð˜3Ó%ØÐ�sÓØ!Ð˜Ó!Ø&(Ð˜˜t™Ó(Ø#'Ð�c˜D‘jÓ'Ø#'Ð�c˜D‘jÓ'Ø€GˆS�4‰ZÓØ€J��d‘
ÓØ&*Ð˜˜t™Ó*Ø()Ð˜3 ™:Ó)Ø"'€N�D˜4‘KÓ'Ø€J�ÓØ#'Ð˜SÓ'Ø#Ð�uÓ#Ø€L�%ÓØ€IˆtÓØ %Ð˜Ó%Ø48€O�^ dÑ*¨TÑ1Ó8Ø €N�DÓ Ø,/Ð�u˜s‘{ TÑ)Ó/Ø#&Ð˜5Ó&Ø!%€N�C˜$‘JÓ%Ø$&Ð�s˜T‘zÓ&Ø$(€K��c‘˜TÑ!Ó(Ø#€L�#˜‘*Ó#Ø#€L�#˜‘*Ó#Ø+/€L�#˜˜S™	‘/ DÑ(Ó/÷(ó (ó    r   c                   ó   • \ rS rSrSrg)ÚDots1RMSNormé�   rT   N©r^   r_   r`   ra   rp   rT   rs   r\   ru   ru   �   ó   † Úrs   ru   c                   ó   • \ rS rSrSrg)ÚDots1RotaryEmbeddingé”   rT   Nrw   rT   rs   r\   rz   rz   ”   rx   rs   rz   c                   ó   • \ rS rSrSrg)ÚDots1Attentioné˜   rT   Nrw   rT   rs   r\   r}   r}   ˜   rx   rs   r}   c                   ó   • \ rS rSrSrg)ÚDots1MLPéœ   rT   Nrw   rT   rs   r\   r€   r€   œ   rx   rs   r€   c                   ó   • \ rS rSrSrg)ÚDots1TopkRouteré    rT   Nrw   rT   rs   r\   rƒ   rƒ       rx   rs   rƒ   c                   ó   • \ rS rSrSrg)ÚDots1MoEé¤   rT   Nrw   rT   rs   r\   r†   r†   ¤   rx   rs   r†   c                   ó   • \ rS rSrSrg)ÚDots1DecoderLayeré¨   rT   Nrw   rT   rs   r\   r‰   r‰   ¨   rx   rs   r‰   c                   ó   • \ rS rSrSrSrg)ÚDots1PreTrainedModelé¬   NrT   )r^   r_   r`   ra   Ú"_keys_to_ignore_on_load_unexpectedrp   rT   rs   r\   rŒ   rŒ   ¬   s   † Ø)-Ó&rs   rŒ   c                   ó   • \ rS rSrSrg)Ú
Dots1Modelé°   rT   Nrw   rT   rs   r\   r�   r�   °   rx   rs   r�   c                   ó:   ^ • \ rS rSrS\\   S\4U 4S jjrSrU =r	$ )ÚDots1ForCausalLMé´   Úsuper_kwargsÚreturnc                 ó$   >• [         TU ]  " S0 UD6$ )aþ  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should either be in `[0, ...,
    config.vocab_size]` or -100 (see `input_ids` docstring). Tokens with indices set to `-100` are ignored
    (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`.

Example:

```python
>>> from transformers import AutoTokenizer, Dots1ForCausalLM

>>> model = Dots1ForCausalLM.from_pretrained("rednote-hilab/dots1.llm1.inst")
>>> tokenizer = AutoTokenizer.from_pretrained("rednote-hilab/dots1.llm1.inst")

>>> prompt = "Hey, are you conscious? Can you talk to me?"
>>> inputs = tokenizer(prompt, return_tensors="pt")

>>> # Generate
>>> generate_ids = model.generate(inputs.input_ids, max_length=30)
>>> tokenizer.batch_decode(generate_ids, skip_special_tokens=True, clean_up_tokenization_spaces=False)[0]
"Hey, are you conscious? Can you talk to me?\nI'm not conscious, but I can talk to you."
```rT   )rV   Úforward)rX   r•   r[   s     €r\   r˜   ÚDots1ForCausalLM.forwardµ   s   ø€ ô4 ‰wŠÑ. Ñ.Ð.rs   rT   )
r^   r_   r`   ra   r   r   r   r˜   rp   rq   rr   s   @r\   r“   r“   ´   s%   ø† ð/àÐ1Ñ2ð/ð 
 ÷/õ /rs   r“   )r   rŒ   r�   r“   N))Úhuggingface_hub.dataclassesr   Úconfiguration_utilsr   Úmodeling_outputsr   Úmodeling_rope_utilsr   Úprocessing_utilsr   Úutilsr	   r
   Ú deepseek_v3.modeling_deepseek_v3r   r   r   r   r   Úqwen3.modeling_qwen3r   r   r   r   r   r   Ú
get_loggerr^   Úloggerr   ru   rz   r}   r€   rƒ   r†   r‰   rŒ   r�   r“   Ú__all__rT   rs   r\   Ú<module>r¥      sñ   ðõ /å 3Ý 6Ý 1Ý &ß ,÷õ ÷÷ ð 
×	Ò	˜HÓ	%€ñ Ð9Ñ:Øôb(Ð"ó b(ó ó ;ðb(ôJ	�<ô 	ô	Ð/ô 	ô	�^ô 	ô	ˆ}ô 	ô	Ð*ô 	ô	ˆ}ô 	ô	Ð.ô 	ô.Ð4ô .ô	�ô 	ô/Ð'ô /ò<�rs   