ó
    OëgjM  ã                  óº  • S SK Jr  S SKrS SKrS SKJrJr  S SKrS SKJ	r	  S SK
JrJr  S SKJr  S SKJr  S SKJr  S SKJr  S S	KJr  S S
KJr  S SKJr  S SKJrJr  S SKJrJ r J!r!  S SKJ"r#  S SK$J%r%  S SK&J'r'  \(       a  S SK(J)r)   S SK*J+r+  \R\                  " \/5      r0 " S S\+5      r1 " S S\5      r2 " S S\25      r3 " S S\2\5      r4g! \, a	    S SK-J+r+   NNf = f)é    )ÚannotationsN)ÚTYPE_CHECKINGÚOptional)ÚBaseModelOutputWithPastÚCausalLMOutputWithPast)ÚPreTrainedModel)Úlogging)Údeprecate_kwarg)Ú	Attention)ÚComba)ÚCombaConfig)Úget_hybrid_attention_spec)ÚCacheÚ"FLAUnsupportedCacheGenerationMixin)ÚFusedCrossEntropyLossÚFusedLinearCrossEntropyLossÚRMSNorm)ÚGatedMLP)Úl2_warp)Úfused_attnres)ÚUnpack)ÚGradientCheckpointingLayerc                  ób   ^ • \ rS rSrSU 4S jjr     S               SS jjrSrU =r$ )Ú
CombaBlocké+   c                ó¦  >• [         TU ]  5         Xl        X l        UR                  (       a  [
        O[        R
                  " UR                  UR                  S9U l	        [        UR                  US9nUb9  [        UR                  US   US   US   US   US   UR                  US9U l        O|[        UR                  UR                  UR                   UR"                  UR$                  UR&                  UR(                  UR*                  UR,                  UR                  US	9U l        UR                  (       a  [
        O[        R
                  " UR                  UR                  S9U l        [1        UR                  UR2                  UR4                  UR6                  UR8                  S
9U l        UR<                  S LU l        U R>                  (       aø  [        R@                  " UR                  SSS9U l!        [        R
                  " UR                  UR                  S9U l"        [        R@                  " UR                  SSS9U l#        [        R
                  " UR                  UR                  S9U l$        UR<                  nSU-  U-  S:H  U l%        SU-  S-   U-  S:H  U l&        SU RB                  l'        SU RF                  l'        g g )N©Úeps)Ú	layer_idxÚ	num_headsÚnum_kv_headsÚqkv_biasÚwindow_sizeÚ
rope_theta)Úhidden_sizer    r!   r"   r#   r$   Úmax_position_embeddingsr   )Úmoder%   Úexpand_vÚhead_dimr    Únum_v_headsÚuse_output_gateÚuse_short_convÚ	conv_sizeÚnorm_epsr   )r%   Úhidden_ratioÚintermediate_sizeÚ
hidden_actÚfuse_swiglué   F©Úin_featuresÚout_featuresÚbias©Únormalized_shaper   é   r   T)(ÚsuperÚ__init__Úconfigr   Ú	fuse_normr   Únnr%   r.   Ú	attn_normr   Úattnr   r&   r   Ú	attn_moder(   r)   r    r*   r+   r,   r-   Úmlp_normÚCombaMLPr/   r0   r1   r2   ÚmlpÚattnres_block_sizeÚuse_attnresÚLinearÚattn_res_projÚattn_res_normÚmlp_res_projÚmlp_res_normÚattnres_is_attn_boundaryÚattnres_is_mlp_boundaryÚ_is_attnres_proj)Úselfr=   r   Ú	attn_specÚ
block_sizeÚ	__class__s        €Ú­C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\fla/models/comba/modeling_comba.pyr<   ÚCombaBlock.__init__-   sd  ø€ Ü‰ÑÔàŒØ"Œà%+×%5×%5�'¼2¿:¹:ÐEÀv×GYÑGYÐ_e×_nÑ_nÑoˆŒÜ-¨f¯k©kÀYÑOˆ	ØÑ Ü!Ø"×.Ñ.Ø# KÑ0Ø& ~Ñ6Ø" :Ñ.Ø% mÑ4Ø$ \Ñ2Ø(.×(FÑ(FØ#ñ	ˆD�Iô Ø×%Ñ%Ø"×.Ñ.ØŸ™ØŸ™Ø ×*Ñ*Ø"×.Ñ.Ø &× 6Ñ 6Ø%×4Ñ4Ø ×*Ñ*ØŸ™Ø#ñˆDŒIð %+×$4×$4�¼"¿*¹*ÐDÀf×FXÑFXÐ^d×^mÑ^mÑnˆŒÜØ×*Ñ*Ø×,Ñ,Ø$×6Ñ6Ø×(Ñ(Ø×*Ñ*ñ
ˆŒð "×4Ñ4¸DÐ@ˆÔØ××Ü!#§¢°v×7IÑ7IÐXYÐ`eÑ!fˆDÔÜ!#§¢¸V×=OÑ=OÐU[×UdÑUdÑ!eˆDÔÜ "§	¢	°f×6HÑ6HÐWXÐ_dÑ eˆDÔÜ "§
¢
¸F×<NÑ<NÐTZ×TcÑTcÑ dˆDÔØ×2Ñ2ˆJØ-.°©]¸jÑ,HÈAÑ,MˆDÔ)Ø,-°	©M¸AÑ,=ÀÑ+KÈqÑ+PˆDÔ(Ø26ˆD×ÑÔ/Ø15ˆD×ÑÕ.ð ó    c           	     óÔ  • U R                   (       a•  UnUc  U R                  U5      nU/nS nOŒ/ UQUPn	U R                  (       a  U	nS n[        U R                  R
                  U	U R                  R
                  U R                  R
                  U R                  R                  S9nOUn
U R                  U5      nU R                  " SUUUUUS.UD6u  pnU R                   (       a‚  Wc  UOX�-   n/ UQUPn	U R                  (       a  U	nS n[        U R                  R
                  U	U R                  R
                  U R                  R
                  U R                  R                  S9nOIU R                  R                  (       a  U R                  UW
S5      u  pOW
U-   nUn
U R                  U5      nU R                  " U40 UD6nU R                   (       a
  Wc  UOX�-   nOW
U-   nXX64nU$ )N©ÚqueryÚ	residualsÚ
rms_weightÚoutput_rms_weightÚrms_eps)Úhidden_statesÚattention_maskÚpast_key_valuesÚ	use_cacheÚoutput_attentionsT© )rG   r@   rM   r   rI   ÚweightrJ   r   rA   rN   rK   rL   rC   r=   r>   rE   )rP   r^   r_   r`   ra   rb   Úattnres_statesÚkwargsÚ
prefix_sumrZ   ÚresidualÚ
attentionsÚoutputss                rT   ÚforwardÚCombaBlock.forwardc   sí  € ð ××Ø&ˆJØÑ%ð
 !%§¡¨zÓ :�Ø", �Ø!‘
à9˜nÐ9¨jÐ9�	Ø×0×0Ø%.�NØ!%�JÜ -Ø×,Ñ,×3Ñ3Ø'Ø#×1Ñ1×8Ñ8Ø&*§n¡n×&;Ñ&;Ø ×.Ñ.×2Ñ2ñ!‘ð %ˆHØ ŸN™N¨=Ó9ˆMØ59·Y²Yð 6
Ø'Ø)Ø+ØØ/ñ6
ð ñ6
Ñ2ˆ ?ð ××Ø*4Ñ*<™À*ÑB\ˆJØ5˜.Ð5¨*Ð5ˆIØ×+×+Ø!*�Ø!�
Ü)Ø×'Ñ'×.Ñ.Ø#Ø×,Ñ,×3Ñ3Ø"&§-¡-×"6Ñ"6Ø×)Ñ)×-Ñ-ñ‰Mð �[‰[×"×"Ø&*§m¡m°MÀ8ÈTÓ&RÑ#ˆM˜8à$ }Ñ4ˆMØ$ˆHØ ŸM™M¨-Ó8ˆMØŸš Ñ9°&Ñ9ˆà××Ø-7Ñ-?™MÀZÑE_‰Mà$ }Ñ4ˆMà ¨oÐNˆàˆrV   )rA   r@   rJ   rI   rM   rN   r=   r   rE   rC   rL   rK   rG   )r=   r   r   Úint)NNFFN)r^   ztorch.Tensorr_   útorch.Tensor | Noner`   ú&Cache | list[torch.FloatTensor] | Nonera   úbool | Nonerb   rp   re   zlist[torch.Tensor] | Nonerf   úUnpack[dict]ÚreturnzLtuple[torch.FloatTensor, tuple[torch.FloatTensor, torch.FloatTensor] | None])Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__r<   rk   Ú__static_attributes__Ú__classcell__©rS   s   @rT   r   r   +   s~   ø† ÷46ðr /3ØBFØ!&Ø).Ø48ðHà#ðHð ,ðHð @ð	Hð
 ðHð 'ðHð 2ðHð ðHð 
V÷Hó HrV   r   c                  óZ   ^ • \ rS rSr\rSrSrS/rSr	U 4S jr
  S     S	S jjrSrU =r$ )
ÚCombaPreTrainedModelé®   ÚmodelTr   c                ó&   >• [         TU ]  " U0 UD6  g ©N)r;   r<   )rP   Úinputsrf   rS   s      €rT   r<   ÚCombaPreTrainedModel.__init__¶   s   ø€ Ü‰Ò˜&Ð+ FÓ+rV   c           	     ó 	  • [        U[        5      (       GaÕ  [        UR                  5       5      R                  R
                  S:w  Ga£  [        R                  " 5          [        UR                  SS5      (       dP  UR                  R                  [        R                  R                  UR                  SSS9R                  5       5        SUR                  l        [        UR                   SS5      (       dÙ  [        R"                  " [        R                  R                  UR                   5      [$        R                  " S5      [$        R                  " S	5      -
  -  [$        R                  " S	5      -   5      R'                  S
S9nU[        R                  " [        R(                  " U* 5      * 5      -   nUR                   R                  U5        SUR                   l        S S S 5        GO][        U[        R*                  [        R,                  45      (       a°  [        USS5      (       a*  [        R                  R/                  UR0                  5        O=[        R                  R3                  UR0                  SU R4                  R6                  S9  UR8                  b)  [        R                  R/                  UR8                  5        O~[        U[        R:                  5      (       a>  [        R                  R3                  UR0                  SU R4                  R6                  S9  O![=        US5      (       a  UR?                  5         UGb  S n[=        US5      (       a  UR@                  R0                  nO'[=        US5      (       a  URB                  R0                  nUbº  US:X  a€  [        R                  RE                  U[$        RF                  " S5      S9  [        R                  " 5          U[$        RF                  " X0R4                  RH                  -  5      -  nS S S 5        g US:X  a   [        R                  R/                  U5        g [K        SU 35      eg g ! , (       d  f       GN#= f! , (       d  f       g = f)NÚmetaÚ_is_hf_initializedFr   é   )ÚaÚbTgš™™™™™¹?gü©ñÒMbP?g-Cëâ6?)ÚminrO   g        )ÚmeanÚstdÚreset_parametersÚo_projÚ	down_projÚrescaleé   )r†   Úzeroz#Invalid prenorm_residual_strategy: )&Ú
isinstancer   ÚnextÚ
parametersÚdeviceÚtypeÚtorchÚno_gradÚgetattrÚA_logÚcopy_r?   ÚinitÚuniform_ÚlogÚ_no_weight_decayÚdt_biasÚexpÚmathÚclampÚexpm1rH   ÚConv1dÚzeros_rd   Únormal_r=   Úinitializer_ranger7   Ú	EmbeddingÚhasattrr‹   rŒ   r�   Úkaiming_uniform_ÚsqrtÚnum_hidden_layersÚ
ValueError)rP   ÚmoduleÚprenorm_residual_strategyÚnum_residuals_per_layerÚdtÚinv_dtÚps          rT   Ú_init_weightsÚ"CombaPreTrainedModel._init_weights¹   s  € ô �fœe×$Ò$¬¨f×.?Ñ.?Ó.AÓ)B×)IÑ)I×)NÑ)NÐRXÔ)XÜ—’•Ü˜vŸ|™|Ð-AÀ5×IÑIØ—L‘L×&Ñ&¤r§w¡w×'7Ñ'7¸¿¹ÈÈRÐ'7Ð'P×'TÑ'TÓ'VÔWØ04�—‘Ô-Ü˜vŸ~™~Ð/CÀU×KÑKÜŸšÜŸ™×(Ñ(¨¯©Ó8¼D¿HºHÀS»MÌDÏHÊHÐUZËOÑ<[Ñ\Ô_c×_gÒ_gÐhmÓ_nÑnóç‘e �e�oð ð  ¤%§)¢)¬U¯[ª[¸"¸Ó-=Ð,=Ó">Ñ>�FØ—N‘N×(Ñ(¨Ô0Ø26�—‘Ô/÷ !‘ô ˜¤§¡¬B¯I©IÐ 6×7Ñ7Ü�vÐ1°5×9Ñ9Ü—‘—‘˜vŸ}™}Õ-ô —‘—‘ §¡°C¸T¿[¹[×=ZÑ=Z�Ñ[Ø�{‰{Ñ&Ü—‘—‘˜vŸ{™{Ô+øÜ˜¤§¡×-Ñ-Ü�G‰G�O‰O˜FŸM™M°¸¿¹×9VÑ9VˆOÒWÜ�VÐ/×0Ñ0Ø×#Ñ#Ô%à$Ò0ð ˆAÜ�v˜x×(Ñ(Ø—M‘M×(Ñ(‘Ü˜ ×-Ñ-Ø×$Ñ$×+Ñ+�Ø‰}ð
 -°	Ó9Ü—G‘G×,Ñ,¨Q´$·)²)¸A³,Ð,Ñ?ÜŸš�ØœTŸYšYÐ'>ÇÁ×A^ÑA^Ñ'^Ó_Ñ_˜÷ )˜à.°&Ó8Ü—G‘G—N‘N 1Õ%ä$Ð'JÐKdÐJeÐ%fÓgÐgð ð 1÷7 !–ú÷\ )�ús   ÁFQÏ.0Q/Ñ
Q,Ñ/
Q=rc   )Nr:   )r®   z	nn.Moduler¯   z
str | Noner°   rm   )rs   rt   ru   rv   r   Úconfig_classÚbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_supports_cache_classr<   r´   rw   rx   ry   s   @rT   r{   r{   ®   sZ   ø† à€LØÐØ&*Ð#Ø%˜ÐØ Ðõ,ð 15Ø'(ð	:hàð:hð $.ð:hð "%÷	:hó :hrV   r{   c                  ó|   ^ • \ rS rSrSU 4S jjrS rS r        S                   S	S jjrSrU =r	$ )
Ú
CombaModeléö   c           	     ó@  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " UR                  UR                  U R                  5      U l        [
        R                  " [        UR                  5       Vs/ s H  n[        X5      PM     sn5      U l        UR                  (       a  [        O[
        R                  " UR                  UR                   S9U l        UR$                  S LU l        U R&                  (       ad  [
        R(                  " UR                  SSS9U l        [
        R                  " UR                  UR                   S9U l        SU R*                  l        SU l        U R3                  5         g s  snf )Nr   r3   Fr4   r8   T)r;   r<   Úpad_token_idÚpadding_idxÚ
vocab_sizer?   r¨   r%   Ú
embeddingsÚ
ModuleListÚranger¬   r   Úlayersr>   r   r.   ÚnormrF   rG   rH   Úres_projÚres_normrO   Úgradient_checkpointingÚ	post_init)rP   r=   r   rS   s      €rT   r<   ÚCombaModel.__init__ø   s+  ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒäŸ,š, v×'8Ñ'8¸&×:LÑ:LÈd×N^ÑN^Ó_ˆŒÜ—m’mÔTYÐZ`×ZrÑZrÔTsÓ$tÒTsÀy¤Z°Ö%BÑTsÑ$tÓuˆŒØ &× 0× 0•W´b·j±jÐ@À&×BTÑBTÐZ`×ZiÑZiÑjˆŒ	à!×4Ñ4¸DÐ@ˆÔØ××ÜŸIšI°&×2DÑ2DÐSTÐ[`ÑaˆDŒMÜŸJšJ¸×8JÑ8JÐPV×P_ÑP_Ñ`ˆDŒMØ-1ˆD�M‰MÔ*à&+ˆÔ#à�‰Õùò %us   ÂFc                ó   • U R                   $ r   ©rÂ   ©rP   s    rT   Úget_input_embeddingsÚCombaModel.get_input_embeddings  s   € Ø�‰ÐrV   c                ó   • Xl         g r   rÍ   ©rP   Úvalues     rT   Úset_input_embeddingsÚCombaModel.set_input_embeddings  s   € Ø�rV   c	           
     ó2  • U(       a  [         R                  " S5        SnUb  UOU R                  R                  nUb  UOU R                  R                  nUb  UO(U R
                  (       d  U R                  R                  OSnUb  UOU R                  R                  nUb  Ub  [        S5      eUc  Uc  [        S5      eUc  U R                  U5      nUn
U(       a+  [        U[        5      (       d  [        R                  " U5      nS nU(       a  SOS nU(       a  SOS nU R                   H/  nU(       a  XÊ4-  nU" U
4UUUUUS.U	D6u  p¯pKU(       d  M*  Xß4-  nM1     U R                  (       ad  / UQU
Pn[        U R                   R"                  UU R$                  R"                  U R&                  R"                  U R$                  R(                  S9n
OU R'                  U
5      n
U(       a  XÊ4-  nU(       d  [+        S X¤XÍ4 5       5      $ [-        U
UUUS	9$ )
NzE`CombaModel` does not `output_attentions` now, setting it to `False`.FzDYou cannot specify both input_ids and inputs_embeds at the same timez5You have to specify either input_ids or inputs_embedsrc   )r_   r`   ra   rb   re   rX   c              3  ó.   #   • U  H  oc  M  Uv •  M     g 7fr   rc   )Ú.0Úis     rT   Ú	<genexpr>Ú%CombaModel.forward.<locals>.<genexpr>Z  s   é € ÐtÒ$b˜qŸ™Ò$bùs   ‚Œ	)Úlast_hidden_stater`   r^   ri   )ÚwarningsÚwarnr=   rb   Úoutput_hidden_statesÚtrainingra   Úuse_return_dictr­   rÂ   r‘   r   Úfrom_legacy_cacherÅ   rG   r   rÇ   rd   rÈ   rÆ   r   Útupler   )rP   Ú	input_idsr_   Úinputs_embedsr`   ra   rb   rß   Úreturn_dictrf   r^   re   Úall_hidden_statesÚ	all_attnsÚlayerri   rZ   s                    rT   rk   ÚCombaModel.forward  s  € ö Ü�MŠMÐaÔbØ %ÐØ1BÑ1NÑ-ÐTX×T_ÑT_×TqÑTqÐØ7KÑ7WÑ3Ð]a×]hÑ]h×]}Ñ]}ÐØ!*Ñ!6‘IÐZ^×Zg×Zg¸T¿[¹[×=RÒ=RÐmrˆ	Ø%0Ñ%<‘kÀ$Ç+Á+×B]ÑB]ˆð Ñ  ]Ñ%>ÜÐcÓdÐdØÑ Ñ!6ÜÐTÓUÐUàÑ Ø ŸO™O¨IÓ6ˆMØ%ˆæœZ¨¼×?Ñ?Ü#×5Ò5°oÓFˆOà48ˆæ"6™B¸DÐÞ+‘B°ˆ	Ø—[”[ˆEÞ#Ø!Ð%5Ñ5Ð!áINØðJà-Ø /Ø#Ø"3Ø-ñJð ñJÑFˆM ÷ !Ð Ø˜]Ñ*’	ñ !ð" ××ð 9˜.Ð8¨-Ð8ˆIÜ)Ø—m‘m×*Ñ*Ø#ØŸ=™=×/Ñ/Ø"&§)¡)×"2Ñ"2ØŸ™×)Ñ)ñ‰Mð !ŸI™I mÓ4ˆMö  ØÐ!1Ñ1ÐæÜÑt ]ÐEVÑ$bÓtÓtÐtÜ&Ø+Ø+Ø+Ø ñ	
ð 	
rV   )	rÂ   rÉ   rÅ   rÆ   rÀ   rÈ   rÇ   rG   rÁ   )r=   r   )NNNNNNNN)rä   útorch.LongTensor | Noner_   zOptional[torch.Tensor]rå   ztorch.FloatTensor | Noner`   ro   ra   rp   rb   rp   rß   rp   ræ   rp   rf   rq   rr   ztuple | BaseModelOutputWithPast)
rs   rt   ru   rv   r<   rÏ   rÔ   rk   rw   rx   ry   s   @rT   r¼   r¼   ö   s¥   ø† ÷ò&ò ð
 .2Ø15Ø26ØBFØ!%Ø)-Ø,0Ø#'ðO
à*ðO
ð /ðO
ð 0ð	O
ð
 @ðO
ð ðO
ð 'ðO
ð *ðO
ð !ðO
ð ðO
ð 
)÷O
ó O
rV   r¼   c                  ó¸   ^ • \ rS rSrS/rU 4S jrS rS rS rS r	S r
S	 r\" S
SSS9          S                       SS jj5       rSrU =r$ )ÚCombaForCausalLMic  zlm_head.weightc                óò   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  UR                  SS9U l        S U l	        U R                  5         g )NF)r7   )r;   r<   r¼   r}   rÁ   r?   rH   r%   Úlm_headÚ	criterionrÊ   )rP   r=   rS   s     €rT   r<   ÚCombaForCausalLM.__init__g  s\   ø€ Ü‰Ñ˜Ô Ü Ó'ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°V×5FÑ5FÈUÑSˆŒØˆŒð 	�‰ÕrV   c                ó.   • U R                   R                  $ r   ©r}   rÂ   rÎ   s    rT   rÏ   Ú%CombaForCausalLM.get_input_embeddingsq  s   € Ø�z‰z×$Ñ$Ð$rV   c                ó$   • XR                   l        g r   ró   rÒ   s     rT   rÔ   Ú%CombaForCausalLM.set_input_embeddingst  s   € Ø %�
‰
ÕrV   c                ó   • U R                   $ r   ©rï   rÎ   s    rT   Úget_output_embeddingsÚ&CombaForCausalLM.get_output_embeddingsw  s   € Ø�|‰|ÐrV   c                ó   • Xl         g r   rø   )rP   Únew_embeddingss     rT   Úset_output_embeddingsÚ&CombaForCausalLM.set_output_embeddingsz  s   € Ø%�rV   c                ó   • Xl         g r   ©r}   )rP   Údecoders     rT   Úset_decoderÚCombaForCausalLM.set_decoder}  s   € Ø�
rV   c                ó   • U R                   $ r   r   rÎ   s    rT   Úget_decoderÚCombaForCausalLM.get_decoder€  s   € Ø�z‰zÐrV   Únum_logits_to_keepz4.50Úlogits_to_keep)ÚversionÚnew_namec                ó6  • Ub  UOU R                   R                  nUb  UOU R                   R                  nU	b  U	OU R                   R                  n	U R                  " SUUUUUUUU	S.UD6nUS   nSu  pïU R                   R
                  (       a  Uc   U R                  U
c  UOUS S 2U
* S 24   5      nUGb�  [        U SS 5      ct  U R                   R
                  (       a  [        U R                   R                  S9nOGU R                   R                  (       a
  [        SS9nO"[        R                  " 5       nOU R                  nUR                  UR                   5      n["        R$                  " USS	S 24   ["        R&                  " US S 2S S	24   UR(                  5      4S	5      nU R                   R
                  (       a3  U" XÕU R                  R*                  U R                  R,                  5      nO^U" UR/                  UR1                  5       S
5      UR/                  S
5      5      nU R                   R                  (       a  [3        Xï5      OUnU	(       d  U4US	S  -   nUb  U4U-   $ U$ [5        UUUR6                  UR8                  UR:                  S9$ )N)rä   r_   rå   r`   ra   rb   rß   ræ   r   )NNrð   )Ú
use_l2warpT)Úinplace_backward.r3   éÿÿÿÿ)ÚlossÚlogitsr`   r^   ri   rc   )r=   rb   rß   rá   r}   Úfuse_linear_cross_entropyrï   r˜   r   r  Úfuse_cross_entropyr   r?   ÚCrossEntropyLossrð   Útor”   r–   ÚcatÚ	full_likeÚignore_indexrd   r7   ÚviewÚnumelr   r   r`   r^   ri   )rP   rä   r_   rå   r`   Úlabelsra   rb   rß   ræ   r  rf   rj   r^   r  r  rð   Úoutputs                     rT   rk   ÚCombaForCausalLM.forwardƒ  sM  € ð 2CÑ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ñ$DÑ È$Ï+É+×JjÑJjð 	ð &1Ñ%<‘kÀ$Ç+Á+×B]ÑB]ˆà—*’*ð 

ØØ)Ø'Ø+ØØ/Ø!5Ø#ñ

ð ñ

ˆð   ™
ˆà!‰ˆØ�{‰{×4×4¸¹Ø—\‘\°>Ñ3I¡-È}Ò]^ÐaoÐ`oÑ`pÐ]pÑOqÓrˆFØÒÜ�t˜[¨$Ó/Ñ7Ø—;‘;×8×8Ü ;ÀtÇ{Á{×G]ÑG]Ñ ^‘IØ—[‘[×3×3Ü 5ÀtÑ L‘Iä "× 3Ò 3Ó 5‘Ià ŸN™N�	Ø—Y‘Y˜}×3Ñ3Ó4ˆFÜ—Y’Y  s¨A©B w¡´·²ÀÊÈ2ÈAÈ2ÈÁÐPY×PfÑPfÓ1gÐhÐjkÓlˆFØ�{‰{×4×4Ù  ¸¿¹×8KÑ8KÈTÏ\É\×M^ÑM^Ó_‘á  §¡¨V¯\©\«^¸RÓ!@À&Ç+Á+ÈbÃ/ÓR�Ø04·±×0F×0F”w˜tÔ,ÈD�æØ�Y ¨¨ Ñ,ˆFØ'+Ñ'7�D�7˜VÑ#ÐC¸VÐCä%ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
rV   )rð   rï   r}   rÁ   )
NNNNNNNNNr   )rä   ztorch.LongTensorr_   rn   rå   rn   r`   ro   r  rë   ra   rp   rb   rp   rß   rp   ræ   rp   r  z
int | Nonerf   rq   rr   ztuple | CausalLMOutputWithPast)rs   rt   ru   rv   Ú_tied_weights_keysr<   rÏ   rÔ   rù   rý   r  r  r
   rk   rw   rx   ry   s   @rT   rí   rí   c  sô   ø† à*Ð+Ðõò%ò&òò&òòñ Ð)°6ÐDTÑUð '+Ø.2Ø-1ØBFØ*.Ø!%Ø)-Ø,0Ø#'Ø%&ðA
à#ðA
ð ,ðA
ð +ð	A
ð
 @ðA
ð (ðA
ð ðA
ð 'ðA
ð *ðA
ð !ðA
ð #ðA
ð ðA
ð 
(ôA
ó VöA
rV   rí   )5Ú
__future__r   r¡   rÝ   Útypingr   r   r–   Útorch.nnr?   Útransformers.modeling_outputsr   r   Útransformers.modeling_utilsr   Útransformers.utilsr	   Útransformers.utils.deprecationr
   Úfla.layers.attnr   Úfla.layers.combar   Ú$fla.models.comba.configuration_combar   Úfla.models.hybridr   Úfla.models.utilsr   r   Úfla.modulesr   r   r   r   rD   Úfla.modules.l2warpr   Úfla.ops.attnresr   Útransformers.processing_utilsr   Útransformers.modeling_layersr   ÚImportErrorÚfla.models.modeling_layersÚ
get_loggerrs   Úloggerr   r{   r¼   rí   rc   rV   rT   Ú<module>r3     s¾   ðõ #ã Û ß *ã Ý ß YÝ 7Ý &Ý :å %Ý "Ý <Ý 7ß Fß SÑ SÝ ,Ý &Ý )æÝ4ðFÝGð 
×	Ò	˜HÓ	%€ô@Ð+ô @ôFEh˜?ô EhôPj
Ð%ô j
ôZb
Ð+Ð-Oõ b
øð}	 ó FßEðFús   ÂC ÃCÃC