ó
    ‡Ž¡jéŒ  ã                   ó¸  • S SK Jr  S SKrS SKJr  SSKJr  SSKJrJ	r	  SSK
Jr  SSKJrJr  SS	KJrJrJrJr  SS
KJr  SSKJr  SSKJrJrJrJr  SSKJr  SSKJ r J!r!  SSK"J#r#J$r$J%r%J&r&J'r'J(r(J)r)J*r*J+r+J,r,  SSK-J.r.J/r/J0r0J1r1J2r2J3r3J4r4  SSK5J6r6J7r7  \Rp                  " \95      r:  SRS\Rv                  S\Rx                  S\Rx                  S\Rx                  S\Rx                  S-  S\=S-  S\=S\\   4S jjr> " S S\$5      r? " S S \*5      r@ " S! S"\+5      rA " S# S$\,5      rB " S% S&\#5      rC " S' S(\&5      rD " S) S*\(5      rE " S+ S,\'5      rF " S- S.\%5      rG " S/ S0\)5      rH\ " S1 S2\5      5       rI " S3 S4\I5      rJ " S5 S6\Rv                  5      rK " S7 S8\Rv                  5      rL " S9 S:\Rv                  5      rM\\ " S; S<\5      5       5       rN " S= S>\Rv                  5      rO " S? S@\Rv                  5      rP " SA SB\25      rQ " SC SD\35      rR " SE SF\05      rS " SG SH\.5      rT " SI SJ\/5      rU " SK SL\15      rV " SM SN\V5      rW " SO SP\V\5      rX/ SQQrYg)Sé    )Ú	dataclassN)Únné   )Úinitialization)ÚCacheÚDynamicCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚBaseModelOutputWithPastÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚCausalLMOutputWithPastÚModelOutput)ÚPreTrainedModel)ÚUnpack)ÚTransformersKwargsÚauto_docstringÚcan_return_tupleÚlogging)Úmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )
ÚEsmAttentionÚEsmEmbeddingsÚ
EsmEncoderÚEsmIntermediateÚEsmLayerÚ	EsmOutputÚ	EsmPoolerÚEsmRotaryEmbeddingÚEsmSelfAttentionÚEsmSelfOutput)ÚLlamaAttentionÚLlamaDecoderLayerÚLlamaMLPÚLlamaPreTrainedModelÚLlamaRMSNormÚLlamaRotaryEmbeddingÚ	repeat_kvé   )ÚEvollaConfigÚSaProtConfigÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚkwargsc                 óÔ  • Uc  UR                  S5      S-  n[        U SS5      n[        X(5      n[        X85      n[        R                  " XR                  SS5      5      U-  n	Ub  X”-   n	[        R                  R                  U	SS9n	[        R                  R                  X–U R                  S9n	[        R                  " X“5      n
U
R                  SS5      R                  5       n
X©4$ )	Néÿÿÿÿç      à¿Únum_key_value_groupsr+   r   r   ©Údim)ÚpÚtraining)ÚsizeÚgetattrr*   ÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr4   r=   Ú
contiguous)r.   r/   r0   r1   r2   r3   r4   r5   Ún_repÚattn_weightsÚattn_outputs              Ú·C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/evolla/modular_evolla.pyÚeager_attention_forwardrJ   C   sÖ   € ð �Ø—*‘*˜R“. DÑ(ˆô
 �FÐ2°AÓ6€EÜ
�CÓ
€CÜ�eÓ#€Eô —<’< §}¡}°Q¸Ó':Ó;¸gÑE€LàÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2Ð(Ð>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$ó    c                   ó(   ^ • \ rS rSrU 4S jrSrU =r$ )ÚEvollaSaProtEmbeddingséf   c                 ó(   >• [         TU ]  U5        U ?g ©N)ÚsuperÚ__init__Úposition_ids©ÚselfÚconfigÚ	__class__s     €rI   rR   ÚEvollaSaProtEmbeddings.__init__g   s   ø€ Ü‰Ñ˜Ô àÑrK   © )Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rR   Ú__static_attributes__Ú__classcell__©rW   s   @rI   rM   rM   f   s   ø† ÷ó rK   rM   c                   ón   ^ • \ rS rSrSS\4U 4S jjjrSS\S\\R                  \	4   4U 4S jjjr
SrU =r$ )ÚEvollaSaProtRotaryEmbeddingém   rV   c                 ó$   >• [         TU ]  U5        g rP   )rQ   rR   )rU   rV   ÚdevicerW   s      €rI   rR   Ú$EvollaSaProtRotaryEmbedding.__init__n   s   ø€ Ü‰Ñ˜Õ rK   Úreturnc                 ó"   >• [         TU ]  U 5      $ rP   )rQ   Úcompute_default_rope_parameters)rV   re   r5   rW   s      €rI   ri   Ú;EvollaSaProtRotaryEmbedding.compute_default_rope_parametersq   s   ø€ Ü‰wÑ6°vÓ>Ð>rK   rY   rP   )rZ   r[   r\   r]   r-   rR   Útupler@   ÚTensorÚfloatri   r^   r_   r`   s   @rI   rb   rb   m   s?   ø† ñ!˜|÷ !ð !ñ?°ð ?ÐX]Ð^c×^jÑ^jÐlqÐ^qÑXr÷ ?ö ?rK   rb   c                   ó   • \ rS rSrSrg)ÚEvollaSaProtSelfAttentionéu   rY   N©rZ   r[   r\   r]   r^   rY   rK   rI   ro   ro   u   ó   † ÚrK   ro   c                   ó   • \ rS rSrSrg)ÚEvollaSaProtSelfOutputéy   rY   Nrq   rY   rK   rI   rt   rt   y   rr   rK   rt   c                   ó   • \ rS rSrSrg)ÚEvollaSaProtAttentioné}   rY   Nrq   rY   rK   rI   rw   rw   }   rr   rK   rw   c                   ó   • \ rS rSrSrg)ÚEvollaSaProtIntermediateé�   rY   Nrq   rY   rK   rI   rz   rz   �   rr   rK   rz   c                   ó   • \ rS rSrSrg)ÚEvollaSaProtOutputé…   rY   Nrq   rY   rK   rI   r}   r}   …   rr   rK   r}   c                   ó   • \ rS rSrSrg)ÚEvollaSaProtLayeré‰   rY   Nrq   rY   rK   rI   r€   r€   ‰   rr   rK   r€   c                   ó   • \ rS rSrSrg)ÚEvollaSaProtEncoderé�   rY   Nrq   rY   rK   rI   rƒ   rƒ   �   rr   rK   rƒ   c                   ó   • \ rS rSrSrg)ÚEvollaSaProtPooleré‘   rY   Nrq   rY   rK   rI   r†   r†   ‘   rr   rK   r†   c                   óš   ^ • \ rS rSr% \\S'   S/rSrSrSr	Sr
\\" \SSS9/\" \SSS9/S	.r\R                   " 5       U 4S
 j5       rSrU =r$ )ÚEvollaSaProtPreTrainedModelé•   rV   r€   Tr+   Ú	attention)ÚindexÚ
layer_nameÚcrossattention)Úhidden_statesÚ
attentionsÚcross_attentionsc                 óÌ   >• [         TU ]  U5        [        U[        5      (       a?  UR	                  UR
                  5      u  p#[        R                  " [        US5      U5        g g )NÚinv_freq)	rQ   Ú_init_weightsÚ
isinstancerb   ri   rV   ÚinitÚcopy_r?   )rU   r.   Úcurr_inv_freqÚ_rW   s       €rI   r”   Ú)EvollaSaProtPreTrainedModel._init_weights¦   sO   ø€ ä‰Ñ˜fÔ%Ü�fÔ9×:Ñ:Ø%×EÑEÀfÇmÁmÓTÑˆMÜ�JŠJ”w˜v zÓ2°MÕBð ;rK   rY   )rZ   r[   r\   r]   r-   Ú__annotations__Ú_no_split_modulesÚ_supports_flash_attnÚ_supports_sdpaÚ_supports_flex_attnÚ_supports_attention_backendr€   r   ro   Ú_can_record_outputsr@   Úno_gradr”   r^   r_   r`   s   @rI   r‰   r‰   •   sv   ø‡ àÓØ,Ð-ÐØÐØ€NØÐØ"&Ðð +Ù%Ð&?ÀqÐU`ÑaÐbáÐ4¸AÐJZÑ[ð
ñÐð ‡]‚]ƒ_ôCó öCrK   r‰   c            
       óÀ   ^ • \ rS rSrS\4U 4S jjrS rS r\\	 SS\
R                  S-  S\
R                  S-  S	\\
R                     \-  4S
 jj5       5       rSrU =r$ )ÚEvollaSaProtProteinEncoderé®   rV   c                 ó    >• [         TU ]  U5        [        U5      U l        [	        US9U l        [        U5      U l        U R                  5         g ©N©rV   )	rQ   rR   rM   Ú
embeddingsrb   Úrotary_embeddingsrƒ   ÚencoderÚ	post_initrT   s     €rI   rR   Ú#EvollaSaProtProteinEncoder.__init__¯   s?   ø€ Ü‰Ñ˜Ô Ü0°Ó8ˆŒÜ!<ÀFÑ!KˆÔÜ*¨6Ó2ˆŒØ�‰ÕrK   c                 ó.   • U R                   R                  $ rP   ©r©   Úword_embeddings©rU   s    rI   Úget_input_embeddingsÚ/EvollaSaProtProteinEncoder.get_input_embeddings¶   s   € Ø�‰×.Ñ.Ð.rK   c                 ó$   • XR                   l        g rP   r¯   ©rU   r1   s     rI   Úset_input_embeddingsÚ/EvollaSaProtProteinEncoder.set_input_embeddings¹   s   € Ø*/�‰Õ'rK   NÚ	input_idsr2   rg   c                 ó¬  • UR                  5       nUu  pVUR                  nUc  [        R                  " XV4US9nU R	                  XS9n[        U R                  UUS9n[        R                  " XgS9R                  S5      n	U R                  X‰5      n
U R                  " U4X*S.UD6nUS   n[        UUR                  UR                  UR                  S9$ )N©re   ©r¸   r2   )rV   Úinputs_embedsr2   r   )r2   Úposition_embeddings)Úlast_hidden_stater�   r�   r‘   )r>   re   r@   Úonesr©   r
   rV   ÚarangeÚ	unsqueezerª   r«   r   r�   r�   r‘   )rU   r¸   r2   r5   Úinput_shapeÚ
batch_sizeÚ
seq_lengthre   r¼   rS   r½   Úencoder_outputsÚsequence_outputs                rI   ÚforwardÚ"EvollaSaProtProteinEncoder.forward¼   sç   € ð  —n‘nÓ&ˆØ!,Ñˆ
à×!Ñ!ˆØÑ!Ü"ŸZšZ¨*Ð)AÈ6ÑRˆNØŸ™°)˜Ð[ˆä2Ø—;‘;Ø'Ø)ñ
ˆô —|’| JÑ>×HÑHÈÓKˆØ"×4Ñ4°]ÓQÐàŸ,š,Øð
Ø*8ñ
Øekñ
ˆð *¨!Ñ,ˆä;Ø-Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ñ	
ð 	
rK   )r©   r«   rª   rP   )rZ   r[   r\   r]   r-   rR   r²   r¶   r   r   r@   rl   rk   r   rÇ   r^   r_   r`   s   @rI   r¤   r¤   ®   sw   ø† ð˜|÷ ò/ò0ð  Øð /3ñ!
à—<‘< $Ñ&ð!
ð Ÿ™ tÑ+ð!
ð
 
ˆu�|‰|Ñ	ÐKÑ	Kô!
ó ó  ö!
rK   r¤   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )Ú!EvollaSequenceCompressorAttentionéâ   c                 óX  >• [         TU ]  5         US-  U l        X0l        X#-  n[        R
                  " U5      U l        [        R
                  " U5      U l        [        R                  " XSS9U l	        [        R                  " XS-  SS9U l
        [        R                  " XASS9U l        g )Nr8   F©Úbiasr   )rQ   rR   ÚscaleÚheadsr   Ú	LayerNormÚ
norm_mediaÚnorm_latentsÚLinearÚto_qÚto_kvÚto_out)rU   r;   Údim_headrÐ   Ú	inner_dimrW   s        €rI   rR   Ú*EvollaSequenceCompressorAttention.__init__ã   s�   ø€ Ü‰ÑÔØ˜t‘^ˆŒ
ØŒ
ØÑ$ˆ	äŸ,š, sÓ+ˆŒÜŸLšL¨Ó-ˆÔä—I’I˜c°5Ñ9ˆŒ	Ü—Y’Y˜s°¡M¸Ñ>ˆŒ
Ü—i’i 	°UÑ;ˆ�rK   c                 ó&  • U R                  U5      nU R                  U5      nU R                  nU R                  U5      n[        R
                  " X4SS9nU R                  U5      R                  SSS9u  pxUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nUR                  UR                  S5      UR                  S5      US5      R                  SSSS5      nXPR                  -  n[        R                  " XWR                  SS5      5      n	X™R                  SSS	9R                  5       -
  n	U	R                   u  p«pÍ[        R"                  " X¼5      R%                  UR&                  5      nUS
S
2S
S
S
S
24   nUS
S
S
2S
S
2S
4   nUU-  nU	R)                  SU-
  R+                  5       S5      n	U	R-                  SS9n[        R                  " UU5      nUR                  SSSS5      nUR/                  UR                  S5      UR                  S5      S5      nU R1                  U5      $ )z¤
Args:
    x (torch.Tensor): image features
        shape (b, n1, D)
    latent (torch.Tensor): latent features
        shape (b, n2, D);  n2: num of latent tokens
éþÿÿÿr:   r   r7   r   r+   r   T©r;   ÚkeepdimNg     ˆÃÀ)rÒ   rÓ   rÐ   rÕ   r@   ÚcatrÖ   ÚchunkÚviewr>   ÚpermuterÏ   rA   rB   ÚamaxÚdetachÚshaper¿   Útore   Úmasked_fillÚboolrD   Úreshaper×   )rU   ÚxÚlatentsÚmaskÚhÚqÚkv_inputÚkÚvÚsimÚbsÚnhÚskdÚokdr¿   Úmask_expÚones_expÚattnÚouts                      rI   rÇ   Ú)EvollaSequenceCompressorAttention.forwardð   s2  € ð �O‰O˜AÓˆØ×#Ñ# GÓ,ˆà�J‰Jˆà�I‰I�gÓˆÜ—9’9˜a˜\¨rÑ2ˆØ�z‰z˜(Ó#×)Ñ)Ø�2ð *ð 
‰ˆð �F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ�F‰F�1—6‘6˜!“9˜aŸf™f Q›i¨¨BÓ/×7Ñ7¸¸1¸aÀÓCˆØ—
‘
‰Nˆô �lŠl˜1Ÿk™k¨"¨bÓ1Ó2ˆØ—H‘H ¨T�HÐ2×9Ñ9Ó;Ñ;ˆØŸ9™9Ñˆ�Ü�zŠz˜"Ó"×%Ñ% d§k¡kÓ2ˆØš˜4 ¢qÐ(Ñ)ˆØ˜ša¢ DÐ(Ñ)ˆØ˜(Ñ"ˆà�o‰o˜q 4™xŸo™oÓ/°Ó6ˆØ�{‰{˜rˆ{Ð"ˆÜ�lŠl˜4 Ó#ˆØ�k‰k˜!˜Q  1Ó%ˆð �k‰k˜#Ÿ(™( 1›+ s§x¡x°£{°BÓ7ˆà�{‰{˜3ÓÐrK   )rÐ   rÓ   rÒ   rÏ   rÖ   r×   rÕ   )é@   é   ©rZ   r[   r\   r]   rR   rÇ   r^   r_   r`   s   @rI   rÊ   rÊ   â   s   ø† ÷<÷) ð ) rK   rÊ   c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )ÚEvollaFeedForwardi  c                 ó  >• [         TU ]  5         [        X-  5      n[        R                  " U5      U l        [        R                  " XSS9U l        [        R                  " 5       U l	        [        R                  " X1SS9U l
        g ©NFrÍ   )rQ   rR   Úintr   rÑ   ÚnormrÔ   Úfc1ÚGELUÚ
activationÚfc2)rU   r;   ÚmultrÙ   rW   s       €rI   rR   ÚEvollaFeedForward.__init__  sZ   ø€ Ü‰ÑÔÜ˜™
“Oˆ	ä—L’L Ó%ˆŒ	Ü—9’9˜S°%Ñ8ˆŒÜŸ'š'›)ˆŒÜ—9’9˜Y°%Ñ8ˆ�rK   c           	      ó~   • U R                  U R                  U R                  U R                  U5      5      5      5      $ rP   )r  r  r  r  )rU   rê   s     rI   rÇ   ÚEvollaFeedForward.forward&  s+   € Ø�x‰x˜Ÿ™¨¯©°·±¸1³Ó(>Ó?Ó@Ð@rK   )r  r  r  r  )é   rþ   r`   s   @rI   r   r     s   ø† ÷9÷Að ArK   r   c                   ó6   ^ • \ rS rSrS\4U 4S jjrS rSrU =r$ )Ú!EvollaSequenceCompressorResampleri*  rV   c                 ó   >• [         TU ]  5         UR                  R                  nUR                  U l        [        R                  " [        R                  " U R
                  U5      SS9U l
        [        R                  " / 5      U l        [        UR                  5       Ha  nU R                  R                  [        R                  " [!        X!R"                  UR$                  S9['        X!R(                  S9/5      5        Mc     [        R*                  " UR                  5      U l        [        R.                  " X!R                  5      U l        g )NT)Úrequires_grad)r;   rØ   rÐ   )r;   r	  )rQ   rR   Úprotein_encoder_configÚhidden_sizeÚresampler_num_latentsÚnum_latentsr   Ú	Parameterr@   Úrandnrë   Ú
ModuleListÚlayersÚrangeÚresampler_depthÚappendrÊ   Úresampler_dim_headÚresampler_headsr   Úresampler_ff_multrÑ   r  rÔ   Úprotein_projector)rU   rV   Úprotein_repr_dimr™   rW   s       €rI   rR   Ú*EvollaSequenceCompressorResampler.__init__+  sö   ø€ Ü‰ÑÔØ!×8Ñ8×DÑDÐØ!×7Ñ7ˆÔÜ—|’|¤E§K¢K°×0@Ñ0@ÐBRÓ$SÐcgÑhˆŒÜ—m’m BÓ'ˆŒÜ�v×-Ñ-Ö.ˆAØ�K‰K×ÑÜ—’ä9Ø 0×;TÑ;TÐ\b×\rÑ\rñô *Ð.>×E]ÑE]Ñ^ð	óö	ñ /ô —L’L ×!3Ñ!3Ó4ˆŒ	Ü!#§¢Ð+;×=OÑ=OÓ!PˆÕrK   c                 ód  • UR                   S   nUR                   u  pE[        R                  " X@R                  5      R	                  UR
                  5      n[        R                  " X&4SS9n[        R                  " U5      R	                  U R                  R
                  5      nU R                  S    UR                  SSS5      -  nUR	                  UR                  5      nU R                   H  u  pšU	" XU5      U-   nU
" U5      U-   nM     U R                  U5      nU R                  U5      $ )Nr   r+   r:   r7   )rå   r@   r¿   r  ræ   re   rß   rë   rá   Údtyper  r   r  )rU   Úembedsrì   Úbró   r™   Úlatent_maskr¿   rë   rù   ÚffÚtransformed_features               rI   rÇ   Ú)EvollaSequenceCompressorResampler.forward@  sø   € Ø�L‰L˜‰Oˆà—
‘
‰ˆÜ—j’j ×%5Ñ%5Ó6×9Ñ9¸$¿+¹+ÓFˆÜ�yŠy˜$Ð,°!Ñ4ˆô �zŠz˜!‹}×Ñ §¡× 3Ñ 3Ó4ˆØ—,‘,˜tÑ$ t§y¡y°°Q¸Ó':Ñ:ˆØ—*‘*˜VŸ\™\Ó*ˆØŸœ‰HˆDÙ˜6¨DÓ1°GÑ;ˆGÙ˜“k GÑ+ŠGñ $ð #×4Ñ4°WÓ=Ðà�y‰yÐ,Ó-Ð-rK   )rë   r  r  r  r   )	rZ   r[   r\   r]   r,   rR   rÇ   r^   r_   r`   s   @rI   r  r  *  s   ø† ðQ˜|÷ Q÷*.ð .rK   r  c                   óÎ   • \ rS rSr% SrSr\R                  S-  \S'   Sr	\R                  S-  \S'   Sr
\\R                  S4   S-  \S'   Sr\\R                  S4   S-  \S'   S	rg)
ÚEvollaProteinEncoderModelOutputiT  a>  
sequence_compressor_output (`torch.FloatTensor` of shape `(batch_size, compressed_seq_len, hidden_size)`, *optional*):
    Compressed sequence representation produced by the sequence compressor module. The sequence length is
    reduced from the original input length to `compressed_seq_len` via learned compression.
NÚsequence_compressor_outputr¾   .r�   r�   rY   )rZ   r[   r\   r]   Ú__doc__r-  r@   ÚFloatTensorr›   r¾   r�   rk   r�   r^   rY   rK   rI   r,  r,  T  st   ‡ ñð <@Ð × 1Ñ 1°DÑ 8Ó?Ø26Ð�u×(Ñ(¨4Ñ/Ó6Ø:>€M�5˜×*Ñ*¨CÐ/Ñ0°4Ñ7Ó>Ø7;€J��e×'Ñ'¨Ð,Ñ-°Ñ4Ö;rK   r,  c                   ót   ^ • \ rS rSrS\4U 4S jjr\S\R                  S\R                  4S j5       r
SrU =r$ )ÚEvollaProteinEncoderic  rV   c                 ón   >• [         TU ]  5         [        UR                  S9U l        [        US9U l        g r§   )rQ   rR   r¤   r  Úmodelr  Úsequence_compressor_resamplerrT   s     €rI   rR   ÚEvollaProteinEncoder.__init__d  s.   ø€ Ü‰ÑÔÜ/°v×7TÑ7TÑUˆŒ
Ü-NÐV\Ñ-]ˆÕ*rK   r¸   r2   c                 ó‚   • U R                  XS9nUR                  nU R                  XR5      n[        UUR                  S9$ )Nr»   )r-  r¾   )r3  r¾   r4  r,  )rU   r¸   r2   r5   Úprotein_outputÚprotein_embedsÚsequence_reprs          rI   rÇ   ÚEvollaProteinEncoder.forwardi  sF   € àŸ™¨i˜ÐWˆØ'×9Ñ9ˆØ×:Ñ:¸>ÓZˆä.Ø'4Ø,×>Ñ>ñ
ð 	
rK   )r3  r4  )rZ   r[   r\   r]   r,   rR   r   r@   Ú
LongTensorr/  rÇ   r^   r_   r`   s   @rI   r1  r1  c  s?   ø† ð^˜|÷ ^ð
 ð
 ×!1Ñ!1ð 
À5×CTÑCTó 
ó ö
rK   r1  c                   ór   ^ • \ rS rSr   S
S\S-  S\S-  S\S-  4U 4S jjjrS r       SS jrS	rU =r	$ )Ú#EvollaSequenceAlignerCrossAttentioniu  NÚprotein_encoder_dimÚstructure_encoder_dimÚmsa_encoder_dimc                 ó¦  >• [         TU ]  5         UR                  U l        UR                  U l        U R                  S-  U l        [        U R                  U R                  -  5      U l        U R                  U R                  -  U l        UR                  nUR                  nUR                  n[        R                  " U R                  U R                  5      U l        UbK  [        R                  " X R                  5      U l        [        R                  " X R                  5      U l        OS U l        S U l        UbK  [        R                  " X0R                  5      U l        [        R                  " X0R                  5      U l        OS U l        S U l        UbK  [        R                  " X@R                  5      U l        [        R                  " X@R                  5      U l        OS U l        S U l        [)        U R                  5      U l        [        R,                  " U5      U l        [        R                  " U R                  U R                  US9U l        [3        U R                  U5      U l        [        R6                  " [8        R:                  " S/5      5      U l        [        R6                  " [8        R:                  " S/5      5      U l        g )Nr8   rÍ   ç        ) rQ   rR   r  Únum_attention_headsrÏ   r  Úattention_head_sizeÚall_head_sizeÚ$aligner_attention_probs_dropout_probÚaligner_enable_biasÚaligner_ffn_multr   rÔ   r/   Úkey_proteinÚvalue_proteinÚkey_structureÚvalue_structureÚkey_msaÚ	value_msaÚEvollaRMSNormÚattention_normÚDropoutr4   Úout_projr   r(  r  r@   ÚtensorÚgate_attentionÚgate_ffw)	rU   rV   r>  r?  r@  Úattention_probs_dropout_probÚenable_biasÚffn_multrW   s	           €rI   rR   Ú,EvollaSequenceAlignerCrossAttention.__init__v  s  ø€ ô 	‰ÑÔà!×-Ñ-ˆÔØ#)×#=Ñ#=ˆÔ Ø×-Ñ-¨tÑ3ˆŒ
Ü#& t×'7Ñ'7¸$×:RÑ:RÑ'RÓ#SˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔà'-×'RÑ'RÐ$Ø×0Ñ0ˆØ×*Ñ*ˆä—Y’Y˜t×/Ñ/°×1CÑ1CÓDˆŒ
ØÑ*Ü!ŸyšyÐ)<×>PÑ>PÓQˆDÔÜ!#§¢Ð+>×@RÑ@RÓ!SˆDÕà#ˆDÔØ!%ˆDÔà Ñ,Ü!#§¢Ð+@×BTÑBTÓ!UˆDÔÜ#%§9¢9Ð-B×DVÑDVÓ#WˆDÕ à!%ˆDÔØ#'ˆDÔ àÑ&ÜŸ9š9 _×6HÑ6HÓIˆDŒLÜŸYšY ×8JÑ8JÓKˆD�NàˆDŒLØ!ˆDŒNä+¨D×,<Ñ,<Ó=ˆÔä—z’zÐ">Ó?ˆŒäŸ	š	 $×"2Ñ"2°D×4DÑ4DÈ;ÑWˆŒä# D×$4Ñ$4°hÓ?ˆŒÜ Ÿlšl¬5¯<ª<¸¸Ó+>Ó?ˆÔÜŸš¤U§\¢\°3°%Ó%8Ó9ˆ�rK   c	                 ó¢  • XgU/n	U	 V
s/ s H	  oªc  M  U
PM     n	n
U	(       d  [        S5      e[        R                  " U	SS9n	U R                  U5      nU R	                  U5      nU R
                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnU R                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnU R                  bA  U R                  b4  UR                  U5      nU R                  U5      nU R                  U5      nOSnSnXÎU/nU V
s/ s H	  oªc  M  U
PM     nn
[        R                  " USS9nXßU/nU V
s/ s H	  oªc  M  U
PM     nn
[        R                  " USS9nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nUR                  5       SS U R                  U R                  4-   nUR                  " U6 R!                  SSSS5      nX°R"                  -  nUcN  [        R$                  " UR                  S5      UR                  S5      5      R                  UR&                  5      nUSS2SSS2S4   U	SS2SSSS24   -  n[        R(                  " UUR+                  SS	5      5      nUUR-                  SS
S9R/                  5       -
  nUR1                  SU-
  R3                  5       [        R4                  " UR6                  5      R8                  5      n[:        R<                  " SS9" U5      n[        R(                  " UU5      nUR!                  SSSS5      R?                  5       nUR                  5       SS	 U R@                  4-   nUR                  " U6 nU RC                  U5      nU$ s  sn
f s  sn
f s  sn
f )zÁ
query_states: text
key_value_states: protein
query_states: [bs, query_seq_len, dim]
key_value_states: [bs, kv_seq_len, dim]
query_attn_mask: [bs, query_seq_len]
kv_attn_mask: [bs, kv_seq_len]
Nz=At least one modality should be provided for cross attention.r+   r:   r7   r   r   r   rÜ   TrÝ   )"Ú
ValueErrorr@   rß   rP  r/   rI  rJ  ræ   rK  rL  rM  rN  r>   rC  rD  rá   râ   rÏ   r¿   re   rA   rB   rã   rä   rç   rè   Úfinfor$  Úminr   ÚSoftmaxrE   rE  rR  )rU   Úquery_statesÚprotein_key_value_statesÚstructure_key_value_statesÚmsa_key_value_statesÚquery_attn_maskÚprotein_kv_attn_maskÚstructure_kv_attn_maskÚmsa_kv_attn_maskÚkv_attn_maskr™   Úquery_layerÚkey_layer_proteinÚvalue_layer_proteinÚkey_layer_structureÚvalue_layer_structureÚkey_layer_msaÚvalue_layer_msaÚ	key_layerÚvalue_layerÚnew_query_layer_shapeÚnew_key_layer_shapeÚnew_value_layer_shaper2   rG   Úattention_scoresÚattention_probsÚcontext_layerÚnew_context_layer_shapes                                rI   Úcross_attentionÚ3EvollaSequenceAlignerCrossAttention.cross_attention©  sK  € ð* -ÐFVÐWˆÙ#/ÓA¢<˜aŸ¡<ˆÐAÞÜÐ\Ó]Ð]Ü—y’y °1Ñ5ˆà×)Ñ)¨,Ó7ˆð —j‘j Ó-ˆà×ÑÑ'¨D×,>Ñ,>Ñ,JØ'?×'BÑ'BÀ<Ó'PÐ$Ø $× 0Ñ 0Ð1IÓ JÐØ"&×"4Ñ"4Ð5MÓ"NÑà $ÐØ"&Ðà×ÑÑ)¨d×.BÑ.BÑ.NØ)C×)FÑ)FÀ|Ó)TÐ&Ø"&×"4Ñ"4Ð5OÓ"PÐØ$(×$8Ñ$8Ð9SÓ$TÑ!à"&ÐØ$(Ð!à�<‰<Ñ#¨¯©Ñ(BØ#7×#:Ñ#:¸<Ó#HÐ Ø ŸL™LÐ)=Ó>ˆMØ"Ÿn™nÐ-AÓB‰Oà ˆMØ"ˆOà&¸]ÐKˆ	Ù )Ó;¢	˜1—Q¡	ˆ	Ð;Ü—I’I˜i¨QÑ/ˆ	à*À?ÐSˆÙ"-Ó?¢+˜Q—q¡+ˆÐ?Ü—i’i °Ñ3ˆà +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
ñ !
Ðð "×&Ò&Ð(=Ð>×FÑFÀqÈ!ÈQÐPQÓRˆà'Ÿn™nÓ.¨s°Ð3Ø×$Ñ$Ø×$Ñ$ð7
ñ 
Ðð —N’NÐ$7Ð8×@Ñ@ÀÀAÀqÈ!ÓLˆ	à +× 0Ñ 0Ó 2°3°BÐ 7Ø×$Ñ$Ø×$Ñ$ð;
ñ !
Ðð "×&Ò&Ð(=Ð>×FÑFÀqÈ!ÈQÐPQÓRˆà!§J¡JÑ.ˆð Ñ"Ü#Ÿjšj¨×):Ñ):¸1Ó)=¸|×?PÑ?PÐQRÓ?SÓT×WÑWÐXd×XkÑXkÓlˆOØ(ª¨D²!°TÐ)9Ñ:¸\Ê!ÈTÐSWÒYZÐJZÑ=[Ñ[ˆä—|’| K°×1DÑ1DÀRÈÓ1LÓMˆØ# l×&7Ñ&7¸BÈÐ&7Ð&M×&TÑ&TÓ&VÑVˆØ'×3Ñ3Ø�Ñ×%Ñ%Ó'¬¯ª°\×5GÑ5GÓ)H×)LÑ)Ló
Ðô Ÿ*š*¨Ò,Ð-=Ó>ˆô Ÿš _°kÓBˆà%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ò*Ð,CÐDˆàŸ™ mÓ4ˆàÐùòq BùòL <ùò @s"   ‰Q“QÅ"QÅ,QÆQÆQc                 óz  • Ubv  UR                   u  pÞnUcc  [        R                  " XÞ5      R                  U	R                  5      U	R                  Xí4S9R                  -  R                  UR                  5      nOS nUby  UR                   u  nnnUce  [        R                  " UU5      R                  U	R                  5      U
R                  UU4S9R                  -  R                  UR                  5      nOS nUby  UR                   u  nnnUce  [        R                  " UU5      R                  U	R                  5      UR                  UU4S9R                  -  R                  UR                  5      nOS nUnUb  UR                  5       (       d0  Ub  UR                  5       (       d  UbŽ  UR                  5       (       ay  UnU R                  UUUUUUUUS9n[        R                  " U R                  5      U-  nUU-   nUnU R                  U5      [        R                  " U R                  5      -  nUU-   nU$ )N)r>   )r_  r`  ra  rb  rc  rd  re  rf  )rå   r@   r¿   ræ   re   ÚexpandÚTÚanyrx  ÚtanhrT  r(  rU  )rU   r_  Úprotein_kv_statesÚstructure_kv_statesÚmsa_kv_statesrc  rd  re  rf  Úprotein_batch_maskÚstructure_batch_maskÚmsa_batch_maskÚpast_key_valuesró   Úprotein_kv_seq_lenr;   Ústructure_kv_seq_lenÚmsa_kv_seq_lenr�   Úresiduals                       rI   rÇ   Ú+EvollaSequenceAlignerCrossAttention.forward  sP  € ð Ñ(Ø*;×*AÑ*AÑ'ˆB CØ#Ñ+ä—J’J˜rÓ6×9Ñ9Ð:L×:SÑ:SÓTØ(×/Ñ/Ð6HÐ5MÐ/ÐN×PÑPñQç‘"Ð&×-Ñ-Ó.ð %øð
 $(Ð àÑ*Ø,?×,EÑ,EÑ)ˆBÐ$ cØ%Ñ-ä—J’J˜rÐ#7Ó8×;Ñ;Ð<N×<UÑ<UÓVØ*×1Ñ1Ð8LÈbÐ7QÐ1ÐR×TÑTñUç‘"Ð(×/Ñ/Ó0ð 'øð
 &*Ð"àÑ$Ø&3×&9Ñ&9Ñ#ˆB� ØÑ'ä—J’J˜r >Ó2×5Ñ5Ð6H×6OÑ6OÓPØ$×+Ñ+°.À"Ð1EÐ+ÐF×HÑHñIç‘"�]×)Ñ)Ó*ð !øð
  $ÐØ$ˆð Ñ*Ð/C×/GÑ/G×/IÑ/IØ#Ñ/Ð4J×4NÑ4N×4PÑ4PØÑ)Ð.>×.BÑ.B×.DÑ.Dà$ˆHØ ×0Ñ0Ø*Ø):Ø+>Ø%2Ø /Ø%9Ø'=Ø!1ð 1ð 	ˆMô "ŸJšJ t×':Ñ':Ó;¸mÑKˆMà$ }Ñ4ˆMà$ˆHØ ŸG™G MÓ2´U·Z²ZÀÇÁÓ5NÑNˆMØ$ }Ñ4ˆMàÐrK   )rE  rD  rP  r4   r(  rT  rU  r  rM  rI  rK  rC  rR  r/   rÏ   rN  rJ  rL  )NNN)NNNNNNN)
rZ   r[   r\   r]   r  rR   rx  rÇ   r^   r_   r`   s   @rI   r=  r=  u  sm   ø† ð +/Ø,0Ø&*ñ1:ð ! 4™Zð1:ð  # T™zð	1:ð
 ˜t™÷1:ð 1:òfnðn "Ø#ØØØ!ØØ÷Gò GrK   r=  c                   ó   • \ rS rSrSrg)rO  ic  rY   Nrq   rY   rK   rI   rO  rO  c  rr   rK   rO  c                   ó   • \ rS rSrSrg)ÚEvollaRotaryEmbeddingig  rY   Nrq   rY   rK   rI   r�  r�  g  rr   rK   r�  c                   ó   • \ rS rSrSrg)Ú	EvollaMLPik  rY   Nrq   rY   rK   rI   r�  r�  k  rr   rK   r�  c                   ó   • \ rS rSrSrg)ÚEvollaAttentionio  rY   Nrq   rY   rK   rI   r‘  r‘  o  rr   rK   r‘  c                   óÐ  ^ • \ rS rSrS\S\4U 4S jjr            SS\R                  S\	\R                  \R                  4   S-  S\R                  S-  S	\R                  S-  S
\S-  S\S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  4S jjrSrU =r$ )ÚEvollaDecoderLayeris  rV   Ú	layer_idxc                 ó²   >• [         TU ]  X5        US-   [        UR                  UR                  -  S5      -  S:X  a  [        UUR                  S9U l        g g )Nr+   r   )r>  )rQ   rR   ÚmaxÚnum_hidden_layersÚaligner_num_add_layersr=  r  Úadapter©rU   rV   r”  rW   s      €rI   rR   ÚEvollaDecoderLayer.__init__t  sY   ø€ Ü‰Ñ˜Ô+Ø˜‰MœS ×!9Ñ!9¸V×=ZÑ=ZÑ!ZÐ\]Ó^Ñ^ÐbcÓcÜ>ØØ$*×$6Ñ$6ñˆD�Lð drK   Nr�   r½   r2   rS   r…  Ú	use_cacher  r€  r�  r‚  rƒ  r„  rc  c                 ó
  • UnU R                  U5      nU R                  " SUUUUUUS.UD6u  nnXñ-   nUnU R                  U5      nU R                  U5      nXñ-   n[	        U S5      (       a  U R                  UUUU	UU
UUS9nU$ )N)r�   r2   rS   r…  rœ  r½   r™  )r_  r  r€  r�  rc  r‚  rƒ  r„  rY   )Úinput_layernormÚ	self_attnÚpost_attention_layernormÚmlpÚhasattrr™  )rU   r�   r½   r2   rS   r…  rœ  r  r€  r�  r‚  rƒ  r„  rc  r5   r‰  r™   s                    rI   rÇ   ÚEvollaDecoderLayer.forward|  sÆ   € ð" !ˆà×,Ñ,¨]Ó;ˆð  Ÿ>š>ð 
Ø'Ø)Ø%Ø+ØØ 3ñ
ð ñ
Ñˆ�qð !Ñ0ˆð !ˆØ×5Ñ5°mÓDˆØŸ™ Ó/ˆØ Ñ0ˆä�4˜×#Ñ#Ø ŸL™LØ*Ø"3Ø$7Ø+Ø /Ø#5Ø%9Ø-ð )ð 	ˆMð ÐrK   )r™  )NNNNFNNNNNNN)rZ   r[   r\   r]   r,   r  rR   r@   rl   rk   r;  r   rè   rÇ   r^   r_   r`   s   @rI   r“  r“  s  s>  ø† ð˜|ð ¸÷ ð IMØ.2Ø04Ø(,Ø!&Ø15Ø37Ø-1Ø26Ø48Ø.2Ø/3ñ3à—|‘|ð3ð # 5§<¡<°·±Ð#=Ñ>ÀÑEð3ð Ÿ™ tÑ+ð	3ð
 ×&Ñ&¨Ñ-ð3ð  ™ð3ð ˜$‘;ð3ð !Ÿ<™<¨$Ñ.ð3ð #Ÿ\™\¨DÑ0ð3ð —|‘| dÑ*ð3ð "ŸL™L¨4Ñ/ð3ð $Ÿl™l¨TÑ1ð3ð Ÿ™ tÑ+ð3ð Ÿ™¨Ñ,÷3ó 3rK   r“  c                   óV   • \ rS rSrSrSrSr/ SQr\R                  " 5       S 5       r
Srg)ÚEvollaPreTrainedModeli²  F)r“  r€   r  r=  c                 óÈ  • U R                   R                  n[        R                  " X5        [	        U[
        5      (       ak  [        R                  " UR                  5        [        R                  " UR                  5        [        R                  " UR                  R                  5        g [	        U[        5      (       a!  [        R                  " UR                  SUS9  g g )NrB  )ÚmeanÚstd)rV   Úinitializer_ranger   r”   r•   r=  r–   Úzeros_rT  rU  Úones_rP  Úweightr  Únormal_rë   )rU   r.   r¨  s      rI   r”   Ú#EvollaPreTrainedModel._init_weights½  s�   € à�k‰k×+Ñ+ˆÜ×%Ò% dÔ3Ü�fÔA×BÑBÜ�KŠK˜×-Ñ-Ô.Ü�KŠK˜Ÿ™Ô(Ü�JŠJ�v×,Ñ,×3Ñ3Õ4Ü˜Ô A×BÑBÜ�LŠL˜Ÿ™¨c°sÓ;ð CrK   rY   N)rZ   r[   r\   r]   r�   rŸ   r    rœ   r@   r¢   r”   r^   rY   rK   rI   r¥  r¥  ²  s6   † Ø ÐØÐØ"'ÐòÐð ‡]‚]ƒ_ñ<ó ó<rK   r¥  c                   óÊ  ^ • \ rS rSrS\4U 4S jjrS rS r\\	\
            SS\R                  S-  S\R                  S-  S	\R                  S-  S
\S-  S\R                  S-  S\S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\\-  4S jj5       5       5       rSrU =r$ )ÚEvollaModeliÉ  rV   c           
      ó8  >• [         TU ]  U5        UR                  U l        UR                  U l        [
        R                  " U R                  UR                  U R                  5      U l        [        US9U l
        [
        R                  " [        UR                  5       Vs/ s H  n[        UUS9PM     sn5      U l        [!        UR                  UR"                  S9U l        ['        USS5      U l        [+        US9U l        U R/                  5         g s  snf )Nr¨   )rV   r”  )ÚepsÚgradient_checkpointingF)rQ   rR   Úpad_token_idÚpadding_idxÚ
vocab_sizer   Ú	Embeddingr  Úembed_tokensr1  Úprotein_encoderr  r  r—  r“  r  rO  Úrms_norm_epsr  r?   r³  r�  Ú
rotary_embr¬   rš  s      €rI   rR   ÚEvollaModel.__init__Ê  sð   ø€ Ü‰Ñ˜Ô Ø!×.Ñ.ˆÔØ ×+Ñ+ˆŒÜŸLšL¨¯©¸&×:LÑ:LÈd×N^ÑN^Ó_ˆÔÜ3¸6ÑBˆÔÜ—m’mô "' v×'?Ñ'?Ô!@óò
 "A�Iô	 #Ø!Ø'ôñ "Añó
ˆŒô " &×"4Ñ"4¸&×:MÑ:MÑNˆŒ	Ü&-¨fÐ6NÐPUÓ&VˆÔ#Ü/°vÑ>ˆŒØ�‰Õùòs   Â#Dc                 ó   • U R                   $ rP   ©r¸  r±   s    rI   r²   Ú EvollaModel.get_input_embeddingsß  s   € Ø× Ñ Ð rK   c                 ó   • Xl         g rP   r¾  rµ   s     rI   r¶   Ú EvollaModel.set_input_embeddingsâ  s   € Ø!ÕrK   Nr¸   r2   rS   r…  r¼   rœ  Úprotein_input_idsÚprotein_attention_maskÚstructure_featsÚ	msa_featsrƒ  r„  rg   c                 óä  • USL USL-  (       a  [        S5      eUc  U R                  U5      nU(       a  Uc  [        U R                  S9nUcU  Ub  UR	                  5       OSn[
        R                  " UR                  S   UR                  S9U-   nUR                  S5      nSnSnUbZ  UbW  U R                  UUS9nUR                  n[
        R                  " UR                  S   UR                  [
        R                  S9n[        U R                  UUUS	9nUnU R                  UUS
9nU R                    H  nU" U4UUUUUU	U
UUUUUS.UD6nM     U R#                  U5      n[%        UUS9nU$ )aÑ  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence in structure-aware tokens. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.
structure_feats (torch.FloatTensor):
    The input IDs for purely structure-based features. Should be of shape `(batch_size, structure_seq_length, structure_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
msa_feats (torch.FloatTensor):
    The input IDs for purely MSA-based features. Should be of shape `(batch_size, msa_seq_length, msa_feat_dim)` and type `torch.FloatTensor`. Dummy input for now.
structure_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely structure-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `structure_feats`. Dummy input for now.
msa_batch_mask (torch.Tensor):
    The batch mask to decide which protein sequences are purely MSA-based. Should be of shape `(batch_size)` and type `torch.Tensor`. Should be paired with `msa_feats`. Dummy input for now.
Nz:You must specify exactly one of input_ids or inputs_embedsr¨   r   r+   rº   r»   )re   r$  )rV   r¼   r2   r…  )rS   )r2   rS   r…  rœ  r  r€  r�  r‚  rƒ  r„  rc  r½   )r¾   r…  )r[  r¸  r   rV   Úget_seq_lengthr@   rÀ   rå   re   rÁ   r¹  r-  r¿   rè   r   r»  r  r  r   )rU   r¸   r2   rS   r…  r¼   rœ  rÂ  rÃ  rÄ  rÅ  rƒ  r„  r5   Úpast_seen_tokensÚprotein_featsr‚  Úprotein_outputsÚcausal_maskr�   r½   Údecoder_layerÚoutputs                          rI   rÇ   ÚEvollaModel.forwardå  sº  € ðB ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMæ˜Ñ0Ü*°$·+±+Ñ>ˆOàÑØCRÑC^˜×=Ñ=Ô?ÐdeÐÜ Ÿ<š<¨×(;Ñ(;¸AÑ(>À}×G[ÑG[Ñ\Ð_oÑoˆLØ'×1Ñ1°!Ó4ˆLàˆØ!ÐàÑ(Ð-CÑ-OØ"×2Ñ2Ø+Ø5ð 3ð ˆOð ,×FÑFˆMÜ!&§¢Ø!×'Ñ'¨Ñ*Ø(×/Ñ/Ü—j‘jñ"Ðô )Ø—;‘;Ø'Ø)Ø+ñ	
ˆð &ˆØ"Ÿo™o¨mÈ,˜oÐWÐà!Ÿ[œ[ˆMÙ)Øðà*Ø)Ø /Ø#Ø"/Ø$3Ø'Ø#5Ø%9Ø-Ø .Ø$7ñð ñŠMñ )ð$ Ÿ	™	 -Ó0ˆä(Ø+Ø+ñ
ˆð ˆrK   )r¸  r³  r  r  rµ  r¹  r»  r¶  )NNNNNNNNNNNN)rZ   r[   r\   r]   r,   rR   r²   r¶   r   r   r   r@   r;  rl   r   r/  rè   rk   r   rÇ   r^   r_   r`   s   @rI   r°  r°  É  sf  ø† ð˜|÷ ò*!ò"ð ØØð .2Ø.2Ø04Ø(,Ø26Ø!%Ø59Ø6:Ø48Ø.2Ø48Ø.2ñ]à×#Ñ# dÑ*ð]ð Ÿ™ tÑ+ð]ð ×&Ñ&¨Ñ-ð	]ð
  ™ð]ð ×(Ñ(¨4Ñ/ð]ð ˜$‘;ð]ð !×+Ñ+¨dÑ2ð]ð !&§¡¨tÑ 3ð]ð ×*Ñ*¨TÑ1ð]ð ×$Ñ$ tÑ+ð]ð $Ÿl™l¨TÑ1ð]ð Ÿ™ tÑ+ð]ð 
Ð(Ñ	(ô]ó ó  ó ö]rK   r°  c                   óB  ^ • \ rS rSrU 4S jrS rS r\\        SS\	R                  S-  S\	R                  S-  S\	R                  S-  S	\	R                  S-  S
\	R                  S-  S\	R                  S-  S\S-  S\\	R                  -  4S jj5       5       rSrU =r$ )ÚEvollaForProteinText2TextiH  c                 óä   >• [         TU ]  U5        [        U5      U l        UR                  U l        [
        R                  " UR                  U R                  SS9U l        U R                  5         g r  )
rQ   rR   r°  r3  r¶  r   rÔ   r  Úlm_headr¬   rT   s     €rI   rR   Ú"EvollaForProteinText2Text.__init__I  sQ   ø€ Ü‰Ñ˜Ô Ü  Ó(ˆŒ
Ø ×+Ñ+ˆŒÜ—y’y ×!3Ñ!3°T·_±_È5ÑQˆŒà�‰ÕrK   c                 ó6   • U R                   R                  5       $ rP   )r3  r²   r±   s    rI   r²   Ú.EvollaForProteinText2Text.get_input_embeddingsQ  s   € Ø�z‰z×.Ñ.Ó0Ð0rK   c                 ó8   • U R                   R                  U5      $ rP   )r3  r¶   rµ   s     rI   r¶   Ú.EvollaForProteinText2Text.set_input_embeddingsT  s   € Ø�z‰z×.Ñ.¨uÓ5Ð5rK   Nr¸   r2   r¼   ÚlabelsrÂ  rÃ  rœ  Úlogits_to_keepc	           
      ól  • U R                   " SUUUUUUS.U	D6n
U
R                  n[        U[        5      (       a  [	        U* S5      OUnU R                  USS2USS24   5      nSnUb  U R                  " SXÔU R                  S.U	D6n[        UUU
R                  U
R                  U
R                  S9nU$ )a|  
protein_input_ids (torch.LongTensor):
    The input IDs for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.LongTensor`.
protein_attention_mask (torch.Tensor):
    The attention mask for the protein sequence. Should be of shape `(batch_size, protein_seq_length)` and type `torch.Tensor`.

Example:

```python
>>> from transformers import EvollaProcessor, EvollaForProteinText2Text
>>> model = EvollaForProteinText2Text.from_pretrained("westlake/Evolla-10B-hf")
>>> processor = EvollaProcessor.from_pretrained("westlake/Evolla-10B-hf")

>>> protein_information = {
    "aa_seq": "your amino acid sequence",
    "foldseek": "your foldseek sequence",
}
>>> question = "What is the function of this protein?"
>>> message = [
    {"role": "system", "content": "You are an AI expert that can answer any questions about protein."},
    {"role": "user", "content": question},
]

>>> inputs = processor(proteins=[protein_information], messages_list=[message], return_tensors="pt", padding="longest")
>>> outputs = model.generate(**inputs)

>>> print(processor.batch_decode(outputs, skip_special_tokens=True))
```)r¸   r2   r¼   rÂ  rÃ  rœ  N)ÚlogitsrØ  r¶  )ÚlossrÛ  r…  r�   r�   rY   )r3  r¾   r•   r  ÚslicerÒ  Úloss_functionr¶  r   r…  r�   r�   )rU   r¸   r2   r¼   rØ  rÂ  rÃ  rœ  rÙ  r5   Úoutputsr�   Úslice_indicesrÛ  rÜ  Ú
lm_outputss                   rI   rÇ   Ú!EvollaForProteinText2Text.forwardW  sÖ   € ðT ,0¯:ª:ð ,
ØØ)Ø'Ø/Ø#9Øñ,
ð ñ,
ˆð  ×1Ñ1ˆä8BÀ>ÔSV×8WÑ8Wœ˜~˜o¨tÔ4Ð]kˆØ—‘˜mªA¨}ºaÐ,?Ñ@ÓAˆàˆØÑØ×%Ò%Ði¨VÈtÏÉÑiÐbhÑiˆDä+ØØØ#×3Ñ3Ø!×/Ñ/Ø×)Ñ)ñ
ˆ
ð ÐrK   )rÒ  r3  r¶  )NNNNNNNr   )rZ   r[   r\   r]   rR   r²   r¶   r   r   r@   r;  rl   r/  rè   r  rÇ   r^   r_   r`   s   @rI   rÐ  rÐ  H  së   ø† õò1ò6ð Øð .2Ø.2Ø26Ø*.Ø59Ø6:Ø!%Ø-.ñBà×#Ñ# dÑ*ðBð Ÿ™ tÑ+ðBð ×(Ñ(¨4Ñ/ð	Bð
 × Ñ  4Ñ'ðBð !×+Ñ+¨dÑ2ðBð !&§¡¨tÑ 3ðBð ˜$‘;ðBð ˜eŸl™lÑ*ôBó ó öBrK   rÐ  )rÐ  r°  r¥  )NrB  )ZÚdataclassesr   r@   r   Ú r   r–   Úcache_utilsr   r   Ú
generationr	   Úmasking_utilsr
   r   Úmodeling_outputsr   r   r   r   Úmodeling_utilsr   Úprocessing_utilsr   Úutilsr   r   r   r   Úutils.genericr   Úutils.output_capturingr   r   Úesm.modeling_esmr   r   r   r   r   r   r    r!   r"   r#   Úllama.modeling_llamar$   r%   r&   r'   r(   r)   r*   Úconfiguration_evollar,   r-   Ú
get_loggerrZ   ÚloggerÚModulerl   rm   rJ   rM   rb   ro   rt   rw   rz   r}   r€   rƒ   r†   r‰   r¤   rÊ   r   r  r,  r1  r=  rO  r�  r�  r‘  r“  r¥  r°  rÐ  Ú__all__rY   rK   rI   Ú<module>rõ     sˆ  ðõ "ã Ý å &ß .Ý )ß J÷ó õ .Ý &÷ó õ 8ß E÷÷ ÷ ÷÷ ñ ÷ =ð 
×	Ò	˜HÓ	%€ð !Øñ %Ø�I‰Ið %à�<‰<ð %ð 
�‰ð %ð �<‰<ð	 %ð
 —L‘L 4Ñ'ð %ð �T‰\ð %ð ð %ð Ð'Ñ(õ %ôF˜]ô ô?Ð"4ô ?ô	Ð 0ô 	ô	˜]ô 	ô	˜Lô 	ô	˜ô 	ô	˜ô 	ô	˜ô 	ô	˜*ô 	ô	˜ô 	ð ôC /ó Có ðCô01
Ð!<ô 1
ôh7 ¨¯	©	ô 7 ôtA˜Ÿ	™	ô Aô'.¨¯	©	ô '.ðT Ø
ô
< kó 
<ó ó ð
<ô
˜2Ÿ9™9ô 
ô$k¨"¯)©)ô kô\	�Lô 	ô	Ð0ô 	ô	�ô 	ô	�nô 	ô<Ð*ô <ô~<Ð0ô <ô.|Ð'ô |ô~SÐ 5°ô Sòl P�rK   