ó
    ˆŽ¡jo ã                   óŽ  • S r SSKrSSKrSSKJr  SSKrSSKJr  SSKJrJ	r	J
r
  SSKJr  SSKJr  SS	KJrJrJr  SS
KJr  SSKJrJr  SSKJr  SSKJrJrJrJrJ r J!r!J"r"  SSK#J$r$J%r%  SSK&J'r'  SSK(J)r)J*r*J+r+J,r,J-r-J.r.  SSK/J0r0J1r1  SSK2J3r3J4r4  SSK5J6r6  \-Rn                  " \85      r9   SES\Rt                  S\Rv                  S\Rv                  S\Rv                  S\Rv                  S-  S\<S-  S\<S\Rv                  S-  S\'\+   4S jjr= " S S \Rt                  5      r> " S! S"\Rt                  5      r? " S# S$\Rt                  5      r@ " S% S&\Rt                  5      rA " S' S(\Rt                  5      rB " S) S*\Rt                  5      rC " S+ S,\Rt                  5      rD " S- S.\5      rE " S/ S0\Rt                  5      rF\, " S1 S2\%5      5       rG " S3 S4\G5      rH\, " S5 S6\G5      5       rI\," S7S89 " S9 S:\G\5      5       rJ\, " S; S<\G5      5       rK\," S=S89 " S> S?\G5      5       rL\, " S@ SA\G5      5       rM\, " SB SC\G5      5       rN/ SDQrOg)FzPyTorch mT5 model.é    N)ÚCallable)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)ÚACT2FN)ÚCacheÚDynamicCacheÚEncoderDecoderCache)ÚGenerationMixin)Úcreate_bidirectional_maskÚcreate_causal_mask)ÚGradientCheckpointingLayer)ÚBaseModelOutputÚ)BaseModelOutputWithPastAndCrossAttentionsÚSeq2SeqLMOutputÚSeq2SeqModelOutputÚ#Seq2SeqQuestionAnsweringModelOutputÚSeq2SeqSequenceClassifierOutputÚTokenClassifierOutput)ÚALL_ATTENTION_FUNCTIONSÚPreTrainedModel)ÚUnpack)ÚDUMMY_INPUTSÚ
DUMMY_MASKÚTransformersKwargsÚauto_docstringÚloggingÚtorch_compilable_check)Úcan_return_tupleÚmerge_with_config_defaults)ÚOutputRecorderÚcapture_outputsé   )Ú	MT5ConfigÚmoduleÚqueryÚkeyÚvalueÚattention_maskÚscalingÚdropoutÚposition_biasÚkwargsc                 óœ  • Uc  UR                  S5      S-  n[        R                  " XR                  SS5      5      U-  n	Ub  X—-   n	Ub  X”-   n	[        R
                  R                  U	SS9n	[        R
                  R                  X–U R                  S9n	[        R                  " X“5      n
U
R                  SS5      R                  5       n
X©4$ )Néÿÿÿÿç      à¿é   r   ©Údim)ÚpÚtrainingr&   )
ÚsizeÚtorchÚmatmulÚ	transposer   Ú
functionalÚsoftmaxr.   r8   Ú
contiguous)r(   r)   r*   r+   r,   r-   r.   r/   r0   Úattn_weightsÚattn_outputs              Ú²C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/mt5/modeling_mt5.pyÚeager_attention_forwardrC   3   s¿   € ð �Ø—*‘*˜R“. DÑ(ˆô —<’< §}¡}°Q¸Ó':Ó;¸gÑE€LàÑ Ø#Ñ3ˆàÑ!Ø#Ñ4ˆä—=‘=×(Ñ(¨¸2Ð(Ð>€LÜ—=‘=×(Ñ(¨È6Ï?É?Ð(Ð[€Lä—,’,˜|Ó3€KØ×'Ñ'¨¨1Ó-×8Ñ8Ó:€KàÐ$Ð$ó    c                   ó2   ^ • \ rS rSrSU 4S jjrS rSrU =r$ )ÚMT5LayerNorméT   c                 óŒ   >• [         TU ]  5         [        R                  " [        R
                  " U5      5      U l        X l        g)zT
Construct a layernorm module in the MT5 style. No bias and no subtraction of mean.
N)ÚsuperÚ__init__r   Ú	Parameterr:   ÚonesÚweightÚvariance_epsilon)ÚselfÚhidden_sizeÚepsÚ	__class__s      €rB   rJ   ÚMT5LayerNorm.__init__U   s/   ø€ ô 	‰ÑÔÜ—l’l¤5§:¢:¨kÓ#:Ó;ˆŒØ #ÕrD   c                 óœ  • UR                  [        R                  5      R                  S5      R	                  SSS9nU[        R
                  " X R                  -   5      -  nU R                  R                  [        R                  [        R                  4;   a%  UR                  U R                  R                  5      nU R                  U-  $ )Nr4   r2   T)Úkeepdim)Útor:   Úfloat32ÚpowÚmeanÚrsqrtrN   rM   ÚdtypeÚfloat16Úbfloat16)rO   Úhidden_statesÚvariances      rB   ÚforwardÚMT5LayerNorm.forward]   s™   € ð !×#Ñ#¤E§M¡MÓ2×6Ñ6°qÓ9×>Ñ>¸rÈ4Ð>ÐPˆØ%¬¯ª°H×?TÑ?TÑ4TÓ(UÑUˆð �;‰;×Ñ¤§¡´·±Ð ?Ó?Ø)×,Ñ,¨T¯[©[×->Ñ->Ó?ˆMà�{‰{˜]Ñ*Ð*rD   )rN   rM   )g�íµ ÷Æ°>)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__rJ   r`   Ú__static_attributes__Ú__classcell__©rR   s   @rB   rF   rF   T   s   ø† ÷$÷+ð +rD   rF   c                   ó6   ^ • \ rS rSrS\4U 4S jjrS rSrU =r$ )ÚMT5DenseActDenseén   Úconfigc                 óX  >• [         TU ]  5         [        R                  " UR                  UR
                  SS9U l        [        R                  " UR
                  UR                  SS9U l        [        R                  " UR                  5      U l
        [        UR                     U l        g ©NF©Úbias)rI   rJ   r   ÚLinearÚd_modelÚd_ffÚwiÚwoÚDropoutÚdropout_rater.   r
   Údense_act_fnÚact©rO   rl   rR   s     €rB   rJ   ÚMT5DenseActDense.__init__o   sn   ø€ Ü‰ÑÔÜ—)’)˜FŸN™N¨F¯K©K¸eÑDˆŒÜ—)’)˜FŸK™K¨¯©¸eÑDˆŒÜ—z’z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�rD   c                 ó  • U R                  U5      nU R                  U5      nU R                  U5      n[        U R                  R
                  [        R                  5      (       a�  UR                  U R                  R
                  R                  :w  aa  U R                  R
                  R                  [        R                  :w  a/  UR                  U R                  R
                  R                  5      nU R	                  U5      nU$ ©N)rt   ry   r.   Ú
isinstanceru   rM   r:   ÚTensorr[   Úint8rV   ©rO   r^   s     rB   r`   ÚMT5DenseActDense.forwardv   s­   € ØŸ™ Ó.ˆØŸ™ Ó/ˆØŸ™ ]Ó3ˆä�t—w‘w—~‘~¤u§|¡|×4Ñ4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ó;Ø—‘—‘×$Ñ$¬¯
©
Ó2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMØŸ™ Ó.ˆØÐrD   )ry   r.   rt   ru   ©	rb   rc   rd   re   r'   rJ   r`   rf   rg   rh   s   @rB   rj   rj   n   s   ø† ð/˜y÷ /÷ð rD   rj   c                   ó6   ^ • \ rS rSrS\4U 4S jjrS rSrU =r$ )ÚMT5DenseGatedActDenseé…   rl   c                 ó¶  >• [         TU ]  5         [        R                  " UR                  UR
                  SS9U l        [        R                  " UR                  UR
                  SS9U l        [        R                  " UR
                  UR                  SS9U l        [        R                  " UR                  5      U l        [        UR                     U l        g rn   )rI   rJ   r   rq   rr   rs   Úwi_0Úwi_1ru   rv   rw   r.   r
   rx   ry   rz   s     €rB   rJ   ÚMT5DenseGatedActDense.__init__†   sŠ   ø€ Ü‰ÑÔÜ—I’I˜fŸn™n¨f¯k©kÀÑFˆŒ	Ü—I’I˜fŸn™n¨f¯k©kÀÑFˆŒ	Ü—)’)˜FŸK™K¨¯©¸eÑDˆŒÜ—z’z &×"5Ñ"5Ó6ˆŒÜ˜&×-Ñ-Ñ.ˆ�rD   c                 ó8  • U R                  U R                  U5      5      nU R                  U5      nX#-  nU R                  U5      n[	        U R
                  R                  [        R                  5      (       a�  UR                  U R
                  R                  R                  :w  aa  U R
                  R                  R                  [        R                  :w  a/  UR                  U R
                  R                  R                  5      nU R                  U5      nU$ r}   )ry   rˆ   r‰   r.   r~   ru   rM   r:   r   r[   r€   rV   )rO   r^   Úhidden_geluÚhidden_linears       rB   r`   ÚMT5DenseGatedActDense.forwardŽ   sÀ   € Ø—h‘h˜tŸy™y¨Ó7Ó8ˆØŸ	™	 -Ó0ˆØ#Ñ3ˆØŸ™ ]Ó3ˆô �t—w‘w—~‘~¤u§|¡|×4Ñ4Ø×#Ñ# t§w¡w§~¡~×';Ñ';Ó;Ø—‘—‘×$Ñ$¬¯
©
Ó2à)×,Ñ,¨T¯W©W¯^©^×-AÑ-AÓBˆMàŸ™ Ó.ˆØÐrD   )ry   r.   rˆ   r‰   ru   rƒ   rh   s   @rB   r…   r…   …   s   ø† ð/˜y÷ /÷ð rD   r…   c                   ó6   ^ • \ rS rSrS\4U 4S jjrS rSrU =r$ )Ú
MT5LayerFFé£   rl   c                 ó  >• [         TU ]  5         UR                  (       a  [        U5      U l        O[        U5      U l        [        UR                  UR                  S9U l	        [        R                  " UR                  5      U l        g )N©rQ   )rI   rJ   Úis_gated_actr…   ÚDenseReluDenserj   rF   rr   Úlayer_norm_epsilonÚ
layer_normr   rv   rw   r.   rz   s     €rB   rJ   ÚMT5LayerFF.__init__¤   s_   ø€ Ü‰ÑÔØ××Ü"7¸Ó"?ˆDÕä"2°6Ó":ˆDÔä& v§~¡~¸6×;TÑ;TÑUˆŒÜ—z’z &×"5Ñ"5Ó6ˆ�rD   c                 óp   • U R                  U5      nU R                  U5      nXR                  U5      -   nU$ r}   )r—   r•   r.   )rO   r^   Úforwarded_statess      rB   r`   ÚMT5LayerFF.forward®   s;   € ØŸ?™?¨=Ó9ÐØ×.Ñ.Ð/?Ó@ÐØ%¯©Ð5EÓ(FÑFˆØÐrD   )r•   r.   r—   rƒ   rh   s   @rB   r�   r�   £   s   ø† ð7˜y÷ 7÷ð rD   r�   c                   ó†   ^ • \ rS rSr   SS\S\S-  S\4U 4S jjjr\SS j5       r	SS jr
    SS	\\   4S
 jjrSrU =r$ )ÚMT5Attentioné¶   Nrl   Ú	layer_idxÚ	is_causalc                 ó  >• [         TU ]  5         Xl        UR                  U l        X@l        X l        UR                  U l        UR                  U l        UR                  U l        UR                  U l
        UR                  U l        UR                  U l        U R                  U R                  -  U l        SU l        X0l        Uc>  U R                  (       a-  [$        R'                  SU R(                  R*                   S35        [,        R.                  " U R                  U R                  SS9U l        [,        R.                  " U R                  U R                  SS9U l        [,        R.                  " U R                  U R                  SS9U l        [,        R.                  " U R                  U R                  SS9U l        U R
                  (       a0  [,        R8                  " U R                  U R                  5      U l        SU l        g )Nç      ð?zInstantiating a decoder z³ without passing `layer_idx` is not recommended and will to errors during the forward call, if caching is used. Please make sure to provide a `layer_idx` when creating this class.Fro   )rI   rJ   rl   Ú
is_decoderr    Úhas_relative_attention_biasÚrelative_attention_num_bucketsÚrelative_attention_max_distancerr   Úd_kvÚkey_value_proj_dimÚ	num_headsÚn_headsrw   r.   Ú	inner_dimr-   rŸ   ÚloggerÚwarning_oncerR   rb   r   rq   ÚqÚkÚvÚoÚ	EmbeddingÚrelative_attention_biasÚgradient_checkpointing)rO   rl   r¤   rŸ   r    rR   s        €rB   rJ   ÚMT5Attention.__init__·   sv  ø€ ô 	‰ÑÔØŒØ ×+Ñ+ˆŒØ"ŒØ+FÔ(Ø.4×.SÑ.SˆÔ+Ø/5×/UÑ/UˆÔ,Ø—~‘~ˆŒØ"(§+¡+ˆÔØ×'Ñ'ˆŒØ×*Ñ*ˆŒØŸ™¨×(?Ñ(?Ñ?ˆŒàˆŒØ"ŒØÑ §§Ü×ÑØ*¨4¯>©>×+BÑ+BÐ*Cð D,ð ,ôô —’˜4Ÿ<™<¨¯©¸eÑDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÑDˆŒÜ—’˜4Ÿ<™<¨¯©¸eÑDˆŒÜ—’˜4Ÿ>™>¨4¯<©<¸eÑDˆŒà×+×+Ü+-¯<ª<¸×8[Ñ8[Ð]a×]iÑ]iÓ+jˆDÔ(à&+ˆÕ#rD   c                 ób  • SnU(       aC  US-  nX@S:„  R                  [        R                  5      U-  -  n[        R                  " U 5      n O,[        R                  " U [        R
                  " U 5      5      * n US-  nX:  nU[        R                  " U R                  5       U-  5      [        R                  " X5-  5      -  X%-
  -  R                  [        R                  5      -   n[        R                  " U[        R                  " XrS-
  5      5      nU[        R                  " X`U5      -  nU$ )aR  
Adapted from Mesh Tensorflow:
https://github.com/tensorflow/mesh/blob/0cb87fe07da627bf0b7e60475d59f95ed6b5be3d/mesh_tensorflow/transformer/transformer_layers.py#L593

Translate relative position to a bucket number for relative attention. The relative position is defined as
memory_position - query_position, i.e. the distance in tokens from the attending position to the attended-to
position. If bidirectional=False, then positive relative positions are invalid. We use smaller buckets for
small absolute relative_position and larger buckets for larger absolute relative_positions. All relative
positions >=max_distance map to the same bucket. All relative positions <=-max_distance map to the same bucket.
This should allow for more graceful generalization to longer sequences than the model has been trained on

Args:
    relative_position: an int32 Tensor
    bidirectional: a boolean - whether the attention is bidirectional
    num_buckets: an integer
    max_distance: an integer

Returns:
    a Tensor with the same shape as relative_position, containing int32 values in the range [0, num_buckets)
r   r4   r&   )rV   r:   ÚlongÚabsÚminÚ
zeros_likeÚlogÚfloatÚmathÚ	full_likeÚwhere)Úrelative_positionÚbidirectionalÚnum_bucketsÚmax_distanceÚrelative_bucketsÚ	max_exactÚis_smallÚrelative_position_if_larges           rB   Ú_relative_position_bucketÚ&MT5Attention._relative_position_bucketÞ   s  € ð, ÐÞØ˜AÑˆKØ°QÑ!6× :Ñ :¼5¿:¹:Ó FÈÑ TÑTÐÜ %§	¢	Ð*;Ó <Ñä!&§¢Ð+<¼e×>NÒ>NÐO`Ó>aÓ!bÐ bÐð   1Ñ$ˆ	Ø$Ñ0ˆð &/Ü�IŠIÐ'×-Ñ-Ó/°)Ñ;Ó<Ü�hŠh�|Ñ/Ó0ñ1àÑ&ñ(÷ ‰"ŒU�Z‰Z‹.ñ	&Ð"ô
 &+§Y¢YØ&¬¯ªÐ8RÐbcÑTcÓ(dó&
Ð"ð 	œEŸKšK¨ÐE_Ó`Ñ`ÐØÐrD   c                 óÖ  • Uc   U R                   R                  R                  n[        R                  " U[        R
                  US9SS2S4   U-   n[        R                  " U[        R
                  US9SSS24   nXe-
  nU R                  UU R                  (       + U R                  U R                  S9nU R                  U5      n	U	R                  / SQ5      R                  S5      n	U	$ )z%Compute binned relative position biasN)r[   Údevice)rÁ   rÂ   rÃ   )r4   r   r&   r   )r³   rM   rË   r:   Úaranger·   rÈ   r£   r¥   r¦   ÚpermuteÚ	unsqueeze)
rO   Úquery_lengthÚ
key_lengthrË   Úpast_seen_tokensÚcontext_positionÚmemory_positionrÀ   Úrelative_position_bucketÚvaluess
             rB   Úcompute_biasÚMT5Attention.compute_bias  sÙ   € à‰>Ø×1Ñ1×8Ñ8×?Ñ?ˆFÜ Ÿ<š<¨¼E¿J¹JÈvÑVÒWXÐZ^ÐW^Ñ_ÐbrÑrÐÜŸ,š, z¼¿¹ÈFÑSÐTXÒZ[ÐT[Ñ\ˆØ+Ñ>ÐØ#'×#AÑ#AØØ#Ÿ™Ô.Ø×;Ñ;Ø×=Ñ=ð	 $Bð $
Ð ð ×-Ñ-Ð.FÓGˆØ—‘¢	Ó*×4Ñ4°QÓ7ˆØˆrD   r0   c                 ó  • UR                   SS n/ UQSPU R                  P7nUb  UR                  U R                  5      OSn	[	        U	[
        R                  5      (       a  U	R                  5       OU	n	USLn
U R                  U5      R                  U5      R                  SS5      nSn[	        U[        5      (       aF  UR                  R                  U R                  5      nU
(       a  UR                  nOUR                  nOUnU
(       a  UOUnU
(       aQ  UbN  U(       aG  UR                   U R                     R"                  nUR                   U R                     R$                  nOØ/ UR                   SS QSPU R                  P7nU R'                  U5      R                  U5      R                  SS5      nU R)                  U5      R                  U5      R                  SS5      nUbU  UR+                  UUU R                  5      u  nnU
(       a.  [	        U[        5      (       a  SUR                  U R                  '   Uc¤  UR                   S   nU R,                  (       de  [
        R.                  " SU R0                  US   U4UR2                  UR4                  S	9nU R6                  (       a  U R8                  (       a  SUl        OU R=                  US   UUR2                  U	S
9n[>        R@                  " U RB                  RD                  [F        5      nU" U UUUU4U R8                  (       d  SOU RH                  U RJ                  US.UD6u  nnURL                  " / UQSP76 RO                  5       nU RQ                  U5      nUUU4$ )zp
Self-attention (if key_value_states is None) or attention over source sentence (provided by key_value_states).
Nr2   r   r&   r4   FTéþÿÿÿ)rË   r[   )rË   rÑ   ç        )r.   r-   r/   ))Úshaper¨   Úget_seq_lengthrŸ   r~   r:   r   Úcloner®   Úviewr<   r   Ú
is_updatedÚgetÚcross_attention_cacheÚself_attention_cacheÚlayersÚkeysrÕ   r¯   r°   Úupdater¤   Úzerosrª   rË   r[   r´   r8   Úrequires_gradrÖ   r   Úget_interfacerl   Ú_attn_implementationrC   r.   r-   Úreshaper?   r±   )rO   r^   ÚmaskÚkey_value_statesr/   Úpast_key_valuesr0   Úinput_shapeÚhidden_shaperÑ   Úis_cross_attentionÚquery_statesrß   Úcurr_past_key_valuesÚcurrent_statesÚ
key_statesÚvalue_statesÚkv_shaperÐ   Úattention_interfacerA   r@   s                         rB   r`   ÚMT5Attention.forward  sA  € ð $×)Ñ)¨#¨2Ð.ˆØB˜ÐB bÐB¨$×*AÑ*AÑBˆØM\ÑMh˜?×9Ñ9¸$¿.¹.ÔIÐnoÐä7AÐBRÔTY×T`ÑT`×7aÑ7aÐ+×1Ñ1Ô3ÐgwÐð .°TÐ9Ðà—v‘v˜mÓ,×1Ñ1°,Ó?×IÑIÈ!ÈQÓOˆð ˆ
Ü�oÔ':×;Ñ;Ø(×3Ñ3×7Ñ7¸¿¹ÓGˆJÞ!à'6×'LÑ'LÑ$à'6×'KÑ'KÑ$à#2Ð æ-?Ñ)À]ˆÞ /Ñ"=Æ*à-×4Ñ4°T·^±^ÑD×IÑIˆJØ/×6Ñ6°t·~±~ÑF×MÑM‰LàP˜×-Ñ-¨c¨rÐ2ÐP°BÐP¸×8OÑ8OÑPˆHØŸ™ Ó/×4Ñ4°XÓ>×HÑHÈÈAÓNˆJØŸ6™6 .Ó1×6Ñ6°xÓ@×JÑJÈ1ÈaÓPˆLàÑ*Ø+?×+FÑ+FÀzÐS_Ðae×aoÑaoÓ+pÑ(�
˜Læ%¬*°_ÔFY×*ZÑ*ZØAE�O×.Ñ.¨t¯~©~Ñ>ð Ñ Ø#×)Ñ)¨"Ñ-ˆJØ×3×3Ü %§¢Ø˜Ÿ™ k°!¡n°jÐAØ'×.Ñ.Ø&×,Ñ,ñ!�ð
 ×.×.°4·=·=Ø26�MÔ/øà $× 1Ñ 1Ø ‘N J°|×7JÑ7JÐ]mð !2ð !�ô )@×(MÒ(MØ�K‰K×,Ñ,Ô.Eó)
Ðñ %8ØØØØØð
%
ð  $Ÿ}Ÿ}‘C°$·,±,Ø—L‘LØ'ñ
%
ð ñ
%
Ñ!ˆ�\ð "×)Ò)Ð;¨;Ð;¸Ò;×FÑFÓHˆØ—f‘f˜[Ó)ˆà˜M¨<Ð7Ð7rD   )rl   rr   r.   r´   r¤   r«   r    r£   r¯   r¨   rŸ   rª   r±   r®   r³   r¦   r¥   r-   r°   )FNF)Té    é€   )Nr   ©NNNN)rb   rc   rd   re   r'   ÚintÚboolrJ   ÚstaticmethodrÈ   rÖ   r   r   r`   rf   rg   rh   s   @rB   r�   r�   ¶   s~   ø† ð %*Ø $Øñ%,àð%,ð ˜‘:ð	%,ð
 ÷%,ð %,ðN ó- ó ð- ô^ð( ØØØñX8ð Ð+Ñ,÷X8ó X8rD   r�   c                   óX   ^ • \ rS rSrSS\S-  4U 4S jjjr   S	S\\   4S jjrSr	U =r
$ )
ÚMT5LayerSelfAttentioni{  NrŸ   c                 óè   >• [         TU ]  5         [        UUUUR                  S9U l        [        UR                  UR                  S9U l        [        R                  " UR                  5      U l        g )N©r¤   rŸ   r    r“   )rI   rJ   r�   r£   ÚSelfAttentionrF   rr   r–   r—   r   rv   rw   r.   ©rO   rl   r¤   rŸ   rR   s       €rB   rJ   ÚMT5LayerSelfAttention.__init__|  s]   ø€ Ü‰ÑÔÜ)ØØ(CØØ×'Ñ'ñ	
ˆÔô ' v§~¡~¸6×;TÑ;TÑUˆŒÜ—z’z &×"5Ñ"5Ó6ˆ�rD   r0   c                 ó†   • U R                  U5      nU R                  " U4UUUS.UD6u  psnXR                  U5      -   nXsU4$ )N)rë   r/   rí   )r—   r  r.   )	rO   r^   r,   r/   rí   r0   Únormed_hidden_statesÚattention_outputr@   s	            rB   r`   ÚMT5LayerSelfAttention.forward‡  sd   € ð  $Ÿ™¨}Ó=ÐØ8<×8JÒ8JØ ð9
àØ'Ø+ñ	9
ð
 ñ9
Ñ5Ð¨ð )¯<©<Ð8HÓ+IÑIÐØ°Ð<Ð<rD   )r  r.   r—   ©FN©NNN©rb   rc   rd   re   rü   rJ   r   r   r`   rf   rg   rh   s   @rB   r   r   {  s>   ø† ñ	7ÈSÐSWÉZ÷ 	7ð 	7ð ØØñ=ð Ð+Ñ,÷=ó =rD   r   c                   óX   ^ • \ rS rSrSS\S-  4U 4S jjjr   S	S\\   4S jjrSr	U =r
$ )
ÚMT5LayerCrossAttentioniœ  NrŸ   c                 óÔ   >• [         TU ]  5         [        USUSS9U l        [	        UR
                  UR                  S9U l        [        R                  " UR                  5      U l        g )NFr  r“   )rI   rJ   r�   ÚEncDecAttentionrF   rr   r–   r—   r   rv   rw   r.   )rO   rl   rŸ   rR   s      €rB   rJ   ÚMT5LayerCrossAttention.__init__�  sU   ø€ Ü‰ÑÔÜ+Ø°ÀÐV[ñ 
ˆÔô ' v§~¡~¸6×;TÑ;TÑUˆŒÜ—z’z &×"5Ñ"5Ó6ˆ�rD   r0   c                 óˆ   • U R                  U5      nU R                  " U4UUUUS.UD6u  p„n	XR                  U5      -   n
X¤U	4$ )N)rë   rì   r/   rí   )r—   r  r.   )rO   r^   rì   r,   r/   rí   r0   r  r  r@   Úlayer_outputs              rB   r`   ÚMT5LayerCrossAttention.forward¥  sf   € ð  $Ÿ™¨}Ó=ÐØ8<×8LÒ8LØ ð9
àØ-Ø'Ø+ñ9
ð ñ9
Ñ5Ð¨ð %§|¡|Ð4DÓ'EÑEˆØ¨LÐ8Ð8rD   )r  r.   r—   r}   r  r  rh   s   @rB   r  r  œ  s=   ø† ñ7¨#°©*÷ 7ð 7ð ØØñ9ð Ð+Ñ,÷9ó 9rD   r  c                   ó^   ^ • \ rS rSrSS\S-  4U 4S jjjr      S	S\\   4S jjrSr	U =r
$ )
ÚMT5Blocki¼  NrŸ   c                 ól  >• [         TU ]  5         UR                  U l        [        R                  " 5       U l        U R
                  R                  [        XUS95        U R                  (       a"  U R
                  R                  [        XS95        U R
                  R                  [        U5      5        g )N©r¤   rŸ   )rŸ   )
rI   rJ   r£   r   Ú
ModuleListÚlayerÚappendr   r  r�   r  s       €rB   rJ   ÚMT5Block.__init__½  s}   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ—]’]“_ˆŒ
Ø�
‰
×ÑÜ! &ÐmvÑwô	
ð �?�?Ø�J‰J×ÑÔ4°VÑQÔRà�
‰
×Ñœ* VÓ,Õ-rD   r0   c                 óp  • U R                   S   " U4UUUS.UD6u  pn
UR                  [        R                  :X  a£  [        R                  " [        R
                  " U5      R                  5       [        R                  " UR                  5      R                  S-
  [        R                  " UR                  5      R                  5      n[        R                  " X* US9nS nU R                  =(       a    US LnU(       aß  U R                   S   " U4UUUUS.UD6u  pn
UR                  [        R                  :X  a£  [        R                  " [        R
                  " U5      R                  5       [        R                  " UR                  5      R                  S-
  [        R                  " UR                  5      R                  5      n[        R                  " X* US9nU R                   S   " U5      nUR                  [        R                  :X  a£  [        R                  " [        R
                  " U5      R                  5       [        R                  " UR                  5      R                  S-
  [        R                  " UR                  5      R                  5      n[        R                  " X* US9nXU4$ )Nr   )r,   r/   rí   iè  )r¹   Úmaxr&   )rì   r,   r/   rí   r2   )r  r[   r:   r\   r¿   ÚisinfÚanyÚfinfor  Úclampr£   )rO   r^   r,   r/   Úencoder_hidden_statesÚencoder_attention_maskÚencoder_decoder_position_biasrí   r0   Úself_attn_position_biasÚ_Úclamp_valueÚcross_attn_position_biasÚdo_cross_attentions                 rB   r`   ÚMT5Block.forwardÉ  s*  € ð 59·J±J¸q²MØð5
à)Ø'Ø+ñ	5
ð
 ñ5
Ñ1ˆ°ð ×Ñ¤%§-¡-Ó/ÜŸ+š+Ü—’˜MÓ*×.Ñ.Ó0Ü—’˜M×/Ñ/Ó0×4Ñ4°tÑ;Ü—’˜M×/Ñ/Ó0×4Ñ4óˆKô
 "ŸKšK¨¸<È[ÑYˆMà#'Ð Ø!Ÿ_™_×RÐ1FÈdÐ1RÐÞØ9=¿¹ÀAºØð:à!6Ø5Ø;Ø /ñ:ð ñ:Ñ6ˆM°Qð ×"Ñ"¤e§m¡mÓ3Ü#ŸkškÜ—K’K Ó.×2Ñ2Ó4Ü—K’K × 3Ñ 3Ó4×8Ñ8¸4Ñ?Ü—K’K × 3Ñ 3Ó4×8Ñ8ó�ô
 !&§¢¨M¸|ÐQ\Ñ ]�ð Ÿ
™
 2š }Ó5ˆð ×Ñ¤%§-¡-Ó/ÜŸ+š+Ü—’˜MÓ*×.Ñ.Ó0Ü—’˜M×/Ñ/Ó0×4Ñ4°tÑ;Ü—’˜M×/Ñ/Ó0×4Ñ4óˆKô
 "ŸKšK¨¸<È[ÑYˆMàÐ7OÐOÐOrD   )r£   r  r
  )NNNNNNr  rh   s   @rB   r  r  ¼  sJ   ø† ñ
.ÈSÐSWÉZ÷ 
.ð 
.ð ØØ"Ø#Ø&*Øñ=Pð Ð+Ñ,÷=Pó =PrD   r  c                   ón   ^ • \ rS rSrSrS\4U 4S jjrS\R                  S\R                  4S jr	Sr
U =r$ )	ÚMT5ClassificationHeadi
  z-Head for sentence-level classification tasks.rl   c                 ó(  >• [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  S9U l        [        R                  " UR                  UR                  5      U l
        g )N)r7   )rI   rJ   r   rq   rr   Údenserv   Úclassifier_dropoutr.   Ú
num_labelsÚout_projrz   s     €rB   rJ   ÚMT5ClassificationHead.__init__  sZ   ø€ Ü‰ÑÔÜ—Y’Y˜vŸ~™~¨v¯~©~Ó>ˆŒ
Ü—z’z F×$=Ñ$=Ñ>ˆŒÜŸ	š	 &§.¡.°&×2CÑ2CÓDˆ�rD   r^   Úreturnc                 óº   • U R                  U5      nU R                  U5      n[        R                  " U5      nU R                  U5      nU R	                  U5      nU$ r}   )r.   r/  r:   Útanhr2  r�   s     rB   r`   ÚMT5ClassificationHead.forward  sN   € ØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆÜŸ
š
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐrD   )r/  r.   r2  )rb   rc   rd   re   Ú__doc__r'   rJ   r:   r   r`   rf   rg   rh   s   @rB   r-  r-  
  s4   ø† Ù7ðE˜y÷ Eð U§\¡\ð °e·l±l÷ ò rD   r-  c                   óÀ   ^ • \ rS rSr% \\S'   SrSrS/rS/r	Sr
SrSrSr\" \SS	9\" \S
S	9\" \S
S	9S.r\S 5       r\R,                  " 5       U 4S j5       rS rSrU =r$ )ÚMT5PreTrainedModeli  rl   ÚtransformerTr  ru   Fr   )Úindexr2   )r^   Ú
attentionsÚcross_attentionsc                 óz   • [         R                  " [        5      n[         R                  " [        5      nUUUS.nU$ )N)Údecoder_input_idsÚ	input_idsÚdecoder_attention_mask)r:   Útensorr   r   )rO   rA  Ú
input_maskÚdummy_inputss       rB   rE  ÚMT5PreTrainedModel.dummy_inputs2  s6   € ä—L’L¤Ó.ˆ	Ü—\’\¤*Ó-ˆ
à!*Ø"Ø&0ñ
ˆð
 ÐrD   c                 ó  >• [         TU ]  U5        U R                  R                  n[	        U[
        5      (       a%  [        R                  " UR                  US-  5        g	[	        U[        [        [        [        45      (       Ga  [        R                  " UR                  R                  SUS-  S9  [        US5      (       aH  U R                  R                   (       d-  [        R                  " UR"                  R                  SUS-  S9  [        US5      (       an  [        R                  " UR$                  R                  SX R                  R&                  S-  -  S9  [        R(                  " UR$                  R*                  5        g	g	[	        U[,        5      (       aj  [        US5      (       aX  [        R                  " UR.                  R                  SUS-  S9  [        R(                  " UR.                  R*                  5        g	g	[	        U[0        5      (       GaA  [        R                  " UR2                  R                  SX R                  R&                  S-  -  S9  [        UR2                  S5      (       aA  UR2                  R*                  b*  [        R(                  " UR2                  R*                  5        [        R                  " UR4                  R                  SX R                  R&                  S-  -  S9  [        UR4                  S5      (       aC  UR4                  R*                  b+  [        R(                  " UR4                  R*                  5        g	g	g	[	        U[6        5      (       GaA  [        R                  " UR8                  R                  SX R                  R&                  S-  -  S9  [        UR8                  S5      (       aA  UR8                  R*                  b*  [        R(                  " UR8                  R*                  5        [        R                  " UR:                  R                  SX R                  R<                  S-  -  S9  [        UR:                  S5      (       aC  UR:                  R*                  b+  [        R(                  " UR:                  R*                  5        g	g	g	[	        U[>        5      (       Gaà  [        R                  " UR@                  R                  SX R                  R&                  S-  -  S9  [        UR@                  S5      (       aA  UR@                  R*                  b*  [        R(                  " UR@                  R*                  5        [        R                  " URB                  R                  SX R                  R&                  S-  -  S9  [        URB                  S5      (       aA  URB                  R*                  b*  [        R(                  " URB                  R*                  5        [        R                  " UR:                  R                  SX R                  R<                  S-  -  S9  [        UR:                  S5      (       aC  UR:                  R*                  b+  [        R(                  " UR:                  R*                  5        g	g	g	[	        U[D        5      (       GaF  U R                  R&                  nU R                  RF                  nU R                  RH                  n[        R                  " URJ                  R                  SX#U-  S-  -  S9  [        R                  " URL                  R                  SX#S-  -  S9  [        R                  " URN                  R                  SX#S-  -  S9  [        R                  " URP                  R                  SX%U-  S-  -  S9  URR                  (       a0  [        R                  " URT                  R                  SX#S-  -  S9  g	g	g	)
zInitialize the weightsr¢   rÚ   )rY   ÚstdÚlm_headÚ
qa_outputsr3   Ú
classifierrp   N)+rI   Ú_init_weightsrl   Úinitializer_factorr~   rF   ÚinitÚ	constant_rM   ÚMT5ModelÚMT5ForConditionalGenerationÚMT5EncoderModelÚMT5ForQuestionAnsweringÚnormal_ÚsharedÚhasattrÚtie_word_embeddingsrI  rJ  rr   Úzeros_rp   ÚMT5ForTokenClassificationrK  r-  r/  r2  rj   rt   ru   rs   r…   rˆ   r‰   r�   r§   r©   r®   r¯   r°   r±   r¤   r³   )rO   r(   Úfactorrr   r¨   rª   rR   s         €rB   rL  Ú MT5PreTrainedModel._init_weights=  s^  ø€ ô 	‰Ñ˜fÔ%Ø—‘×/Ñ/ˆÜ�fœl×+Ñ+Ü�NŠN˜6Ÿ=™=¨&°3©,Õ7ÜØÜÔ2´OÔE\Ð]÷
ò 
ô �LŠL˜Ÿ™×-Ñ-°C¸VÀc¹\ÒJÜ�v˜y×)Ñ)°$·+±+×2Q×2QÜ—’˜VŸ^™^×2Ñ2¸À&È3Á,ÒOÜ�v˜|×,Ñ,Ü—’˜V×.Ñ.×5Ñ5¸CÀV×P[ÑP[×PcÑPcÐhlÑOlÑEmÒnÜ—’˜F×-Ñ-×2Ñ2Õ3ð -ô ˜Ô 9×:Ñ:Ü�v˜|×,Ñ,Ü—’˜V×.Ñ.×5Ñ5¸CÀVÈcÁ\ÒRÜ—’˜F×-Ñ-×2Ñ2Õ3ð -ô ˜Ô 5×6Ò6Ü�LŠL˜Ÿ™×,Ñ,°3¸FÇ{Á{×GZÑGZÐ_cÑFcÑ<dÒeÜ�v—|‘| V×,Ñ,°·±×1BÑ1BÑ1NÜ—’˜FŸL™L×-Ñ-Ô.Ü�LŠL˜Ÿ™×/Ñ/°c¸vÏ+É+×J]ÑJ]ÐbfÑIfÑ?gÒhÜ�v—‘¨×/Ñ/°F·O±O×4HÑ4HÑ4TÜ—’˜FŸO™O×0Ñ0Õ1ð 5UÐ/ä˜Ô 0×1Ò1Ü�LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DWÑDWÐ\`ÑC`Ñ9aÒbÜ�v—y‘y &×)Ñ)¨f¯i©i¯n©nÑ.HÜ—’˜FŸI™IŸN™NÔ+Ü�LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DTÑDTÐY]ÑC]Ñ9^Ò_Ü�v—y‘y &×)Ñ)¨f¯i©i¯n©nÑ.HÜ—’˜FŸI™IŸN™NÕ+ð /IÐ)ä˜Ô 5×6Ò6Ü�LŠL˜Ÿ™×+Ñ+°#¸6ÇkÁk×FYÑFYÐ^bÑEbÑ;cÒdÜ�v—{‘{ F×+Ñ+°·±×0@Ñ0@Ñ0LÜ—’˜FŸK™K×,Ñ,Ô-Ü�LŠL˜Ÿ™×+Ñ+°#¸6ÇkÁk×FYÑFYÐ^bÑEbÑ;cÒdÜ�v—{‘{ F×+Ñ+°·±×0@Ñ0@Ñ0LÜ—’˜FŸK™K×,Ñ,Ô-Ü�LŠL˜Ÿ™×)Ñ)°¸ÇKÁK×DTÑDTÐY]ÑC]Ñ9^Ò_Ü�v—y‘y &×)Ñ)¨f¯i©i¯n©nÑ.HÜ—’˜FŸI™IŸN™NÕ+ð /IÐ)ä˜¤×-Ò-Ø—k‘k×)Ñ)ˆGØ!%§¡×!1Ñ!1ÐØ—k‘k×+Ñ+ˆGÜ�LŠL˜Ÿ™Ÿ™¨s¸ÐM_ÑC_ÐdhÑBhÑ8iÒjÜ�LŠL˜Ÿ™Ÿ™¨s¸È4Á-Ñ8PÒQÜ�LŠL˜Ÿ™Ÿ™¨s¸È4Á-Ñ8PÒQÜ�LŠL˜Ÿ™Ÿ™¨s¸ÐM_ÑC_ÐdhÑBhÑ8iÒjØ×1×1Ü—’˜V×;Ñ;×BÑBÈÐRXÐimÑ\mÑRnÓoð 2ð .rD   c                 ó:  • U R                   R                  nU R                   R                  nUc  [        S5      eUR	                  UR
                  5      nUSS S24   R                  5       USSS 24'   X$S'   Uc  [        S5      eUR                  US:H  U5        U$ )NzŒself.model.config.decoder_start_token_id has to be defined. In MT5 it is usually set to the pad_token_id. See MT5 docs for more information..r2   r&   ).r   z1self.model.config.pad_token_id has to be defined.éœÿÿÿ)rl   Údecoder_start_token_idÚpad_token_idÚ
ValueErrorÚ	new_zerosrÛ   rÝ   Úmasked_fill_)rO   rA  r^  r_  Úshifted_input_idss        rB   Ú_shift_rightÚMT5PreTrainedModel._shift_rightu  s­   € Ø!%§¡×!CÑ!CÐØ—{‘{×/Ñ/ˆà!Ñ)Üð5óð ð
 &×/Ñ/°	·±Ó@ÐØ%.¨s°C°R°C¨xÑ%8×%>Ñ%>Ó%@Ð˜#˜q™r˜'Ñ"Ø$:˜&Ñ!àÑÜÐPÓQÐQà×&Ñ&Ð'8¸DÑ'@À,ÔOà Ð rD   © )rb   rc   rd   re   r'   Ú__annotations__Úbase_model_prefixÚsupports_gradient_checkpointingÚ_no_split_modulesÚ_keep_in_fp32_modulesÚ_supports_flash_attnÚ_supports_flex_attnÚ_supports_sdpaÚ_can_compile_fullgraphr$   r  r   r  Ú_can_record_outputsÚpropertyrE  r:   Úno_gradrL  rd  rf   rg   rh   s   @rB   r:  r:    s¥   ø‡ ð ÓØ%ÐØ&*Ð#à#˜ÐØ!˜FÐð !ÐØÐØ€NØ!Ðñ (¨¸Ñ:Ù$Ð%:À"ÑEÙ*Ð+AÈÑLñÐð ñó ðð ‡]‚]ƒ_ô5pó ð5p÷n!ð !rD   r:  c                   ór   ^ • \ rS rSrU 4S jrS r\\\       SS\	\
   4S jj5       5       5       rSrU =r$ )ÚMT5StackiŒ  c                 ó  >• [         TU ]  U5        [        R                  " UR                  UR
                  5      U l        UR                  U l        [        R                  " [        UR                  5       Vs/ s H  n[        U[        US:H  5      US9PM     sn5      U l        [        UR
                  UR                  S9U l        [        R"                  " UR$                  5      U l        U R)                  5         SU l        g s  snf )Nr   r  r“   F)rI   rJ   r   r²   Ú
vocab_sizerr   Úembed_tokensr£   r  ÚrangeÚ
num_layersr  rý   ÚblockrF   r–   Úfinal_layer_normrv   rw   r.   Ú	post_initr´   )rO   rl   ÚirR   s      €rB   rJ   ÚMT5Stack.__init__�  sÈ   ø€ Ü‰Ñ˜Ô äŸLšL¨×):Ñ):¸F¿N¹NÓKˆÔØ ×+Ñ+ˆŒä—]’]Ü^cÐdj×duÑduÔ^vÓwÒ^vÐYZŒX�f¼$¸qÀA¹v»,ÐRSÔTÑ^vÑwó
ˆŒ
ô !-¨V¯^©^À×AZÑAZÑ [ˆÔÜ—z’z &×"5Ñ"5Ó6ˆŒð 	�‰ÔØ&+ˆÕ#ùò xs   Á9!Dc                 ó   • Xl         g r}   )rw  ©rO   Únew_embeddingss     rB   Úset_input_embeddingsÚMT5Stack.set_input_embeddings�  s   € Ø*ÕrD   r0   c           	      óì  • US L US L-  (       a  [        S5      eUc  U R                  U5      nUSL a   U R                  (       d  [        SU  S35      eU R                  (       ah  U(       a`  Uc]  U R                  R                  (       a/  [        [        U R                  S9[        U R                  S95      nO'[        U R                  S9nOU R                  (       d  S nU R                  R                  (       a8  [        U R                  UU[        U[
        5      (       a  UR                  OUS9nO[        U R                  UUS9nS n	U R                  (       a  Ub  [        U R                  UUUS9n	S n
S nU R                  U5      nU R                   H2  nU" UUU
UU	U4S	U0UD6u  pÎnUn
U R                  (       d  M+  Uc  M0  UnM4     U R                  U5      nU R                  U5      n[        UUS
9$ )Nz:You must specify exactly one of input_ids or inputs_embedsTz)`use_cache` can only be set to `True` if z is used as a decoder)rl   )rl   Úinputs_embedsr,   rí   )rl   r…  r,   )rl   r…  r,   r#  rí   )Úlast_hidden_staterí   )r`  rw  r£   rl   Úis_encoder_decoderr   r   r   r~   râ   r   r.   rz  r{  r   )rO   rA  r,   r#  r$  r…  rí   Ú	use_cacher0   Úencoder_extended_attention_maskr/   r%  r^   Úlayer_moduleÚself_attention_position_biasÚcross_attention_position_biass                   rB   r`   ÚMT5Stack.forward   só  € ð ˜Ð -°tÐ";×<ÜÐYÓZÐZàÑ Ø ×-Ñ-¨iÓ8ˆMà˜ÒØ—?—?Ü Ð#LÈTÈFÐRgÐ!hÓiÐià�?�?Þ˜_Ñ4Ø—;‘;×1×1Ü&9Ü$¨D¯K©KÑ8¼,ÈdÏkÉkÑ:Zó'‘Oô '3¸$¿+¹+Ñ&F�OøØ——ð #ˆOà�;‰;×!×!Ü/Ø—{‘{Ø+Ø-ä˜oÔ/B×CÑCð !0× DÒ Dà$ñ‰Nô 7Ø—{‘{Ø+Ø-ñˆNð +/Ð'Ø�?�?Ð4Ñ@Ü.GØ—{‘{Ø+Ø5Ø&;ñ	/Ð+ð ˆØ(,Ð%àŸ™ ]Ó3ˆà ŸJœJˆLÙYeØØØØ%Ø/Ø-ñ	Zð !0ð	Zð ñ	ZÑVˆMÐ9Vð 9ˆMØ��‰Ð#8Ó#DØ0MÒ-ñ 'ð" ×-Ñ-¨mÓ<ˆØŸ™ ]Ó3ˆä8Ø+Ø+ñ
ð 	
rD   )rz  r.   rw  r{  r´   r£   )NNNNNNN)rb   rc   rd   re   rJ   r‚  r#   r%   r   r   r   r`   rf   rg   rh   s   @rB   rt  rt  Œ  s[   ø† õ,ò +ð  ØØð ØØ"Ø#ØØØñW
ð Ð+Ñ,ôW
ó ó ó  öW
rD   rt  c                   ó”  ^ • \ rS rSr% SrSr\\S'   S/rSSS.r	S\4U 4S jjr
S	 rS
 r\\         SS\R                   S-  S\R"                  S-  S\R                   S-  S\R$                  S-  S\\\R"                        S-  S\S-  S\R*                  S-  S\R*                  S-  S\S-  S\\   S\4S jj5       5       rSrU =r$ )rP  iý  aG  
Examples:

```python
>>> from transformers import MT5Model, AutoTokenizer

>>> model = MT5Model.from_pretrained("google/mt5-small")
>>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
>>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
>>> summary = "Weiter Verhandlung in Syrien."
>>> inputs = tokenizer(article, return_tensors="pt")
>>> labels = tokenizer(text_target=summary, return_tensors="pt")

>>> outputs = model(input_ids=inputs["input_ids"], decoder_input_ids=labels["input_ids"])
>>> hidden_states = outputs.last_hidden_state
```Úmt5rl   úFdecoder.block.0.layer.1.EncDecAttention.relative_attention_bias.weightúshared.weight©úencoder.embed_tokens.weightúdecoder.embed_tokens.weightc                 óˆ  >• [         TU ]  U5        [        R                  " UR                  UR
                  5      U l        [        R                  " U5      nSUl	        SUl
        [        U5      U l        [        R                  " U5      nSUl	        UR                  Ul        [        U5      U l        U R!                  5         g ©NFT)rI   rJ   r   r²   rv  rr   rU  ÚcopyÚdeepcopyr£   rˆ  rt  ÚencoderÚnum_decoder_layersry  Údecoderr|  ©rO   rl   Úencoder_configÚdecoder_configrR   s       €rB   rJ   ÚMT5Model.__init__  s’   ø€ Ü‰Ñ˜Ô Ü—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü Ó/ˆŒäŸš vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü Ó/ˆŒð 	�‰ÕrD   c                 ó   • U R                   $ r}   ©rU  ©rO   s    rB   Úget_input_embeddingsÚMT5Model.get_input_embeddings+  ó   € Ø�{‰{ÐrD   c                 ó|   • Xl         U R                  R                  U5        U R                  R                  U5        g r}   ©rU  r™  r‚  r›  r€  s     rB   r‚  ÚMT5Model.set_input_embeddings/  ó+   € Ø$ŒØ�‰×)Ñ)¨.Ô9Ø�‰×)Ñ)¨.Õ9rD   NrA  r,   r@  rB  Úencoder_outputsrí   r…  Údecoder_inputs_embedsrˆ  r0   r4  c
                 óÊ  • Uc  U R                   " S	UUUS.U
D6nOK[        U[        5      (       d6  [        US   [        U5      S:”  a  US   OS[        U5      S:”  a  US   OSS9nUS   nU R                  " S	UUUUUUU	S.U
D6n[        UR                  UR                  UR                  UR                  UR                  UR                  UR                  UR                  S9$ )
aD	  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
    should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5
    Training](./mt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.

Example:

```python
>>> from transformers import AutoTokenizer, MT5Model

>>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
>>> model = MT5Model.from_pretrained("google/mt5-small")

>>> input_ids = tokenizer(
...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
... ).input_ids  # Batch size 1
>>> decoder_input_ids = tokenizer("Studies show that", return_tensors="pt").input_ids  # Batch size 1

>>> # preprocess: Prepend decoder_input_ids with start token which is pad token for MT5Model.
>>> # This is not needed for torch's MT5ForConditionalGeneration as it does this internally using labels arg.
>>> decoder_input_ids = model._shift_right(decoder_input_ids)

>>> # forward pass
>>> outputs = model(input_ids=input_ids, decoder_input_ids=decoder_input_ids)
>>> last_hidden_states = outputs.last_hidden_state
```N©rA  r,   r…  r   r&   r4   ©r†  r^   r=  ©rA  r,   r…  rí   r#  r$  rˆ  )r†  rí   Údecoder_hidden_statesÚdecoder_attentionsr>  Úencoder_last_hidden_stater#  Úencoder_attentionsrf  )r™  r~   r   Úlenr›  r   r†  rí   r^   r=  r>  )rO   rA  r,   r@  rB  rª  rí   r…  r«  rˆ  r0   r^   Údecoder_outputss                rB   r`   ÚMT5Model.forward4  s  € ðD Ñ"Ø"Ÿlšlð Ø#Ø-Ø+ñð ñ	‰Oô ˜O¬_×=Ñ=Ü-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ó4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÓ1I˜?¨1Ò-ÈtñˆOð (¨Ñ*ˆð Ÿ,š,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øñ	
ð ñ	
ˆô "Ø-×?Ñ?Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ñ	
ð 		
rD   )r›  r™  rU  ©	NNNNNNNNN)rb   rc   rd   re   r8  Ú
model_typer'   rg  Ú"_keys_to_ignore_on_load_unexpectedÚ_tied_weights_keysrJ   r£  r‚  r"   r   r:   Ú
LongTensorÚFloatTensorÚ
BoolTensorÚtupler   r   rý   r   r   r   r`   rf   rg   rh   s   @rB   rP  rP  ý  sU  ø‡ ñð" €JØÓØ*rÐ)sÐ&à'6Ø'6ñÐð˜y÷ ò$ò:ð
 Øð .2Ø37Ø59Ø:>ØBFØ(,Ø-1Ø59Ø!%ñd
à×#Ñ# dÑ*ðd
ð ×)Ñ)¨DÑ0ðd
ð !×+Ñ+¨dÑ2ð	d
ð
 !&× 0Ñ 0°4Ñ 7ðd
ð ˜u U×%6Ñ%6Ñ7Ñ8¸4Ñ?ðd
ð  ™ðd
ð —|‘| dÑ*ðd
ð  %Ÿ|™|¨dÑ2ðd
ð ˜$‘;ðd
ð Ð+Ñ,ðd
ð 
ôd
ó ó öd
rD   rP  z;
    MT5 Model with a `language modeling` head on top.
    )Úcustom_introc                   óØ  ^ • \ rS rSr% SrSr\\S'   S/rSSSS.r	S\4U 4S jjr
S	 rS
 r\\          SS\R                   S-  S\R"                  S-  S\R                   S-  S\R$                  S-  S\\\R(                        S-  S\S-  S\R"                  S-  S\R"                  S-  S\R                   S-  S\S-  S\\   S\4S jj5       5       rS\R(                  4S jrSrU =r$ )rQ  iž  aï  
Examples:

```python
>>> from transformers import MT5ForConditionalGeneration, AutoTokenizer

>>> model = MT5ForConditionalGeneration.from_pretrained("google/mt5-small")
>>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
>>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
>>> summary = "Weiter Verhandlung in Syrien."
>>> inputs = tokenizer(article, text_target=summary, return_tensors="pt")

>>> outputs = model(**inputs)
>>> loss = outputs.loss
```r�  rl   r�  r‘  )r“  r”  zlm_head.weightc                 ó  >• [         TU ]  U5        UR                  U l        [        R
                  " UR                  UR                  5      U l        [        R                  " U5      nSUl
        SUl        [        U5      U l        [        R                  " U5      nSUl
        UR                  Ul        [        U5      U l        [        R"                  " UR                  UR                  SS9U l        U R'                  5         g )NFTro   )rI   rJ   rr   Ú	model_dimr   r²   rv  rU  r—  r˜  r£   rˆ  rt  r™  rš  ry  r›  rq   rI  r|  rœ  s       €rB   rJ   Ú$MT5ForConditionalGeneration.__init__¾  s»   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü Ó/ˆŒäŸš vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü Ó/ˆŒä—y’y §¡°×1BÑ1BÈÑOˆŒð 	�‰ÕrD   c                 ó   • U R                   $ r}   r¡  r¢  s    rB   r£  Ú0MT5ForConditionalGeneration.get_input_embeddingsÓ  r¥  rD   c                 ó|   • Xl         U R                  R                  U5        U R                  R                  U5        g r}   r§  r€  s     rB   r‚  Ú0MT5ForConditionalGeneration.set_input_embeddingsÖ  r©  rD   NrA  r,   r@  rB  rª  rí   r…  r«  Úlabelsrˆ  r0   r4  c                 óØ  • Uc  U R                   " SUUUS.UD6nOK[        U[        5      (       d6  [        US   [        U5      S:”  a  US   OS[        U5      S:”  a  US   OSS9nUS   nU	b  Uc  Uc  U R	                  U	5      nU R
                  " SUUUUUUU
S.UD6nUS   nU R                  U5      nSnU	b[  [        SS	9nU	R                  UR                  5      n	U" UR                  S
UR                  S
5      5      U	R                  S
5      5      n[        UUUR                  UR                  UR                  UR                   UR"                  UR                  UR                  S9	$ )aµ
  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
    should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5
    Training](./mt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[-100, 0, ...,
    config.vocab_size - 1]`. All labels set to `-100` are ignored (masked), the loss is only computed for
    labels in `[0, ..., config.vocab_size]`

Examples:

```python
>>> from transformers import AutoTokenizer, MT5ForConditionalGeneration

>>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
>>> model = MT5ForConditionalGeneration.from_pretrained("google/mt5-small")

>>> # training
>>> input_ids = tokenizer("The <extra_id_0> walks in <extra_id_1> park", return_tensors="pt").input_ids
>>> labels = tokenizer("<extra_id_0> cute dog <extra_id_1> the <extra_id_2>", return_tensors="pt").input_ids
>>> outputs = model(input_ids=input_ids, labels=labels)
>>> loss = outputs.loss
>>> logits = outputs.logits

>>> # inference
>>> input_ids = tokenizer(
...     "summarize: studies have shown that owning a dog is good for you", return_tensors="pt"
... ).input_ids  # Batch size 1
>>> outputs = model.generate(input_ids)
>>> print(tokenizer.decode(outputs[0], skip_special_tokens=True))
>>> # studies have shown that owning a dog is good for you.
```Nr­  r   r&   r4   r®  r¯  r]  ©Úignore_indexr2   ©	ÚlossÚlogitsrí   r°  r±  r>  r²  r#  r³  rf  )r™  r~   r   r´  rd  r›  rI  r   rV   rË   rÞ   r9   r   rí   r^   r=  r>  r†  )rO   rA  r,   r@  rB  rª  rí   r…  r«  rÈ  rˆ  r0   r^   rµ  Úsequence_outputÚ	lm_logitsrÍ  Úloss_fcts                     rB   r`   Ú#MT5ForConditionalGeneration.forwardÛ  s   € ðR Ñ"à"Ÿlšlð Ø#Ø-Ø+ñð ñ	‰Oô ˜O¬_×=Ñ=Ü-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ó4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÓ1I˜?¨1Ò-ÈtñˆOð (¨Ñ*ˆàÑÐ"3Ñ";Ð@UÑ@]à $× 1Ñ 1°&Ó 9Ðð Ÿ,š,ð 	
Ø'Ø1Ø/Ø+Ø"/Ø#1Øñ	
ð ñ	
ˆð *¨!Ñ,ˆà—L‘L Ó1ˆ	àˆØÑÜ'°TÑ:ˆHà—Y‘Y˜y×/Ñ/Ó0ˆFÙ˜IŸN™N¨2¨y¯~©~¸bÓ/AÓBÀFÇKÁKÐPRÃOÓTˆDäØØØ+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ñ

ð 
	
rD   c                 ó$   • U R                  U5      $ r}   )rd  )rO   rÈ  s     rB   Ú%prepare_decoder_input_ids_from_labelsÚAMT5ForConditionalGeneration.prepare_decoder_input_ids_from_labels]  s   € Ø× Ñ  Ó(Ð(rD   )r›  r™  rI  rÂ  rU  ©
NNNNNNNNNN)rb   rc   rd   re   r8  r¸  r'   rg  r¹  rº  rJ   r£  r‚  r"   r   r:   r»  r¼  r½  r¾  r   r   rý   r   r   r   r`   rÔ  rf   rg   rh   s   @rB   rQ  rQ  ž  s‰  ø‡ ñð  €JØÓØ*rÐ)sÐ&à'6Ø'6Ø)ñÐð˜y÷ ò*ò:ð
 Øð .2Ø37Ø59Ø:>Ø=AØ(,Ø26Ø:>Ø*.Ø!%ñ}
à×#Ñ# dÑ*ð}
ð ×)Ñ)¨DÑ0ð}
ð !×+Ñ+¨dÑ2ð	}
ð
 !&× 0Ñ 0°4Ñ 7ð}
ð ˜u U§\¡\Ñ2Ñ3°dÑ:ð}
ð  ™ð}
ð ×(Ñ(¨4Ñ/ð}
ð  %×0Ñ0°4Ñ7ð}
ð × Ñ  4Ñ'ð}
ð ˜$‘;ð}
ð Ð+Ñ,ð}
ð 
ô}
ó ó ð}
ð@)¸E¿L¹L÷ )ò )rD   rQ  c                   óè   ^ • \ rS rSr% SrSr\\S'   SS0rS\4U 4S jjr	S r
S	 r\\   SS\R                  S
-  S\R                   S
-  S\R                   S
-  S\\   S\4
S jj5       5       rSrU =r$ )rR  ia  a¶  
Examples:

```python
>>> from transformers import MT5EncoderModel, AutoTokenizer

>>> model = MT5EncoderModel.from_pretrained("google/mt5-small")
>>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
>>> article = "UN Offizier sagt, dass weiter verhandelt werden muss in Syrien."
>>> input_ids = tokenizer(article, return_tensors="pt").input_ids
>>> outputs = model(input_ids)
>>> hidden_state = outputs.last_hidden_state
```r�  rl   r“  r‘  c                 óä   >• [         TU ]  U5        [        R                  " UR                  UR
                  5      U l        UnSUl        SUl        [        U5      U l
        U R                  5         g )NF)rI   rJ   r   r²   rv  rr   rU  rˆ  r‡  rt  r™  r|  )rO   rl   r�  rR   s      €rB   rJ   ÚMT5EncoderModel.__init__x  sY   ø€ Ü‰Ñ˜Ô Ü—l’l 6×#4Ñ#4°f·n±nÓEˆŒàˆØ#(ˆÔ Ø,1ˆÔ)Ü Ó/ˆŒð 	�‰ÕrD   c                 ó   • U R                   $ r}   r¡  r¢  s    rB   r£  Ú$MT5EncoderModel.get_input_embeddings…  r¥  rD   c                 óF   • Xl         U R                  R                  U5        g r}   )rU  r™  r‚  r€  s     rB   r‚  Ú$MT5EncoderModel.set_input_embeddings‰  s   € Ø$ŒØ�‰×)Ñ)¨.Õ9rD   NrA  r,   r…  r0   r4  c                 ó2   • U R                   " SUUUS.UD6nU$ )aÂ  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
    should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training).

Example:

```python
>>> from transformers import AutoTokenizer, MT5EncoderModel

>>> tokenizer = AutoTokenizer.from_pretrained("google/mt5-small")
>>> model = MT5EncoderModel.from_pretrained("google/mt5-small")
>>> input_ids = tokenizer(
...     "Studies have been shown that owning a dog is good for you", return_tensors="pt"
... ).input_ids  # Batch size 1
>>> outputs = model(input_ids=input_ids)
>>> last_hidden_states = outputs.last_hidden_state
```r­  rf  )r™  )rO   rA  r,   r…  r0   rª  s         rB   r`   ÚMT5EncoderModel.forward�  s3   € ðB ,0¯<ª<ð ,
ØØ)Ø'ñ,
ð ñ	,
ˆð ÐrD   )r™  rU  r  )rb   rc   rd   re   r8  r¸  r'   rg  rº  rJ   r£  r‚  r"   r   r:   r»  r¼  r   r   r   r`   rf   rg   rh   s   @rB   rR  rR  a  s¶   ø‡ ñð €JØÓà% ðÐð

˜y÷ 
òò:ð Øð .2Ø37Ø26ñ	%à×#Ñ# dÑ*ð%ð ×)Ñ)¨DÑ0ð%ð ×(Ñ(¨4Ñ/ð	%ð
 Ð+Ñ,ð%ð 
ô%ó ó ö%rD   rR  z„
    MT5 model with a sequence classification/head on top (a linear layer on top of the pooled output) e.g. for GLUE
    tasks.
    c                   óx  ^ • \ rS rSrS/rS\4U 4S jjr\\         SS\	R                  S-  S\	R                  S-  S\	R                  S-  S	\	R                  S-  S
\\	R                     S-  S\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S\\   S\4S jj5       5       rSrU =r$ )ÚMT5ForSequenceClassificationi¸  r�  rl   c                 ó„   >• [         TU ]  U5        [        U5      U l        [	        U5      U l        U R                  5         g r}   )rI   rJ   rP  r;  r-  Úclassification_headr|  rz   s     €rB   rJ   Ú%MT5ForSequenceClassification.__init__Â  s6   ø€ Ü‰Ñ˜Ô Ü# FÓ+ˆÔÜ#8¸Ó#@ˆÔ ð 	�‰ÕrD   NrA  r,   r@  rB  rª  r…  r«  rÈ  rˆ  r0   r4  c
                 ó
  • Ub  Sn	Uc%  Ub"  [        SU R                  R                   35      eUc"  Uc  Uc  [        S5      eU R	                  U5      nU R
                  " U4UUUUUUU	S.U
D6nUS   nUR                  U R                  R                  5      R                  UR                  5      n[        [        R                  " UR                  S5      5      R                  5       S:H  S5        UR                   u  pïnXÍSS24   n[        UR                   S   U-  S:¬  S	5        UR#                  US
U5      SS2S
SS24   nU R%                  U5      nSnUGbÓ  UR                  UR                  5      nU R                  R&                  c¥  U R                  R(                  S:X  a  SU R                  l        OyU R                  R(                  S:”  aN  UR*                  [        R,                  :X  d  UR*                  [        R.                  :X  a  SU R                  l        OSU R                  l        U R                  R&                  S:X  aT  [1        5       nU R                  R(                  S:X  a&  U" UR3                  5       UR3                  5       5      nO˜U" UU5      nOŽU R                  R&                  S:X  aG  [5        5       nU" UR#                  S
U R                  R(                  5      UR#                  S
5      5      nO-U R                  R&                  S:X  a  [7        5       nU" UU5      n[9        UUUR:                  UR<                  UR>                  UR@                  URB                  URD                  URF                  S9	$ )aò  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
    should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./mt5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    MT5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [MT5
    Training](./mt5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
NFz8Passing input embeddings is currently not supported for ú°If no `decoder_input_ids` or `decoder_inputs_embeds` are passed, `input_ids` cannot be `None`. Please pass either `input_ids` or `decoder_input_ids` or `decoder_inputs_embeds`.)r,   r@  rB  rª  r…  r«  rˆ  r   r&   z7All examples must have the same number of <eos> tokens.z3Each example must contain at least one <eos> token.r2   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationrÌ  )$ÚNotImplementedErrorrR   rb   r`  rd  r;  Úeqrl   Úeos_token_idrV   rË   r!   r:   Úunique_consecutiveÚsumÚnumelrÛ   rÞ   rã  Úproblem_typer1  r[   r·   rü   r   Úsqueezer   r   r   rí   r°  r±  r>  r²  r#  r³  )rO   rA  r,   r@  rB  rª  r…  r«  rÈ  rˆ  r0   ÚoutputsrÏ  Úeos_maskÚ
batch_sizer'  rP   ÚselectedÚsentence_representationrÎ  rÍ  rÑ  s                         rB   r`   Ú$MT5ForSequenceClassification.forwardÊ  s+  € ð^ ÑØˆIàÑ Ñ!:Ü%ØJÈ4Ï>É>×KbÑKbÐJcÐdóð ð Ñ$Ð)>Ñ)FØÑ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðà&*×&6Ò&6Øð
'
à)Ø/Ø#9Ø+Ø'Ø"7Øñ
'
ð ñ
'
ˆð " !™*ˆà—<‘< §¡× 8Ñ 8Ó9×<Ñ<¸_×=SÑ=SÓTˆäÜ×$Ò$ X§\¡\°!£_Ó5×;Ñ;Ó=ÀÑBØEô	
ð &5×%:Ñ%:Ñ"ˆ
�{Ø"ªQ ;Ñ/ˆÜØ�N‰N˜1Ñ Ñ+¨qÑ0ØAô	
ð #+§-¡-°
¸BÀÓ"LÊQÐPRÒTUÈXÑ"VÐØ×)Ñ)Ð*AÓBˆàˆØÒØ—Y‘Y˜vŸ}™}Ó-ˆFØ�{‰{×'Ñ'Ñ/Ø—;‘;×)Ñ)¨QÓ.Ø/;�D—K‘KÕ,Ø—[‘[×+Ñ+¨aÓ/°V·\±\ÄUÇZÁZÓ5OÐSY×S_ÑS_Ôch×clÑclÓSlØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ó7Ü"›9�Ø—;‘;×)Ñ)¨QÓ.Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# F¨FÓ3‘DØ—‘×)Ñ)Ð-JÓJÜ+Ó-�Ù §¡¨B°·±×0FÑ0FÓ GÈÏÉÐUWËÓY‘Ø—‘×)Ñ)Ð-IÓIÜ,Ó.�Ù ¨Ó/�ä.ØØØ#×3Ñ3Ø")×"?Ñ"?Ø&×9Ñ9Ø$×5Ñ5Ø&-×&GÑ&GØ")×"?Ñ"?Ø&×9Ñ9ñ

ð 
	
rD   )rã  r;  r·  )rb   rc   rd   re   r¹  r'   rJ   r"   r   r:   r»  r   Úlistr¼  rý   r   r   r   r`   rf   rg   rh   s   @rB   rá  rá  ¸  s2  ø† ð +sÐ)sÐ&ð˜y÷ ð Øð .2Ø.2Ø59Ø:>Ø:>Ø26Ø:>Ø*.Ø!%ñ}
à×#Ñ# dÑ*ð}
ð Ÿ™ tÑ+ð}
ð !×+Ñ+¨dÑ2ð	}
ð
 !&× 0Ñ 0°4Ñ 7ð}
ð ˜e×/Ñ/Ñ0°4Ñ7ð}
ð ×(Ñ(¨4Ñ/ð}
ð  %×0Ñ0°4Ñ7ð}
ð × Ñ  4Ñ'ð}
ð ˜$‘;ð}
ð Ð+Ñ,ð}
ð 
)ô}
ó ó ö}
rD   rá  c                   óà   ^ • \ rS rSrS\4U 4S jjr\\    SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\
\   S
\4S jj5       5       rSrU =r$ )rY  iM  rl   c                 ó0  >• [         TU ]  U5        UR                  U l        [        U5      U l        [
        R                  " UR                  5      U l        [
        R                  " UR                  UR                  5      U l        U R                  5         g r}   )rI   rJ   r1  rR  r;  r   rv   r0  r.   rq   rP   rK  r|  rz   s     €rB   rJ   Ú"MT5ForTokenClassification.__init__P  sj   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä*¨6Ó2ˆÔÜ—z’z &×";Ñ";Ó<ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrD   NrA  r,   r…  rÈ  r0   r4  c                 ó>  • U R                   " U4UUS.UD6nUS   nU R                  U5      nU R                  U5      nSn	Ub<  [        5       n
U
" UR	                  SU R
                  5      UR	                  S5      5      n	[        U	UUR                  UR                  S9$ )aî  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. MT5 is a model with relative position embeddings so you
    should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [MT5 Training](./t5#training).
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
)r,   r…  r   Nr2   )rÍ  rÎ  r^   r=  )	r;  r.   rK  r   rÞ   r1  r   r^   r=  )rO   rA  r,   r…  rÈ  r0   rò  r^   rÎ  rÍ  rÑ  s              rB   r`   Ú!MT5ForTokenClassification.forward[  s®   € ð2 $(×#3Ò#3Øð$
à)Ø'ñ$
ð ñ	$
ˆð   ™
ˆØŸ™ ]Ó3ˆØ—‘ Ó/ˆàˆØÑÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDä$ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
rD   )rK  r.   r1  r;  rû   )rb   rc   rd   re   r'   rJ   r"   r   r:   r   r   r   r   r`   rf   rg   rh   s   @rB   rY  rY  M  s�   ø† ð	˜y÷ 	ð Øð *.Ø.2Ø-1Ø&*ñ+
à—<‘< $Ñ&ð+
ð Ÿ™ tÑ+ð+
ð —|‘| dÑ*ð	+
ð
 —‘˜tÑ#ð+
ð Ð+Ñ,ð+
ð 
ô+
ó ó ö+
rD   rY  c                   ó´  ^ • \ rS rSrS/rSSS.rS\4U 4S jjrS rS r	\
\          SS
\R                  S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\\\R"                        S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\R                  S	-  S\S	-  S\\   S\4S jj5       5       rSrU =r$ )rS  iŒ  r�  r‘  r’  rl   c                 ó,  >• [         TU ]  U5        UR                  U l        [        R
                  " UR                  UR                  5      U l        [        R                  " U5      nSUl
        SUl        [        U5      U l        [        R                  " U5      nSUl
        UR                  Ul        [        U5      U l        UR"                  U l        [        R$                  " UR&                  UR"                  5      U l        U R+                  5         g r–  )rI   rJ   rr   rÂ  r   r²   rv  rU  r—  r˜  r£   rˆ  rt  r™  rš  ry  r›  r1  rq   rP   rJ  r|  rœ  s       €rB   rJ   Ú MT5ForQuestionAnswering.__init__•  sÈ   ø€ Ü‰Ñ˜Ô ØŸ™ˆŒä—l’l 6×#4Ñ#4°f·n±nÓEˆŒäŸš vÓ.ˆØ$)ˆÔ!Ø#(ˆÔ Ü Ó/ˆŒäŸš vÓ.ˆØ$(ˆÔ!Ø$*×$=Ñ$=ˆÔ!Ü Ó/ˆŒà ×+Ñ+ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrD   c                 ó   • U R                   $ r}   r¡  r¢  s    rB   r£  Ú,MT5ForQuestionAnswering.get_input_embeddings¬  r¥  rD   c                 ó|   • Xl         U R                  R                  U5        U R                  R                  U5        g r}   r§  r€  s     rB   r‚  Ú,MT5ForQuestionAnswering.set_input_embeddings°  r©  rD   NrA  r,   r@  rB  rª  Ústart_positionsÚend_positionsr…  r«  rˆ  r0   r4  c                 ó¾  • Ub  Ub  Sn
Uc"  U	c  Uc  [        S5      eU R                  U5      nUc  U R                  " SUUUS.UD6nOK[        U[        5      (       d6  [	        US   [        U5      S:”  a  US   OS[        U5      S:”  a  US   OSS9nUS   nU R                  " SUUU	SUUU
S	.UD6nUS   nU R                  U5      nUR                  SS
S9u  nnUR                  S
5      R                  5       nUR                  S
5      R                  5       nSnUbé  Ubæ  [        UR                  5       5      S:”  a*  UR                  S
5      R                  UR                  5      n[        UR                  5       5      S:”  a*  UR                  S
5      R                  UR                  5      nUR                  S5      nUR                  SU5      nUR                  SU5      n[        US9nU" UU5      nU" UU5      nUU-   S-  n[!        UUUUR"                  UR$                  UR&                  UR(                  UR*                  UR$                  UR&                  S9
$ )aÚ  
input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
    Indices of input sequence tokens in the vocabulary. T5 is a model with relative position embeddings so you
    should be able to pad the inputs on both the right and the left.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for detail.

    [What are input IDs?](../glossary#input-ids)

    To know more on how to prepare `input_ids` for pretraining take a look a [T5 Training](./t5#training).
decoder_input_ids (`torch.LongTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Indices of decoder input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are decoder input IDs?](../glossary#decoder-input-ids)

    T5 uses the `pad_token_id` as the starting token for `decoder_input_ids` generation. If `past_key_values`
    is used, optionally only the last `decoder_input_ids` have to be input (see `past_key_values`).

    To know more on how to prepare `decoder_input_ids` for pretraining take a look at [T5
    Training](./t5#training).
decoder_attention_mask (`torch.BoolTensor` of shape `(batch_size, target_sequence_length)`, *optional*):
    Default behavior: generate a tensor that ignores pad tokens in `decoder_input_ids`. Causal mask will also
    be used by default.
NFræ  r­  r   r&   r4   r®  r¯  r2   r5   rÊ  )
rÍ  Ústart_logitsÚ
end_logitsrí   r°  r±  r>  r²  r#  r³  rf  )r`  rd  r™  r~   r   r´  r›  rJ  Úsplitrñ  r?   r9   rV   rË   r"  r   r   rí   r^   r=  r>  r†  )rO   rA  r,   r@  rB  rª  r  r  r…  r«  rˆ  r0   r^   rµ  rÏ  rÎ  r  r	  Ú
total_lossÚignored_indexrÑ  Ú
start_lossÚend_losss                          rB   r`   ÚMT5ForQuestionAnswering.forwardµ  s¦  € ðZ Ñ&¨=Ñ+DØˆIð
 Ñ$Ð)>Ñ)FØÑ Ü ðUóð ð
 !%× 1Ñ 1°)Ó <Ðð Ñ"Ø"Ÿlšlð Ø#Ø-Ø+ñð ñ	‰Oô ˜O¬_×=Ñ=Ü-Ø"1°!Ñ"4Ü47¸Ó4HÈ1Ó4L˜o¨aÒ0ÐRVÜ14°_Ó1EÈÓ1I˜?¨1Ò-ÈtñˆOð (¨Ñ*ˆð Ÿ,š,ð 	
Ø'Ø1Ø/Ø Ø"/Ø#1Øñ	
ð ñ	
ˆð *¨!Ñ,ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ð#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÑ&¨=Ñ+Dä�?×'Ñ'Ó)Ó*¨QÓ.Ø"1×"9Ñ"9¸"Ó"=×"@Ñ"@À×ATÑATÓ"U�Ü�=×%Ñ%Ó'Ó(¨1Ó,Ø -× 5Ñ 5°bÓ 9× <Ñ <¸Z×=NÑ=NÓ O�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÑCˆHÙ! ,°Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJä2ØØ%Ø!Ø+×;Ñ;Ø"1×"?Ñ"?Ø.×9Ñ9Ø,×=Ñ=Ø&5×&GÑ&GØ"1×"?Ñ"?Ø.×9Ñ9ñ
ð 	
rD   )r›  r™  rÂ  r1  rJ  rU  rÖ  )rb   rc   rd   re   r¹  rº  r'   rJ   r£  r‚  r"   r   r:   r»  r¼  r½  r¾  r   rý   r   r   r   r`   rf   rg   rh   s   @rB   rS  rS  Œ  se  ø† à*rÐ)sÐ&à'6Ø'6ñÐð˜y÷ ò.ò:ð
 Øð .2Ø37Ø59Ø:>Ø=AØ37Ø15Ø26Ø:>Ø!%ñy
à×#Ñ# dÑ*ðy
ð ×)Ñ)¨DÑ0ðy
ð !×+Ñ+¨dÑ2ð	y
ð
 !&× 0Ñ 0°4Ñ 7ðy
ð ˜u U§\¡\Ñ2Ñ3°dÑ:ðy
ð ×)Ñ)¨DÑ0ðy
ð ×'Ñ'¨$Ñ.ðy
ð ×(Ñ(¨4Ñ/ðy
ð  %×0Ñ0°4Ñ7ðy
ð ˜$‘;ðy
ð Ð+Ñ,ðy
ð 
-ôy
ó ó öy
rD   rS  )rR  rQ  rS  rá  rY  rP  r:  )NrÚ   N)Pr8  r—  r½   Úcollections.abcr   r:   r   Útorch.nnr   r   r   Ú r	   rN  Úactivationsr
   Úcache_utilsr   r   r   Ú
generationr   Úmasking_utilsr   r   Úmodeling_layersr   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   r   Úprocessing_utilsr   Úutilsr   r   r   r   r    r!   Úutils.genericr"   r#   Úutils.output_capturingr$   r%   Úconfiguration_mt5r'   Ú
get_loggerrb   r¬   ÚModuler   r¼   rC   rF   rj   r…   r�   r�   r   r  r  r-  r:  rt  rP  rQ  rR  rá  rY  rS  Ú__all__rf  rD   rB   Ú<module>r"     s   ðñ ã Û Ý $ã Ý ß AÑ Aå &Ý !ß CÑ CÝ )ß JÝ 9÷÷ ñ ÷ GÝ &ß r× rß Iß EÝ (ð 
×	Ò	˜HÓ	%€ð !ØØ)-ñ%Ø�I‰Ið%à�<‰<ð%ð 
�‰ð%ð �<‰<ð	%ð
 —L‘L 4Ñ'ð%ð �T‰\ð%ð ð%ð —<‘< $Ñ&ð%ð Ð'Ñ(õ%ôB+�2—9‘9ô +ô4�r—y‘yô ô.˜BŸI™Iô ô<�—‘ô ô&A8�2—9‘9ô A8ôJ=˜BŸI™Iô =ôB9˜RŸY™Yô 9ô@JPÐ)ô JPô\˜BŸI™Iô ð$ ôj!˜ó j!ó ðj!ô\n
Ð!ô n
ðb ô]
Ð!ó ]
ó ð]
ñ@ ðñô
{)Ð"4°oó {)óð
{)ð| ôSÐ(ó Só ðSñl ðñôL
Ð#5ó L
óðL
ð^ ô;
Ð 2ó ;
ó ð;
ð| ôd
Ð0ó d
ó ðd
òN�rD   