ó
    ˆŽ¡j?½  ã                   ó´  • S r SSKrSSKrSSKJr  SSKJrJrJr  SSKJ	r
  SSKJr  SSKJr  SS	KJrJrJrJrJrJrJr  SS
KJr  SSKJrJr  SSKJr  SSKJrJ r J!r!J"r"J#r#J$r$  \RJ                  " \&5      r' " S S\RP                  5      r) " S S\RP                  5      r* " S S\RP                  5      r+ " S S\RP                  5      r, " S S\RP                  5      r- " S S\RP                  5      r. " S S\RP                  5      r/ " S S\RP                  5      r0 " S S \RP                  5      r1\ " S! S"\5      5       r2\ " S# S$\25      5       r3\ " S% S&\25      5       r4 " S' S(\RP                  5      r5\" S)S*9 " S+ S,\25      5       r6\ " S- S.\25      5       r7\ " S/ S0\25      5       r8 " S1 S2\RP                  5      r9\ " S3 S4\25      5       r:S7S5 jr;/ S6Qr<g)8zPyTorch I-BERT model.é    N)Únn)ÚBCEWithLogitsLossÚCrossEntropyLossÚMSELossé   )Úinitialization)Úgelu)Úcreate_bidirectional_mask)Ú)BaseModelOutputWithPastAndCrossAttentionsÚ,BaseModelOutputWithPoolingAndCrossAttentionsÚMaskedLMOutputÚMultipleChoiceModelOutputÚQuestionAnsweringModelOutputÚSequenceClassifierOutputÚTokenClassifierOutput)ÚPreTrainedModel)Úauto_docstringÚloggingé   )ÚIBertConfig)ÚIntGELUÚIntLayerNormÚ
IntSoftmaxÚQuantActÚQuantEmbeddingÚQuantLinearc                   ó>   ^ • \ rS rSrSrU 4S jr SS jrS rSrU =r	$ )ÚIBertEmbeddingsé.   zN
Same as BertEmbeddings with a tiny tweak for positional embeddings indexing.
c                 ó`  >• [         TU ]  5         UR                  U l        SU l        SU l        SU l        SU l        SU l        [        UR                  UR                  UR                  U R                  U R                  S9U l        [        UR                  UR                  U R                  U R                  S9U l        [        R                   " ["        R$                  " UR&                  5      R)                  S5      SS	9U l        UR                  U l        [        UR&                  UR                  U R,                  U R                  U R                  S9U l        [1        U R                  U R                  S
9U l        [1        U R                  U R                  S
9U l        [7        UR                  UR8                  U R                  U R                  UR:                  S9U l        [1        U R
                  U R                  S
9U l        [        R@                  " URB                  5      U l"        g )Né   é   é   é    )Úpadding_idxÚ
weight_bitÚ
quant_mode)r&   r'   ©r   éÿÿÿÿF)Ú
persistent©r'   ©ÚepsÚ
output_bitr'   Úforce_dequant)#ÚsuperÚ__init__r'   Úembedding_bitÚembedding_act_bitÚact_bitÚln_input_bitÚln_output_bitr   Ú
vocab_sizeÚhidden_sizeÚpad_token_idÚword_embeddingsÚtype_vocab_sizeÚtoken_type_embeddingsr   ÚBufferÚtorchÚarangeÚmax_position_embeddingsÚexpandÚposition_idsr%   Úposition_embeddingsr   Úembeddings_act1Úembeddings_act2r   Úlayer_norm_epsr/   Ú	LayerNormÚoutput_activationÚDropoutÚhidden_dropout_probÚdropout©ÚselfÚconfigÚ	__class__s     €Ú¶C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/ibert/modeling_ibert.pyr1   ÚIBertEmbeddings.__init__3   s¾  ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆÔØ!#ˆÔØˆŒØˆÔØˆÔä-Ø×ÑØ×ÑØ×+Ñ+Ø×)Ñ)Ø—‘ñ 
ˆÔô &4Ø×"Ñ" F×$6Ñ$6À4×CUÑCUÐbf×bqÑbqñ&
ˆÔ"ô
 ŸIšI¤e§l¢l°6×3QÑ3QÓ&R×&YÑ&YÐZaÓ&bÐotÑuˆÔð "×.Ñ.ˆÔÜ#1Ø×*Ñ*Ø×ÑØ×(Ñ(Ø×)Ñ)Ø—‘ñ$
ˆÔ ô  (¨×(>Ñ(>È4Ï?É?Ñ[ˆÔÜ'¨×(>Ñ(>È4Ï?É?Ñ[ˆÔä%Ø×ÑØ×%Ñ%Ø×)Ñ)Ø—‘Ø ×.Ñ.ñ
ˆŒô "*¨$¯,©,À4Ç?Á?Ñ!SˆÔÜ—z’z &×"<Ñ"<Ó=ˆ�ó    c                 ó–  • UcD  Ub0  [        XR                  U5      R                  UR                  5      nOU R	                  U5      nUb  UR                  5       nOUR                  5       S S nUc8  [        R                  " U[        R                  U R                  R                  S9nUc  U R                  U5      u  pGOS nU R                  U5      u  p‰U R                  UUUU	S9u  p«U R                  U5      u  pÍU R                  U
UUUS9u  p«U R                  X«5      u  p«U R                  U
5      n
U R!                  X«5      u  p«X«4$ )Nr)   ©ÚdtypeÚdevice©ÚidentityÚidentity_scaling_factor)Ú"create_position_ids_from_input_idsr%   ÚtorV   Ú&create_position_ids_from_inputs_embedsÚsizer>   ÚzerosÚlongrB   r:   r<   rD   rC   rG   rK   rH   )rM   Ú	input_idsÚtoken_type_idsrB   Úinputs_embedsÚpast_key_values_lengthÚinput_shapeÚinputs_embeds_scaling_factorr<   Ú$token_type_embeddings_scaling_factorÚ
embeddingsÚembeddings_scaling_factorrC   Ú"position_embeddings_scaling_factors                 rP   ÚforwardÚIBertEmbeddings.forwardb   sq  € ð ÑØÑ$äAØ×/Ñ/Ð1Gó ç‘"�Y×%Ñ%Ó&ñ ð  $×JÑJÈ=ÓY�àÑ Ø#Ÿ.™.Ó*‰Kà'×,Ñ,Ó.¨s°Ð3ˆKàÑ!Ü"Ÿ[š[¨¼E¿J¹JÈt×O`ÑO`×OgÑOgÑhˆNàÑ Ø:>×:NÑ:NÈyÓ:YÑ7ˆMÐ7à+/Ð(ØFJ×F`ÑF`ÐaoÓFpÑCÐà04×0DÑ0DØØ(Ø*Ø$Hð	 1Eð 1
Ñ-ˆ
ð CG×BZÑBZÐ[gÓBhÑ?ÐØ04×0DÑ0DØØ%Ø(Ø$Fð	 1Eð 1
Ñ-ˆ
ð 15·±¸zÓ0eÑ-ˆ
Ø—\‘\ *Ó-ˆ
Ø04×0FÑ0FÀzÓ0mÑ-ˆ
ØÐ4Ð4rR   c                 ó  • UR                  5       SS nUS   n[        R                  " U R                  S-   X0R                  -   S-   [        R                  UR
                  S9nUR                  S5      R                  U5      $ )z¯
We are provided embeddings directly. We cannot infer which are padded so just generate sequential position ids.

Args:
    inputs_embeds: torch.Tensor

Returns: torch.Tensor
Nr)   r   rT   r   )r]   r>   r?   r%   r_   rV   Ú	unsqueezerA   )rM   rb   rd   Úsequence_lengthrB   s        rP   r\   Ú6IBertEmbeddings.create_position_ids_from_inputs_embeds�   s~   € ð $×(Ñ(Ó*¨3¨BÐ/ˆØ% a™.ˆä—|’|Ø×Ñ˜qÑ  /×4DÑ4DÑ"DÀqÑ"HÔPU×PZÑPZÐcp×cwÑcwñ
ˆð ×%Ñ% aÓ(×/Ñ/°Ó<Ð<rR   )rG   r4   rK   r3   r2   rD   rE   r5   r6   rH   r%   rC   rB   r'   r<   r:   )NNNNr   )
Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r1   rj   r\   Ú__static_attributes__Ú__classcell__©rO   s   @rP   r   r   .   s%   ø† ñõ->ð` rsô,5÷\=ð =rR   r   c                   ó6   ^ • \ rS rSrU 4S jr  SS jrSrU =r$ )ÚIBertSelfAttentioné¢   c           
      ó$  >• [         TU ]  5         UR                  UR                  -  S:w  a7  [	        US5      (       d&  [        SUR                   SUR                   S35      eUR                  U l        SU l        SU l        SU l	        UR                  U l        [        UR                  UR                  -  5      U l        U R                  U R                  -  U l        [        UR                  U R                  SU R                  U R                  U R                  SS	9U l        [        UR                  U R                  SU R                  U R                  U R                  SS	9U l        [        UR                  U R                  SU R                  U R                  U R                  SS	9U l        [#        U R                  U R                  S
9U l        [#        U R                  U R                  S
9U l        [#        U R                  U R                  S
9U l        [#        U R                  U R                  S
9U l        [,        R.                  " UR0                  5      U l        [5        U R                  U R                  UR6                  S9U l        g )Nr   Úembedding_sizezThe hidden size (z6) is not a multiple of the number of attention heads (Ú)r!   r$   T©Úbiasr&   Úbias_bitr'   Úper_channelr+   ©r'   r/   )r0   r1   r8   Únum_attention_headsÚhasattrÚ
ValueErrorr'   r&   r€   r4   ÚintÚattention_head_sizeÚall_head_sizer   ÚqueryÚkeyÚvaluer   Úquery_activationÚkey_activationÚvalue_activationrH   r   rI   Úattention_probs_dropout_probrK   r   r/   ÚsoftmaxrL   s     €rP   r1   ÚIBertSelfAttention.__init__£   s  ø€ Ü‰ÑÔØ×Ñ × :Ñ :Ñ:¸aÓ?ÌÐPVÐXh×HiÑHiÜØ# F×$6Ñ$6Ð#7ð 8Ø ×4Ñ4Ð5°Qð8óð ð !×+Ñ+ˆŒØˆŒØˆŒØˆŒà#)×#=Ñ#=ˆÔ Ü#& v×'9Ñ'9¸F×<VÑ<VÑ'VÓ#WˆÔ Ø!×5Ñ5¸×8PÑ8PÑPˆÔô !Ø×ÑØ×ÑØØ—‘Ø—]‘]Ø—‘Øñ
ˆŒ
ô Ø×ÑØ×ÑØØ—‘Ø—]‘]Ø—‘Øñ
ˆŒô !Ø×ÑØ×ÑØØ—‘Ø—]‘]Ø—‘Øñ
ˆŒ
ô !)¨¯©À$Ç/Á/Ñ RˆÔÜ& t§|¡|ÀÇÁÑPˆÔÜ (¨¯©À$Ç/Á/Ñ RˆÔÜ!)¨$¯,©,À4Ç?Á?Ñ!SˆÔä—z’z &×"EÑ"EÓFˆŒä! $§,¡,¸4¿?¹?ÐZ`×ZnÑZnÑoˆ�rR   c                 ól  • U R                  X5      u  pVU R                  X5      u  pxU R                  X5      u  pšU R                  XV5      u  p¼U R	                  Xx5      u  pÞU R                  Xš5      u  nnUR                  S S n/ UQSPU R                  P7nUR                  U5      R                  SS5      nUR                  U5      R                  SS5      nUR                  U5      R                  SS5      n[        R                  " X½R                  SS5      5      n[        R                  " U R                  5      nUU-  nU R                  (       a  XÎ-  U-  nOS nUb  UU-   nU R                  UU5      u  nnU R!                  U5      n[        R                  " UU5      nUb  UU-  nOS nUR#                  SSSS5      R%                  5       nUR'                  5       S S U R(                  4-   nUR                  " U6 nU R+                  UU5      u  nnU(       a  UU4OU4nU(       a  UU4OU4nUU4$ )Nr)   r   é   éþÿÿÿr   r   )r‰   rŠ   r‹   rŒ   r�   rŽ   Úshaper‡   ÚviewÚ	transposer>   ÚmatmulÚmathÚsqrtr'   r�   rK   ÚpermuteÚ
contiguousr]   rˆ   rH   )rM   Úhidden_statesÚhidden_states_scaling_factorÚattention_maskÚoutput_attentionsÚmixed_query_layerÚ mixed_query_layer_scaling_factorÚmixed_key_layerÚmixed_key_layer_scaling_factorÚmixed_value_layerÚ mixed_value_layer_scaling_factorÚquery_layerÚquery_layer_scaling_factorÚ	key_layerÚkey_layer_scaling_factorÚvalue_layerÚvalue_layer_scaling_factorrd   Úhidden_shapeÚattention_scoresÚscaleÚattention_scores_scaling_factorÚattention_probsÚattention_probs_scaling_factorÚcontext_layerÚcontext_layer_scaling_factorÚnew_context_layer_shapeÚoutputsÚoutput_scaling_factors                                rP   rj   ÚIBertSelfAttention.forwardÚ   sp  € ð ?C¿j¹jÈÓ>uÑ;ÐØ:>¿(¹(À=Ó:oÑ7ˆØ>B¿j¹jÈÓ>uÑ;Ðð 37×2GÑ2GØó3
Ñ/ˆð /3×.AÑ.AÀ/Ó.rÑ+ˆ	Ø26×2GÑ2GØó3
Ñ/ˆÐ/ð
 $×)Ñ)¨#¨2Ð.ˆØC˜ÐC bÐC¨$×*BÑ*BÑCˆØ!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆØ—N‘N <Ó0×:Ñ:¸1¸aÓ@ˆ	Ø!×&Ñ& |Ó4×>Ñ>¸qÀ!ÓDˆô !Ÿ<š<¨×5HÑ5HÈÈRÓ5PÓQÐÜ—	’	˜$×2Ñ2Ó3ˆØ+¨eÑ3ÐØ�?�?Ø.HÑ.cÐfkÑ.kÑ+à.2Ð+àÑ%à/°.Ñ@Ðð ;?¿,¹,ØÐ=ó;
Ñ7ˆÐ7ð Ÿ,™, Ó7ˆäŸš _°kÓBˆØ)Ñ5Ø+IÐLfÑ+fÑ(à+/Ð(à%×-Ñ-¨a°°A°qÓ9×DÑDÓFˆØ"/×"4Ñ"4Ó"6°s¸Ð";¸t×?QÑ?QÐ>SÑ"SÐØ%×*Ò*Ð,CÐDˆð 7;×6LÑ6LØÐ7ó7
Ñ3ˆÐ3ö 7H�= /Ñ2ÈmÐM]ˆö !ð *Ð+IÑJà.Ð0ð 	ð Ð-Ð-Ð-rR   )r4   rˆ   r‡   r€   rK   rŠ   r�   rƒ   rH   r'   r‰   rŒ   r�   r‹   rŽ   r&   ©NF©rp   rq   rr   rs   r1   rj   ru   rv   rw   s   @rP   ry   ry   ¢   s   ø† õ5pðv Ø÷H.ò H.rR   ry   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚIBertSelfOutputi%  c           
      ót  >• [         TU ]  5         UR                  U l        SU l        SU l        SU l        SU l        SU l        [        UR                  UR                  SU R                  U R
                  U R                  SS9U l
        [        U R                  U R                  S9U l        [        UR                  UR                  U R                  U R                  UR                  S9U l        [        U R                  U R                  S9U l        [$        R&                  " UR(                  5      U l        g ©Nr!   r$   r#   Tr~   r+   r,   )r0   r1   r'   r4   r&   r€   r5   r6   r   r8   Údenser   Úln_input_actr   rF   r/   rG   rH   r   rI   rJ   rK   rL   s     €rP   r1   ÚIBertSelfOutput.__init__&  sú   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆŒØˆŒØˆŒØˆÔØˆÔä Ø×ÑØ×ÑØØ—‘Ø—]‘]Ø—‘Øñ
ˆŒ
ô % T×%6Ñ%6À4Ç?Á?ÑSˆÔÜ%Ø×ÑØ×%Ñ%Ø×)Ñ)Ø—‘Ø ×.Ñ.ñ
ˆŒô "*¨$¯,©,À4Ç?Á?Ñ!SˆÔÜ—z’z &×"<Ñ"<Ó=ˆ�rR   c                 óÄ   • U R                  X5      u  pU R                  U5      nU R                  UUUUS9u  pU R                  X5      u  pU R	                  X5      u  pX4$ ©NrW   ©r¿   rK   rÀ   rG   rH   ©rM   r�   rž   Úinput_tensorÚinput_tensor_scaling_factors        rP   rj   ÚIBertSelfOutput.forwardC  óz   € Ø6:·j±jÀÓ6mÑ3ˆØŸ™ ]Ó3ˆØ6:×6GÑ6GØØ(Ø!Ø$?ð	 7Hð 7
Ñ3ˆð 7;·n±nÀ]Ó6qÑ3ˆà6:×6LÑ6LØó7
Ñ3ˆð Ð:Ð:rR   ©rG   r4   r€   r¿   rK   rÀ   r5   r6   rH   r'   r&   rº   rw   s   @rP   r¼   r¼   %  ó   ø† õ>÷:;ð ;rR   r¼   c                   ó6   ^ • \ rS rSrU 4S jr  SS jrSrU =r$ )ÚIBertAttentioniT  c                 ó„   >• [         TU ]  5         UR                  U l        [        U5      U l        [        U5      U l        g ©N)r0   r1   r'   ry   rM   r¼   ÚoutputrL   s     €rP   r1   ÚIBertAttention.__init__U  s3   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ& vÓ.ˆŒ	Ü% fÓ-ˆ�rR   c                 óŽ   • U R                  UUUU5      u  pVU R                  US   US   X5      u  pxU4USS  -   n	U4USS  -   n
Xš4$ )Nr   r   )rM   rÐ   )rM   r�   rž   rŸ   r    Úself_outputsÚself_outputs_scaling_factorÚattention_outputÚattention_output_scaling_factorr¶   Úoutputs_scaling_factors              rP   rj   ÚIBertAttention.forward[  s~   € ð 59·I±IØØ(ØØó	5
Ñ1ˆð =A¿K¹KØ˜‰OÐ8¸Ñ;¸]ó=
Ñ9Ðð $Ð%¨°Q°RÐ(8Ñ8ˆØ"AÐ!CÐFaÐbcÐbdÐFeÑ!eÐØÐ.Ð.rR   )rÐ   r'   rM   r¹   rº   rw   s   @rP   rÍ   rÍ   T  s   ø† õ.ð Ø÷/ò /rR   rÍ   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚIBertIntermediateip  c           
      ó¼  >• [         TU ]  5         UR                  U l        SU l        SU l        SU l        [        UR                  UR                  SU R                  U R
                  U R                  SS9U l	        UR                  S:w  a  [        S5      e[        U R                  UR                  S9U l        [        U R                  U R                  S9U l        g )	Nr!   r$   Tr~   r	   z3I-BERT only supports 'gelu' for `config.hidden_act`r‚   r+   )r0   r1   r'   r4   r&   r€   r   r8   Úintermediate_sizer¿   Ú
hidden_actr…   r   r/   Úintermediate_act_fnr   rH   rL   s     €rP   r1   ÚIBertIntermediate.__init__q  s¶   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆŒØˆŒØˆŒÜ Ø×ÑØ×$Ñ$ØØ—‘Ø—]‘]Ø—‘Øñ
ˆŒ
ð ×Ñ Ó&ÜÐRÓSÐSÜ#*°d·o±oÐU[×UiÑUiÑ#jˆÔ Ü!)¨$¯,©,À4Ç?Á?Ñ!SˆÕrR   c                 óz   • U R                  X5      u  pU R                  X5      u  pU R                  X5      u  pX4$ rÏ   )r¿   rÞ   rH   )rM   r�   rž   s      rP   rj   ÚIBertIntermediate.forward…  sL   € Ø6:·j±jÀÓ6mÑ3ˆØ6:×6NÑ6NØó7
Ñ3ˆð
 7;×6LÑ6LØó7
Ñ3ˆð Ð:Ð:rR   )r4   r€   r¿   rÞ   rH   r'   r&   rº   rw   s   @rP   rÚ   rÚ   p  s   ø† õT÷(
;ð 
;rR   rÚ   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚIBertOutputi’  c           
      ót  >• [         TU ]  5         UR                  U l        SU l        SU l        SU l        SU l        SU l        [        UR                  UR                  SU R                  U R
                  U R                  SS9U l        [        U R                  U R                  S9U l        [        UR                  UR                  U R                  U R                  UR                   S9U l        [        U R                  U R                  S9U l        [&        R(                  " UR*                  5      U l        g r¾   )r0   r1   r'   r4   r&   r€   r5   r6   r   rÜ   r8   r¿   r   rÀ   r   rF   r/   rG   rH   r   rI   rJ   rK   rL   s     €rP   r1   ÚIBertOutput.__init__“  sú   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆŒØˆŒØˆŒØˆÔØˆÔä Ø×$Ñ$Ø×ÑØØ—‘Ø—]‘]Ø—‘Øñ
ˆŒ
ô % T×%6Ñ%6À4Ç?Á?ÑSˆÔÜ%Ø×ÑØ×%Ñ%Ø×)Ñ)Ø—‘Ø ×.Ñ.ñ
ˆŒô "*¨$¯,©,À4Ç?Á?Ñ!SˆÔÜ—z’z &×"<Ñ"<Ó=ˆ�rR   c                 óÄ   • U R                  X5      u  pU R                  U5      nU R                  UUUUS9u  pU R                  X5      u  pU R	                  X5      u  pX4$ rÃ   rÄ   rÅ   s        rP   rj   ÚIBertOutput.forward°  rÉ   rR   rÊ   rº   rw   s   @rP   rã   rã   ’  rË   rR   rã   c                   ó<   ^ • \ rS rSrU 4S jr  SS jrS rSrU =r$ )Ú
IBertLayeriÁ  c                 óL  >• [         TU ]  5         UR                  U l        SU l        SU l        [        U5      U l        [        U5      U l        [        U5      U l
        [        U R                  U R                  S9U l        [        U R                  U R                  S9U l        g )Nr!   r   r+   )r0   r1   r'   r4   Úseq_len_dimrÍ   Ú	attentionrÚ   Úintermediaterã   rÐ   r   Úpre_intermediate_actÚpre_output_actrL   s     €rP   r1   ÚIBertLayer.__init__Â  s}   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒØˆŒàˆÔÜ'¨Ó/ˆŒÜ-¨fÓ5ˆÔÜ! &Ó)ˆŒä$,¨T¯\©\ÀdÇoÁoÑ$VˆÔ!Ü& t§|¡|ÀÇÁÑPˆÕrR   c                 ó~   • U R                  UUUUS9u  pVUS   nUS   nUSS  n	U R                  Xx5      u  p«U
4U	-   n	U	$ )N)r    r   r   )rì   Úfeed_forward_chunk)rM   r�   rž   rŸ   r    Úself_attention_outputsÚ%self_attention_outputs_scaling_factorrÕ   rÖ   r¶   Úlayer_outputÚlayer_output_scaling_factors               rP   rj   ÚIBertLayer.forwardÏ  sz   € ð IMÏÉØØ(ØØ/ð	 IWð I
ÑEÐð 2°!Ñ4ÐØ*OÐPQÑ*RÐ'à(¨¨Ð,ˆà48×4KÑ4KØó5
Ñ1ˆð  �/ GÑ+ˆàˆrR   c                 ó¢   • U R                  X5      u  pU R                  X5      u  p4U R                  X45      u  p4U R                  X4X5      u  pVXV4$ rÏ   )rî   rí   rï   rÐ   )rM   rÕ   rÖ   Úintermediate_outputÚ"intermediate_output_scaling_factorrõ   rö   s          rP   rò   ÚIBertLayer.feed_forward_chunkè  su   € Ø<@×<UÑ<UØó=
Ñ9Ðð CG×BSÑBSØóC
Ñ?Ðð CG×BUÑBUØóC
Ñ?Ðð 59·K±KØÐEUó5
Ñ1ˆð Ð8Ð8rR   )r4   rì   rí   rÐ   rî   rï   r'   rë   r¹   )	rp   rq   rr   rs   r1   rj   rò   ru   rv   rw   s   @rP   ré   ré   Á  s    ø† õQð" Øô÷29ð 9rR   ré   c                   ó:   ^ • \ rS rSrU 4S jr    SS jrSrU =r$ )ÚIBertEncoderiù  c                 óê   >• [         TU ]  5         Xl        UR                  U l        [        R
                  " [        UR                  5       Vs/ s H  n[        U5      PM     sn5      U l	        g s  snf rÏ   )
r0   r1   rN   r'   r   Ú
ModuleListÚrangeÚnum_hidden_layersré   Úlayer)rM   rN   Ú_rO   s      €rP   r1   ÚIBertEncoder.__init__ú  sT   ø€ Ü‰ÑÔØŒØ ×+Ñ+ˆŒÜ—]’]ÄÀf×F^ÑF^Ô@_Ó#`Ò@_¸1¤J¨vÖ$6Ñ@_Ñ#`Óaˆ�
ùÒ#`s   ÁA0c                 ó2  • U(       a  SOS nU(       a  SOS nS n	[        U R                  5       H2  u  p«U(       a  Xq4-   nU" UUUU5      nUS   nU(       d  M*  XŒS   4-   nM4     U(       a  Xq4-   nU(       d  [        S UUUU	4 5       5      $ [        UUUU	S9$ )N© r   r   c              3   ó0   #   • U  H  nUc  M  Uv •  M     g 7frÏ   r  )Ú.0Úvs     rP   Ú	<genexpr>Ú'IBertEncoder.forward.<locals>.<genexpr>   s"   é € ð 	ò�Að ÷ ‘òùs   ‚�	)Úlast_hidden_stater�   Ú
attentionsÚcross_attentions)Ú	enumerater  Útupler   )rM   r�   rž   rŸ   r    Úoutput_hidden_statesÚreturn_dictÚall_hidden_statesÚall_self_attentionsÚall_cross_attentionsÚiÚlayer_moduleÚlayer_outputss                rP   rj   ÚIBertEncoder.forward   sÔ   € ö #7™B¸DÐÞ$5™b¸4ÐØ#Ðä(¨¯©Ö4‰OˆAÞ#Ø$5Ð8HÑ$HÐ!á(ØØ,ØØ!ó	ˆMð *¨!Ñ,ˆMß Ð Ø&9È1Ñ=MÐ<OÑ&OÒ#ñ  5ö  Ø 1Ð4DÑ DÐæÜñ 	ð "Ø%Ø'Ø(ñ	ó	ó 	ð 	ô 9Ø+Ø+Ø*Ø1ñ	
ð 	
rR   )rN   r  r'   )NFFTrº   rw   s   @rP   rý   rý   ù  s!   ø† õbð ØØ"Ø÷/
ò /
rR   rý   c                   ó.   ^ • \ rS rSrU 4S jrS rSrU =r$ )ÚIBertPooleri2  c                 óØ   >• [         TU ]  5         UR                  U l        [        R                  " UR
                  UR
                  5      U l        [        R                  " 5       U l        g rÏ   )	r0   r1   r'   r   ÚLinearr8   r¿   ÚTanhÚ
activationrL   s     €rP   r1   ÚIBertPooler.__init__3  sF   ø€ Ü‰ÑÔØ ×+Ñ+ˆŒÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ'š'›)ˆ�rR   c                 ó\   • US S 2S4   nU R                  U5      nU R                  U5      nU$ ©Nr   )r¿   r  )rM   r�   Úfirst_token_tensorÚpooled_outputs       rP   rj   ÚIBertPooler.forward9  s6   € ð +ª1¨a¨4Ñ0ÐØŸ
™
Ð#5Ó6ˆØŸ™¨Ó6ˆØÐrR   )r  r¿   r'   rº   rw   s   @rP   r  r  2  s   ø† õ$÷ð rR   r  c                   ój   ^ • \ rS rSr% \\S'   Sr\R                  " 5       U 4S j5       r	SS jr
SrU =r$ )ÚIBertPreTrainedModeliB  rN   Úibertc                 ó²  >• [         TU ]  U5        [        U[        5      (       aß  [        R
                  " UR                  SU R                  R                  S9  UR                  b   [        R                  " UR                  5        [        USS5      b@  [        R                  " UR                  5        [        R                  " UR                  5        [        USS5      b!  [        R                  " UR                  5        gg[        U[        5      (       aÚ  [        R
                  " UR                  SU R                  R                  S9  UR                   bI  [        UR                  SS5      (       d-  [        R                  " UR                  UR                      5        [        USS5      bA  [        R                  " UR"                  5        [        R                  " UR                  5        gg[        U[$        5      (       a0  [        US	S5      b!  [        R                  " UR&                  5        gg[        U[(        5      (       a!  [        R                  " UR                  5        g[        U[*        5      (       a\  [        R,                  " UR.                  [0        R2                  " UR.                  R4                  S
   5      R7                  S5      5        g[        U[8        5      (       ac  [        R:                  " UR<                  S5        [        R:                  " UR>                  S5        [        R                  " UR@                  5        gg)zInitialize the weightsg        )ÚmeanÚstdNÚweight_integerÚbias_integerÚ_is_hf_initializedFÚweight_scaling_factorÚshiftr)   r(   gñhãˆµøä¾gñhãˆµøä>)!r0   Ú_init_weightsÚ
isinstancer   ÚinitÚnormal_ÚweightrN   Úinitializer_ranger   Úzeros_Úgetattrr,  Úfc_scaling_factorr-  r   r%   r/  r   r0  ÚIBertLMHeadr   Úcopy_rB   r>   r?   r•   rA   r   Ú	constant_Úx_minÚx_maxÚact_scaling_factor)rM   ÚmodulerO   s     €rP   r1  Ú"IBertPreTrainedModel._init_weightsG  s2  ø€ ô 	‰Ñ˜fÔ%Ü�fœk×*Ñ*Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6SÒTØ�{‰{Ñ&Ü—’˜FŸK™KÔ(Ü�vÐ/°Ó6ÑBÜ—’˜F×1Ñ1Ô2Ü—’˜F×4Ñ4Ô5Ü�v˜~¨tÓ4Ñ@Ü—’˜F×/Ñ/Õ0ð Aä˜¤×/Ñ/Ü�LŠL˜Ÿ™¨S°d·k±k×6SÑ6SÒTà×!Ñ!Ñ-´g¸f¿m¹mÐMaÐch×6iÑ6iÜ—’˜FŸM™M¨&×*<Ñ*<Ñ=Ô>Ü�vÐ6¸Ó=ÑIÜ—’˜F×8Ñ8Ô9Ü—’˜F×1Ñ1Õ2ð Jô ˜¤×-Ñ-Ü�v˜w¨Ó-Ñ9Ü—’˜FŸL™LÕ)ð :ä˜¤×,Ñ,Ü�KŠK˜Ÿ™Õ$Ü˜¤×0Ñ0Ü�JŠJ�v×*Ñ*¬E¯LªL¸×9LÑ9L×9RÑ9RÐSUÑ9VÓ,W×,^Ñ,^Ð_fÓ,gÕhÜ˜¤×)Ñ)Ü�NŠN˜6Ÿ<™<¨Ô/Ü�NŠN˜6Ÿ<™<¨Ô.Ü�KŠK˜×1Ñ1Õ2ð *rR   c                 ó   • [        S5      e)Nz6`resize_token_embeddings` is not supported for I-BERT.)ÚNotImplementedError)rM   Únew_num_tokenss     rP   Úresize_token_embeddingsÚ,IBertPreTrainedModel.resize_token_embeddingsh  s   € Ü!Ð"ZÓ[Ð[rR   r  rÏ   )rp   rq   rr   rs   r   Ú__annotations__Úbase_model_prefixr>   Úno_gradr1  rE  ru   rv   rw   s   @rP   r'  r'  B  s4   ø‡ àÓØÐà
‡]‚]ƒ_ô3ó ð3÷@\ò \rR   r'  c                   ó<  ^ • \ rS rSrSrSU 4S jjrS rS r\        SS\	R                  S-  S\	R                  S-  S	\	R                  S-  S
\	R                  S-  S\	R                  S-  S\S-  S\S-  S\S-  S\\\	R                     -  4S jj5       rSrU =r$ )Ú
IBertModelil  aš  

The model can behave as an encoder (with only self-attention) as well as a decoder, in which case a layer of
cross-attention is added between the self-attention layers, following the architecture described in [Attention is
all you need](https://huggingface.co/papers/1706.03762) by Ashish Vaswani, Noam Shazeer, Niki Parmar, Jakob Uszkoreit,
Llion Jones, Aidan N. Gomez, Lukasz Kaiser and Illia Polosukhin.

c                 óä   >• [         TU ]  U5        Xl        UR                  U l        [	        U5      U l        [        U5      U l        U(       a  [        U5      OSU l	        U R                  5         g)z^
add_pooling_layer (bool, *optional*, defaults to `True`):
    Whether to add a pooling layer
N)r0   r1   rN   r'   r   rg   rý   Úencoderr  ÚpoolerÚ	post_init)rM   rN   Úadd_pooling_layerrO   s      €rP   r1   ÚIBertModel.__init__w  sX   ø€ ô
 	‰Ñ˜Ô ØŒØ ×+Ñ+ˆŒä)¨&Ó1ˆŒÜ# FÓ+ˆŒæ->”k &Ô)ÀDˆŒð 	�‰ÕrR   c                 ó.   • U R                   R                  $ rÏ   ©rg   r:   ©rM   s    rP   Úget_input_embeddingsÚIBertModel.get_input_embeddingsˆ  s   € Ø�‰×.Ñ.Ð.rR   c                 ó$   • XR                   l        g rÏ   rS  )rM   r‹   s     rP   Úset_input_embeddingsÚIBertModel.set_input_embeddings‹  s   € Ø*/�‰Õ'rR   Nr`   rŸ   ra   rB   rb   r    r  r  Úreturnc	           	      óN  • Ub  UOU R                   R                  nUb  UOU R                   R                  nUb  UOU R                   R                  nUb  Ub  [	        S5      eUb"  U R                  X5        UR                  5       n
O"Ub  UR                  5       S S n
O[	        S5      eU
u  p¼Ub  UR                  OUR                  nUc  [        R                  " X¼4US9nUc$  [        R                  " U
[        R                  US9nU R                  UUUUS9u  pï[        U R                   UUS9nU R                  UUUUUUS9nUS	   nU R                  b  U R                  U5      OS nU(       d
  UU4US
S  -   $ [!        UUUR"                  UR$                  UR&                  S9$ )NzDYou cannot specify both input_ids and inputs_embeds at the same timer)   z5You have to specify either input_ids or inputs_embeds)rV   rT   )r`   rB   ra   rb   )rN   rb   rŸ   )rŸ   r    r  r  r   r   )r  Úpooler_outputr�   r  r  )rN   r    r  r  r…   Ú%warn_if_padding_and_no_attention_maskr]   rV   r>   Úonesr^   r_   rg   r
   rM  rN  r   r�   r  r  )rM   r`   rŸ   ra   rB   rb   r    r  r  Úkwargsrd   Ú
batch_sizeÚ
seq_lengthrV   Úembedding_outputÚembedding_output_scaling_factorÚencoder_outputsÚsequence_outputr$  s                      rP   rj   ÚIBertModel.forwardŽ  s×  € ð 2CÑ1NÑ-ÐTX×T_ÑT_×TqÑTqÐà$8Ñ$DÑ È$Ï+É+×JjÑJjð 	ð &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆàÑ  ]Ñ%>ÜÐcÓdÐdØÑ"Ø×6Ñ6°yÔQØ#Ÿ.™.Ó*‰KØÑ&Ø'×,Ñ,Ó.¨s°Ð3‰KäÐTÓUÐUà!,Ñˆ
Ø%.Ñ%:�×!Ò!À×@TÑ@TˆàÑ!Ü"ŸZšZ¨*Ð)AÈ6ÑRˆNØÑ!Ü"Ÿ[š[¨¼E¿J¹JÈvÑVˆNà<@¿O¹OØØ%Ø)Ø'ð	 =Lð =
Ñ9Ðô 3Ø—;‘;Ø*Ø)ñ
ˆð Ÿ,™,ØØ+Ø)Ø/Ø!5Ø#ð 'ð 
ˆð *¨!Ñ,ˆØ8<¿¹Ñ8O˜Ÿ™ OÔ4ÐUYˆæØ# ]Ð3°oÀaÀbÐ6IÑIÐIä;Ø-Ø'Ø)×7Ñ7Ø&×1Ñ1Ø,×=Ñ=ñ
ð 	
rR   )rN   rg   rM  rN  r'   )T)NNNNNNNN)rp   rq   rr   rs   rt   r1   rU  rX  r   r>   Ú
LongTensorÚFloatTensorÚboolr   r  rj   ru   rv   rw   s   @rP   rK  rK  l  sû   ø† ñ÷ò"/ò0ð ð .2Ø37Ø26Ø04Ø26Ø)-Ø,0Ø#'ñE
à×#Ñ# dÑ*ðE
ð ×)Ñ)¨DÑ0ðE
ð ×(Ñ(¨4Ñ/ð	E
ð
 ×&Ñ&¨Ñ-ðE
ð ×(Ñ(¨4Ñ/ðE
ð   $™;ðE
ð # T™kðE
ð ˜D‘[ðE
ð 
6¸¸e×>OÑ>OÑ8PÑ	PôE
ó öE
rR   rK  c                   ó^  ^ • \ rS rSrSSS.rU 4S jrS rS r\         SS	\	R                  S-  S
\	R                  S-  S\	R                  S-  S\	R                  S-  S\	R                  S-  S\	R                  S-  S\S-  S\S-  S\S-  S\\\	R                     -  4S jj5       rSrU =r$ )ÚIBertForMaskedLMi×  z(ibert.embeddings.word_embeddings.weight$zlm_head.bias)zlm_head.decoder.weightzlm_head.decoder.biasc                 ó‚   >• [         TU ]  U5        [        USS9U l        [	        U5      U l        U R                  5         g ©NF)rP  )r0   r1   rK  r(  r:  Úlm_headrO  rL   s     €rP   r1   ÚIBertForMaskedLM.__init__Þ  s6   ø€ Ü‰Ñ˜Ô ä ¸%Ñ@ˆŒ
Ü" 6Ó*ˆŒð 	�‰ÕrR   c                 ó.   • U R                   R                  $ rÏ   )rn  ÚdecoderrT  s    rP   Úget_output_embeddingsÚ&IBertForMaskedLM.get_output_embeddingsç  s   € Ø�|‰|×#Ñ#Ð#rR   c                 óZ   • XR                   l        UR                  U R                   l        g rÏ   )rn  rq  r   )rM   Únew_embeddingss     rP   Úset_output_embeddingsÚ&IBertForMaskedLM.set_output_embeddingsê  s   € Ø-�‰ÔØ*×/Ñ/ˆ�‰ÕrR   Nr`   rŸ   ra   rB   rb   Úlabelsr    r  r  rZ  c
                 óœ  • U	b  U	OU R                   R                  n	U R                  UUUUUUUU	S9nUS   nU R                  U5      nSnUbF  [	        5       nU" UR                  SU R                   R                  5      UR                  S5      5      nU	(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  S9$ )az  
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ...,
    config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the
    loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]`
N©rŸ   ra   rB   rb   r    r  r  r   r)   r“   ©ÚlossÚlogitsr�   r  )
rN   r  r(  rn  r   r–   r7   r   r�   r  )rM   r`   rŸ   ra   rB   rb   rx  r    r  r  r_  r¶   re  Úprediction_scoresÚmasked_lm_lossÚloss_fctrÐ   s                    rP   rj   ÚIBertForMaskedLM.forwardî  s÷   € ð( &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ð 	
ˆð " !™*ˆØ ŸL™L¨Ó9ÐàˆØÑÜ'Ó)ˆHÙ%Ð&7×&<Ñ&<¸RÀÇÁ×AWÑAWÓ&XÐZ`×ZeÑZeÐfhÓZiÓjˆNæØ'Ð)¨G°A°B¨KÑ7ˆFØ3AÑ3M�^Ð%¨Ñ.ÐYÐSYÐYäØØ$Ø!×/Ñ/Ø×)Ñ)ñ	
ð 	
rR   )r(  rn  ©	NNNNNNNNN)rp   rq   rr   rs   Ú_tied_weights_keysr1   rr  rv  r   r>   rg  rh  ri  r   r  rj   ru   rv   rw   s   @rP   rk  rk  ×  s  ø† ð #MØ .ñÐõ
ò$ò0ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñ0
à×#Ñ# dÑ*ð0
ð ×)Ñ)¨DÑ0ð0
ð ×(Ñ(¨4Ñ/ð	0
ð
 ×&Ñ&¨Ñ-ð0
ð ×(Ñ(¨4Ñ/ð0
ð × Ñ  4Ñ'ð0
ð   $™;ð0
ð # T™kð0
ð ˜D‘[ð0
ð 
˜% × 1Ñ 1Ñ2Ñ	2ô0
ó ö0
rR   rk  c                   ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )r:  i"  z)I-BERT Head for masked language modeling.c                 ó°  >• [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  UR                  S9U l        [        R                  " UR                  UR                  5      U l
        [        R                  " [        R                  " UR                  5      5      U l        g )N)r-   )r0   r1   r   r  r8   r¿   rG   rF   Ú
layer_normr7   rq  Ú	Parameterr>   r^   r   rL   s     €rP   r1   ÚIBertLMHead.__init__%  s‡   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
ÜŸ,š, v×'9Ñ'9¸v×?TÑ?TÑUˆŒä—y’y ×!3Ñ!3°V×5FÑ5FÓGˆŒÜ—L’L¤§¢¨V×->Ñ->Ó!?Ó@ˆ�	rR   c                 ó‚   • U R                  U5      n[        U5      nU R                  U5      nU R                  U5      nU$ rÏ   )r¿   r	   r†  rq  )rM   Úfeaturesr_  Úxs       rP   rj   ÚIBertLMHead.forward-  s;   € Ø�J‰J�xÓ ˆÜ�‹GˆØ�O‰O˜AÓˆð �L‰L˜‹OˆàˆrR   )r   rq  r¿   r†  ©	rp   rq   rr   rs   rt   r1   rj   ru   rv   rw   s   @rP   r:  r:  "  s   ø† Ù3õA÷ð rR   r:  zž
    I-BERT Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled
    output) e.g. for GLUE tasks.
    )Úcustom_introc                   óH  ^ • \ rS rSrU 4S jr\         SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\	S-  S\	S-  S\	S-  S\
\\R                     -  4S jj5       rSrU =r$ )ÚIBertForSequenceClassificationi8  c                 ó¤   >• [         TU ]  U5        UR                  U l        [        USS9U l        [        U5      U l        U R                  5         g rm  )r0   r1   Ú
num_labelsrK  r(  ÚIBertClassificationHeadÚ
classifierrO  rL   s     €rP   r1   Ú'IBertForSequenceClassification.__init__?  sC   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ñ@ˆŒ
Ü1°&Ó9ˆŒð 	�‰ÕrR   Nr`   rŸ   ra   rB   rb   rx  r    r  r  rZ  c
                 ó.  • U	b  U	OU R                   R                  n	U R                  UUUUUUUU	S9nUS   nU R                  U5      nSnUGbŽ  U R                   R                  c‘  U R
                  S:X  a  SU R                   l        OoU R
                  S:”  aN  UR                  [        R                  :X  d  UR                  [        R                  :X  a  SU R                   l        OSU R                   l        U R                   R                  S:X  aI  [        5       nU R
                  S:X  a&  U" UR                  5       UR                  5       5      nOŒU" XÖ5      nOƒU R                   R                  S:X  a=  [        5       nU" UR                  SU R
                  5      UR                  S5      5      nO,U R                   R                  S:X  a  [        5       nU" XÖ5      nU	(       d  U4US	S -   nUb  U4U-   $ U$ [        UUUR                   UR"                  S
9$ )ae  
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the sequence classification/regression loss. Indices should be in `[0, ...,
    config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If
    `config.num_labels > 1` a classification loss is computed (Cross-Entropy).
Nrz  r   r   Ú
regressionÚsingle_label_classificationÚmulti_label_classificationr)   r“   r{  )rN   r  r(  r”  Úproblem_typer’  rU   r>   r_   r†   r   Úsqueezer   r–   r   r   r�   r  ©rM   r`   rŸ   ra   rB   rb   rx  r    r  r  r_  r¶   re  r}  r|  r€  rÐ   s                    rP   rj   Ú&IBertForSequenceClassification.forwardI  sÓ  € ð( &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ð 	
ˆð " !™*ˆØ—‘ Ó1ˆàˆØÒØ�{‰{×'Ñ'Ñ/Ø—?‘? aÓ'Ø/;�D—K‘KÕ,Ø—_‘_ qÓ(¨f¯l©l¼e¿j¹jÓ.HÈFÏLÉLÔ\a×\eÑ\eÓLeØ/L�D—K‘KÕ,à/K�D—K‘KÔ,à�{‰{×'Ñ'¨<Ó7Ü"›9�Ø—?‘? aÓ'Ù# F§N¡NÓ$4°f·n±nÓ6FÓG‘Dá# FÓ3‘DØ—‘×)Ñ)Ð-JÓJÜ+Ó-�Ù §¡¨B°·±Ó @À&Ç+Á+ÈbÃ/ÓR‘Ø—‘×)Ñ)Ð-IÓIÜ,Ó.�Ù Ó/�ÞØ�Y ¨¨ Ñ,ˆFØ)-Ñ)9�T�G˜fÑ$ÐE¸vÐEä'ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
rR   )r”  r(  r’  r‚  )rp   rq   rr   rs   r1   r   r>   rg  rh  ri  r   r  rj   ru   rv   rw   s   @rP   r�  r�  8  s  ø† õð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñA
à×#Ñ# dÑ*ðA
ð ×)Ñ)¨DÑ0ðA
ð ×(Ñ(¨4Ñ/ð	A
ð
 ×&Ñ&¨Ñ-ðA
ð ×(Ñ(¨4Ñ/ðA
ð × Ñ  4Ñ'ðA
ð   $™;ðA
ð # T™kðA
ð ˜D‘[ðA
ð 
" E¨%×*;Ñ*;Ñ$<Ñ	<ôA
ó öA
rR   r�  c                   óH  ^ • \ rS rSrU 4S jr\         SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\	S-  S\	S-  S\	S-  S\
\\R                     -  4S jj5       rSrU =r$ )ÚIBertForMultipleChoiceiŽ  c                 óú   >• [         TU ]  U5        [        U5      U l        [        R
                  " UR                  5      U l        [        R                  " UR                  S5      U l
        U R                  5         g )Nr   )r0   r1   rK  r(  r   rI   rJ   rK   r  r8   r”  rO  rL   s     €rP   r1   ÚIBertForMultipleChoice.__init__�  sV   ø€ Ü‰Ñ˜Ô ä Ó'ˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸Ó:ˆŒð 	�‰ÕrR   Nr`   ra   rŸ   rx  rB   rb   r    r  r  rZ  c
                 óZ  • U	b  U	OU R                   R                  n	Ub  UR                  S   OUR                  S   nUb!  UR                  SUR	                  S5      5      OSnUb!  UR                  SUR	                  S5      5      OSnUb!  UR                  SUR	                  S5      5      OSnUb!  UR                  SUR	                  S5      5      OSnUb1  UR                  SUR	                  S5      UR	                  S5      5      OSnU R                  UUUUUUUU	S9nUS   nU R                  U5      nU R                  U5      nUR                  SU5      nSnUb  [        5       nU" UU5      nU	(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  S9$ )a›  
input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`):
    Indices of input sequence tokens in the vocabulary.

    Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and
    [`PreTrainedTokenizer.__call__`] for details.

    [What are input IDs?](../glossary#input-ids)
token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0,
    1]`:

    - 0 corresponds to a *sentence A* token,
    - 1 corresponds to a *sentence B* token.

    [What are token type IDs?](../glossary#token-type-ids)
labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*):
    Labels for computing the multiple choice classification loss. Indices should be in `[0, ...,
    num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See
    `input_ids` above)
position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*):
    Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0,
    config.max_position_embeddings - 1]`.

    [What are position IDs?](../glossary#position-ids)
inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*):
    Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This
    is useful if you want more control over how to convert `input_ids` indices into associated vectors than the
    model's internal embedding lookup matrix.
Nr   r)   r”   )rB   ra   rŸ   rb   r    r  r  r“   r{  )rN   r  r•   r–   r]   r(  rK   r”  r   r   r�   r  )rM   r`   ra   rŸ   rx  rB   rb   r    r  r  r_  Únum_choicesÚflat_input_idsÚflat_position_idsÚflat_token_type_idsÚflat_attention_maskÚflat_inputs_embedsr¶   r$  r}  Úreshaped_logitsr|  r€  rÐ   s                           rP   rj   ÚIBertForMultipleChoice.forwardš  sÞ  € ðX &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆØ,5Ñ,A�i—o‘o aÒ(À}×GZÑGZÐ[\ÑG]ˆàCLÑCX˜Ÿ™¨¨I¯N©N¸2Ó,>Ô?Ð^bˆØLXÑLd˜L×-Ñ-¨b°,×2CÑ2CÀBÓ2GÔHÐjnÐØR`ÑRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐØR`ÑRl˜n×1Ñ1°"°n×6IÑ6IÈ"Ó6MÔNÐrvÐð Ñ(ð ×Ñ˜r =×#5Ñ#5°bÓ#9¸=×;MÑ;MÈbÓ;QÔRàð 	ð —*‘*ØØ*Ø.Ø.Ø,Ø/Ø!5Ø#ð ð 	
ˆð   ™
ˆàŸ™ ]Ó3ˆØ—‘ Ó/ˆØ Ÿ+™+ b¨+Ó6ˆàˆØÑÜ'Ó)ˆHÙ˜O¨VÓ4ˆDæØ%Ð'¨'°!°"¨+Ñ5ˆFØ)-Ñ)9�T�G˜fÑ$ÐE¸vÐEä(ØØ"Ø!×/Ñ/Ø×)Ñ)ñ	
ð 	
rR   )r”  rK   r(  r‚  )rp   rq   rr   rs   r1   r   r>   rg  rh  ri  r   r  rj   ru   rv   rw   s   @rP   rŸ  rŸ  Ž  s  ø† õð ð .2Ø26Ø37Ø*.Ø04Ø26Ø)-Ø,0Ø#'ñV
à×#Ñ# dÑ*ðV
ð ×(Ñ(¨4Ñ/ðV
ð ×)Ñ)¨DÑ0ð	V
ð
 × Ñ  4Ñ'ðV
ð ×&Ñ&¨Ñ-ðV
ð ×(Ñ(¨4Ñ/ðV
ð   $™;ðV
ð # T™kðV
ð ˜D‘[ðV
ð 
# U¨5×+<Ñ+<Ñ%=Ñ	=ôV
ó öV
rR   rŸ  c                   óH  ^ • \ rS rSrU 4S jr\         SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\	S-  S\	S-  S\	S-  S\
\\R                     -  4S jj5       rSrU =r$ )ÚIBertForTokenClassificationiô  c                 ó.  >• [         TU ]  U5        UR                  U l        [        USS9U l        [
        R                  " UR                  5      U l        [
        R                  " UR                  UR                  5      U l        U R                  5         g rm  )r0   r1   r’  rK  r(  r   rI   rJ   rK   r  r8   r”  rO  rL   s     €rP   r1   Ú$IBertForTokenClassification.__init__ö  sk   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ñ@ˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŒÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrR   Nr`   rŸ   ra   rB   rb   rx  r    r  r  rZ  c
                 óª  • U	b  U	OU R                   R                  n	U R                  UUUUUUUU	S9nUS   nU R                  U5      nU R	                  U5      nSnUb<  [        5       nU" UR                  SU R                  5      UR                  S5      5      nU	(       d  U4USS -   nUb  U4U-   $ U$ [        UUUR                  UR                  S9$ )zÃ
labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*):
    Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`.
Nrz  r   r)   r“   r{  )rN   r  r(  rK   r”  r   r–   r’  r   r�   r  rœ  s                    rP   rj   Ú#IBertForTokenClassification.forward  só   € ð$ &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ð 	
ˆð " !™*ˆàŸ,™, Ó7ˆØ—‘ Ó1ˆàˆØÑÜ'Ó)ˆHÙ˜FŸK™K¨¨D¯O©OÓ<¸f¿k¹kÈ"»oÓNˆDæØ�Y ¨¨ Ñ,ˆFØ)-Ñ)9�T�G˜fÑ$ÐE¸vÐEä$ØØØ!×/Ñ/Ø×)Ñ)ñ	
ð 	
rR   )r”  rK   r(  r’  r‚  )rp   rq   rr   rs   r1   r   r>   rg  rh  ri  r   r  rj   ru   rv   rw   s   @rP   r¬  r¬  ô  sù   ø† õ	ð ð .2Ø37Ø26Ø04Ø26Ø*.Ø)-Ø,0Ø#'ñ1
à×#Ñ# dÑ*ð1
ð ×)Ñ)¨DÑ0ð1
ð ×(Ñ(¨4Ñ/ð	1
ð
 ×&Ñ&¨Ñ-ð1
ð ×(Ñ(¨4Ñ/ð1
ð × Ñ  4Ñ'ð1
ð   $™;ð1
ð # T™kð1
ð ˜D‘[ð1
ð 
  u×'8Ñ'8Ñ!9Ñ	9ô1
ó ö1
rR   r¬  c                   ó2   ^ • \ rS rSrSrU 4S jrS rSrU =r$ )r“  i6  z-Head for sentence-level classification tasks.c                 ó,  >• [         TU ]  5         [        R                  " UR                  UR                  5      U l        [        R                  " UR                  5      U l        [        R                  " UR                  UR                  5      U l
        g rÏ   )r0   r1   r   r  r8   r¿   rI   rJ   rK   r’  Úout_projrL   s     €rP   r1   Ú IBertClassificationHead.__init__9  s`   ø€ Ü‰ÑÔÜ—Y’Y˜v×1Ñ1°6×3EÑ3EÓFˆŒ
Ü—z’z &×"<Ñ"<Ó=ˆŒÜŸ	š	 &×"4Ñ"4°f×6GÑ6GÓHˆ�rR   c                 óÒ   • US S 2SS S 24   nU R                  U5      nU R                  U5      n[        R                  " U5      nU R                  U5      nU R	                  U5      nU$ r"  )rK   r¿   r>   Útanhr³  )rM   rŠ  r_  r�   s       rP   rj   ÚIBertClassificationHead.forward?  s^   € Ø ¢ A¢q Ñ)ˆØŸ™ ]Ó3ˆØŸ
™
 =Ó1ˆÜŸ
š
 =Ó1ˆØŸ™ ]Ó3ˆØŸ™ mÓ4ˆØÐrR   )r¿   rK   r³  r�  rw   s   @rP   r“  r“  6  s   ø† Ù7õI÷ð rR   r“  c                   óh  ^ • \ rS rSrU 4S jr\          SS\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S\R                  S-  S	\R                  S-  S
\R                  S-  S\	S-  S\	S-  S\	S-  S\
\\R                     -  4S jj5       rSrU =r$ )ÚIBertForQuestionAnsweringiI  c                 óä   >• [         TU ]  U5        UR                  U l        [        USS9U l        [
        R                  " UR                  UR                  5      U l        U R                  5         g rm  )
r0   r1   r’  rK  r(  r   r  r8   Ú
qa_outputsrO  rL   s     €rP   r1   Ú"IBertForQuestionAnswering.__init__K  sU   ø€ Ü‰Ñ˜Ô Ø ×+Ñ+ˆŒä ¸%Ñ@ˆŒ
ÜŸ)š) F×$6Ñ$6¸×8IÑ8IÓJˆŒð 	�‰ÕrR   Nr`   rŸ   ra   rB   rb   Ústart_positionsÚend_positionsr    r  r  rZ  c                 ó"  • U
b  U
OU R                   R                  n
U R                  UUUUUUU	U
S9nUS   nU R                  U5      nUR	                  SSS9u  nnUR                  S5      R                  5       nUR                  S5      R                  5       nS nUb¶  Ub³  [        UR                  5       5      S:”  a  UR                  S5      n[        UR                  5       5      S:”  a  UR                  S5      nUR                  S5      nUR                  SU5      nUR                  SU5      n[        US9nU" Xö5      nU" UU5      nUU-   S-  nU
(       d  UU4USS  -   nUb  U4U-   $ U$ [        UUUUR                  UR                  S9$ )	Nrz  r   r   r)   ©Údim)Úignore_indexr“   )r|  Ústart_logitsÚ
end_logitsr�   r  )rN   r  r(  r»  Úsplitr›  rœ   Úlenr]   Úclampr   r   r�   r  )rM   r`   rŸ   ra   rB   rb   r½  r¾  r    r  r  r_  r¶   re  r}  rÃ  rÄ  Ú
total_lossÚignored_indexr€  Ú
start_lossÚend_lossrÐ   s                          rP   rj   Ú!IBertForQuestionAnswering.forwardU  s½  € ð &1Ñ%<‘kÀ$Ç+Á+×BYÑBYˆà—*‘*ØØ)Ø)Ø%Ø'Ø/Ø!5Ø#ð ð 	
ˆð " !™*ˆà—‘ Ó1ˆØ#)§<¡<°°r <Ð#:Ñ ˆ�jØ#×+Ñ+¨BÓ/×:Ñ:Ó<ˆØ×'Ñ'¨Ó+×6Ñ6Ó8ˆ
àˆ
ØÑ&¨=Ñ+Dä�?×'Ñ'Ó)Ó*¨QÓ.Ø"1×"9Ñ"9¸"Ó"=�Ü�=×%Ñ%Ó'Ó(¨1Ó,Ø -× 5Ñ 5°bÓ 9�à(×-Ñ-¨aÓ0ˆMØ-×3Ñ3°A°}ÓEˆOØ)×/Ñ/°°=ÓAˆMä'°]ÑCˆHÙ! ,Ó@ˆJÙ 
¨MÓ:ˆHØ$ xÑ/°1Ñ4ˆJæØ" JÐ/°'¸!¸"°+Ñ=ˆFØ/9Ñ/E�Z�M FÑ*ÐQÈ6ÐQä+ØØ%Ø!Ø!×/Ñ/Ø×)Ñ)ñ
ð 	
rR   )r(  r’  r»  )
NNNNNNNNNN)rp   rq   rr   rs   r1   r   r>   rg  rh  ri  r   r  rj   ru   rv   rw   s   @rP   r¹  r¹  I  s  ø† õð ð .2Ø37Ø26Ø04Ø26Ø37Ø15Ø)-Ø,0Ø#'ñ=
à×#Ñ# dÑ*ð=
ð ×)Ñ)¨DÑ0ð=
ð ×(Ñ(¨4Ñ/ð	=
ð
 ×&Ñ&¨Ñ-ð=
ð ×(Ñ(¨4Ñ/ð=
ð ×)Ñ)¨DÑ0ð=
ð ×'Ñ'¨$Ñ.ð=
ð   $™;ð=
ð # T™kð=
ð ˜D‘[ð=
ð 
&¨¨e×.?Ñ.?Ñ(@Ñ	@ô=
ó ö=
rR   r¹  c                 óº   • U R                  U5      R                  5       n[        R                  " USS9R	                  U5      U-   U-  nUR                  5       U-   $ )a1  
Replace non-padding symbols with their position numbers. Position numbers begin at padding_idx+1. Padding symbols
are ignored. This is modified from fairseq's *utils.make_positions*.

Args:
input_ids (`torch.LongTensor`):
       Indices of input sequence tokens in the vocabulary.

Returns: torch.Tensor
r   rÀ  )Úner†   r>   ÚcumsumÚtype_asr_   )r`   r%   rc   ÚmaskÚincremental_indicess        rP   rZ   rZ   –  sW   € ð �<‰<˜Ó$×(Ñ(Ó*€DÜ Ÿ<š<¨°!Ñ4×<Ñ<¸TÓBÐE[Ñ[Ð_cÑcÐØ×#Ñ#Ó%¨Ñ3Ð3rR   )rk  rŸ  r¹  r�  r¬  rK  r'  )r   )=rt   r™   r>   r   Útorch.nnr   r   r   Ú r   r3  Úactivationsr	   Úmasking_utilsr
   Úmodeling_outputsr   r   r   r   r   r   r   Úmodeling_utilsr   Úutilsr   r   Úconfiguration_ibertr   Úquant_modulesr   r   r   r   r   r   Ú
get_loggerrp   ÚloggerÚModuler   ry   r¼   rÍ   rÚ   rã   ré   rý   r  r'  rK  rk  r:  r�  rŸ  r¬  r“  r¹  rZ   Ú__all__r  rR   rP   Ú<module>rà     s  ðñ" ã ã Ý ß AÑ Aå &Ý Ý 6÷÷ ñ õ .ß ,Ý ,ß c× cð 
×	Ò	˜HÓ	%€ôq=�b—i‘iô q=ôh@.˜Ÿ™ô @.ôF,;�b—i‘iô ,;ô^/�R—Y‘Yô /ô8;˜Ÿ	™	ô ;ôD,;�"—)‘)ô ,;ô^59�—‘ô 59ôp6
�2—9‘9ô 6
ôr�"—)‘)ô ð  ô&\˜?ó &\ó ð&\ðR ôg
Ð%ó g
ó ðg
ðT ôG
Ð+ó G
ó ðG
ôT�"—)‘)ô ñ, ðñôM
Ð%9ó M
óðM
ð` ôb
Ð1ó b
ó ðb
ðJ ô>
Ð"6ó >
ó ð>
ôB˜bŸi™iô ð& ôI
Ð 4ó I
ó ðI
ôX4ò"�rR   