jSrSSKJr SSKJr SSKrSSKJr SSKJrJ r J r SSK J r SS KJrJr SS KJrJrJr SS KJr SS KJrJr SS KJr SSKJrJrJrJ r J!r!J"r"J#r#J$r$ SSK%J&r&J'r' SSK(J)r) SSK*J+r+ SSK,J-r-J.r.J/r/J0r0 SSK1J2r2J3r3 SSK4J5r5 SSK6J7r7 \0Rp"\95r:"SS\Rv5r<SVS\RvS\RzS\RzS\RzS\RzS-S\>S-S\>S \)\.4S!jjr?"S"S#\Rv5r@"S$S%\Rv5rA"S&S'\Rv5rB"S(S)\Rv5rC"S*S+\Rv5rD"S,S-\Rv5rE"S.S/\5rF"S0S1\Rv5rG"S2S3\Rv5rH"S4S5\Rv5rI\/"S6S7\'55rJ\/"S8S99\"S:S;\-555rK\/"S<S=\J55rL"S>S?\Rv5rM"S@SA\Rv5rN\/"SBS99"SCSD\J55rO\/"SES99"SFSG\J55rP\/"SHS99"SISJ\J55rQ\/"SKS99"SLSM\J55rR\/"SNSO\J55rS\/"SPSQ\J55rT\/"SRS99"SSST\J\55rU/SUQrVg)WzPyTorch ELECTRA model.)Callable) dataclassN)nn)BCEWithLogitsLossCrossEntropyLossMSELoss)initialization)ACT2FNget_activation)Cache DynamicCacheEncoderDecoderCache)GenerationMixin)create_bidirectional_maskcreate_causal_mask)GradientCheckpointingLayer)"BaseModelOutputWithCrossAttentions)BaseModelOutputWithPastAndCrossAttentions!CausalLMOutputWithCrossAttentionsMaskedLMOutputMultipleChoiceModelOutputQuestionAnsweringModelOutputSequenceClassifierOutputTokenClassifierOutput)ALL_ATTENTION_FUNCTIONSPreTrainedModel)Unpack)apply_chunking_to_forward) ModelOutputTransformersKwargsauto_docstringlogging)can_return_tuplemerge_with_config_defaults)capture_outputs) ElectraConfigc ^\rSrSrSrU4SjrS S\RS-S\RS-S\RS-S\RS-S \ S \R4 S jjr S r U=r $)ElectraEmbeddings8zGConstruct the embeddings from word, position and token_type embeddings.c.>[TU]5 [R"URUR UR S9Ul[R"URUR 5Ul [R"URUR 5Ul [R"UR URS9Ul [R"UR5Ul[R""[$R&"UR5R)S5SS9Ul[R""[$R,"UR*R/5[$R0S9SS9Ulg)N) padding_idxepsr'F) persistentdtype)super__init__r Embedding vocab_sizeembedding_size pad_token_idword_embeddingsmax_position_embeddingsposition_embeddingstype_vocab_sizetoken_type_embeddings LayerNormlayer_norm_epsDropouthidden_dropout_probdropoutBuffertorcharangeexpand position_idszerossizelongtoken_type_idsselfconfig __class__s ںC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/electra/modeling_electra.pyr6ElectraEmbeddings.__init__;s !||F,=,=v?T?Tbhbubuv#%<<0N0NPVPePe#f %'\\&2H2H&J_J_%`"f&;&;AVAVWzz&"<"<= IIell63Q3Q&R&Y&YZa&botu ii D4E4E4J4J4LTYT^T^(_lqrN input_idsrMrI inputs_embedspast_key_values_lengthreturncUbUR5nOUR5SSnUupxUcURSS2XXU-24nUc[US5(aQURR UR SS5n [ R"U SUS9n U R Xx5nO8[ R"U[ RURRS9nUcURU5nURU5n XJ-n URU5n X-n URU 5n URU 5n U $)Nr1rMrr')dimindex)r4device)rKrIhasattrrMrHshaperFgatherrJrLr\r;r?r=r@rD) rOrUrMrIrVrW input_shape batch_size seq_lengthbuffered_token_type_idsr? embeddingsr=s rRforwardElectraEmbeddings.forwardIsG  #..*K',,.s3K!,  ,,Q0FVlIl0l-lmL  !t-..*.*=*=*D*D\EWEWXYEZ\^*_'*/,,7NTU]i*j'!8!?!? !W!&[ SWSdSdSkSk!l   00;M $ : :> J": "66|D5 ^^J/ \\*- rT)r@rDr=rIr?rMr;)NNNNr)__name__ __module__ __qualname____firstlineno____doc__r6rF LongTensor FloatTensorintTensorre__static_attributes__ __classcell__rQs@rRr*r*8sQ s .2260426&' (##d*(((4/(&&- ( ((4/ ( !$ ( ((rTr*modulequerykeyvalueattention_maskscalingrDkwargsc UcURS5S-n[R"XRSS55U-nUbX-n[R R USS9n[R RXURS9n[R"X5n U RSS5R5n X4$)Nr1r rZ)ptrainingr') rKrFmatmul transposer functionalsoftmaxrDr contiguous) rsrtrurvrwrxrDry attn_weights attn_outputs rReager_attention_forwardrus**R.D(<<}}Q':;gEL!#4 ==((2(>L==((6??([L,,|3K''1-88:K  $$rTc ^\rSrSrS U4SjjrS S\R S\RS-S\S-S\ \ S\ \R 4 S jjr S r U=r$) ElectraSelfAttentionNcN>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul URS-Ul [R"URUR5Ul [R"URUR5Ul[R"URUR5Ul[R "UR"5UlUR&UlX lX0lgNrr9zThe hidden size (z6) is not a multiple of the number of attention heads ()r{)r5r6 hidden_sizenum_attention_headsr] ValueErrorrPrnattention_head_size all_head_sizerxrLinearrtrurvrBattention_probs_dropout_probrD is_decoder is_causal layer_idxrOrPrrrQs rRr6ElectraSelfAttention.__init__sM     : : :a ?PVXhHiHi#F$6$6#78 445Q8  #)#=#= #&v'9'9F??*9*N*N'&=%C%CI\`\j\j%k "I(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((rT) rrrPrDrrrurrrtrxrvFN)NNrgrhrirjr6rFrormr rr!tuplererprqrrs@rRrrsl#648(, ')||')))D0') ') +, ') u||  ')')rTrc^\rSrSrS U4SjjrS S\R S\RS-S\RS-S\S-S\ \ S \ \R 4 S jjr S r U=r$)ElectraCrossAttentionNc,>[TU]5 URUR-S:wa7[ US5(d&[ SURSURS35eXlURUl[URUR- 5UlURUR-Ul URS-Ul [R"URUR5Ul [R"URUR5Ul[R"URUR5Ul[R "UR"5UlX lX0lgr)r5r6rrr]rrPrnrrrxrrrtrurvrBrrDrrrs rRr6ElectraCrossAttention.__init__s@     : : :a ?PVXhHiHi#F$6$6#78 445Q8  #)#=#= #&v'9'9FB**4>>:(?(M(M KK , ,.E) %8      % $}}C$,,..LL %  % ! "));;;;FFH ((rT) rrrPrDrrurrrtrxrvr)NNN)rgrhrirjr6rFrormrrr!rrerprqrrs@rRrrs#4;?376: 1)||1) %00471)))D0 1) -t3 1) +, 1) u|| 1)1)rTrcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $)ElectraSelfOutputi#c(>[TU]5 [R"URUR5Ul[R "URURS9Ul[R"UR5Ul gNr.) r5r6rrrdenser@rArBrCrDrNs rRr6ElectraSelfOutput.__init__$s` YYv1163E3EF f&8&8f>S>STzz&"<"<= rTr input_tensorrXcpURU5nURU5nURX-5nU$NrrDr@rOrrs rRreElectraSelfOutput.forward*5 =1  ]3 }'CD rTr@rrD rgrhrirjr6rFrorerprqrrs@rRrr#6> U\\RWR^R^rTrc^\rSrSrS U4SjjrSS\R S\RS-S\RS-S\RS-S\S-S \ \ S \ \R 4S jjr S r U=r$)ElectraAttentioni2Nc>[TU]5 X@lU(a[O[nU"XUS9Ul[ U5Ulg)Nrr)r5r6is_cross_attentionrrrOroutput)rOrPrrrattention_classrQs rRr6ElectraAttention.__init__3s9 "43E/K_#F9U '/ rTrrwrencoder_attention_maskrryrXc UR(dUOUnUR"U4UUUS.UD6upxURXq5nXx4$)N)rrwr)rrOr) rOrrwrrrryattention_outputrs rRreElectraAttention.forward:s\04/F/FLb)- * "7)+ *  * & ;;'7G--rT)rrrO)FNFNNNNrrrs@rRrr2s048:>;?(, .||.))D0. %0047 . !& 1 1D 8 .  .+,. u|| ..rTrcb^\rSrSrU4SjrS\R S\R 4SjrSrU=r $)ElectraIntermediateiPc>[TU]5 [R"URUR 5Ul[UR[5(a[URUl gURUl gr) r5r6rrrintermediate_sizerr hidden_actstrr intermediate_act_fnrNs rRr6ElectraIntermediate.__init__Qs` YYv1163K3KL f'' - -'-f.?.?'@D $'-'8'8D $rTrrXcJURU5nURU5nU$rrr)rOrs rRreElectraIntermediate.forwardYs& =1 00? rTrrrrs@rRrrPs(9U\\ellrTrcz^\rSrSrU4SjrS\R S\R S\R 4SjrSrU=r $) ElectraOutputi`c(>[TU]5 [R"URUR 5Ul[R"UR URS9Ul[R"UR5Ul gr) r5r6rrrrrr@rArBrCrDrNs rRr6ElectraOutput.__init__as` YYv779K9KL f&8&8f>S>STzz&"<"<= rTrrrXcpURU5nURU5nURX-5nU$rrrs rRreElectraOutput.forwardgrrTrrrrs@rRrr`rrTrc^\rSrSrSU4SjjrSS\R S\RS-S\RS-S\RS-S\S-S \ \ S \R 4S jjr S r S r U=r$) ElectraLayerioNc>[TU]5 URUlSUl[ XR US9UlUR UlURUlUR(a0UR (d[US35e[ USUSS9Ul [U5Ul [U5Ul g)Nr'rz> should be used as a decoder model if cross attention is addedFT)rrr)r5r6chunk_size_feed_forward seq_len_dimrr attentionadd_cross_attentionrcrossattentionr intermediaterr)rOrPrrQs rRr6ElectraLayer.__init__ps '-'E'E$)&>  $ ,$  $   1 ??4@4!122 =dVDDD )-(;(;%%& ) !0 )  ) % " 6 0  # #T%A%A4CSCSUe rTcJURU5nURX!5nU$r)rr)rOrintermediate_outputrs rRrElectraLayer.feed_forward_chunks)"//0@A{{#6I rT)rrrrrrrrrr)rgrhrirjr6rFrormr rr!rerrprqrrs@rRrros,,48:>;?(, %||%))D0% %0047 % !& 1 1D 8 %  %+,% %NrTrc^\rSrSrU4SjrSS\R S\RS-S\RS-S\RS-S\S-S \ S-S \ \ S \ \R \ -4S jjrS rU=r$)ElectraEncoderic >[TU]5 Xl[R"[ UR 5Vs/sH n[XS9PM sn5Ulgs snf)N)r) r5r6rPr ModuleListrangenum_hidden_layersrlayer)rOrPirQs rRr6ElectraEncoder.__init__sH  ]]uU[UmUmOn#oOn!L$EOn#op #osANrrwrrr use_cacheryrXc [UR5HupU "UUU4UUS.UD6nM [UU(aUS9$SS9$)N)rrlast_hidden_stater) enumeraterr) rOrrwrrrr ryr layer_modules rRreElectraEncoder.forwardsg )4OA(%(> /   M 59+/8O  >B  rT)rPr)NNNNN)rgrhrirjr6rFrormr boolrr!rrrerprqrrs@rRrrsq48:>;?(,!% || ))D0  %0047  !& 1 1D 8    $; +,  u|| H H  rTrc2^\rSrSrSrU4SjrSrSrU=r$)ElectraDiscriminatorPredictionsizEPrediction module for the discriminator, made up of two dense layers.c>[TU]5 [R"URUR5Ul[ UR5Ul[R"URS5Ul Xl gNr') r5r6rrrrr r activationdense_predictionrPrNs rRr6(ElectraDiscriminatorPredictions.__init__s\ YYv1163E3EF ():):; " &*<*?  6 &&}5==bA rT)rrPrr rgrhrirjrkr6rerprqrrs@rRrrsOrTrc2^\rSrSrSrU4SjrSrSrU=r$)ElectraGeneratorPredictionsizAPrediction module for the generator, made up of two dense layers.c>[TU]5 [S5Ul[R "UR URS9Ul[R"URUR 5Ul g)Ngelur.) r5r6r rrr@r9rArrrrNs rRr6$ElectraGeneratorPredictions.__init__sV (0f&;&;AVAVWYYv1163H3HI rTclURU5nURU5nURU5nU$r)rrr@)rOgenerator_hidden_statesrs rRre#ElectraGeneratorPredictions.forwards3 #:;  6 }5 rT)r@rrrrrs@rRr r sKJrTr cP^\rSrSr\rSrSrSrSr Sr Sr \ \ \S.rU4SjrSrU=r$)ElectraPreTrainedModelielectraT)r attentionscross_attentionscF>[TU]U5 [U[5(a|[R "UR [R"UR RS5RS55 [R"UR5 gg)Nr1r0) r5 _init_weightsrr*initcopy_rIrFrGr^rHzeros_rM)rOrsrQs rRr-$ElectraPreTrainedModel._init_weightssp f% f/ 0 0 JJv**ELL9L9L9R9RSU9V,W,^,^_f,g h KK-- . 1rT)rgrhrirjr( config_classbase_model_prefixsupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attn_supports_attention_backendrrr_can_record_outputsr-rprqrrs@rRr(r(sF L!&*#N"&%*1 //rTr(z3 Output type of [`ElectraForPreTraining`]. ) custom_introc\rSrSr%SrSr\RS-\S'Sr \RS-\S'Sr \ \RS-\S'Sr \ \RS-\S'Sr g) ElectraForPreTrainingOutputi a loss (*optional*, returned when `labels` is provided, `torch.FloatTensor` of shape `(1,)`): Total loss of the ELECTRA objective. logits (`torch.FloatTensor` of shape `(batch_size, sequence_length)`): Prediction scores of the head (scores for each token before SoftMax). Nlossrrr*r2)rgrhrirjrkr>rFrm__annotations__rrrr*rpr2rTrRr=r= sg&*D%  d ")'+FE   $+59M5**+d2926Je''(4/6rTr=c^\rSrSrU4SjrSrSr\\\ SS\ RS-S\ RS-S\ RS-S \ RS-S \ RS-S \ RS-S \ RS-S \ \ RS-S\S-S\\S\\ R\-4Sjj555rSrSrU=r$) ElectraModeli c2>[TU]U5 [U5UlURUR :wa0[ R"URUR 5Ul[U5Ul Xl SUl UR5 g)NF)r5r6r*rdr9rrrembeddings_projectrencoderrPgradient_checkpointing post_initrNs rRr6ElectraModel.__init__"sr  +F3  F$6$6 6&(ii0E0EvGYGY&ZD #%f-  &+# rTc.URR$rrdr;rOs rRget_input_embeddings!ElectraModel.get_input_embeddings/s...rTc$XRlgrrI)rOrvs rRset_input_embeddings!ElectraModel.set_input_embeddings2s */'rTNrUrwrMrIrVrrrr ryrXc USLUSL- (a [S5eURR(aU bU OURRn OSn U (abUc_UcURR(a.[ [ URS9[ URS95O[ URS9nUbUR5OSn URUUUUU S9n [US5(aURU 5n URUUU UUS9up'UR"U 4UUUUU US.U D6n [U RU RS 9$) Nz:You must specify exactly one of input_ids or inputs_embedsF)rPr)rUrIrMrVrWrC)rwrembedding_outputrr)rwrrrr rIr )rrPrr is_encoder_decoderrrget_seq_lengthrdr]rC_create_attention_masksrDrr r)rOrUrwrMrIrVrrrr ryrWrQencoder_outputss rRreElectraModel.forward5su  -t"; <YZ Z ;; ! !%.%:  @U@UII 0)4 8V8V$L $DlZ^ZeZeFfg!5  FUE`!?!?!Afg??%)'#9 +  4- . .#667GH 151M1M)#9-"7+ 2N2 .,,   )"7#9+%    9-??+;;  rTcURR(a[URUUUS9nO[URUUS9nUb[URUUUS9nX4$)N)rPrVrwr)rPrVrw)rPrVrwr)rPrrr)rOrwrrQrrs rRrT$ElectraModel._create_attention_masksysr ;; ! !/{{.- / N7{{.-N " -%>{{.5&; & "55rT)rPrdrCrDrE) NNNNNNNNN)rgrhrirjr6rKrNr%r&r"rFrolistrmrrr!rrrerTrprqrrs@rRrArA s2 /0 *..2.2,0-1596::>!%> <<$&>  t+>  t+ > llT) > ||d* >  %||d2> !& t 3> e//047> $;> +,>  u|| A A>  > B66rTrAc2^\rSrSrSrU4SjrSrSrU=r$)ElectraClassificationHeadiz-Head for sentence-level classification tasks.c>[TU]5 [R"URUR5UlUR b UR O URn[S5Ul [R"U5Ul [R"URUR5Ul g)Nr")r5r6rrrrclassifier_dropoutrCr rrBrD num_labelsout_projrOrPr]rQs rRr6"ElectraClassificationHead.__init__s YYv1163E3EF )/)B)B)NF % %TZTnTn )0zz"45  &"4"4f6G6GH rTc USS2SSS24nURU5nURU5nURU5nURU5nURU5nU$)Nr)rDrrr_)rOfeaturesryxs rRre!ElectraClassificationHead.forwardsZ Q1W  LLO JJqM OOA  LLO MM! rT)rrrDr_rrrs@rRr[r[s7IrTr[c^\rSrSrSrS\4U4SjjrS S\RS\RS-S\R4S jjr S r U=r $) ElectraSequenceSummaryia Compute a single vector summary of a sequence hidden states. Args: config ([`ElectraConfig`]): The config used by the model. Relevant arguments in the config class of the model are (refer to the actual config class of your model for the default values it uses): - **summary_type** (`str`) -- The method to use to make this summary. Accepted values are: - `"last"` -- Take the last token hidden state (like XLNet) - `"first"` -- Take the first token hidden state (like Bert) - `"mean"` -- Take the mean of all tokens hidden states - `"cls_index"` -- Supply a Tensor of classification token position (GPT/GPT-2) - `"attn"` -- Not implemented now, use multi-head attention - **summary_use_proj** (`bool`) -- Add a projection after the vector extraction. - **summary_proj_to_labels** (`bool`) -- If `True`, the projection outputs to `config.num_labels` classes (otherwise to `config.hidden_size`). - **summary_activation** (`Optional[str]`) -- Set to `"tanh"` to add a tanh activation to the output, another string or `None` will add no activation. - **summary_first_dropout** (`float`) -- Optional dropout probability before the projection and activation. - **summary_last_dropout** (`float`)-- Optional dropout probability after the projection and activation. rPc>[TU]5 [USS5UlURS:Xa[e[ R "5Ul[US5(aUR(aq[US5(a.UR(aURS:a URnO URn[ R"URU5Ul[USS5nU(a [U5O[ R "5Ul[ R "5Ul[US5(a5UR"S:a%[ R$"UR"5Ul[ R "5Ul[US 5(a7UR(S:a&[ R$"UR(5Ulggg) N summary_typelastattnsummary_use_projsummary_proj_to_labelsrsummary_activationsummary_first_dropoutsummary_last_dropout)r5r6getattrriNotImplementedErrorrIdentitysummaryr]rlrmr^rrr r first_dropoutrorB last_dropoutrp)rOrP num_classesactivation_stringrQs rRr6ElectraSequenceSummary.__init__sa #FNFC    && %{{} 6- . .63J3Jv788V=Z=Z_e_p_pst_t$// $00 99V%7%7EDL#F,@$GIZN3D$E`b`k`k`m[[] 62 3 38T8TWX8X!#F,H,H!ID KKM 61 2 2v7R7RUV7V " 6+F+F GD 8W 2rTNr cls_indexrXcURS:Xa USS2S4nGOURS:Xa USS2S4nGOURS:XaURSS9nOURS :XaUc?[R"US SS2SS24URS S- [R S 9nOXUR S5R S5nURS UR5S- -URS54-5nURS U5RS 5nOURS:Xa[eURW5nURU5nURU5nUR!U5nU$)a# Compute a single vector summary of a sequence hidden states. Args: hidden_states (`torch.FloatTensor` of shape `[batch_size, seq_len, hidden_size]`): The hidden states of the last layer. cls_index (`torch.LongTensor` of shape `[batch_size]` or `[batch_size, ...]` where ... are optional leading dimensions of `hidden_states`, *optional*): Used if `summary_type == "cls_index"` and takes the last token of the sequence as classification token. Returns: `torch.FloatTensor`: The summary of the sequence hidden states. rjNr1firstrmeanr'r}rz.r3)r1rk)rir}rF full_liker^rL unsqueezerHrZrKr_rrrrurtrrv)rOrrzrs rRreElectraSequenceSummary.forwardsn    &"1b5)F   ' )"1a4(F   & ("''A'.F   + - !OO!#rr1*-!''+a/** &//3==bA %,,Uimmo6I-JmN`N`acNdMf-fg "))"i8@@DF   & (% %##F+f%(""6* rT)rrurvrtrir) rgrhrirjrkr(r6rFrmrlrerprqrrs@rRrgrgsV2H}H<VZ)"..);@;K;Kd;R)   ))rTrgz ELECTRA Model transformer with a sequence classification/regression head on top (a linear layer on top of the pooled output) e.g. for GLUE tasks. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$) ElectraForSequenceClassificationic>[TU]U5 URUlXl[ U5Ul[ U5UlUR5 gr) r5r6r^rPrAr)r[ classifierrFrNs rRr6)ElectraForSequenceClassification.__init__sF   ++ #F+ 3F; rTNrUrwrMrIrVlabelsryrXc UR"U4UUUUSS.UD6nUSn URU 5n Sn UGbURRcURS:XaSURlOoURS:aNUR [ R:XdUR [ R:XaSURlOSURlURRS:XaI[5n URS:Xa&U "U R5UR55n OU "X5n OURRS:Xa=[5n U "U RS UR5URS 55n O,URRS:Xa[5n U "X5n [U U URUR S 9$) ae labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the sequence classification/regression loss. Indices should be in `[0, ..., config.num_labels - 1]`. If `config.num_labels == 1` a regression loss is computed (Mean-Square loss), If `config.num_labels > 1` a classification loss is computed (Cross-Entropy). TrwrMrIrV return_dictrNr' regressionsingle_label_classificationmulti_label_classificationr1r>rrr*)r)rrP problem_typer^r4rFrLrnrrrrrrrr*) rOrUrwrMrIrVrryrsequence_outputrr>loss_fcts rRre(ElectraForSequenceClassification.forward&s$'+ll ' ))%' ' ' #6a81  {{''/??a'/;DKK,__q(fllejj.HFLL\a\e\eLe/LDKK,/KDKK,{{''<7"9??a'#FNN$4fnn6FGD#F3D))-JJ+- B @&++b/R))-II,./'5CC2==   rT)rrPr)r^NNNNNN)rgrhrirjr6r$r"rFrorr!rrrerprqrrs@rRrrs*..2.2,0-1&*9 <<$&9  t+9  t+ 9 llT) 9 ||d* 9  t#9 +,9  u|| 7 79 9 rTrz Electra model with a binary classification head on top as used during pretraining for identifying generated tokens. It is recommended to load the discriminator checkpoint into that model. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)ElectraForPreTrainingidc>[TU]U5 [U5Ul[ U5UlUR 5 gr)r5r6rAr)rdiscriminator_predictionsrFrNs rRr6ElectraForPreTraining.__init__ls3  #F+ )H)P& rTNrUrwrMrIrVrryrXc UR"U4UUUUSS.UD6nUSn URU 5n Sn Ub[R"5n Ub_UR SU R S5S:Hn U R SU R S5U nXmnU "XR 55n O4U "U R SU R S5UR 55n [U U URURS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the ELECTRA loss. Input should be a sequence of tokens (see `input_ids` docstring) Indices should be in `[0, 1]`: - 0 indicates the token is an original token, - 1 indicates the token was replaced. Examples: ```python >>> from transformers import ElectraForPreTraining, AutoTokenizer >>> import torch >>> discriminator = ElectraForPreTraining.from_pretrained("google/electra-base-discriminator") >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-discriminator") >>> sentence = "The quick brown fox jumps over the lazy dog" >>> fake_sentence = "The quick brown fox fake over the lazy dog" >>> fake_tokens = tokenizer.tokenize(fake_sentence, add_special_tokens=True) >>> fake_inputs = tokenizer.encode(fake_sentence, return_tensors="pt") >>> discriminator_outputs = discriminator(fake_inputs) >>> predictions = torch.round((torch.sign(discriminator_outputs[0]) + 1) / 2) >>> fake_tokens ['[CLS]', 'the', 'quick', 'brown', 'fox', 'fake', 'over', 'the', 'lazy', 'dog', '[SEP]'] >>> predictions.squeeze().tolist() [0.0, 0.0, 0.0, 0.0, 0.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0] ```TrrNr1r'r) r)rrrrr^floatr=rr*)rOrUrwrMrIrVrryrdiscriminator_sequence_outputrr>r active_loss active_logits active_labelss rRreElectraForPreTraining.forwardts)V'+ll ' ))%' ' ' #)DA(F%//0MN  ++-H),11"6S6Y6YZ[6\]abb & B0M0S0STU0V WXc d & 3  /B/B/DE B0M0S0STU0V WY_YeYeYgh*5CC2==   rT)rr)r)rgrhrirjr6r$r"rFrorr!rr=rerprqrrs@rRrrds*..2.2,0-1&*F <<$&F  t+F  t+ F llT) F ||d* F  t#F +,F  u|| : :F F rTrz Electra model with a language modeling head on top. Even though both the discriminator and generator may be loaded into this model, the generator is the only model of the two to have been trained for the masked language modeling task. cL^\rSrSrSS0rU4SjrSrSr\\ SS\ RS-S \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ \ S\\ R\-4Sjj55rSrU=r$)ElectraForMaskedLMigenerator_lm_head.weight)electra.embeddings.word_embeddings.weightc>[TU]U5 [U5Ul[ U5Ul[ R"URUR5Ul UR5 gr) r5r6rAr)r generator_predictionsrrr9r8generator_lm_headrFrNs rRr6ElectraForMaskedLM.__init__sR  #F+ %@%H"!#6+@+@&BSBS!T rTcUR$rrrJs rRget_output_embeddings(ElectraForMaskedLM.get_output_embeddings%%%rTcXlgrr)rOr;s rRset_output_embeddings(ElectraForMaskedLM.set_output_embeddingss!0rTNrUrwrMrIrVrryrXc nUR"U4UUUUSS.UD6nUSn URU 5n URU 5n Sn UbQ[R"5n U "U R SUR R5UR S55n [U U URURS9$)az labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the masked language modeling loss. Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` TrrNr1r) r)rrrrrrPr8rrr*) rOrUrwrMrIrVrryr%generator_sequence_outputprediction_scoresr>rs rRreElectraForMaskedLM.forwards$#',, # ))%' # # %! 667PQ 223DE  **,H-222t{{7M7MNPVP[P[\^P_`D$1??.99   rTr)rrr)rgrhrirj_tied_weights_keysr6rrr$r"rFrorr!rrrerprqrrs@rRrrs56ab&1*..2.2,0-1&*) <<$&)  t+)  t+ ) llT) ) ||d* )  t#) +,)  u|| ~ -) ) rTrz Electra model with a token classification head on top. Both the discriminator and generator may be loaded into this model. c8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)ElectraForTokenClassificationicf>[TU]U5 URUl[U5UlUR b UR O UR n[R"U5Ul [R"URUR5Ul UR5 gr)r5r6r^rAr)r]rCrrBrDrrrrFr`s rRr6&ElectraForTokenClassification.__init__s   ++#F+ )/)B)B)NF % %TZTnTn zz"45 ))F$6$68I8IJ rTNrUrwrMrIrVrryrXc DUR"U4UUUUSS.UD6nUSn URU 5n URU 5n Sn Ub<[5n U "U R SUR 5UR S55n [ U U URURS9$)z labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the token classification loss. Indices should be in `[0, ..., config.num_labels - 1]`. TrrNr1r) r)rDrrrr^rrr*) rOrUrwrMrIrVrryrrrr>rs rRre%ElectraForTokenClassification.forwards '+ll ' ))%' ' ' #)DA(F%(, 5R(S%!>?  ')HFKKDOO[TU]U5 URUl[U5Ul[ R "URUR5UlUR5 gr) r5r6r^rAr)rrr qa_outputsrFrNs rRr6$ElectraForQuestionAnswering.__init__MsS   ++#F+ ))F$6$68I8IJ rTNrUrwrMrIrVstart_positions end_positionsryrXc UR"U4UUUUSS.UD6n U Sn URU 5n U RSSS9upU RS5R 5n U RS5R 5n SnUbUb[ UR 55S:aURS5n[ UR 55S:aURS5nU R S5nURSU5nURSU5n[US9nU"X5nU"X5nUU-S- n[UU U U RU RS 9$) NTrrr'r1r}) ignore_indexr|)r> start_logits end_logitsrr*) r)rsplitrrlenrKclamprrrr*)rOrUrwrMrIrVrrryrrrrr total_loss ignored_indexr start_lossend_losss rRre#ElectraForQuestionAnswering.forwardWsz'+ll ' ))%' ' ' #6a81#)<<r<#: #++B/::< ''+668   &=+D?'')*Q."1"9"9""==%%'(1, - 5 5b 9 (--a0M-33A}EO)//=AM']CH!,@J :H$x/14J+%!5CC2==   rT)r)r^r)NNNNNNN)rgrhrirjr(r3r4r6r$r"rFrorr!rrrerprqrrs@rRrrHs L!*..2.2,0-1/3-13 <<$&3  t+3  t+ 3 llT) 3 ||d* 3 ,3 ||d*3 +,3  u|| ; ;3 3 rTrc8^\rSrSrU4Sjr\\SS\RS-S\RS-S\RS-S\RS-S\RS-S \RS-S \ \ S \ \R\ -4S jj55r S rU=r$)ElectraForMultipleChoiceic>[TU]U5 [U5Ul[ U5Ul[ R"URS5Ul UR5 gr) r5r6rAr)rgsequence_summaryrrrrrFrNs rRr6!ElectraForMultipleChoice.__init__sM  #F+ 6v >))F$6$6: rTNrUrwrMrIrVrryrXc UbURSOURSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb!URSURS55OSnUb1URSURS5URS55OSnUR"U4UUUUSS.UD6n U Sn UR U 5n UR U 5n U RSU5n SnUb[ 5nU"X5n[UU U RU RS9$) a input_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`): Indices of input sequence tokens in the vocabulary. Indices can be obtained using [`AutoTokenizer`]. See [`PreTrainedTokenizer.encode`] and [`PreTrainedTokenizer.__call__`] for details. [What are input IDs?](../glossary#input-ids) token_type_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Segment token indices to indicate first and second portions of the inputs. Indices are selected in `[0, 1]`: - 0 corresponds to a *sentence A* token, - 1 corresponds to a *sentence B* token. [What are token type IDs?](../glossary#token-type-ids) position_ids (`torch.LongTensor` of shape `(batch_size, num_choices, sequence_length)`, *optional*): Indices of positions of each input sequence tokens in the position embeddings. Selected in the range `[0, config.max_position_embeddings - 1]`. [What are position IDs?](../glossary#position-ids) inputs_embeds (`torch.FloatTensor` of shape `(batch_size, num_choices, sequence_length, hidden_size)`, *optional*): Optionally, instead of passing `input_ids` you can choose to directly pass an embedded representation. This is useful if you want more control over how to convert `input_ids` indices into associated vectors than the model's internal embedding lookup matrix. labels (`torch.LongTensor` of shape `(batch_size,)`, *optional*): Labels for computing the multiple choice classification loss. Indices should be in `[0, ..., num_choices-1]` where `num_choices` is the size of the second dimension of the input tensors. (See `input_ids` above) Nr'r1r~Trrr) r^rrKr)rrrrrr*)rOrUrwrMrIrVrry num_choicesrr pooled_outputrreshaped_logitsr>rs rRre ElectraForMultipleChoice.forwardsT-6,Aiooa(}GZGZ[\G] >G>SINN2y~~b'9:Y] M[Mg,,R1D1DR1HImqM[Mg,,R1D1DR1HImqGSG_|((\->->r-BCei (   r=#5#5b#9=;M;Mb;Q R  '+ll ' ))%' ' ' #6a8--o> / ++b+6  ')HO4D("5CC2==   rT)rr)rr)rgrhrirjr6r$r"rFrorr!rrrerprqrrs@rRrrs*..2.2,0-1&*N <<$&N  t+N  t+ N llT) N ||d* N  t#N +,N  u|| 8 8N N rTrzS ELECTRA Model with a `language modeling` head on top for CLM fine-tuning. c^\rSrSrSS0rU4SjrSrSr\\ SS\ RS-S \ RS-S \ RS-S \ RS-S \ RS-S \ RS-S\ RS-S\ RS-S\ S-S\ S-S\\ R-S\\S\\ R\-4Sjj55rSrU=r$)ElectraForCausalLMirrc0>[TU]U5 UR(d[R S5 [ U5Ul[U5Ul[R"URUR5Ul UR5 g)NzOIf you want to use `ElectraForCausalLM` as a standalone, add `is_decoder=True.`)r5r6rloggerwarningrAr)r rrrr9r8rrFrNs rRr6ElectraForCausalLM.__init__sh    NNl m#F+ %@%H"!#6+@+@&BSBS!T rTcUR$rrrJs rRr(ElectraForCausalLM.get_output_embeddingsrrTcXlgrr)rOnew_embeddingss rRr(ElectraForCausalLM.set_output_embeddingss!/rTNrUrwrMrIrVrrrrr logits_to_keepryrXc UbSn UR"U4UUUUUUU U SS. U D6n U Rn[U [5(a [ U *S5OU nUR UR USS2USS2455nSnUb)UR"SUXRRS.U D6n[UUU RU RU RU RS9$)a labels (`torch.LongTensor` of shape `(batch_size, sequence_length)`, *optional*): Labels for computing the left-to-right language modeling loss (next word prediction). Indices should be in `[-100, 0, ..., config.vocab_size]` (see `input_ids` docstring) Tokens with indices set to `-100` are ignored (masked), the loss is only computed for the tokens with labels in `[0, ..., config.vocab_size]` Example: ```python >>> from transformers import AutoTokenizer, ElectraForCausalLM, ElectraConfig >>> import torch >>> tokenizer = AutoTokenizer.from_pretrained("google/electra-base-generator") >>> config = ElectraConfig.from_pretrained("google/electra-base-generator") >>> config.is_decoder = True >>> model = ElectraForCausalLM.from_pretrained("google/electra-base-generator", config=config) >>> inputs = tokenizer("Hello, my dog is cute", return_tensors="pt") >>> outputs = model(**inputs) >>> prediction_logits = outputs.logits ```NFT) rwrMrIrVrrrr r)rrr8)r>rrrr*r+r2)r)r rrnslicerr loss_functionrPr8rrrr*r+)rOrUrwrMrIrVrrrrr rryoutputsr slice_indicesrr>s rRreElectraForCausalLM.forwardsN  I=A\\  > ))%'"7#9+ >  >  11 8B>SV8W8W~ot4]k ''(B(B=QRTacdQdCe(fg  %%pVF{{OeOepiopD0#33!//))$55   rTr) NNNNNNNNNNr)rgrhrirjrr6rrr$r"rFror rrnrr!rrrerprqrrs@rRrrs\ 56ab &0*..2.2,0-1596:&*(,!%-.F <<$&F  t+F  t+ F llT) F ||d* F  %||d2F !& t 3F  t#F F $;F ell*F +,F  u|| @ @F F rTr) rrrrrrrrAr()Nr)Wrkcollections.abcr dataclassesrrFrtorch.nnrrrr r. activationsr r cache_utilsr rr generationr masking_utilsrrmodeling_layersrmodeling_outputsrrrrrrrrmodeling_utilsrrprocessing_utilsr pytorch_utilsrutilsr r!r"r# utils.genericr$r%utils.output_capturingr&configuration_electrar( get_loggerrgrModuler*rorrrrrrrrrrrr r(r=rAr[rgrrrrrrr__all__r2rTrRrsH$! AA&1CC)J9   G&6 J50   H %9 9F!% II% <<% % << % LL4' % T\ %%'(%:@)299@)HI)BIII)Z  .ryy.< "))  BII >->D RYY @bii&"))$/_//*   7+ 7  7v6)v6v6r 0`RYY`F F '=F  F RQ 2Q Q h> /> > B6 $:6 6 rC "8C C L[ 5[ [ | ] /]  ] @ rT