j-SrSSKrSSKJr SSKJr SSKJr SSK J r SSK J r SSK Jr SS KJr SS KJr S S KJrJrJrJrJrJrJr S SKJr S r"SS\R<5r"SS\5r "SS\5r!"SS\R<5r""SS\5r#"SS\5r$\"SS\55r%"SS\\%5r&"SS \5r'"S!S"\5r(/S#Qr)g)$zPyTorch Hubert model.N)initialization)ACT2FN)is_deepspeed_zero3_enabled)BaseModelOutput)PreTrainedModel)auto_docstring)OutputRecorder)Wav2Vec2EncoderWav2Vec2EncoderStableLayerNormWav2Vec2FeatureEncoderWav2Vec2ForCTC!Wav2Vec2ForSequenceClassification Wav2Vec2ModelWav2Vec2SamePadLayer) HubertConfigc.^\rSrSrU4SjrSrSrU=r$)HubertPositionalConvEmbedding)c4>[TU]5 [R"URURUR UR S-UR S9UlSUl[USS5(a'[R"UR5UlGO[RRn[[RRS5(a$[RRRn[5(Ga%SSKnUR"R%URR&SS9 U"URSSS 9UlSSS5 [URS 5(aUURRR&R(nURRR&R*nO,URR,nURR.nUR"R1X5 UR"R1X5 OU"URSSS 9Ul[3UR 5Ul[6UR8Ulg!,(df  GN,=f) Nr ) kernel_sizepaddinggroupsconv_pos_batch_normF weight_normr modifier_rankweight)namedimparametrizations)super__init__nnConv1d hidden_sizenum_conv_pos_embeddingsnum_conv_pos_embedding_groupsconv batch_normgetattr BatchNorm1dutilsrhasattrr#r deepspeedzeroGatheredParametersr original0 original1weight_gweight_vregister_external_parameterHubertSamePadLayerrrfeat_extract_activation activation)selfconfigrr1r6r7 __class__s ڷC:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\transformers/models/hubert/modular_hubert.pyr%&HubertPositionalConvEmbedding.__init__*s II      6622a777    60% 8 8 nnV-?-?@DO((..Krxx00-@@ hh77CC )++ ^^66tyy7G7GWX6Y +DIIH! LDIZ499&899#yy99@@JJH#yy99@@JJH#yy11H#yy11H::4J::4J' aH )&*H*HI  !?!?@ZYs J JcURSS5nURbURU5nURU5nURU5nUR U5nURSS5nU$)Nrr ) transposer,r+rr;r< hidden_statess r?forward%HubertPositionalConvEmbedding.forwardOsn%//15 ?? & OOM:M -0  ]3  6 %//15 )r;r,r+r__name__ __module__ __qualname____firstlineno__r%rE__static_attributes__ __classcell__r>s@r?rr)s#AJ  rGrc\rSrSrSrg)r9[NrIrJrKrLrMrRrGr?r9r9[rGr9c\rSrSrSrg)HubertFeatureEncoder_rRNrSrRrGr?rVrV_rTrGrVc.^\rSrSrU4SjrSrSrU=r$)HubertFeatureProjectionccx>[TU]5 URUlUR(a1[R"UR SUR S9Ul[R"UR SUR5Ul [R"UR5Ul g)N)eps)r$r%feat_proj_layer_normr& LayerNormconv_dimlayer_norm_eps layer_normLinearr( projectionDropoutfeat_proj_dropoutdropoutr<r=r>s r?r% HubertFeatureProjection.__init__ds} $*$?$?!  $ $ ll6??2+>FDYDYZDO))FOOB$79K9KLzz&":":; rGcUR(aURU5nURU5nURU5nU$)N)r^rbrdrgrCs r?rEHubertFeatureProjection.forwardls;  $ $ OOM:M 6  ]3 rG)rgr^rbrdrHrOs@r?rYrYcs<rGrYc\rSrSrSrg) HubertEncoderurRNrSrRrGr?rmrmurTrGrmc\rSrSrSrg)HubertEncoderStableLayerNormyrRNrSrRrGr?rprpyrTrGrpc^\rSrSr%\\S'SrSrSrSS/r Sr Sr Sr Sr \\/\"\S S S 9S .r\R("5U4S j5rS\R,\-4SjrS\S\R,4SjrSrU=r$)HubertPreTrainedModel}r=hubert input_valuesaudioHubertEncoderLayerParametrizedConv1dTrencoder)index layer_name)rD attentionscV>[TU]U5 [U[R5(Ga,[ 5(aSSKn[US5(ak[US5(aZURRURUR/SS9 [R"UR5 SSS5 OnURRURSS9 [R"UR5 SSS5 O [R"UR5 URb![R "UR5 gg[U["5(a3[US5(a![R$"UR&5 gg[U[(5(aN[US5(a<[R*"UR,SUR.R0S -- 5 ggg!,(df  N=f!,(df  N=f) zInitialize the weightsrNr7r6rmasked_spec_embed layer_weightsg?r)r$ _init_weights isinstancer&r'rr1r0r2r3r7r6initkaiming_normal_r biaszeros_ HubertModeluniform_rHubertForSequenceClassification constant_rr=num_hidden_layers)r<moduler1r>s r?r#HubertPreTrainedModel._init_weightss f% fbii ( ()++ 6:..76:3N3N"::FOOV__;]mn:o,,V]];po#::6==XY:Z,,V]];[Z$$V]]3{{& FKK('  , ,v233 f6674  ? @ @v//v33SDKK._conv_out_lengths 99\7wWZ[[ [rG)zipr= conv_kernel conv_stride)r<rrrrs r? _get_feat_extract_output_lengths6HubertPreTrainedModel._get_feat_extract_output_lengthssG  \ $'t{{'>'> @W@W#X K,]PM$YrGfeature_vector_lengthattention_maskcURURS55R[R5nUR Sn[R "XA4URURS9nSU[R"UR SURS9US- 4'URS/5RS5RS/5R5nU$)Nr\r)dtypedevicer)r) rsumtorlongshapezerosrrarangeflipcumsumbool)r<rroutput_lengths batch_sizes r?"_get_feature_vector_attention_mask8HubertPreTrainedModel._get_feature_vector_attention_masks>>~?Q?QRT?UVYYZ_ZdZde#))!,   /~7K7KTbTiTi uv ^%9%9!%<^EZEZ[]kno]opq',,bT299"=BBB4HMMOrGrR)rIrJrKrLr__annotations__base_model_prefixmain_input_nameinput_modalities_no_split_modulessupports_gradient_checkpointing_supports_flash_attn_supports_sdpa_supports_flex_attnrx!HubertEncoderLayerStableLayerNormr HubertAttention_can_record_outputsrno_gradr LongTensorintrrrMrNrOs@r?rsrs}s  $O-/CD&*#N,.OP$_A)T  ]]_``2 e>N>NQT>T   ]b]m]m  rGrsc^\rSrSrS\4U4SjjrSrSS\RS-S\RS-S\RS-S \ S-S \ S-S \ S-S \ \ -4S jjr SrU=r$)rr=c>[TU]U5 Xl[U5Ul[ U5UlURS:dURS:aG[R"[R"UR5R55UlUR (a[#U5UlO['U5UlUR)5 U?g)Ng)r$r%r=rVfeature_extractorrYfeature_projectionmask_time_probmask_feature_probr& ParameterrTensorr(rrdo_stable_layer_normrprzrm post_initadapterrhs r?r%HubertModel.__init__s   !5f!="9&"A  3 &&*B*BS*H%'\\%,,v?Q?Q2R2[2[2]%^D "  & &7?DL(0DL  LrGc[S5e)NzNot needed for Hubert)AttributeError)r<s r?freeze_feature_encoder"HubertModel.freeze_feature_encoders455rGNrvrmask_time_indicesoutput_attentionsoutput_hidden_states return_dictreturnc UbUOURRnUbUOURRnUbUOURRnUR U5nUR SS5nUbUR URSU5nURU5n URXS9n URU UUUUS9n U Sn U(d U 4U SS-$[U U RU RS9$)a mask_time_indices (`torch.BoolTensor` of shape `(batch_size, sequence_length)`, *optional*): Indices to mask extracted features for contrastive loss. When in training mode, model learns to predict masked extracted features in *config.proj_codevector_dim* space. Example: ```python >>> from transformers import AutoProcessor, HubertModel >>> from datasets import load_dataset >>> processor = AutoProcessor.from_pretrained("facebook/hubert-large-ls960-ft") >>> model = HubertModel.from_pretrained("facebook/hubert-large-ls960-ft") >>> def map_to_array(example): ... example["speech"] = example["audio"]["array"] ... return example >>> ds = load_dataset("hf-internal-testing/librispeech_asr_dummy", "clean", split="validation") >>> ds = ds.map(map_to_array) >>> input_values = processor(ds["speech"][0], return_tensors="pt").input_values # Batch size 1 >>> hidden_states = model(input_values).last_hidden_state ```Nrr )r)rrrrr)last_hidden_staterDr})r=rrrrrBrrr_mask_hidden_statesrzrrDr}) r<rvrrrrrkwargsextract_featuresrDencoder_outputss r?rEHubertModel.forwards)H2C1N-TXT_T_TqTq$8$D $++JjJj &1%rs &!@/-#4/!"/BII/d -  1 bii$ O  #A BOBBJ\ -!6\ ~ >  &G  frG