oMjASSKrSSKrSSKJr SSKJs Jr SSKrSSK J r J r SSK J r SSKJr SSKJr SSKJr SS KJr S S KJrJrJr \ R4"\5r"S S \R:5r"SS\R:5r"SS\R:5r "SS\R:5r!"SS\R:5r""SS\R:5r#"SS\R:5r$"SS\R:5r%"SS\R:5r&"SS\R:5r'"S S!\R:5r("S"S#\\\ 5r)g)$N) ConfigMixinregister_to_config)logging)apply_forward_hook)get_activation)AutoencoderKLOutput) ModelMixin)AutoencoderMixin DecoderOutputDiagonalGaussianDistributionc^\rSrSrSS\S\S\\\\\4-S\\\\\4-S\\\\\4-S\\\\\4-S\S \S S 4U4S jjjrS \ RS \ R4Sjr Sr U=r $)HunyuanImageRefinerCausalConv3d" in_channels out_channels kernel_sizestridepaddingdilationbiaspad_modereturnNc >[T U]5 [U[5(aX3U4OUnXlUSS-USS-USS-USS-USS- S4Ul[ R"XX4XVUS9Ulg)Nrrr )r) super__init__ isinstanceintrtime_causal_paddingnnConv3dconv) selfrrrrrrrr __class__s C:\Users\johnn\AppData\Local\Packages\PythonSoftwareFoundation.Python.3.13_qbz5n2kfra8p0\LocalCache\local-packages\Python313\site-packages\diffusers/models/autoencoders/autoencoder_kl_hunyuanimage_refiner.pyr(HunyuanImageRefinerCausalConv3d.__init__#s AKKY\A]A]{=cn  Na  Na  Na  Na  NQ  $  IIkgfjk hidden_statescv[R"XRURS9nUR U5$)N)mode)Fpadr!rr$)r%r*s r'forward'HunyuanImageRefinerCausalConv3d.forward>s+m-E-EDMMZ yy''r))r$rr!)rr rr T replicate)__name__ __module__ __qualname____firstlineno__r tupleboolstrrtorchTensorr/__static_attributes__ __classcell__r&s@r'rr"s 34-..//0#lll5c3// l eCcM** l uS#s]++ lc3m,,lll ll6(U\\(ell((r)rc N^\rSrSrSrS S\S\S\S\SS4 U4S jjjrS rS r U=r $) HunyuanImageRefinerRMS_normCa A custom RMS normalization layer. Args: dim (int): The number of dimensions to normalize over. channel_first (bool, optional): Whether the input tensor has channels as the first dimension. Default is True. images (bool, optional): Whether the input represents image data. Default is True. bias (bool, optional): Whether to include a learnable bias term. Default is False. dim channel_firstimagesrrNcR>[TU]5 U(dSOSnU(aU/UQ7OU4nX lUS-Ul[R "[ R"U55UlU(a0[R "[ R"U55Ul gSUl g)N)r r r )r r g?g) rrrBscaler" Parameterr9onesgammazerosr)r%rArBrCrbroadcastable_dimsshaper&s r'r$HunyuanImageRefinerRMS_norm.__init__Osw .4Y&.;*)*#*#X \\%**U"34 8# UHo[TR5;v M g7fN)r8dtype).0txs r' 6HunyuanImageRefinerRMS_norm.forward..Zs!Q '=!QWW '=s$')float4_float8_r rA) rPr9float16bfloat16anyr- normalizefloatrBtorErHr)r%rSneeds_fp32_normalize normalizeds ` r'r/#HunyuanImageRefinerRMS_norm.forwardYs ww5==%..*II SQ '=Q N [[.BUYUgUgPQmoqtt GG JJ&3dii??r))rrBrHrE)TTF) r2r3r4r5__doc__r r7rr/r;r<r=s@r'r?r?CsJ FCFFTFX\FimFF@@r)r?cj^\rSrSrS\4U4SjjrS\RS\R4SjrSr U=r $)HunyuanImageRefinerAttnBlockdrc>[TU]5 Xl[USS9Ul[ R "XSS9Ul[ R "XSS9Ul[ R "XSS9Ul [ R "XSS9Ul g)NFrCr r) rrrr?normr"r#to_qto_kto_vproj_out)r%rr&s r'r%HunyuanImageRefinerAttnBlock.__init__esj &/ EJ IIkAF IIkAF IIkAF  +J r)rSrcUnURU5nURU5nURU5nURU5nURupgpn UR XgX-U -5R SSS5RS5R5nUR XgX-U -5R SSS5RS5R5nUR XgX-U -5R SSS5RS5R5n[RRX4USS9nURS5R XhXU5R SSSSS5nURU5nX-$)Nrrr ) attn_maskr)rjrkrlrmrKreshapepermute unsqueeze contiguousr" functionalscaled_dot_product_attentionsqueezern) r%rSidentityquerykeyvalue batch_sizechannelsframesheightwidths r'r/$HunyuanImageRefinerAttnBlock.forwardpsm IIaL ! iil ! 6;kk3 fe jFOe4KLTTUVXY[\]gghijuuwkk*%0GHPPQRTUWXYccdefqqs jFOe4KLTTUVXY[\]gghijuuw MM 6 6u5TX 6 Y IIaL VH M U UVWYZ\]_`bc d MM! |r))rrjrnrlrkrm r2r3r4r5r rr9r:r/r;r<r=s@r'rereds1 KC K%,,r)recr^\rSrSrS S\S\S\4U4Sjjjr\S Sj5rS\ R4Sjr S r U=r $) HunyuanImageRefinerUpsampleDCAErradd_temporal_upsamplec>[TU]5 U(aSOSn[XU-SS9UlX0lXB-U-Ulg)Nrrrri)rrrr$rrepeats)r%rrrfactorr&s r'r(HunyuanImageRefinerUpsampleDCAE.__init__sA 33KPVAVdef %:", ; r)c URupEpgnX-U-n XY-n URXAX#XXx5nURSSSSSSSS5nURXJXa-Xr-X-5$) z Convert (b, r1*r2*r3*c, f, h, w) -> (b, c, r1*f, r2*h, r3*w) Args: tensor: Input tensor of shape (b, r1*r2*r3*c, f, h, w) r1: temporal upsampling factor r2: height upsampling factor r3: width upsampling factor rrrr rrrKviewrtrs) tensorr1r2r3bpacked_cfhwrcs r'_dcae_upsample_rearrange8HunyuanImageRefinerUpsampleDCAE._dcae_upsample_rearrangesr &||Q12  QBA!71aAq!Q7~~aAFAFAF;;r)rScUR(aSOSnURU5nUR(a`URUSSSS9nUSS2SURSS-24nURUSSSS9nUR UR S-SS9nX4-$URX2SSS9nUR UR SS9nURXBSSS9nX4-$)Nrr rrrrrA)rr$rrKrepeat_interleaver)r%rSrrshortcuts r'r/'HunyuanImageRefinerUpsampleDCAE.forwards,,Q! IIaL  % %--aA!-BA!&qwwqzQ&&'A44Q1q4IH11$,,!:KQR1SH |--a1-CA**4<[TU]5 U(aSOSnX$-S:Xde[XU-SS9UlX0lXA-U-Ulg)Nrrrrrri)rrrr$r group_size)r%rrrrr&s r'r*HunyuanImageRefinerDownsampleDCAE.__init__sR 59$)))3KQWAWefg '>$ .,>r)c URupEpgnXa-Xr-X-pn URXEXXX5nURSSSSSSSS5nURXAU-U-U-XU 5$) z Convert (b, c, r1*f, r2*h, r3*w) -> (b, r1*r2*r3*c, f, h, w) This packs spatial/temporal dimensions into channels (opposite of upsample) rrrrr rrrrr) rrrrrrpacked_fpacked_hpacked_wrrrs r'_dcae_downsample_rearrange[TU]5 U=(d Un[U5Ul[ USS9Ul[ XSS9Ul[ USS9Ul[ X"SS9Ul SUl X:wa[R"XSSSS9Ul gg)NFrhrrir r)rrr) rrr nonlinearityr?norm1rconv1norm2conv2 conv_shortcutr"r#)r%rrrr&s r'r'HunyuanImageRefinerResnetBlock.__init__s #2{ *=90UK 4[\]^ 0eL 4\]^_ !  &!#;RS\]gh!iD  'r)r*cUnURU5nURU5nURU5nURU5nURU5nUR U5nUR bUR U5nX-$rO)rrrrrr)r%r*residuals r'r/&HunyuanImageRefinerResnetBlock.forwards  =1 ))-8  =1  =1 ))-8  =1    )))(3H''r))rrrrrr)Nswish) r2r3r4r5r r8rr9r:r/r;r<r=s@r'rrse$($ jjDjj j  jj*(U\\(ell((r)rc ~^\rSrSrS S\S\S\SS4U4SjjjrS\RS\R4S jr S r U=r $) HunyuanImageRefinerMidBlockir num_layers add_attentionrNc>[TU]5 X0l[UUS9/n/n[ U5HYnUR(aUR [ U55 OUR S5 UR [UUS95 M[ [R"U5Ul [R"U5Ul SUl g)NrrF) rrrrrangeappendrer" ModuleList attentionsresnetsgradient_checkpointing)r%rrrrr_r&s r'r$HunyuanImageRefinerMidBlock.__init__s * +'(    z"A!!!!">{"KL!!$' NN. +!,  #-- 3}}W- &+#r)r*cURS"U5n[URURSS5Hup#UbU"U5nU"U5nM U$)Nrr )rzipr)r%r*attnresnets r'r/#HunyuanImageRefinerMidBlock.forward3sY Q 6 ab1ABLD $] 3 "=1MC r))rrrr)r T r2r3r4r5r r7rr9r:r/r;r<r=s@r'rrs\" ",",", ",  ",",HU\\ellr)rc ^\rSrSrS S\S\S\S\S-S\SS4 U4S jjjrS \RS\R4S jrS r U=r $)HunyuanImageRefinerDownBlock3Di>Nrrrdownsample_out_channelsrrc4>[TU]5 /n[U5H&nUS:XaUOUnUR[ UUS95 M( [ R "U5UlUb&[ R "[UUUS9/5Ul OSUl SUl g)Nrr)rrF) rrrrrr"rrr downsamplersr) r%rrrrrrir&s r'r'HunyuanImageRefinerDownBlock3D.__init__?s z"A)*a+\K NN. +!- #}}W- " . " 5$%<0G!D !%D &+#r)r*cURH nU"U5nM URbURH nU"U5nM U$rO)rr)r%r*r downsamplers r'r/&HunyuanImageRefinerDownBlock3D.forwarddsJllF"=1M#    (#00 +M :  1r))rrrr NTrr=s@r'rr>sw .2'+ #,#,#, #, "%t #, "% #, #,#,JU\\ellr)rc ^\rSrSrS S\S\S\S\S-S\SS4 U4S jjjrS \RS\R4S jr S r U=r $)HunyuanImageRefinerUpBlock3DioNrrrupsample_out_channelsrrc4>[T U]5 /n[U5H&nUS:XaUOUnUR[ UUS95 M( [ R "U5UlUb&[ R "[UUUS9/5Ul OSUl SUl g)Nrr)rrF) rrrrrr"rrr upsamplersr) r%rrrrrrrinput_channelsr&s r'r%HunyuanImageRefinerUpBlock3D.__init__ps z"A,-F[ N NN. .!- #}}W- , mm3$%:.CDO#DO&+#r)r*c,[R"5(a6UR(a%URHnUR X!5nM OURH nU"U5nM UR bUR H nU"U5nM U$rO)r9is_grad_enabledrr_gradient_checkpointing_funcr)r%r*r upsamplers r'r/$HunyuanImageRefinerUpBlock3D.forwards}  " "t'B'B,, $ A A& X ',, &} 5 ' ?? &!__ )- 8 -r))rrrrrr=s@r'rrosw ,0&* $,$,$, $, #Tz $, $ $, $,$,L U\\ ell  r)rc^\rSrSrSrSS\S\S\\S4S\S\S \S \S S 4U4S jjjrS\ RS \ R4Sjr Sr U=r $)HunyuanImageRefinerEncoder3Diz) 3D vae encoder for HunyuanImageRefiner. rrblock_out_channels.layers_per_blocktemporal_compression_ratiospatial_compression_ratiodownsample_match_channelrNc >[TU]5 XlX lUSUR-Ul[ XSSS9UlSUl[R"/5Ul USn[[U55Hn U [R"U5:n X9n U (d[UUU SSS9n U nO:U [R"Xe-5:n U(aX9S-OU n[UUU UU S9n UnURR!U 5 M [#USS9Ul[%USSS 9Ul[R("5Ul[ USUSS9UlSUlg) NrXrrriF)rrrrrr rrh)rrrrrrconv_in mid_blockr"r down_blocksrlennplog2rrrr?norm_outSiLUconv_actconv_outr)r%rrrrrrr input_channelradd_spatial_downsampleoutput_channel down_blockrrr&s r'r%HunyuanImageRefinerEncoder3D.__init__su &(,R0D4E4EE6{WXDYghi ==,*1- s-./A%&1J)K%K "/2N);/ -!/,0,1  !/ *+rww7P7n/o*o'G_*<U*Ces';/ -!/,C,C  !8    # #J /1045ASTVAWX34Fr4JSXY   78J28NP\jkl &+#r)r*c>URU5n[R"5(aRUR(aAURHnUR X!5nM UR UR U5nO,URH nU"U5nM UR U5nURup4pVnURUSURXVU5RSS9nURU5nURU5nURU5nX- nU$)NrXrrY)rr9rrrrrrKrrrr r r ) r%r*rr~rframerr short_cuts r'r/$HunyuanImageRefinerEncoder3D.forwards ]3  " "t'B'B".. $ A A* \ /!==dnnm\M".. *= 9 /!NN=9M/<.A.A+ ue!&&z2tW\]bbghbi  m4  m4  m4 " r)) r rr rrrrrr r)r@rrrrTr2r3r4r5rcr r6r7rr9r:r/r;r<r=s@r'rrs .I !*+)+)-5,5,5,"#s(O 5,  5, %( 5,$'5,#'5, 5,5,nU\\ellr)rc^\rSrSrSrSS\S\S\\S4S\S\S \S \4U4S jjjrS \ RS \ R4Sjr Sr U=r $)HunyuanImageRefinerDecoder3DizJ Causal decoder for 3D video-like data used for HunyuanImage-2.1 Refiner. rrr.rrrupsample_match_channelc ">[TU]5 X@lXlX lUSUR-Ul[ URUSSS9Ul[R"/5Ul [USS9Ul USn[[U55Hn X9n U [R "U5:n U [R "U5:n U (dU (a-U(aX9S-OU n [#URS-UU U U S9nU nO[#URS-UU SSS9nU nURR%U5 M ['USSS 9Ul[R*"5Ul[ USUSS9UlSUlg) Nrrrirr )rrrrrFrXrh)rrrrrrepeatrrr"r up_blocksrrrrrrrrr?r r r r r)r%rrrrrrr r rradd_spatial_upsamplerrup_blockr&s r'r%HunyuanImageRefinerDecoder3D.__init__s  0&((+t/?/?? 6t7G7GI[\]I^lmn r*5ASTUAVW+1- s-./A/2N#$rww/H'I#I $%0J(K$K !#'[T U]5 [UUS-UUUUUS9Ul[ UU[ [ U55UUUU S9UlX`lXpl SUl SUl SUl SUl SUlSUlSUlg)Nr)rrrrrrr)rrrrrrr Frg?)rrrencoderrlistreverseddecoderrr use_slicing use_tilingtile_sample_min_heighttile_sample_min_widthtile_sample_stride_heighttile_sample_stride_widthtile_overlap_factor) r%rrr,rrrrrr r-r&s r'r)AutoencoderKLHunyuanImageRefiner.__init__\s 3#(1,1-'A&?%=  4'%#H-?$@A-'A&?#9  *C&*D'!  '*#%("*-&(+%#' r)r6r7r8r9r:cSUlU=(d URUlU=(d URUlU=(d URUlU=(d URUlU=(d UR Ulg)a Enable tiled VAE decoding. When this option is enabled, the VAE will split the input tensor into tiles to compute decoding and encoding in several steps. This is useful for saving a large amount of memory and to allow processing larger images. Args: tile_sample_min_height (`int`, *optional*): The minimum height required for a sample to be separated into tiles across the height dimension. tile_sample_min_width (`int`, *optional*): The minimum width required for a sample to be separated into tiles across the width dimension. tile_sample_stride_height (`int`, *optional*): The minimum amount of overlap between two consecutive vertical tiles. This is to ensure that there are no tiling artifacts produced across the height dimension. tile_sample_stride_width (`int`, *optional*): The stride between two consecutive horizontal tiles. This is to ensure that there are no tiling artifacts produced across the width dimension. TN)r5r6r7r8r9r:)r%r6r7r8r9r:s r' enable_tiling.AutoencoderKLHunyuanImageRefiner.enable_tilingsj2&<&[@[@[#%:%Xd>X>X")B)ddFdFd&(@(aDDaDa%#6#R$:R:R r)rScURu p#nUR(a/X@R:dX0R:aUR U5$UR U5nU$rO)rKr5r7r6 tiled_encoder0)r%rSrrrs r'_encode(AutoencoderKLHunyuanImageRefiner._encodesT!"1a ??(B(B BfOjOjFj$$Q' ' LLOr) return_dictc>UR(aYURSS:aFURS5Vs/sHo0RU5PM nn[R "U5nOURU5n[ U5nU(dU4$[US9$s snf)a Encode a batch of images into latents. Args: x (`torch.Tensor`): Input batch of images. return_dict (`bool`, *optional*, defaults to `True`): Whether to return a [`~models.autoencoder_kl.AutoencoderKLOutput`] instead of a plain tuple. Returns: The latent representations of the encoded videos. If `return_dict` is True, a [`~models.autoencoder_kl.AutoencoderKLOutput`] is returned, otherwise a plain `tuple` is returned. rr ) latent_dist)r4rKsplitrAr9rrr )r%rSrCx_sliceencoded_slicesr posteriors r'encode'AutoencoderKLHunyuanImageRefiner.encodes    QCD771:N:ll73:NN .)A QA03 < "y99OsBzcURu p#nURUR-nURUR-nUR(aXF:dX5:aUR U5$UR U5nU$rO)rKr6rr7r5 tiled_decoder3)r%rLrrrtile_latent_min_heighttile_latent_min_widthdecs r'_decode(AutoencoderKLHunyuanImageRefiner._decodesv!"1a!%!\>\ \ ?? =A`$$Q' 'll1o r)c(UR(aYURSS:aFURS5Vs/sHo0RU5PM nn[R "U5nOURU5nU(dU4$[ US9$s snf)a Decode a batch of images. Args: z (`torch.Tensor`): Input batch of latent vectors. return_dict (`bool`, *optional*, defaults to `True`): Whether to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. Returns: [`~models.vae.DecoderOutput`] or `tuple`: If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is returned. rr )sample)r4rKrFrRr9rr)r%rLrCz_slicedecoded_slicesdecodeds r'decode'AutoencoderKLHunyuanImageRefiner.decodesu    QCD771:N:ll73:NNii/Gll1oG: G,,OsBar blend_extentc [URSURSU5n[U5HInUSS2SS2SS2U*U-SS24SXC- - -USS2SS2SS2USS24XC- --USS2SS2SS2USS24'MK U$)Nr minrKr)r%r[rr\ys r'blend_v(AutoencoderKLHunyuanImageRefiner.blend_vs1772; \B |$A Aq<-!*;Q!>?1qGWCWX[\]^`acdfgij]j[k \ AaAq!m %r)c[URSURSU5n[U5HInUSS2SS2SS2SS2U*U-4SXC- - -USS2SS2SS2SS2U4XC- --USS2SS2SS2SS2U4'MK U$)NrXr r_r%r[rr\rSs r'blend_h(AutoencoderKLHunyuanImageRefiner.blend_h s1772; \B |$A Aq!l]Q->!>?1qGWCWX[\]^`acdfgij]j[k \ AaAq!m %r)c [URSURSU5n[U5HInUSS2SS2U*U-SS2SS24SXC- - -USS2SS2USS2SS24XC- --USS2SS2USS2SS24'MK U$)Nr r_res r'blend_t(AutoencoderKLHunyuanImageRefiner.blend_ts1772; \B |$A A }q'8!Q!>?1qGWCWX[\]^`acdfgij]j[k \ AaAq!m %r)c  URu p#nURUR-nURUR-n[ USUR - -5n[ USUR - -5n[ XPR -5n [ X`R -5n XY- n Xj- n /n [ SX75Hxn/n[ SXH5HRnUSS2SS2SS2XUR-2UUUR-24nURU5nURU5 MT U RU5 Mz /n[U 5Hup/n[U5HiunnUS:aURXS- UUU 5nUS:aURUUS- UU 5nURUSS2SS2SS2SU 2SU 245 Mk UR[R"USS95 M [R"USS9nU$)zEncode a batch of images using a tiled encoder. Args: x (`torch.Tensor`): Input batch of videos. Returns: `torch.Tensor`: The latent representation of the encoded videos. r rNrXrYr^)rKr6rr7r r:rr0r enumeraterbrfr9r)r%rSrrrrOrPoverlap_height overlap_width blend_height blend_widthrow_limit_heightrow_limit_widthrowsrrowjtile result_rows result_rowmomentss r'r@-AutoencoderKLHunyuanImageRefiner.tiled_encodes"#1a!%!\>\ \3q4;S;S7STU1Q9Q9Q5QRS 14L4LLM /2J2JJK 1@/=q&1AC1e3D7777D6666 8||D) 4 4 KK 2 oFAJ$S>4q5<<U AlKDq5<<AE D+FD!!$q!Q0A1A0ACSOCS'S"TU *   uyy< =&))KR0r)c URu p#nURUR-nURUR-n[ USUR - -5n[ USUR - -5n[ XPR -5n [ X`R -5n XY- n Xj- n /n [ SX75Hdn/n[ SXH5H>nUSS2SS2SS2XU-2UUU-24nURU5nURU5 M@ U RU5 Mf /n[U 5Hup/n[U5HiunnUS:aURXS- UUU 5nUS:aURUUS- UU 5nURUSS2SS2SS2SU 2SU 245 Mk UR[R"USS95 M [R"USS9nU$)a Decode a batch of images using a tiled decoder. Args: z (`torch.Tensor`): Input batch of latent vectors. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`~models.vae.DecoderOutput`] instead of a plain tuple. Returns: [`~models.vae.DecoderOutput`] or `tuple`: If return_dict is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is returned. r rNrXrYr^)rKr6rr7r r:rr3rrmrbrfr9r)r%rLrrrrOrPrnrorprqrrrsrtrrurvrwrXrxryrQs r'rN-AutoencoderKLHunyuanImageRefiner.tiled_decodeKs "#1a!%!\>\ \3q4;S;S7STU1Q9Q9Q5QRS 14L4LLM /2J2JJK 1@/=q&1AC1e3222111 3,,t, 7#4 KK 2 oFAJ$S>4q5<<U AlKDq5<<AE D+FD!!$q!Q0A1A0ACSOCS'S"TU *   uyy< =&ii , r)rUsample_posterior generatorcUnURU5RnU(aURUS9nOUR5nUR XsS9nU$)a Args: sample (`torch.Tensor`): Input sample. sample_posterior (`bool`, *optional*, defaults to `False`): Whether to sample from the posterior. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`DecoderOutput`] instead of a plain tuple. generator (`torch.Generator`, *optional*): A [`torch.Generator`](https://pytorch.org/docs/stable/generated/torch.Generator.html) to make sampling deterministic. Returns: [`~models.vae.DecoderOutput`] or `tuple`: If `return_dict` is True, a [`~models.vae.DecoderOutput`] is returned, otherwise a plain `tuple` is returned. )r)rC)rJrErUr,rY) r%rUr~rCrrSrIrLrQs r'r/(AutoencoderKLHunyuanImageRefiner.forwardsS. KKN..   9 5A Akk!k5 r)) r3r0rrr:r6r7r8r9r4r5) rrr)rrrrrTTgfk}Ж?)NNNNNr)FTN) r2r3r4r5rc _supports_gradient_checkpointingrr r6r7r^rr=r9r:rArr rrJrRrrYrbrfrjr@rN Generatorr/r;r<r=s@r'r+r+Qs(,$!.I !)+*+)-'+ '7(7(7( 7( "#s(O 7(  7($'7(%(7(#'7(!%7(7( 7(7(v.2,02615,0 S #d S #TzS$)4< S #($, S #T\ S S@%,,37::,0: u%AB B::6  %,, - -4-=SXS_S_C_--2%,,cell%,,cell%,,cell0ell0u||0d4ell4u||4r"' ,0    ??T)   % r)r+)*numpyrr9torch.nnr"torch.nn.functionalrwr-torch.utils.checkpointconfiguration_utilsrrutilsrutils.accelerate_utilsr activationsr modeling_outputsr modeling_utilsr vaer rr get_loggerr2loggerModulerr?rerrrrrrrrr+r)r'rs  B8(2'NN   H %(bii(B@"))@B"299"J+bii+\- -`$(RYY$(N-"))-`.RYY.b42994nT299TnQ299QhNz3C[Nr)