Mj'<ddlZddlmZddlZddlmZddlmZmZddlmZddl Z ddl Z ddl m Z ddl Z ddlZddlZddlZddlZddlmZddlmZddlmZmZdZd d gd fd d gd fddgd fddgd fddgd fddgd fddgd fddgd fddgd fddgd fdd gd fdd gd fdd gd fd d gd fdd gd fdZidddgd fdddgd fddd gd fd!dd"gd fd#ddgd fd$dd gd fd%ddgd fd&dd gd fd'ddgd fd(d dgd fd)ddgd fd*ddgd fd+d dgd fd,ddgd fd-d.dgd fd/d"dgd fd0d1dgd fd2dgd fd3dgd fd4dgd fd5dgd fd6Zd7Zdnd9Zdod:Z d;Z!dpd=Z"d>Z#Gd?d@ej$Z%dqdCZ& drdIZ'dsdLZ(GdMdNZ)dOej*dPej*fdQZ+ dtdTej*dUej*dVe)dWe,dXe,f dYZ-dZej*d[ej*d\e,d]ej*fd^Z.ddlmZdd_l/m0Z0dd`l1m2Z2m3Z3 dudZej*dbej*dcej*d\e,dde4dee5dfe5d]ej*fdgZ6dhZ7diZ8djZ9dkZ:dlZ;dmZcL|\}}|\}}||z ||z |zz }|||z z|z} | Srr) column source_range target_rangeepsilon source_min source_maxnew_minnew_max normalizedscaleds rnormalize_and_scaler$,sH)J #GW:%*z*AG*KLJ 7W, - 7F Mrc~|j}|||j}d|jddzz }||z}|dd}|dd}|j}|d|d}|d|jdzdz dz}t |d|z } tj|| } d|D} | D]} | |ddz} | d}~ ||}||}t|D]W\}}|d }||z}| d| z }| | |X~g}t| D]`\}}tj |d }|d|ddd}| |atj |d S) N??r i4*cg|]}gSrr).0_s r z5calculate_x_ref_attn_map_per_head..Cs000!2000rNNN.dimr)dtypetodeviceshape transposeviewintrsplitsoftmax enumeratesumappendconcatsqueeze)visual_qref_kref_target_masksref_images_count attn_biasr2scalevisual_q_shape number_chunks chunk_sizechunks maps_listsq_chunkattnx_ref_attn_map_source class_idxref_target_mask x_ref_attnmapx_ref_attn_maps maps_list attn_map_fuses r!calculate_x_ref_attn_map_per_headrT7sS NE HHUOO  x / /E (.$+ +E%H!!!Q''H OOAq ! !E^N}}R!355H"2&u{26(BQFMnR(=899J K* - -F00/000J " "R000 $ R 0 0 +..u55 5 8 8 ? ?*34D*E*E 8 8 &I-.CDO1OCM)--b11O4G4G4I4IIM y ! ( ( 7 7 7 7 ! !O )* 5 500 9 YR888 %**1nQ.?DDLLQOO  //// <Q / / //rc|j}|||j}d|jddzz }||z}|dd}|dd}||ddz}|d}~g}||}||}t |D]}\} } | d} || z} | d| z } | ddd} | d} | | ~~tj |d S) Nr&r'r(r r)r*r/rr0) r2r3r4r5r6r:r;r<permutemeanr=rr>) r@rArBrCr2rErLrMrQrNrOrPs rcalculate_x_ref_attn_maprX\sy NE HHUOO  x / /E (.$+ +E%H!!!Q''H OOAq ! !E eoob"-- -D LL,, O'**511144U;;&/0@&A&A.." ?)*?@-? %))"--0C0C0E0EE %--aA66 %**2.. }---- <Q / / //r c |\}}}||z} | dkrd}n | dkrdnd}|ddd| f}|dkrO|j} || d|d}|dd|df}|j| ddg| d dR}|j\} } } } |j\}} tj|| |j|j }| |z}|d krTt |D]C}t|dddd||d zddf|dddd||d zddf||}||z }Dn_t |D]O}t|dddd||z|d z|zddf|dddd||z|d z|zddf||}||z }P||z}|S) zArgs: query (torch.tensor): B M H K key (torch.tensor): B M H K shape (tuple): (N_t, N_h, N_w) ref_target_masks: [B, N_h * N_w] irYi(Nrr'r*)r2r4r ) r5reshaperzerosr2r4rangerTrX)r@rAr5rB split_numrCN_tN_hN_w x_seqlensrFr-seq_lensheads class_numrQ split_chunkix_ref_attn_maps_perheads rget_attn_map_with_targetrkwsMCcc ID #t++BB aaa)m$E!!##N1$5sB??AAA/0001#8#N1$5rPN233x111aP[m]^_`]`bm\mNmopopopHp?qsxyzyzyz|}|}|}@ABM@MOPQRORT_N_@_abababzbtceuwG'H'H # 6 6OOy O rct|dd}|d\}}tj| |fd}t|dS)Nz... (d r) -> ... d rr)rr'r0z... d r -> ... (d r))runbindrstack)xx1x2s r rotate_halfrtsY!+q111A XX"X  FB bS"I2&&&A Q. / //rc,eZdZfdZdZddZxZS)RotaryPositionalEmbedding1Dcdt||_d|_dS)Ni')super__init__head_dimbase)selfrz __class__s rryz$RotaryPositionalEmbedding1D.__init__s-    rcNd|jtjd|jdd|jdz|jz zz }||j}tjd||}t|dd}|S)Nr&rr)z..., f -> ... fz... n -> ... (n r)rm) r{rarangerzfloatr3r4einsumr)r| pos_indicesfreqss rprecompute_freqs_cis_1dz3RotaryPositionalEmbedding1D.precompute_freqs_cis_1dstyU\!T]A%F%FG]$-[\J\G]%^%d%d%f%fimiv%vwx+,, . 0A0A0C0CUKKu2a888 rNc`|d}|td}||d}|||x}||<|dd|dd}}|j} |tj } d}t| } | |z} | |z} | | z } ~ | | } | S)z1D RoPE. Args: query (torch.tensor): [B, head, seq, head_dim] pos_indices (torch.tensor): [seq,] Returns: query with the same shape as input. rmultitalk_ropeN) clearrgetrcos unsqueezesinr2r3rrrt) r|qlistr cache_entryxqcache freqs_cisrrxq_dtypexq_outxq_rots rforwardz#RotaryPositionalEmbedding1D.forwards !H *++99[$//  -1-I-I+-V-V VIk*==??,,Q//99!<W>WXY>Z>Z>d>def>g>gS8u{## V$$# # & 8$$ rr)__name__ __module__ __qualname__ryrr __classcell__)r}s@rrvrvs[rrvctjtj|d}|r|dsd|z}||z }|S)Nzutf-8.)binasciib2a_hexosurandomdecode startswith)lengthsuffixnames r rand_namers_  BJv.. / / 6 6w ? ?D   %% "6\F  Krr Tr'r c |#tjdt|n|}d} t|D]} |t t }tjfd| dDd dddd}|d z tj  }tj||d d d d g} |D]} | | | |cSdS)Nz/tmp)rcVg|]%}tj|&S))nrow normalize value_range) torchvisionutils make_grid)r,urrrs rr.zcache_video..sN!++DI;,PPrr)r r0rlibx264rY-crf10)fpscodecquality ffmpeg_params)ospjoinrr_clampminmaxrrprorVtypeuint8cpuimageio get_writernumpy append_dataclose) tensor save_filerrrrrretry cache_fileerrorr-writerframes ``` r cache_videors%,&)###2; E 5\\c+..K0@0@AA q)) &' ((( )01a(;(;  3,$$U[115577#JCyRTekmqdrsss\\^^ & &E   u % % % % #rFc@dd}|dz}|r)t|d||dddn|dzd z }|dd d d}t j|d zdd t j}||||| |j \} } } } | |z } |d z} dd|dd| | g} tj | d|dz}|rRddd|d| ddddddddd|g} tj | dtj |tj | dSddd|d| ddddd|g } tj | dtj |tj | dS)N ctj||||}t|dD]+}tj|}||,|dS)N)rrrz Saving video)desc)rrrnparrayrr)frames save_pathrrrrrs r save_videoz%save_video_ffmpeg..save_videosu# 3}   &~666 & &EHUOOE   u % % % % rz -temp.mp4rr Tr)rrrrrrr)rr)rrz-cropaudio.wavffmpegz-iz-t)checkr z-yz-c:vrr0z-presetveryslowz-c:aaacz -shortest)rN)rrrVrrrclipastyperr5 subprocessrunrremove)gen_video_samplesrvocal_audio_listrrhigh_quality_saver save_path_tmp video_audior-Tdurationsave_path_crop_audio final_commands rsave_video_ffmpegrs5 +M I,66q99+" '      )*A- !))!Q15599;;AACC gkC/C88??II  ; 3HHHH#(JAq!Q3wH$'77    MN=----F"I!(   - & I C z E     }D1111 -    &'''''              }D1111 -    &'''''rc0eZdZdefdZdejfdZdS)MomentumBuffermomentumc"||_d|_dS)Nrrrunning_average)r|rs rryzMomentumBuffer.__init__Ts   r update_valuec8|j|jz}||z|_dSrr)r|r new_averages rupdatezMomentumBuffer.updateXs$md&:: +k9rN)rrrrryrTensorrrrrrrSsL!!!!!:5<::::::rrv0v1cL|j}||}}tjj|gd}||zgdd|z}||z }||||fS)Nr'r*r0T)r1keepdim)r2doublernn functionalrr<r3)rrr2 v0_parallel v0_orthogonals rprojectr^s HE YY[["))++B   & &r/?/?/? & @ @B7--$4$4$4d-CCbHK$M >>% -"2"25"9"9 99r7diff pred_condmomentum_bufferetanorm_thresholdc>||||j}|dkr]tj|}|dgdd}t d|tj|||z }||z}t||\}} | ||zz} | S)Nrr)rT)pr1rz diff_norm: )rrr ones_likenormprintminimumr) r r r r r ones diff_norm scale_factor diff_paralleldiff_orthogonalnormalized_updates radaptive_projected_guidancerjs"t$$$.t$$II'7'7'7IFF  'I''(((}T>I+EFF l"%,T9%=%="M?'# *== r source_chunkreference_imagestrengthreturnc |dkr|Sd|cxkrdksntd||j}|j}|ddddd}|ddddd}|dzdz }|dzdz }tj|dd}tj|dd} tj |} n,#t$r} td | d |cYd } ~ Sd } ~ wwxYwg} t|j dD]} || } tj | }nB#t$r5} td | d | d| | Yd } ~ [d } ~ wwxYw|}tdD]}|d d d d |f|d d d d |f}}| d d d d |f| d d d d |f}}|dkr ||d d d d |f<|d d d d |f|z ||z z|z|d d d d |f< tj|}nC#t$r6} td| d| d| | Yd } ~ d } ~ wwxYwtj|dd}d|z | z||zz}| |tj| d}|dzdz }t)j|ddddd}|}|||}|S)a Matches the color of a source video chunk to a reference image and blends with the original. Args: source_chunk (torch.Tensor): The video chunk to be color-corrected (B, C, T, H, W) in range [-1, 1]. Assumes B=1 (batch size of 1). reference_image (torch.Tensor): The reference image (B, C, 1, H, W) in range [-1, 1]. Assumes B=1 and T=1 (single reference frame). strength (float): The strength of the color correction (0.0 to 1.0). 0.0 means no correction, 1.0 means full correction. Returns: torch.Tensor: The color-corrected and blended video chunk. rr&*Strength must be between 0.0 and 1.0, got rr r)r@z3Warning: Could not convert reference image to Lab: z+. Skipping color correction for this chunk.Nz(Warning: Could not convert source frame z to Lab: . Using original frame.z+Warning: Could not convert corrected frame z back to RGB: axisr4r2) ValueErrorr4r2r?rVrrrrrrgb2labrr_r5r=copyrWstdlab2rgbrpr from_numpyr contiguousr3)rrrr4r2 source_npref_np source_np_01 ref_np_01ref_labecorrected_frames_np_01risource_frame_rgb_01 source_labcorrected_lab_framejmean_srcstd_srcmean_refstd_reffully_corrected_frame_rgb_01blended_frame_rgb_01corrected_chunk_np_01corrected_chunk_np_minus1_1corrected_chunk_tensor output_tensors rmatch_and_blend_colorsrB~s"3 ( ! ! ! !c ! ! ! !PhPPQQQ  F  E$$Q''//1a;;??AAGGIII  $ $Q ' ' / / 2 2 : :1a C C G G I I O O Q QFOs*L#$I7<c22L 3,,I- **  rArrrsss   <%a( ) )&<&<*1o ':;;JJ    cQccccc d d d " ) )*= > > > HHHH  )oo//q z zA *111aaa7 3 8 8 : :Jqqq!!!Qw > > HHHH  (*w/KSRU'V'V$!"H 0CChQmFmm%%&:;;;;H%;!DDD$93#>#"E#-.IJJRRSTVWYZ\]^^hhijkk3>>@@*--V5-IIM sN%D:: E#EE#E#F## G"-+GG"K L'+LL)ndimage)binary_erosiondistance_transform_edt correctedreference_videomask copy_modesource_border_distancereference_border_distancec |dkr|Sd|cxkrdksntd||dvrtd||j}|j}|j\} } } } } |dkrS|ddd| d| | | }n|d kr@|dd| d| | | }n-|d kr|d}|dddd d }|dddd d }|dd }|dzd z }|dzd z }tj |dd}tj |dd}g}t| D]Y}||}||}||}t|||}|d }|d } t||}t||}nB#t$r5}t!d|d|d||Yd}~d}~wwxYwt%|||||}|dkrt'|||d}nx|dkrt'|||d}n_|dkrYt)||||d |d|d |}t'|||d}t+||||d |d|}||[tj|d}|d zdz } t/j| d dddd}!|!}!|!||}"|"S)u4 Matches the color of a source video chunk to a reference video using mask-based region sampling. Args: source_chunk (torch.Tensor): The video chunk to be color-corrected (B, C, T, H, W) in range [-1, 1]. Assumes B=1 (batch size of 1). reference_video (torch.Tensor): The reference video (B, C, T, H, W) in range [-1, 1]. Must have same temporal dimension as source_chunk. mask (torch.Tensor): Binary mask (B, 1, T, H, W) or (T, H, W) or (H, W) with values 0 and 1. Color correction is applied to pixels where mask=1. strength (float): The strength of the color correction (0.0 to 1.0). 0.0 means no correction, 1.0 means full correction. copy_mode (str): What to do with mask=0 pixels: "corrected" (keep original), "reference", "source", "progressive_blend" (double-sided progressive blending near borders). source_border_distance (int): Distance in pixels from mask border to sample source video (mask=1 side). reference_border_distance (int): Distance in pixels from mask border to sample reference video (mask=0 side). For "progressive_blend" mode, this also defines the blending falloff distance. Returns: torch.Tensor: The color-corrected and blended video chunk. Notes: - Color statistics are sampled from border regions to determine source and reference tints - Progressive blending creates smooth double-sided transitions: * mask=1 side: 60% source + 40% reference at border → 100% source deeper in * mask=0 side: 60% reference + 40% source at border → 100% reference deeper in rr&r )rF referencesourceprogressive_blendzRcopy_mode must be 'corrected', 'reference', 'source', or 'progressive_blend', got r)rr rr! source_regionreference_regionz6Warning: Could not compute color statistics for frame z: r"NrMrNrOsource_distancesreference_distancesr#r%)r&r4r2r5r1rexpandr?rVrrrrr_get_border_sampling_regionscompute_color_statsrr= apply_color_correction_with_maskapply_copy_with_mask)apply_progressive_blend_in_corrected_area)apply_progressive_blend_in_reference_arearprr+r,r3)#rrGrHrrIrJrKr4r2BCrHWr- reference_npmask_npr/reference_np_01r3t source_framereference_frame frame_maskborder_regionssource_sample_regionreference_sample_region source_statsreference_statsr2corrected_framer>r?r@rAs# r match_and_blend_colors_with_maskrmsL3 ( ! ! ! !c ! ! ! !PhPPQQQQQQynwyyzzz  F  E &MAq!Q xxzzQ~~a  **1--77::AA!Q1aPP q~~a  **1--44Q1aCC q~~a  $$Q''//1a;;??AAGGIII"**1--55aAqAAEEGGMMOOLll1oo%%a((,,..4466GOs*L#c)S0O7<c22LgosC88O 1XX-7-7#A )!,QZ 5ZAWYrss-o>"01C"D .|=QRRL1/CZ[[OO    j1jjPQjjj k k k " ) ), 7 7 7 HHHH  ; *lOX    # #2?OU_almmOO ( " "2?LR\^fggOO - - -G*/@R1S124JO 3?OU_almmOGz12NCX4Y)O %%o6666H%;!DDD$93#>#"E"-.IJJRRSTVWYZ\]^^hhijkk3>>@@*--V5-IIM s? K  L*+LLc|t}t|}t|}|||kz}|||kz}||||dS)a Create regions for sampling near mask borders with separate distances for source and reference. Args: mask: Binary mask (H, W) with 0s and 1s source_border_distance: Distance from border to include in source sampling (mask=1 side) reference_border_distance: Distance from border to include in reference sampling (mask=0 side) Returns: Dict with sampling regions and distance maps for blending )rQrRrSrT)rboolrE)rHrJrK mask_booldist_from_mask0dist_from_mask1rQrRs rrVrVqsy D!!I-Y77O-iZ88O4J!JKM# :S'ST',+.   rc*tj|std tj|}n$#t$r}td|d}~wwxYw||}tj|dtj|dd}|S)a/ Compute color statistics (mean and std) for Lab channels in the sampling region. Args: image: RGB image (H, W, C) in range [0, 1] sample_region: Boolean mask (H, W) indicating pixels to sample Returns: Dict with 'mean' and 'std' for Lab components zNo pixels in sampling regionz Could not convert image to Lab: Nrr#)rWr))ranyr&rr'rWr))image sample_region image_labr2sampled_pixelsstatss rrWrWs 6- 97888AM%(( AAA?A??@@@A}-NQ///vn1---  E Ls: AAAc tj|}n,#t$r}td|d|cYd}~Sd}~wwxYw|}|dk}t dD]c} |d| } |d| } |d| } |d| } | dkr| ||| f<H||| f| z | | z z| z||| f<d tj|}n,#t$r}td |d|cYd}~Sd}~wwxYwtj|d d }|}d|z ||z|||zz||<|S) a Apply color correction to pixels where mask=1. Args: source_frame: RGB image (H, W, C) in range [0, 1] mask: Binary mask (H, W) source_stats: Color statistics from source sampling region reference_stats: Color statistics from reference sampling region strength: Blending strength Returns: Corrected RGB image (H, W, C) z0Warning: Could not convert source frame to Lab: r"Nr rrWr)rz8Warning: Could not convert corrected frame back to RGB: rr&) rr'r&rr(r_r*rr)rdrHrjrkrr5r2 corrected_labcorrection_regioncr8r9r:r;fully_corrected_rgbresults rrXrXs]<00  [[[[\\\OO%%M1XX  '*u%a("6*1-!%(+ a<<2:M+Q. / /0!34x?GgDUVYaa +Q. / /#mM::  ccccddd '"5sC@@   F X&788&'899 :  Ms5 A;AAC&& D0D D Dc,|}|}tj|rhddlm} | |dktjd} |dk| z} tj| r!tjtj|} tjtj| } ddlm }|| | d}tj |d }tj ||}tj ||}||krq||}||z ||z z }d d |z z}t| D]F\}\}}||}| |\}}||}d |z |||fz||||fzz|||f<G|S) a Apply progressive blending in the corrected area (mask=1) near the border. Args: corrected_frame: RGB image (H, W, C) - the color-corrected source frame reference_frame: RGB image (H, W, C) - the reference frame mask: Binary mask (H, W) source_region: Boolean mask (H, W) indicating the source blending region (mask=1 near border) source_distances: Distance map (H, W) into mask=1 area from mask=0 border reference_region: Boolean mask (H, W) indicating the reference sampling region (mask=0 near border) source_border_distance: Maximum distance for source blending Returns: Blended RGB image (H, W, C) Notes: - Each source pixel blends with its closest reference border pixel (for speed) - At mask border: 60% source + 40% reference - Deeper into mask=1 area: 100% corrected source rbinary_dilationr rr structurecdist euclideanmetricr#皙?r&r(rrt scipy.ndimagerr column_stackwherescipy.spatial.distancerargminrrr;)rlrerHrQrSrRrJr blend_regionrmask_1_dilated border_pixels source_coords border_coordsrdistances_matrixclosest_border_indicesmin_distance_in_regionmax_distance_in_regionsource_dist_valuesnormalized_distances blend_weightsrisource_ysource_xclosest_border_idxborder_yborder_xweights rrZrZs* ! ! # #F!L vl' 211111)bgfooNNNn4 6-  OBH\,B,BCCMOBH],C,CDDM 5 4 4 4 4 4$u]M+VVV %'Y/?a%H%H%H "&(V,<\,J%K%K "%'V,<\,J%K%K "%(>>>%5l%C"(:=S(SXnrHYH(I$ #s-A'A B 09/G/G  +A+()?)B&)67I)J&Hh*1-Fv89K)LL81C!DDE8X-.. Mrc,|}|}tj|rhddlm}||dktjd} |dk| z} tj| r!tjtj|} tjtj| } ddlm } | | | d}tj |d }tj ||}tj ||}||krq||}||z ||z z }d d |z z}t| D]F\}\}}||}| |\}}||}d |z |||fz||||fzz|||f<G|S) aB Apply progressive blending in the reference area (mask=0) near the border. Args: reference_frame: RGB image (H, W, C) - the reference frame with copied reference pixels source_frame: RGB image (H, W, C) - the original source frame mask: Binary mask (H, W) reference_region: Boolean mask (H, W) indicating the reference blending region (mask=0 near border) reference_distances: Distance map (H, W) into mask=0 area from mask=1 border reference_border_distance: Maximum distance for reference blending Returns: Blended RGB image (H, W, C) Notes: - Each reference pixel blends with its closest source border pixel (for speed) - At mask border: 60% reference + 40% source - Deeper into mask=0 area: 100% reference rrrrr rrrr#rr&r)rerdrHrRrTrKrrrmask_0_dilatedsource_border_pixelsreference_coordssource_border_coordsrrclosest_source_indicesrrreference_dist_valuesrrriref_yref_xclosest_source_idxrrrs rr[r[4s( ! ! # #F$L vl&111111)bgfooNNN $ ^; 6& ' ' !rx /E/EFF #%?28>>(;L(I%(=@V(V[quK\K(L$ #s-A'A B *33C)D)D  %A~u)?)B&)=>P)Q&Hh*1-Fv)FFh.@!AAB5%<(( Mrc\|}|dk}|dkr ||||<|S)a6 Copy pixels to mask=0 regions based on copy_source parameter. Args: source_frame: RGB image (H, W, C) reference_frame: RGB image (H, W, C) mask: Binary mask (H, W) copy_source: "reference" or "source" Returns: Combined RGB image (H, W, C) rrM)r()rdrerH copy_sourcer mask_0_regions rrYrYxs?   FQYMk!! / >} Mr)rr)NrYr)rr)Nrr rTrr)rrF)Nrr)rFrYrY)=reinopsrrtorch.nnrr functoolsrruuidrrrr soundfilesfrros.pathpathrskimager mmgp.offloadrr VID_EXTENSIONSASPECT_RATIO_627ASPECT_RATIO_960rr$rTrXrkrtModulervrrrrrrrrrBscipyrCrrDrEstrr8rmrVrWrXrZr[rYrrrrs  $$$$$$$$ 000000001Dk1 d Q'73*a\_ad[eghZiCj!_c A3*a\_ad[eghZiCj!_c A3*a\`be[fhiZjSk1 s Q'74+qAQ SS sDk1 %d Q'79?3+qAQSY]`bf\gij[k sDk1 %d Q'79?3+qAQSY]`bf\gij[k sCj!_%c{A&69?$a@PSY\`be[fhiZj tSk1   &s Q'7 :@4+qAQ TZ]acf\gij[k   tSk1   *.s Q'74+qAQ]acf\gij[kSk1   #0#0#0#0J0006''''T000.....")...d#    DL(L(L(L(^:::::::: : L : L : : : :-1"$  \*   (ff f`efjojvffffR@@@@@@@@!"$%'BB,B\B ,B B  B  B #B \BBBBJ   F@777tCCCLAAAHr