jC.dZddlZddlmcmZddlmZmZm Z ddl m Z m Z m Z mZmZ ddlmZn#eeef$rdZYnwxYw ddlmZn#eeef$rdZYnwxYw ddlmZn#eeef$rdZYnwxYw dd lmZn#eef$rdZYnwxYw dd lmZn>#eeef$rdZYn2e$r+Ze e!d sdZYdZ[ndZ[wwxYwej"fdd d Z#dZ$dZ%dddZ&dZ'dddZ(dIdddddZ)ddddZ*dJdddZ+dZ,dddd Z-d!Z.dd"d#Z/dKd$Z0dddd%d&Z1dId'Z2d(Z3d)Z4d*Z5d+Z6d,Z7d-Z8iZ9dLd.Z:d/Z;dddddd0d1Zd4Z?dd5d6Z@d7ZAd8ZBd9ZCdd:d;ZDd<ZEdMd=ZFdd>d?ZGd@ZHdJddAdBZIdCZJdIdd:dDZKdEZLdFZMdGZNdJdHZOdS)NaActivation operations shared by denoisers. Owned inputs use x_list through the dispatch boundary. In-place operations borrow their destination; residuals, weights and conditioning are always borrowed. Math contracts are explicit: split-half RMS/RoPE rounds after normalization, matrix RoPE keeps FP32 until its final cast, and complex RoPE rounds before rotation. Whole-head normalization and compact rotary layouts remain model supplied. N) int8_backend kernel_policy native_ops)choosecuda_inferencedevice_supported packed_fmatake)denoiser_triton)denoiser_bits_triton)qk_quant_triton) adaln_triton)rms_rope_split_half_zinfer_schema(func):dtypesactc2tj||||S)zyWhether layer(x) can run through the INT8 backend's fused kernel, with the input operation act ('gelu' needs WGPKernels).r)rcan_fuse_linear)layerxrrs :C:\Users\johnn\OneDrive\Documents\AI\shared\kernels\ops.py can_linearr,s  'qS I I IIc v|s|t|Stj|t|fi|S)zzproject_many..;s-@@fJvq))@@@@@@rc3fK|]+}|jgjdd|jdRV,dS)Nr')reshapeshape)r$outrs rr&zproject_many..=sJRR3[S[>!'#2#,> " >>>RRRRRRrc3.K|]}|VdSr"r#s rr&zproject_many..>s+11v111111r) r torchcompiler is_compiling is_contiguousallr linear_multir)r*tuple)modulesroutputsrs @r project_manyr78s V A N ' ' ) )Saoo.?.?S@@@@@@@@@S+GQYYr172;5O5OPPRRRR'RRRRRR 1111111 1 11rF approximatectduoF|jtjko1t |d|ot |jtj S)zuA BF16 activation on a CUDA GPU that runs FP16 kernels but is below the BF16 floor (Turing): the bits variants apply.N))capability_floorr9) _bits_tritondtyper.bfloat16rr device)rr9s r_bf16_bits_eligiblerBAsP  $ ?EN)B ?q6{SSS ?$QXu~>>>@rc|r|dd|ddz nddko=t|z|ko'tfdt|DS)z~Segments (start, stop, row) that tile the rows in equal spans with rows 0, 1, 2...: one modulation row per batch row or frame.rrc3VK|]#\}\}}}||zko||zko||kV$dSr"r-)r$indexstartstoprowspans rr&z(_consecutive_segments..LskJJXqX]_q`egkmpEUT\)Sdedl.BSse|JJJJJJr)lenr2 enumerate)segmentstotalrIs @r_consecutive_segmentsrNHs.6 =8A;q>HQKN * *AD 1H JH -6 JJJJJu~@HvIvIJJJJJKr)r?c6t|}|j}|X|Utt|ddtjjjur tj r t||j drt||dd|ddz } }| || |j dgd}}t||d||djd|d||djd|j || zdS|o4|dus |j|jko |j|jko|j ddk} t*duo)t-|t/|jjo| } | oC|duo?| o=t3|o.|jtjtjfvo|j|jk} t9d| p| r3| rt;j|||||nt+j||||||S||}|||}||j||j}}|D]E\} }}|| |d ||z ||F|S) afnorm(x), cast to dtype when given (an FP32 modulation after a 16-bit norm), then per segment (start, stop, row) of rows x * (1 + scale[row]) + shift[row] with PyTorch's roundings in the modulated dtype. norm may be weightless. The Triton kernel is bit-identical to the eager path except on the packed-FMA architecture, where it counts as approximate.Nforwardrrr' r8zSegmented Norm Modulation owned_tritong?)!r weightgetattrtyper.nn LayerNormrPrallow_approximaterNr*rJview_layer_norm_modulatetor? unsqueezeepsr1rA_tritonrr rErBfloat32r@rr> norm_modulatemul_add_)rnormshiftscalerLr?rrTspansrowslayoutfusedbitsrFrGrHs rraraOs V A [F &.WT$ZZD-Q-QUZU]UgUo-o-ouBuTuVuV-o%h ;;.p (mmXa[^hqk!n%DtVVE4556#FE&5&M,<,> ^CH;RWR]C]  ) FFFVNRV|)!T5%JJJX_Xmnoquw|DFNPUYVYV V QA  DDKK88AG$$ehhqw&7&75E$>>tS %* 3s+,,11%*==== HrT)in_placeframes fused_addc t| |p1 jdko&|jdko|jdd jdfv}|o| o jdko|jdko|duo|j|jko| ogt fd||fDoI||ko|ddko|p9|d|jd|dzko|jd jdko|jd jdko~|jddkom jd|jdzdkoN|jd jdkr|p|jd|jdzdk}|ot duot d }|o| ot d } t|rd nd |p| r&| rtnt |||S j} |r|jd jdkrg jd|jdd jd | d}|dn| d}|r2tj|d|z |r nd | S|r d|zn d|zz ||r |n |z | S)zCNative dtype arithmetic; optionally broadcast modulation per frame.rNc3XK|]$}|jjko|jjkV%dSr"r?rAr$trs rr&zscale_shift..qs;+p+p\]AGqw,>,W18qxCW+p+p+p+p+p+prrr'iTr8zFrame Scale/Shiftz Scale/ShiftrR)r+)r ndimr*r1r2strider_rrBrr> scale_shiftr)r]r.addcmulrbrc) rrfrerlrmrn broadcastrirjrkr*rs @rrxrxls V Aa16Q;`5:?`u{1~RSUVU\]^U_Q`?`Iw MwafkwejAowRW_cRcwhmhsw|xCiCw//##w(++p+p+p+pbginao+p+p+p(p(pw,,..ELLNN2w7<||A!7Kw&&((_ELLOOu{1~PUP\P\]^P_P_?_,_w+a.AGAJ. w49;r?agbk3Q w +a.1$ w *+ek!n)D)I w ;q>QWQZ/IE4G4G4I4IuU[YZ^^c^ijk^lMlpuMu    Rwd* R~aT/R/R/RE  L%i L$7t$L$L$LD V>!!U]VZ[[[Z $1 '>>q%PXYYY GE >%+a.AGAJ.. IIagaj%+a."agbk B B"" 5??1+=+=^}UAIq86MaaNNNVVW\]]]%:q5y1E ?A % 4AFF5MMM1u9 99U  rrlrncd|t|}|gd}}t|||||S)zKeep the supplied norm's reduction/affine contract, then modulate. Unlike fused AdaLN this retains the native norm reduction and its rounding. Residual inputs may be borrowed by passing [residual]; the norm allocates. Nr{)r rx)rrdrfrerlrnrs rnorm_scale_shiftr}s= T&\\ATAF vueh) T T TTrư>)rmct|}|j}|r|jdkr|jdkr|jdd|jdfvrv|jd}||d|z|d|z|d}|dd|d|dd|d}}|gd}}t |||||S)alayer_norm(x) * (1 + scale) + shift over the last dimension; scale and shift broadcast over x, or, with frames, hold one row per group of tokens (batch, groups, dim) for x (batch, tokens, dim) with groups dividing the tokens (a modulation per latent frame).rprrrur'N)r r*rvrZr)r[)rrerfr^rmrr*groupss rlayer_norm_modulaters V A GE V%*//afkkek!nQPQPWXYPZO6[6[Q FF58f$eAh&&8%( C C}}RE!H55u}}RERSH7U7UuTAF uc : : ? ? F FFrct|ttjtjfdobjdvoYoEdko-jddkotfd||fD}td|otj j |otdu }|d kr"tj|||S|d krtj|||St#jjdf| g|jjkdc}}t)|||| S)NT)rr9)rurprr'rQc 3vK|]}|jdko|jjko|jjko|jjkos|jdjdkoW|ddko>t dt |jj|j dDVdS)rr'rc34K|]\}}|dkp||kVdSrNr-)r$abs rr&z1_layer_norm_modulate...s3ZZ$!QQ!V-qAvZZZZZZrN)rvr?rAr*rwr2ziprss rr&z'_layer_norm_modulate..stt_`VaZ[AFaf$4[AG9K[PQPX\]\dPd[ 3[89 8I[ZZ#agqwPQPVwxxGX:Y:YZZZZZttttttrAdaLN) wgpkernelstritonrrr^)rl)r rr.r@float16rvr1numelr*r2rr availablerA _adaln_tritonimplementationadalnF layer_normr?rx)rrerfr^validfamilyrlrs @rr[r[s V A Au~u}&ESW X X Xt6!t&'oo&7&7t<=GGIIMtNOgVXk]aNatttttejlqdrttttt G(P*2Fqx2P2P >]$%>@@@F (..q%DDD "1eUC888 QC000A#u{ag5tFHa vueh ? ? ??r)packedrlc|r't|dd\}}n|\}}|tj|||S)zNative SiLU rounding, followed by native multiplication, without concat. Packed input is gate|value. Separate projections pass [gate, value]; both are consumed. No approximate kernel is promoted to a new model contract. rur')inplace)r chunkclearrsilurb)rrrlgatevalues rswiglursf 6ll((B// ee e  6$ ) ) ) . .u 5 55rct|}|||}|||~|gd}}t||S)zSeparate projections with native activation rounding and module hooks. Evaluate activation before up_proj so its source and result do not coexist with the up projection. All three projections keep their MMGP/LoRA hooks. N)r rbr)r gate_projup_proj down_proj activationrr gate_lists r gated_mlprs^ V A :iill # #DIIggajj fdtI )Y ' ''rrc|r't|dd\}}n|\}}|dtj|dz ztj|zdtj|dz zzS)zBounded SiLU gate and saturated value, preserving native rounding. Packed input is value|gate; separate projections pass [gate, value]. rur'g@g9@)r rrr.tanhsigmoid)rrrrs rsiturs 6ll((B// tt e  %*TCZ(( (5=+>+> >4%*UZ]aUaJbJbCb ccrct||drt||d|||S|t|}|(||||S)Nrr input_actresidualresidual_scaler+)rrrrbrc)rrrroutputs r swiglu_linearrs%##CeVx(%3CCC C U6&>> " "F N##((222 Mr)rrjrLct|}|r|n|}||d|jd|d|jd}}|D]8\} } } | | kr,|| | || | || 9|S|r|||S||||S)a=residual += gate * x in place, x handed off in x_list; gate broadcastable to x, or with segments [(start, stop, row)] over the rows of (batch * tokens) a table of gate rows (rows, dim) applied span by span, without a per-token gate tensor. tanh applies to the gate first; fused adds in one rounding (addcmul).Nr')r rrZr*addcmul_rbrc) rrrrrjrLrrhsourcerFrGrHs rgated_residual_rs V A (499;;;DD}}R);<U>Uf ( I I E4e||U4Z ))&t*T>T>V>V 9^hinqtv{p|^}^}K%*e02EQRQYZZZZ-$$U]]2->->-A-A!'-J-J eaS8FPXYYYY aA  }}}Q,6usEl,K,K x8==%= ( ( (QYQbQbcdfkfnfnopovfwfwQxQxxrc t|}t||o#|j|jko|}|D]\}}} ||kr|||} |rUt ||||g| || |j| } | | ur| | m| |||||| |j|S)zresidual[start:stop] += gate[row] * layer(x[start:stop]) for each segment (start, stop, row), written into residual: the INT8 backend's residual epilogue (one rounding) where the layer supports it, else the layer's output scaled and added in place.r)r rr?r1rr\copy_r) rrrrrLrrjrFrGrHtargetr+s r gated_linearrs  V A ua [X^qw%> [8CYCYC[C[E$TTtS %<<eDj)F TUQuTz]OfUYZ]U^UaUabcbiUjUjpvwwwf$$LL%%%ad m 4 4d3ill8>6R6RSSS Orig?c|jtjn|j}|tvr,tj|jt|<tt|tz|z}|rt|| tz }tt|tztz| tz }t||S)a_Rows per chunk of mlp(): the chunk's working set (expanded_row_bytes per row: the fc1 output, plus the up projection or the concatenated fc2 input of a prefix MLP) within MLP_L2_FRACTION of the device's L2 cache, and, for those double working sets, within a MLP_MIN_CHUNKS-th of the tokens; never fewer than MLP_MIN_ROWS nor more than MLP_MAX_CHUNKS chunks per call; the whole input when it is that short. A larger floor (2048 rows) used to override the budget for wide FFNs and raised their transient above the former per-model loops (Flux single blocks: 113 MB against 43 MB at 1024x1024).)rEr.cudacurrent_device_l2_cache_bytesget_device_properties L2_cache_sizerMLP_L2_FRACTIONminMLP_MIN_CHUNKSmax MLP_MIN_ROWSMLP_ROW_MULTIPLEMLP_MAX_CHUNKS)tokensexpanded_row_bytesrAdoublerEbudgetrhs rmlp_chunk_rowsr s ,2<+?EJ % % ' ' 'V\E O##!&!A!A%!H!H!V '/9 : :>P PF ;Vw.89:: |V'77:JJvgYgNgLh i iD vt  rcjddjdjdjdzcfdt|DS)zmlp() segments for a gate of shape (batch or 1, groups, dim) over an input (batch, tokens, dim): groups is 1, or a number of frames that divides the tokens. A gate per token does not fit segments: apply it after mlp().rr'rcg|]I}tD]7}|z|zz|z|dzzzjddkr|nd|ff8JS)rr)ranger*)r$rgrrrIrs r z!gate_segments..5s W W WmnINOUIVIV W WDEQZ!d( "AJ!a%4$?4:VW=[\K\K\aabcefFfAg h W W W Wr)r)r*r)rbatchrrrIs` `@@r gate_segmentsr0s << 1 r4:b> : :D:a=&DJqM"9LFD W W W W W W Wrwx}r~r~ W W WWr)upprefixrrrLc t|} | j| d| jd} } || n |d| jd} |dn |d|jd}|j|dn|jz|dn|jd|jzz} t | jd| | z| j|dup|du} |||dvr|nd}|d| jd|fgn|D]9\}}}|dn,|d| j }t||| D]}t|| z|}|| ||}|rt|||r|nd}||dkr&|$t||| ||gd n|dkrt|gn|$t||| ||gd nm|d krt|gnW|d kr$tjj|n-tjj||d krd nd}| tj||||fd}| ||}|j |j kr0||,t||rt+||g||dn||||t+||g|n ||}~|||||||||;| | S)alfc2(act(fc1(x))) by chunks of tokens (mlp_chunk_rows): each chunk's output replaces its input rows, or is added to residual scaled by gate and written into residual. x, handed off in x_list, and residual have shape (..., dim) and are contiguous. act: 'gelu' (tanh approximation), 'gelu_erf', 'silu', 'situ' (fc1 rows value | gate), or 'swiglu' (fc1 rows gate | value). With up, fc1 is the gate and up is the value projection for situ/swiglu. prefix: rows (..., p) borrowed and concatenated before the activated fc1 output as fc2's input (fc2 reads p + the activation's width: the single-stream blocks of Flux, whose second linear takes the attention output and the MLP together). gate is one vector per channel for all tokens, or segments [(start, stop, vector)] over the flattened tokens (frames, spans, batch rows: gate_segments). Where the INT8 backend supports it, the activation is fused into fc2's kernel (gelu, packed swiglu, without prefix) and the gated residual into its epilogue, rounded once; otherwise the activation runs in place and the residual update follows. Returns x, or residual.r'Nr)r)gelur)rrFrrrrrnoner8dimr)r)r r*rZrr element_sizerAr)r\r?rrrrrr.opsatensilu_gelu_catrrrcr)rfc1fc2rrrrrrLrr*r+widthrhfusablerFrGrffirstlasthrjrrs rmlpr8s V Awr172;//1E!!X]]2qwr{%C%CC^TTRb9I)J)JF  RZR_ EfnbhbnoqbruxvFcF GE !'!*eann.>.>&>QS[_Q_Quciqucu v v vDZFNs>P7P7PccVZG9A9I171:t455x**tU 5==+<+<+?+? +J+J5$-- * *Eut|T**DAeDjM""A&T:c1'+J+J+JTGGPTE}CF&==UWUcT1bb5://0????vy~DwDwDimopnqiririrJL.&!RR%* %6%6!7FFFFmpt|m|m|^dfgeh^i^i^i58F]]%)...q111 H\H\]^vy~DwDwDlrlrJPH\IQIQ!IveDj1152>>>t_F|qw&&E,=(BV[efikl[m[mBVsQC58CS44Y_pu|BCCCC5:5FsQC51111CCPQFFA Q A5))))+ *, 88E??rct|r$tdtdutdudkr|S|d|d}}|t j|| ||fdjg|jddRfS) NzSplit-Half RMSNorm + RoPE)rSkitchenr.r.rr'rru)rrr__kitchen_rms_roper.stackr)r*)ropecss rprepare_split_half_roperes $  6*ET[cgTgrCKOrO$P$P$PT]$]$] <fqA ;a!Q]333;KQWKaKKKK KKrctr |dustdusto&tfd|j|jfDS)Nc3XK|]$}|jjko|jjkV%dSr")rAr?)r$prs rr&z*can_split_half_rms_rope..ns:hhAAH(?QW-?hhhhhhr)rr_rBr2rT)rrq_normk_norms` rcan_split_half_rms_roperlsh A   NF$$6$M':M hRefgRhRh ihhhh&-Y_YfIghhhhhjrc|D]\}}tdd|d|zz}td|jd|D]1}|dd|||zf}|||2||jd}|d|d} }d} |D]\}} |dd|f|d|d |zf} } | | j| jkrt j| } | | | | | | d | | | | dSdS) Nrrrrrr.rur'r) rrrrr*rr. empty_likerbr)tensorsrtensorrdrhrFpartpairscosinesinescratch_rseconds r_native_split_half_norm_rope_rqs## 1yfTl&8&8&:&:V=P=P=R=R&RSTT1fl1ot44 # #E!!!U54<//0D JJttDzz " " " " #  2F|T&\  8 8IFA"3;/U1u9_8L1M6E'-5;">">*511 MM% JJv   ' 'B ' ? ? ? KK   ( ($ 7 7 7 7 8 8r) allow_fusionc||n|}|(|r&t||||rt||||||Std||f||ffD|||fS)Nc g|] \}}|||f Sr"r-)r$rrds rrz)split_half_norm_rope_..s&"h"h"hDZ[ZgAt9ZgZgZgr)rsplit_half_rms_rope_r)qkrrrrrrs rsplit_half_norm_rope_rs~ ]A L-DQPVX^-_-_#Aq$GGG!"h"ha[1f+r!r_r) rrrrrrrrdhalfs rrrs~ am  Avv}fm)/V\"=MPQ=QSSS SK!V- = =4 9   !'!*q..71:?D a111ete na111dee nfdkSWS^'+xb9IA9M O O O O O  # # =  "1dk4 B B B B   adDH = = = = *At9+t < < < < a4Krc2|dSt|trfd|DtfdDrtdt jtj}t|D]\}\}}}||||<t jdDdd |fSt|trg|dRS|j |j #t j tj j n|j dfS) a:(weight, eps, weight_rows) of a per-head RMSNorm given as a module (.weight, .eps), as a (weight, eps) tuple, or as segments [(start, stop, module or tuple)] over the tokens with their own weights (stacked into (segments, D) rows, weight_rows the row of each token; one eps); (None, 0.0, None) for no norm.N)NgNc:g|]\}}}t|Sr-)_norm_parameters)r$rsegmentrArs rrz$_norm_parameters..s,UUU}q!W!'66::UUUrc3HK|]}|dddkVdS)rrNr-)r$rpartss rr&z#_norm_parameters..s488$tAw%(1+%888888rz5Segments of a fused q/k norm must share their epsilonrrcg|] }|d S)rr-)r$rs rrz$_norm_parameters..s666DG666rrr)rlistany ValueErrorr.zerosint32rKrr4rTr^finfor`) rdrrArhrErFrGrr/s `` @rr,r,s? |$KUUUUUPTUUU 8888%888 8 8 VTUU U{6VDDD'0 % % #E#E4$Dt  {6666677q!dJJ${d{{ ;$(:J EM2266PTPXZ^ ^^rrc ||n|fd||fD\\}} } \} } } t1t|dr!||jdnd}|SjddvsDt t jt jfrtfd|| fDsdSddg}||d\}}}}| d }|tj || || || n!| |jd dd}tj || || ||||||| |d <~|.|d\}}}}tj |||| ||||||  |d <t|S)aThe staged SageAttention 2 quantization of q and / or k (batch, tokens, heads, head_dim; the other None) fused with their per-head RMSNorm (a module with .weight and .eps, a (weight, eps) tuple, None, or segments [(start, stop, module or tuple)] over the tokens, each with its own weights, in the activation dtype or FP32: a joint attention's streams) and RoPE (rope: (1 | batch, tokens, 1, pairs, 2) cos / sin or (..., pairs, 2, 2) rotation matrices [[cos, -sin], [sin, cos]], read through their strides; None for no rotation; channel p with p + pairs, or interleaved 2p with 2p + 1): (q_quantized, k_quantized) as the core's staged_quantize_q / staged_quantize_k return them, quantized from the FP32 rotated rows (one rounding fewer than the in-place kernel followed by the core's quantizer), k's mean from FP32 partial sums. q is left as it is (staged_attention writes the output into it) and k is not modified (the caller releases it). None where the fusion does not apply (a core without fused_qk_layouts or a quantization layout the kernels do not reproduce, no CUDA inference, head_dim not 64 / 128, norm weights on another device / dtype): the caller runs norm_rope and the core's quantizers.Nc3ZK|]%}t|jdjV&dSr)r,r*rA)r$rdrs rr&z.split_half_norm_rope_quant_..sW.shDDXYuvvCqx18+SAGU];S0SvCvCvCvCDDrrk_mean)r  weight_rowsr)meansm_scale scale_epsr r?rr)rArBr r?) _qk_quanthasattrr9r*rr.r@rr2getnorm_rope_meanexpandnorm_rope_quantizer4)rrrrrcoresettingsr q_weightq_epsq_rowsk_weightk_epsk_rowslayouts quantizedriblockrArBr@rs @rsplit_half_norm_rope_quant_rTsc]AG>SX_`dfxXyXy>Sd##Hagbk:::@DG172;i77~aY^YginivXw?x?x?x7DDDD^fhp]qDDDDD8tt I}-4S\*x||H%%rvr~y'8T5kgmnnnnEIEPEPQRQXYZQ[]_acEdEd 3AxufV[bfqyFO@KY_aaa ! }-4S\*x 3AxufV[fn{D@KY_aaa !   rcttduot|rtj||||Sddlm}|||||t jj ||S)NrRr)rms_norm_rope_) rr_rr!shared.sol_attnrVr.autogradgraphincrement_version)rrTrr^rVs r sol_rms_rope_r[s 7$.D>!3D3DEEE7 FD#666......N1fdC((( N**1--- Hrcvtduotd}| otd}|s|oމddkoʼndjdkot |jdvodt |jcxkodkncoid |jcxkot |jkncoBjd dkp1d jdjdzk}|j r tj nj |o*tfd g|j|jRD}td | sd S|rt nt||||tjjdS)zTry compact split RoPE without materializing token-sized tables. False leaves x untouched for the model's layout-specific PyTorch fallback. Norm over all heads and its prepared statistics are deliberately external. NTr8r'rru)rrprprc3vK|]3}|jjko|jko|V4dSr")rAr?r1)r$rtr?rs rr&z compact_rope_..scII(ah.Y17e3CYHYHYIIIIIIrzCompact Split RoPErRF)r_rrBrwr*rJaxis_cos grid_sizespad_sizeuse_fp32_freqsr.r`r?r2axis_sinrr> split_roperXrYrZ) rcacheri head_startdim_headnativerkrr?s ` @r compact_rope_rhs D H^A4%H%H%HF: B-aTBBBDnM""M'(xx{{agaj'@MV_%%/M45V=N9O9O4T4T4T4TST4T4T4T4TMen;;;;s6?';';;;;;M'!*/KQXXa[[AGAJ4K%K  #1 >EMMqwE  IcIIIII#GV_#Gv#G#GIIIIIE &U ; ; ;u&\\w221eVZQYZZZ N**1--- 4r)offset_in_weight_dtypect|}|r\|jrU|jtjtjfvr6|j|jkr&t j||jdf|dz|S|r|dz n| dz}t j| |jdf|| |jS)aFP32 RMSNorm with the weight stored around zero: (weight + 1) added in FP32, or in the weight's dtype first (Krea 2's `scale + 1.0` on a BF16 parameter), then the product in FP32 and one final rounding. With the offset in the weight's dtype on a 16-bit CUDA tensor, PyTorch's fused kernel does that arithmetic in FP32 without materializing FP32 copies of x (two of them cost a Krea 2 block 25 MiB of peak).r'r) r is_cudar?r.r@rrrms_normr*rr\)rrTr^rirrfs rzero_center_rms_normrm s V A>!)>ENEM;Z0Z0Z_e_kopov_v_vz!agbk^VaZ===$: RVaZ    QR@RE :aggii!'"+ < < ? ? H HHrcnt|}||S)zzx *= sigmoid(gate) in place, with PyTorch's native sigmoid rounding; the gate is consumed (its sigmoid is taken in place).)r rbsigmoid_)rrrs r sigmoid_gate_rps'  ??D 66$--// " ""r)manualc0t|}|j}|}|r3|tjtj|dzdd|zz}n#t j||jdf|}| |S)zUnweighted FP32 RMSNorm, rounded before a caller's weight multiply. Flux uses explicit square/mean/rsqrt; SenseNova uses native rms_norm. Keep those reductions distinct, and leave weight promotion to the norm module. rur'T)rkeepdimr) r r?rr.rsqrtr@rrlr*r\)rr^rqrr?s rrms_norm_roundedrus V A GE  A 3  EJqAv2tDDDsJKK K Jq172;.c 2 2 2 44;;rcPt|}|j|j}}|jg|dddddR}|d|dz}|d|dz}||||S)zFFP32 matrix rotation with separate products, followed by a final cast.Nr'rrurr)r r*r?rr)rcr\)rfreqsrr*r?rs r matrix_roperx,s V A7AG5E 05":0r010a000A 6]QvY &F f & !A ;;q>> ! !% ( ( + +E 2 22rc|r|ddddf|ddddf}}n|dd\}}||}n9|td|jD}||||||d |||||S) zJNative multiply then addcmul; borrow the destination and optional scratch..rNrurr'rc36K|]}td|VdSrN)slice)r$sizes rr&zpaired_rope_..?s*GG4aGGGGGGrr )rcloner4r*rrbr)rrrrr rrs rr"r"6s+#qt!t) aQTT lvr** v++--%GG5;GGGGGH e JJvB777 KK  $/// Hrct|}|j}|}fddD\}}t|||d|S)zGQwen Image's FP32 multiply/addcmul rotation after native norm rounding.c3xK|]4}d|fddV5dS).rruN)r])r$irws rr&z#interleaved_rope..KsGLLE#q&M++A..88;;LLLLLLr)rrTr)r r?rr"r\)rrwrr?rrs ` rinterleaved_roperFsd V A GE  ALLLLVLLLLFD 64T : : : = =e D DDrct|}|d|f}||||S)zDNative-dtype rotation with model-supplied partner indices and signs..)r rbr)rrrpartnerrrotateds r indexed_roperOs: V AW oG 66&>> " "7D 1 11rct|}|||j}||Stj|jg|jddddR}|| dtj | dS)Nr'rurrp) r r\r?r.view_as_complexrr)r*rbr] view_as_realflatten)rrdrwrrs rcomplex_norm_roperVs V A Q 17A }#$5AGGII$5$Kqwss|$KR$K$K$K$KLLG LL##$$$  g & & . .q 1 11rct|}|j}|}||dd|z|z}|ddd}|d|dz|d|dzz d |S)Nr'T)rs)r'rur rr) r r?rsquarer@addrt unflattenr]rr\)rrTrwr^rr?rs rmatrix_rms_roper`s V A GE  A AHHJJOOBO - - 1 1# 6 6 < < > >>OA KKG $ $ . .r 2 2E &ME&M )E&ME&M,I I R RSU V V Y YZ_ ` ``rr")r~)NN)Fr{)P__doc__r.torch.nn.functionalrW functionalrrrrdispatchrrr r r r r_ ImportErrorOSError RuntimeErrorr r>rrCrr comfy_kitchenrrr3excstr startswithr@rrr7rBrNrarxr}rr[rrrrrrrrrrrrrrrrrrrrr$rr,rTr[rhrmrprurxr"rrrrr-rrrs 5555555555PPPPPPPPPPPPPP,,,,,,,Wl+GGG6666666Wl+LLL.......Wl+III///////WMMMGGGGGGGWl+ 3s88  4 5 5  %*N#4$JJJJJ JJJ222+0@@@@@KKKBF     :tEUZ8>BUUUUUU G% G G G G G@@@&"D 6 6 6 6 6 ( ( ( d d d d d5:QU$ y y y y"      WWW&*$D[_*****ZLLLjjj 888&OS(,___&\aB   .IN I I I I I###       333  u      EEE222222aaaaaas`0 ==A AAA A-,A-1A88 BBB C C !CC