/* * Copyright (c) Lynne * * This file is part of FFmpeg. * * FFmpeg is free software; you can redistribute it and/or * modify it under the terms of the GNU Lesser General Public * License as published by the Free Software Foundation; either * version 2.1 of the License, or (at your option) any later version. * * FFmpeg is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU * Lesser General Public License for more details. * * You should have received a copy of the GNU Lesser General Public * License along with FFmpeg; if not, write to the Free Software * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA */ #include "libavutil/aarch64/asm.S" /* Open `doc/transforms.md` to see the code upon which the transforms here were * based upon. * * File conventions: * GPRs: x0-x3 - arguments, untouched * x4 - Lookup table base pointer * x5-x6 - macro ld1 temps/function scratch * x7-x9 - FFT table state * x10-x17 - lookup table/macro scratch * w19-w20 - current/target length when needed * x21-x22 - len*2, len*6 * * Vectors: v0-v7 - coefficients * v8-v15 - coefficients when needed, otherwise untouched * v16-v30 - used as needed * v31 - -1.0, +1.0, -1.0, +1.0. Never touched after loading. * * Stack: backup for v8-v15 and x19-x22 when needed, and transform lengths */ #define M_SQRT1_2 0.707106781186547524401 #define COS16_1 0.92387950420379638671875 #define COS16_3 0.3826834261417388916015625 /* We only ever load this once at the start, and then live with losing an * entire register as we need to lug this all the time everywhere. * Clearly should be integrated into an fsadd and fmlsa, but "muh RISC!". */ const subadd, align=4 .float -1.0, 1.0, -1.0, 1.0 endconst .macro LOAD_SUBADD movrel x5, subadd ld1 { v31.4s }, [x5] .endm .macro SETUP_LUT no_lut=0 .if \no_lut == 0 ldr x4, [x0, #8] .endif .endm .macro LOAD_INPUT dst1, dst2, dst3, dst4, src, no_lut=0, discont=0 .if \no_lut == 1 .if \discont == 1 ldp q\dst1\(), q\dst2\(), [\src\()] ldp q\dst3\(), q\dst4\(), [\src\(), #32] add \src\(), \src\(), #64 .else ld1 { v\dst1\().4s, v\dst2\().4s, v\dst3\().4s, v\dst4\().4s }, [\src], #64 .endif .else ldp w10, w11, [x4, #0 ] ldp w12, w13, [x4, #8 ] ldp w14, w15, [x4, #16] ldp w16, w17, [x4, #24] add x4, x4, #32 ldr d\dst1, [\src, x10, lsl #3] add x11, \src, x11, lsl #3 ldr d\dst2, [\src, x12, lsl #3] add x13, \src, x13, lsl #3 ldr d\dst3, [\src, x14, lsl #3] add x15, \src, x15, lsl #3 ldr d\dst4, [\src, x16, lsl #3] add x17, \src, x17, lsl #3 ld1 { v\dst1\().d }[1], [x11] ld1 { v\dst2\().d }[1], [x13] ld1 { v\dst3\().d }[1], [x15] ld1 { v\dst4\().d }[1], [x17] .endif .endm .macro FFT4 e0, o0, standalone fadd v16.4s, \e0\().4s, \o0\().4s // r1..4 fsub \e0\().4s, \e0\().4s, \o0\().4s // t1..4 rev64 v18.4s, \e0\().4s zip2 \o0\().2d, v16.2d, \e0\().2d zip1 v17.2d, v16.2d, \e0\().2d mov \o0\().d[1], v18.d[1] fadd \e0\().4s, v17.4s, \o0\().4s // a1,2 b1,4 fsub v16.4s, v17.4s, \o0\().4s // a3,4 b3,2 mov \o0\().16b, v16.16b // Swap once again... mov \o0\().s[3], \e0\().s[3] mov \e0\().s[3], v16.s[3] .if \standalone == 0 uzp2 \o0\().2d, \e0\().2d, \o0\().2d uzp1 \e0\().2d, \e0\().2d, v16.2d .endif .endm // Identical to FFT4, but does 2 transforms in parallel, with no deinterleaving .macro FFT4_X2 e0, o0, e1, o1, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 fadd \t0\().4s, \e0\().4s, \o0\().4s // r1234 fadd \t2\().4s, \e1\().4s, \o1\().4s // r1234 fsub \e0\().4s, \e0\().4s, \o0\().4s // t1234 fsub \e1\().4s, \e1\().4s, \o1\().4s // t1234 rev64 \t4\().4s, \e0\().4s rev64 \t5\().4s, \e1\().4s zip2 \o0\().2d, \t0\().2d, \t4\().2d // t3,4 r3,4 zip2 \o1\().2d, \t2\().2d, \t5\().2d // t3,4 r3,4 fneg \t3\().4s, \t4\().4s fneg \t4\().4s, \t5\().4s mov \o0\().s[3], \t3\().s[3] mov \o1\().s[3], \t4\().s[3] zip1 \t1\().2d, \t0\().2d, \e0\().2d // t1,2 r1,2 zip1 \t3\().2d, \t2\().2d, \e1\().2d // t1,2 r1,2 fsub \t4\().4s, \t1\().4s, \o0\().4s // a34 b32 fadd \t5\().4s, \t1\().4s, \o0\().4s // a12 b14 fsub \t2\().4s, \t3\().4s, \o1\().4s // a34 b32 fadd \t3\().4s, \t3\().4s, \o1\().4s // a12 b14 zip1 \e0\().2d, \t5\().2d, \t4\().2d // a1234 zip2 \o0\().2d, \t5\().2d, \t4\().2d // b1432 zip1 \e1\().2d, \t3\().2d, \t2\().2d // a1234 zip2 \o1\().2d, \t3\().2d, \t2\().2d // b1432 .endm const tab_8pt, align=4 .float M_SQRT1_2, -M_SQRT1_2, -M_SQRT1_2, M_SQRT1_2 endconst .macro FFT8 e0, e1, o0, o1, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 movrel x5, tab_8pt fsub \t1\().4s, \e1\().4s, \o1\().4s // j1234 fadd \o1\().4s, \e1\().4s, \o1\().4s // k1234 fsub \t0\().4s, \e0\().4s, \o0\().4s // r1234 fadd \o0\().4s, \e0\().4s, \o0\().4s // q1234 ld1 { \t5\().4s }, [x5] ext \t4\().16b, \o1\().16b, \o1\().16b, #12 rev64 \t4\().4s, \t4\().4s ext \t2\().16b, \o0\().16b, \t4\().16b, #8 // o0[0,1], o1[3,2] mov \o0\().d[1], \t4\().d[1] // o0[3, 4]; o1[1, 4] fsub \e1\().4s, \o0\().4s, \t2\().4s // s34, g43 fadd \t2\().4s, \o0\().4s, \t2\().4s // s12, g12 rev64 \t6\().4s, v31.4s // 1, -1, 1, -1 dup \o0\().2d, \t0\().d[0] // r1212 dup \o1\().2d, \t0\().d[1] // r3434 rev64 \t4\().4s, \e1\().4s // xxg34 rev64 \o1\().4s, \o1\().4s // r4343 ext \t6\().16b, v31.16b, \t6\().16b, #8 // -1, 1, 1, -1 zip1 \t3\().2d, \t2\().2d, \e1\().2d // s1234 zip2 \t2\().2d, \t2\().2d, \t4\().2d // g1234 fadd \e0\().4s, \t3\().4s, \t2\().4s // out_e1 fsub \e1\().4s, \t3\().4s, \t2\().4s // out_e2 fmul \t1\().4s, \t1\().4s, \t5\().4s // j * +--+M_SQRT1_2 fmls \o0\().4s, \o1\().4s, \t6\().4s // z1234 rev64 \t4\().4s, \t1\().4s // j2143 fmla \t1\().4s, \t4\().4s, v31.4s // l2143 rev64 \t4\().4s, \t1\().4s // l1234 ext \t4\().16b, \t4\().16b, \t4\().16b, #8 // l3412 fmla \t4\().4s, \t1\().4s, v31.4s // t1234 fadd \o1\().4s, \o0\().4s, \t4\().4s // out_o2 fsub \o0\().4s, \o0\().4s, \t4\().4s // out_o1 .endm // Identical as FFT8, but does 2 transforms in parallel .macro FFT8_X2 e0, e1, o0, o1, e2, e3, o2, o3 movrel x5, tab_8pt fadd v19.4s, \e3\().4s, \o3\().4s // k1234 fadd v17.4s, \e1\().4s, \o1\().4s // k1234 fadd v18.4s, \e2\().4s, \o2\().4s // q1234 fadd v16.4s, \e0\().4s, \o0\().4s // q1234 ld1 { v23.4s }, [x5] ext v22.16b, v19.16b, v19.16b, #12 ext v21.16b, v17.16b, v17.16b, #12 rev64 v22.4s, v22.4s rev64 v21.4s, v21.4s ext v19.16b, v18.16b, v22.16b, #8 ext v17.16b, v16.16b, v21.16b, #8 mov v18.d[1], v22.d[1] mov v21.d[0], v16.d[0] fadd v22.4s, v18.4s, v19.4s // s12, g12 fsub v19.4s, v18.4s, v19.4s // s34, g43 fsub v18.4s, v21.4s, v17.4s // s34, g43 fadd v16.4s, v21.4s, v17.4s // s12, g12 fsub \e0\().4s, \e0\().4s, \o0\().4s // r1234 fsub v20.4s, \e1\().4s, \o1\().4s // j1234 fsub \e2\().4s, \e2\().4s, \o2\().4s // r1234 fsub v21.4s, \e3\().4s, \o3\().4s // j1234 rev64 v24.4s, v31.4s // 1, -1, 1, -1 zip1 v17.2d, v16.2d, v18.2d // s1234 zip1 \e1\().2d, v22.2d, v19.2d // s1234 rev64 v18.4s, v18.4s // xxg34 rev64 v19.4s, v19.4s // xxg34 zip2 v16.2d, v16.2d, v18.2d // g1234 zip2 \e3\().2d, v22.2d, v19.2d // g1234 dup \o0\().2d, \e0\().d[0] // r1212 dup \o1\().2d, \e0\().d[1] // r3434 dup \o2\().2d, \e2\().d[0] // r1212 dup \o3\().2d, \e2\().d[1] // r3434 fadd \e2\().4s, \e1\().4s, \e3\().4s // out_e1 fsub \e3\().4s, \e1\().4s, \e3\().4s // out_e2 fadd \e0\().4s, v17.4s, v16.4s // out_e1 fsub \e1\().4s, v17.4s, v16.4s // out_e2 ext v24.16b, v31.16b, v24.16b, #8 // -1, 1, 1, -1 rev64 \o1\().4s, \o1\().4s // r4343 rev64 \o3\().4s, \o3\().4s // r4343 fmul v19.4s, v20.4s, v23.4s // j * +--+M_SQRT1_2 fmul v21.4s, v21.4s, v23.4s // j * +--+M_SQRT1_2 rev64 v20.4s, v19.4s // j2143 rev64 v18.4s, v21.4s // j2143 fmls \o0\().4s, \o1\().4s, v24.4s // z1234 fmls \o2\().4s, \o3\().4s, v24.4s // z1234 fmla v19.4s, v20.4s, v31.4s // l2143 fmla v21.4s, v18.4s, v31.4s // l2143 rev64 v20.4s, v19.4s // l1234 rev64 v18.4s, v21.4s // l1234 ext v20.16b, v20.16b, v20.16b, #8 // l3412 ext v18.16b, v18.16b, v18.16b, #8 // l3412 fmla v20.4s, v19.4s, v31.4s // t1234 fmla v18.4s, v21.4s, v31.4s // t1234 fadd \o1\().4s, \o0\().4s, v20.4s // out_o2 fadd \o3\().4s, \o2\().4s, v18.4s // out_o2 fsub \o0\().4s, \o0\().4s, v20.4s // out_o1 fsub \o2\().4s, \o2\().4s, v18.4s // out_o1 .endm const tab_16pt, align=4 .float -COS16_1, COS16_1, -COS16_3, COS16_3 // Could be +-+- too .float COS16_3, COS16_3, COS16_1, COS16_1 .float 1.0, 1.0, M_SQRT1_2, M_SQRT1_2 endconst // 16-point FFT // t3, t4, t5, t6 must be sequential .macro FFT16 e0, e1, e2, e3, o0, o1, o2, o3, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, t6=v22 FFT8 \e0, \e1, \e2, \e3, \t0, \t1, \t2, \t3, \t4, \t5, \t6 FFT4_X2 \o0, \o1, \o2, \o3, \t0, \t1, \t2, \t3, \t4, \t5, \t6 movrel x5, tab_16pt rev64 \t0\().4s, \o0\().4s // z[ 8, 9].imre rev64 \t1\().4s, \o2\().4s // z[10,11].imre ins \t0\().d[0], xzr ins \t1\().d[0], xzr ld1 { \t4\().4s, \t5\().4s, \t6\().4s }, [x5] // TODO: We could derive \t4\() or \t5\() from either, but it seems cheaper to load fmla \o2\().4s, \t1\().4s, v31.4s // s[4567] fmls \o0\().4s, \t0\().4s, v31.4s // s[0123] fmul \t2\().4s, \o1\().4s, \t4\().4s fmul \t3\().4s, \o3\().4s, \t4\().4s rev64 \o3\().4s, \o3\().4s rev64 \o1\().4s, \o1\().4s fmla \t3\().4s, \o3\().4s, \t5\().4s // s[12, 13, 14, 15] fmls \t2\().4s, \o1\().4s, \t5\().4s // s[ 8, 9, 10, 11] fmul \t1\().4s, \o2\().4s, \t6\().4s // s[4567] * mult fmul \t0\().4s, \o0\().4s, \t6\().4s // s[0123] * mult mov \o1\().16b, \t3\().16b mov \o2\().16b, \t1\().16b fsub \t3\().4s, \t3\().4s, \t2\().4s // y34, u34 fsub \t1\().4s, \t1\().4s, \t0\().4s // w34, x34 fadd \t2\().4s, \t2\().4s, \o1\().4s // y56, u56 rev64 \t3\().4s, \t3\().4s fadd \t0\().4s, \t0\().4s, \o2\().4s // w56, x56 rev64 \t1\().4s, \t1\().4s fmul \t2\().4s, \t2\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s fadd \o3\().4s, \e3\().4s, \t3\().4s fsub \o2\().4s, \e3\().4s, \t3\().4s fsub \o1\().4s, \e2\().4s, \t2\().4s fadd \o0\().4s, \e2\().4s, \t2\().4s fsub \e2\().4s, \e0\().4s, \t0\().4s fadd \e0\().4s, \e0\().4s, \t0\().4s fsub \e3\().4s, \e1\().4s, \t1\().4s fadd \e1\().4s, \e1\().4s, \t1\().4s .endm function ff_tx_fft2_float_neon, export=1 ld2r { v0.2d, v1.2d }, [x2] fneg v2.2s, v1.2s mov v2.d[1], v1.d[0] fsub v2.4s, v0.4s, v2.4s st1 { v2.4s }, [x1] ret endfunc .macro FFT4_FN name, inv function ff_tx_fft4_\name\()_float_neon, export=1 ld1 {v0.4s, v1.4s}, [x2] .if \inv == 1 mov v2.d[0], v0.d[1] mov v0.d[1], v1.d[1] mov v1.d[1], v2.d[0] .endif FFT4 v0, v1, 1 st1 { v0.4s, v1.4s }, [x1] ret endfunc .endm FFT4_FN fwd, 0 FFT4_FN inv, 1 .macro FFT8_FN name, no_perm function ff_tx_fft8_\name\()_neon, export=1 SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_SUBADD FFT8 v0, v1, v2, v3 zip1 v16.2d, v0.2d, v2.2d zip2 v17.2d, v0.2d, v2.2d zip1 v18.2d, v1.2d, v3.2d zip2 v19.2d, v1.2d, v3.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1] ret endfunc .endm FFT8_FN float, 0 FFT8_FN ns_float, 1 .macro FFT16_FN name, no_perm function ff_tx_fft16_\name\()_neon, export=1 SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 5, 6, 7, x2, \no_perm LOAD_SUBADD FFT16 v0, v1, v2, v3, v4, v5, v6, v7 zip1 v20.2d, v0.2d, v4.2d zip2 v21.2d, v0.2d, v4.2d zip1 v22.2d, v1.2d, v6.2d zip2 v23.2d, v1.2d, v6.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64 zip1 v24.2d, v2.2d, v5.2d zip2 v25.2d, v2.2d, v5.2d zip1 v26.2d, v3.2d, v7.2d zip2 v27.2d, v3.2d, v7.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1] ret endfunc .endm FFT16_FN float, 0 FFT16_FN ns_float, 1 const tab_15pt, align=4 .float 1.0, 1.0, -1.0, -1.0 endconst // Tab_53 twiddles (v28..v30) duplicated/pre-signed once, instead of per // transform: v8/v9 = -+tab[8,9]/[10,11], v25/v28/v29 = tab[0,1]/[2,3]/[4,5], // v10 = +-tab[6,7]. v30 keeps tab[8..11]. Callers preserve d8-d10. .macro FFT15_DERIVE_CONSTS dup v8.2d, v30.d[0] dup v9.2d, v30.d[1] dup v10.2d, v29.d[1] dup v25.2d, v28.d[0] dup v28.2d, v28.d[1] dup v29.2d, v29.d[0] fmul v8.4s, v8.4s, v31.4s fmul v9.4s, v9.4s, v31.4s fmul v10.4s, v10.4s, v24.4s .endm .macro FFT15_LOAD no_perm, advance=0 .if \no_perm == 1 // Writebacks leave x2 a whole transform (120B) ahead for the PFA loop ld1 { v0.4s }, [x2], #16 // in[0,1] ld1r { v1.2d }, [x2], #8 // in[2] duplicated ld1 { v2.4s, v3.4s, v4.4s }, [x2], #48 // in[3..8] ld1 { v5.4s, v6.4s, v7.4s }, [x2], #48 // in[9..14] .else ldp w10, w11, [x4] // lut[0,1] ldr w12, [x4, #8] // lut[2] ldp w13, w14, [x4, #12] // lut[3,4] ldp w15, w16, [x4, #20] // lut[5,6] ldr d0, [x2, x10, lsl #3] add x10, x2, x11, lsl #3 add x12, x2, x12, lsl #3 ld1 { v0.d }[1], [x10] ld1r { v1.2d }, [x12] ldr d2, [x2, x13, lsl #3] add x13, x2, x14, lsl #3 ldr d3, [x2, x15, lsl #3] add x15, x2, x16, lsl #3 ld1 { v2.d }[1], [x13] ld1 { v3.d }[1], [x15] ldp w10, w11, [x4, #28] // lut[7,8] ldp w12, w13, [x4, #36] // lut[9,10] ldp w14, w15, [x4, #44] // lut[11,12] ldp w16, w17, [x4, #52] // lut[13,14] .if \advance == 1 add x4, x4, #60 .endif ldr d4, [x2, x10, lsl #3] add x10, x2, x11, lsl #3 ldr d5, [x2, x12, lsl #3] add x12, x2, x13, lsl #3 ldr d6, [x2, x14, lsl #3] add x14, x2, x15, lsl #3 ldr d7, [x2, x16, lsl #3] add x16, x2, x17, lsl #3 ld1 { v4.d }[1], [x10] ld1 { v5.d }[1], [x12] ld1 { v6.d }[1], [x14] ld1 { v7.d }[1], [x16] .endif .endm // Single 15-point FFT (see doc/transforms.md and the AVX2 FFT15); each ymm // becomes a pair of quads holding 2 complex each. Uses the derived constants // and tab_15pt in v24 (dc fold sign); with hoist_strides=1 the caller // provides x6/x7 = stride*3/*5. .macro FFT15_CORE hoist_strides=0 .if \hoist_strides == 0 add x6, x3, x3, lsl #1 // stride*3 add x7, x3, x3, lsl #2 // stride*5 .endif add x8, x1, x7 // &out[5] add x9, x8, x7 // &out[10] // 4x parallel 3pt over in[3..14] (the in[11..14] -+ signs are folded // into the twiddles: k = in[11..14] + Q4 -+ Q0), interleaved with the // dc 3pt over in[0..2] ([dc] tagged, v0 = dc[0] dup, v1 = dc[1,2]) fsub v16.4s, v2.4s, v4.4s // q[0,1]raw = in[3,4]-in[7,8] ext v26.16b, v0.16b, v0.16b, #8 // [dc] (in1, in0) fsub v17.4s, v3.4s, v5.4s // q[2,3]raw = in[5,6]-in[9,10] fadd v27.4s, v0.4s, v26.4s // [dc] pc[1]raw = in0+in1 fadd v2.4s, v2.4s, v4.4s // q[4,5]raw fsub v20.4s, v0.4s, v26.4s // [dc] (in0-in1, in1-in0) fadd v3.4s, v3.4s, v5.4s // q[6,7]raw rev64 v20.4s, v20.4s // [dc] pc[0]raw in hi half rev64 v16.4s, v16.4s // q[0,1]raw re/im-swapped ext v21.16b, v20.16b, v27.16b, #8 // [dc] (pc[0], pc[1]) rev64 v17.4s, v17.4s // q[2,3]raw re/im-swapped fadd v0.4s, v1.4s, v27.4s // [dc] dc[0] = in2 + pc[1]raw (dup) fadd v22.4s, v6.4s, v2.4s // y[0,1] = in[11,12] + q[4,5] fmul v21.4s, v21.4s, v30.4s // [dc] pc[0,1] scaled by tab[8..11] fadd v23.4s, v7.4s, v3.4s // y[2,3] = in[13,14] + q[6,7] fmul v16.4s, v16.4s, v8.4s // Q0[0,1] ext v26.16b, v21.16b, v21.16b, #8 // [dc] (pc[1], pc[0]) fmul v17.4s, v17.4s, v8.4s // Q0[2,3] fmla v21.4s, v26.4s, v24.4s // [dc] (dc[1]_int, dc[2]_int) fmla v6.4s, v2.4s, v9.4s // M[0,1] = in[11,12] + q*Q4mult fmla v7.4s, v3.4s, v9.4s // M[2,3] = in[13,14] + q*Q4mult fmla v1.4s, v21.4s, v31.4s // [dc] v1 = (dc[1], dc[2]) — DC done fsub v4.4s, v6.4s, v16.4s // k[0,1] = M[0,1] - Q0[0,1] fsub v5.4s, v7.4s, v17.4s // k[2,3] = M[2,3] - Q0[2,3] fadd v2.4s, v6.4s, v16.4s // k[4,5] = M[0,1] + Q0[0,1] fadd v3.4s, v7.4s, v17.4s // k[6,7] = M[2,3] + Q0[2,3] // 4pt butterflies on y (v22,v23), k[0..3] (v4,v5), k[4..7] (v2,v3); // one shared swapped operand per pair leaves the hi t's half-swapped, // which the dup-symmetric twiddles absorb and the output stage uses ext v16.16b, v23.16b, v23.16b, #8 // (y3, y2) ext v17.16b, v5.16b, v5.16b, #8 // (k3, k2) ext v20.16b, v3.16b, v3.16b, #8 // (k7, k6) fsub v21.4s, v22.4s, v16.4s // (t3, t2) fadd v22.4s, v22.4s, v16.4s // (t0, t1) fsub v26.4s, v4.4s, v17.4s // (t7, t6) fadd v4.4s, v4.4s, v17.4s // (t4, t5) fsub v27.4s, v2.4s, v20.4s // (t11, t10) fadd v2.4s, v2.4s, v20.4s // (t8, t9) // the 3 direct outputs: out[0,10,5] = dc[0,1,2] + t[0,4,8] + t[1,5,9] ext v16.16b, v22.16b, v22.16b, #8 // (t1, t0) zip1 v17.2d, v4.2d, v2.2d // (t4, t8) zip2 v20.2d, v4.2d, v2.2d // (t5, t9) fadd v16.4s, v16.4s, v22.4s // t[0]+t[1] fadd v17.4s, v17.4s, v20.4s // (t[4]+t[5], t[8]+t[9]) fadd v16.4s, v16.4s, v0.4s // out[0] fadd v17.4s, v17.4s, v1.4s // (out[10], out[5]) st1 { v16.d }[0], [x1] st1 { v17.d }[1], [x8] st1 { v17.d }[0], [x9] // twiddles; swap(t * tab) = swap(t) * tab as every multiplier is // dup-symmetric. lo chunks seed the accumulator with dc[] (= the // output stage's dc preadd): m = dc + t*v25 - swap(t)*v28; hi chunks // r = t*v29 + swap(t)*v10, v10's +- giving r[3] += t[2]/r[2] -= t[3] // in half-swapped order. Accumulates are spread out for the A53 ext v16.16b, v22.16b, v22.16b, #8 // (t1, t0) mov v6.16b, v0.16b // m0 = dc[0] ext v17.16b, v21.16b, v21.16b, #8 // (t2, t3) fmul v7.4s, v21.4s, v29.4s // (r3, r2) fmla v6.4s, v22.4s, v25.4s // m0 += t[0,1]*r_lo dup v18.2d, v1.d[0] // m1 = dc[1] fmla v18.4s, v4.4s, v25.4s // m1 += t[4,5]*r_lo fmls v6.4s, v16.4s, v28.4s // m0 -= swap*nt_lo ext v16.16b, v4.16b, v4.16b, #8 // (t5, t4) fmla v7.4s, v17.4s, v10.4s // (r3, r2) += swap*nt_hi ext v17.16b, v26.16b, v26.16b, #8 // (t6, t7) fmls v18.4s, v16.4s, v28.4s // m1 -= swap*nt_lo fmul v23.4s, v26.4s, v29.4s // (r7, r6) dup v19.2d, v1.d[1] // m2 = dc[2] fmla v19.4s, v2.4s, v25.4s // m2 += t[8,9]*r_lo ext v16.16b, v2.16b, v2.16b, #8 // (t9, t8) fmla v23.4s, v17.4s, v10.4s // (r7, r6) ext v17.16b, v27.16b, v27.16b, #8 // (t10, t11) fmul v5.4s, v27.4s, v29.4s // (r11, r10) fmls v19.4s, v16.4s, v28.4s // m2 -= swap*nt_lo fmla v5.4s, v17.4s, v10.4s // (r11, r10) // output butterflies around rot(x) = (x.im, -x.re): out = m +- rot(r_hi). // The half-swap makes rev(r_hi) a plain rev64, and u = rev*v31 is // exact (+-1.0), so each +-rot pair is one non-destructive fsub/fadd rev64 v16.4s, v7.4s // (r3.im, r3.re, r2.im, r2.re) rev64 v17.4s, v23.4s // (r7.im, r7.re, r6.im, r6.re) rev64 v20.4s, v5.4s // (r11.im, ..., r10.re) fmul v16.4s, v16.4s, v31.4s // u0 fmul v17.4s, v17.4s, v31.4s // u1 fmul v20.4s, v20.4s, v31.4s // u2 fsub v7.4s, v6.4s, v16.4s // (out6, out3) = m0 + rot fadd v6.4s, v6.4s, v16.4s // (out9, out12) = m0 - rot fsub v23.4s, v18.4s, v17.4s // (out1, out13) fadd v22.4s, v18.4s, v17.4s // (out4, out7) fsub v5.4s, v19.4s, v20.4s // (out11, out8) fadd v4.4s, v19.4s, v20.4s // (out14, out2) add x10, x1, x6, lsl #1 // &out[6] add x11, x1, x6 // &out[3] st1 { v7.d }[0], [x10] st1 { v7.d }[1], [x11] add x12, x8, x3, lsl #2 // &out[9] add x13, x1, x6, lsl #2 // &out[12] st1 { v6.d }[0], [x12] st1 { v6.d }[1], [x13] add x10, x1, x3 // &out[1] add x11, x9, x6 // &out[13] st1 { v23.d }[0], [x10] st1 { v23.d }[1], [x11] add x12, x1, x3, lsl #2 // &out[4] add x13, x8, x3, lsl #1 // &out[7] st1 { v22.d }[0], [x12] st1 { v22.d }[1], [x13] add x10, x9, x3 // &out[11] add x11, x1, x3, lsl #3 // &out[8] st1 { v5.d }[0], [x10] st1 { v5.d }[1], [x11] add x12, x9, x3, lsl #2 // &out[14] add x13, x1, x3, lsl #1 // &out[2] st1 { v4.d }[0], [x12] st1 { v4.d }[1], [x13] .endm .macro FFT15_FN name, no_perm function ff_tx_fft15_\name\()_neon, export=1 stp d8, d9, [sp, #-32]! str d10, [sp, #16] SETUP_LUT \no_perm movrel x5, X(ff_tx_tab_53_float) ld1 { v28.4s, v29.4s, v30.4s }, [x5] // 5pt cos, 5pt sin, 3pt movrel x5, tab_15pt ld1 { v24.4s }, [x5] // sign mask LOAD_SUBADD FFT15_DERIVE_CONSTS FFT15_LOAD \no_perm FFT15_CORE ldr d10, [sp, #16] ldp d8, d9, [sp], #32 ret endfunc .endm FFT15_FN float, 0 FFT15_FN ns_float, 1 // 15xM PFA (len = 15*M, M a power of two), like the x86 fft_pfa_15xM: // dim1 = M 15pt transforms scattered into s->tmp at sub_map[i], spaced M // apart; dim2 = 15 in-place M-pt subtransforms (plain blr, the uniform ABI // needs no asm-call variant); post = out[i*stride] = s->tmp[out_map[i]]. // AVTXContext offsets: len=0, map=8, tmp=24, sub=32, fn[0]=40. .macro PFA_15_FN name, no_perm function ff_tx_fft_pfa_15xM_\name\()_neon, export=1 AARCH64_SIGN_LINK_REGISTER stp x29, x30, [sp, #-128]! mov x29, sp stp x19, x20, [sp, #16] stp x21, x22, [sp, #32] stp x23, x24, [sp, #48] stp x25, x26, [sp, #64] stp x27, x28, [sp, #80] stp d8, d9, [sp, #96] str d10, [sp, #112] mov x25, x0 // root context mov x26, x1 // user out mov x27, x3 // user stride (bytes) ldr w19, [x0, #0] // len = 15*M ldr x20, [x0, #24] // s->tmp ldr x23, [x0, #32] // s->sub ldr w24, [x23, #0] // M ldr x28, [x23, #8] // sub_map lsl x3, x24, #3 // M*8 = 15pt output stride .if \no_perm == 0 ldr x4, [x0, #8] // in_map, advanced by the loads .endif movrel x5, X(ff_tx_tab_53_float) ld1 { v28.4s, v29.4s, v30.4s }, [x5] movrel x5, tab_15pt ld1 { v24.4s }, [x5] LOAD_SUBADD FFT15_DERIVE_CONSTS add x6, x3, x3, lsl #1 // stride*3 add x7, x3, x3, lsl #2 // stride*5 1: ldr w10, [x28], #4 // sub_map[i] add x1, x20, x10, lsl #3 FFT15_LOAD \no_perm, advance=1 FFT15_CORE hoist_strides=1 subs w19, w19, #15 b.gt 1b ldr x28, [x25, #40] // ctx->fn[0] mov x21, x20 // column base mov w19, #15 2: mov x0, x23 mov x1, x21 mov x2, x21 mov x3, #8 blr x28 // M-point FFT, in-place add x21, x21, x24, lsl #3 subs w19, w19, #1 b.gt 2b // out[i*stride] = s->tmp[out_map[i]], unrolled by 4 (len % 60 == 0) ldr w19, [x25, #0] // len ldr x22, [x25, #8] // s->map add x22, x22, x19, lsl #2 // out_map = map + len mov x10, x26 lsl x11, x27, #1 // 2*stride add x12, x27, x11 // 3*stride 3: ldp w13, w14, [x22], #8 // out_map[i, i+1] ldp w15, w16, [x22], #8 // out_map[i+2, i+3] ldr d0, [x20, x13, lsl #3] ldr d1, [x20, x14, lsl #3] ldr d2, [x20, x15, lsl #3] ldr d3, [x20, x16, lsl #3] str d0, [x10] str d1, [x10, x27] str d2, [x10, x11] str d3, [x10, x12] add x10, x10, x11, lsl #1 subs w19, w19, #4 b.gt 3b ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] ldp x23, x24, [sp, #48] ldp x25, x26, [sp, #64] ldp x27, x28, [sp, #80] ldp d8, d9, [sp, #96] ldr d10, [sp, #112] ldp x29, x30, [sp], #128 AARCH64_VALIDATE_LINK_REGISTER ret endfunc .endm PFA_15_FN float, 0 PFA_15_FN ns_float, 1 // Inverse MDCT (see ff_tx_mdct_inv): pre-rotation, in-place N/2pt FFT, // post-rotation. out is the N/2-complex buffer z, in the N-real input. // Where ld2 can split the re/im planes the multiplies are planar and fused // (fmul + fmls/fmla); the gather and the odd tail use interleaved pairs // (trn1/trn2 broadcast, rev64 swap, the v31 fmla). // AVTXContext offsets: len=0, map=8 (doubled in init), exp=16, sub=32, fn=40. function ff_tx_mdct_inv_float_neon, export=1 AARCH64_SIGN_LINK_REGISTER stp x29, x30, [sp, #-48]! mov x29, sp stp x19, x20, [sp, #16] stp x21, x22, [sp, #32] mov x19, x1 // z mov x20, x0 // ctx ldr w21, [x0, #0] // N ldr x22, [x0, #16] // exp ldr x4, [x0, #8] // map (doubled) LOAD_SUBADD sub x5, x21, #1 madd x14, x5, x3, x2 // in2 = in + (N-1)*stride lsr w7, w21, #1 // len2 cmp x3, #4 // contiguous input and b.ne 6f // len2 % 4 == 0 takes the tst w21, #7 // single-sweep path b.eq 3f 6: // pre-rotation via gather: z[i] = (in2[-k], in1[k])*exp[i], 2 cx/iter mov x13, x19 mov x15, x22 1: ldp w5, w6, [x4], #8 // k0, k1 madd x8, x5, x3, x2 // &in[k0] (im0) msub x9, x5, x3, x14 // &in2[-k0] (re0) madd x10, x6, x3, x2 // &in[k1] (im1) msub x11, x6, x3, x14 // &in2[-k1] (re1) ldr s0, [x9] ld1 { v0.s }[1], [x8] ld1 { v0.s }[2], [x11] ld1 { v0.s }[3], [x10] // (re0, im0, re1, im1) = tmp ld1 { v1.4s }, [x15], #16 // exp[i,i+1] trn1 v2.4s, v0.4s, v0.4s // re dup trn2 v3.4s, v0.4s, v0.4s // im dup rev64 v4.4s, v1.4s // exp swapped fmul v5.4s, v2.4s, v1.4s fmul v6.4s, v3.4s, v4.4s fmla v5.4s, v6.4s, v31.4s // z = tmp*exp st1 { v5.4s }, [x13], #16 subs w7, w7, #2 b.gt 1b b 4f // pre-rotation, contiguous: the input is effectively (im, re, im, ...) // interleaved, so sweep it once from both ends with the planes // ld2-split: low ims pair with high res (outputs p, p+1) and high ims // with low res (outputs p'-1, p', p' = len2-1-p), scattered through // the inverse map (2nd half of s->map) with the natural twiddles 3: add x15, x22, x21, lsl #2 // exp + len2 (asc) add x4, x4, x21, lsl #1 // map + len2 (asc) sub x10, x14, #12 // &in[N-4] (desc) add x12, x15, x21, lsl #2 sub x12, x12, #16 // exp + (N-2) (desc) add x13, x4, x21, lsl #1 sub x13, x13, #8 // map + (N-2) (desc) 5: ld2 { v6.2s, v7.2s }, [x2], #16 // (im_p, im_p1) (re_p', re_p'm1) ld2 { v16.2s, v17.2s }, [x10] // (im_p'm1, im_p') (re_p1, re_p) sub x10, x10, #16 rev64 v17.2s, v17.2s // (re_p, re_p1) ld2 { v1.2s, v2.2s }, [x15], #16 // A: er, ei rev64 v7.2s, v7.2s // (re_p'm1, re_p') ld2 { v3.2s, v4.2s }, [x12] // B: er, ei sub x12, x12, #16 fmul v5.2s, v17.2s, v1.2s // A: re*er fmul v18.2s, v17.2s, v2.2s // A: re*ei fmul v19.2s, v7.2s, v3.2s // B: re*er fmul v20.2s, v7.2s, v4.2s // B: re*ei fmls v5.2s, v6.2s, v2.2s // A: z.re plane fmla v18.2s, v6.2s, v1.2s // A: z.im plane fmls v19.2s, v16.2s, v4.2s // B: z.re plane fmla v20.2s, v16.2s, v3.2s // B: z.im plane ldp w16, w17, [x4], #8 // inv_map[p, p+1] zip1 v0.2s, v5.2s, v18.2s // z_p zip2 v1.2s, v5.2s, v18.2s // z_p1 str d0, [x19, w16, uxtw #3] str d1, [x19, w17, uxtw #3] ldp w16, w17, [x13] // inv_map[p'-1, p'] sub x13, x13, #8 zip1 v2.2s, v19.2s, v20.2s // z_p'm1 zip2 v3.2s, v19.2s, v20.2s // z_p' str d2, [x19, w16, uxtw #3] str d3, [x19, w17, uxtw #3] subs w7, w7, #4 b.gt 5b 4: ldr x5, [x20, #40] // fn[0] ldr x0, [x20, #32] // sub[0] mov x1, x19 mov x2, x19 mov x3, #8 blr x5 // N/2pt FFT, in-place // post-rotation over symmetric pairs (i0 = len4+i, i1 = len4-1-i), // 2 pairs/iter, planes ld2-split: r = swap(z)*swap(e), the im parts // crossing partners (z[i0] = (r_i0.re, r_i1.im) and vice versa), so // each side stores its re plane zipped with the other's reversed ims add x15, x22, x21, lsl #2 // exp_post = exp + len2 add x8, x19, x21, lsl #1 // p_i0 = z + len4 sub x9, x8, #16 // p_i1 = z + len4 - 2 add x10, x15, x21, lsl #1 // e_i0 = exp_post + len4 sub x11, x10, #16 // e_i1 = exp_post + len4 - 2 lsr w7, w21, #2 // len4 lsr w6, w7, #1 // pairs of pairs cbz w6, 8f // N == 4: lone middle pair 2: ld2 { v0.2s, v1.2s }, [x8] // z asc (i0, i0+1): re, im planes ld2 { v2.2s, v3.2s }, [x9] // z desc (i1-1, i1) ld2 { v4.2s, v5.2s }, [x10], #16 // e asc: er, ei ld2 { v6.2s, v7.2s }, [x11] // e desc sub x11, x11, #16 fmul v16.2s, v1.2s, v5.2s // asc: z.im*e.im fmul v18.2s, v1.2s, v4.2s // asc: z.im*e.re fmul v20.2s, v3.2s, v7.2s // desc: z.im*e.im fmul v22.2s, v3.2s, v6.2s // desc: z.im*e.re fmls v16.2s, v0.2s, v4.2s // re plane (r_i0, r_i0+1) fmla v18.2s, v0.2s, v5.2s // im plane (r_i0, r_i0+1) fmls v20.2s, v2.2s, v6.2s // re plane (r_i1-1, r_i1) fmla v22.2s, v2.2s, v7.2s // im plane (r_i1-1, r_i1) rev64 v22.2s, v22.2s // (r_i1.im, r_i1-1.im) rev64 v18.2s, v18.2s // (r_i0+1.im, r_i0.im) zip1 v0.4s, v16.4s, v22.4s // (z[i0], z[i0+1]) zip1 v2.4s, v20.4s, v18.4s // (z[i1-1], z[i1]) st1 { v0.4s }, [x8], #16 st1 { v2.4s }, [x9] sub x9, x9, #16 subs w6, w6, #1 b.gt 2b 8: // odd len4 (N % 8 == 4): one leftover pair, (i0, i1) = (len2-1, 0). // x8/x10 already point at it; x9/x11 sit one complex below their slot. tbz w7, #0, 9f LOAD_SUBADD // v31 (clobbered by the FFT) add x9, x9, #8 add x11, x11, #8 ldr d0, [x8] // z[i0] ld1 { v0.d }[1], [x9] // v0 = (z[i0], z[i1]) ldr d1, [x10] // exp[i0] ld1 { v1.d }[1], [x11] // v1 = (exp[i0], exp[i1]) rev64 v2.4s, v0.4s // swap(z) = a rev64 v3.4s, v1.4s // swap(exp) = b trn1 v4.4s, v2.4s, v2.4s // a.re dup trn2 v5.4s, v2.4s, v2.4s // a.im dup fmul v6.4s, v4.4s, v3.4s // a.re*b fmul v7.4s, v5.4s, v1.4s // a.im*b_swap (b_swap = orig exp) fmla v6.4s, v7.4s, v31.4s // v6 = (r0.re, r0.im, r1.re, r1.im) mov v16.16b, v6.16b ins v16.s[1], v6.s[3] // (r0.re, r1.im, r1.re, r1.im) ins v16.s[3], v6.s[1] // (r0.re, r1.im, r1.re, r0.im) st1 { v16.d }[0], [x8] // z[i0] = (r0.re, r1.im) st1 { v16.d }[1], [x9] // z[i1] = (r1.re, r0.im) 9: ldp x19, x20, [sp, #16] ldp x21, x22, [sp, #32] ldp x29, x30, [sp], #48 AARCH64_VALIDATE_LINK_REGISTER ret endfunc .macro SETUP_SR_RECOMB len, re, im, dec ldr w5, =(\len - 4*7) movrel \re, X(ff_tx_tab_\len\()_float) add \im, \re, x5 mov \dec, #-32 .if \len > 32 mov x21, #2*\len add x22, x21, x21, lsl #1 .endif .endm .macro SR_COMBINE e0, e1, e2, e3, e4, e5, e6, e7, \ o0, o1, o2, o3, o4, o5, o6, o7, \ re, im, dec, swap_im, \ t0=v16, t1=v17, t2=v18, t3=v19, t4=v20, t5=v21, \ t6=v22, t7=v23, t8=v24, t9=v25, ta=v26, tb=v27 ld1 { \t8\().4s, \t9\().4s }, [\im], \dec ld1 { \t0\().4s, \t1\().4s }, [\re], #32 .if \swap_im == 1 ext \t2\().16b, \t9\().16b, \t9\().16b, #8 ext \t3\().16b, \t8\().16b, \t8\().16b, #8 .else ext \t2\().16b, \t8\().16b, \t8\().16b, #8 ext \t3\().16b, \t9\().16b, \t9\().16b, #8 .endif trn1 \t4\().4s, \t0\().4s, \t0\().4s // cos0022 trn2 \t0\().4s, \t0\().4s, \t0\().4s // cos4466 trn1 \t5\().4s, \t1\().4s, \t1\().4s // cos1133 trn2 \t1\().4s, \t1\().4s, \t1\().4s // cos5577 rev64 \t6\().4s, \o0\().4s // E m2[0,1].imre rev64 \t7\().4s, \o2\().4s // O m2[0,1].imre rev64 \t8\().4s, \o4\().4s // E m2[2,3].imre rev64 \t9\().4s, \o6\().4s // O m2[2,3].imre fmul \t6\().4s, \t6\().4s, \t4\().4s // E m2[0,1].imre*t1[0,2] fmul \t7\().4s, \t7\().4s, \t0\().4s // O m2[0,1].imre*t1[0,2] fmul \t8\().4s, \t8\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2] fmul \t9\().4s, \t9\().4s, \t0\().4s // O m2[2,3].imre*t1[0,2] rev64 \ta\().4s, \o1\().4s // E m3[0,1].imre rev64 \tb\().4s, \o3\().4s // O m3[0,1].imre rev64 \t4\().4s, \o5\().4s // E m3[2,3].imre rev64 \t0\().4s, \o7\().4s // O m3[2,3].imre fmul \ta\().4s, \ta\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6] fmul \tb\().4s, \tb\().4s, \t1\().4s // O m3[0,1].imre*t1[4,6] fmul \t4\().4s, \t4\().4s, \t5\().4s // E m3[2,3].imre*t1[4,6] fmul \t0\().4s, \t0\().4s, \t1\().4s // O m3[2,3].imre*t1[4,6] trn1 \t5\().4s, \t3\().4s, \t3\().4s // wim2200 trn2 \t3\().4s, \t3\().4s, \t3\().4s // wim3311 trn1 \t1\().4s, \t2\().4s, \t2\().4s // wim6644 trn2 \t2\().4s, \t2\().4s, \t2\().4s // wim7755 fmul \t5\().4s, \t5\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s fmul \t2\().4s, \t2\().4s, v31.4s fmla \t7\().4s, \o2\().4s, \t5\().4s // O w0123 fmls \t9\().4s, \o6\().4s, \t5\().4s // O j0123 fmla \t6\().4s, \o0\().4s, \t3\().4s // E w0123 fmls \t8\().4s, \o4\().4s, \t3\().4s // E j0123 fmla \ta\().4s, \o1\().4s, \t2\().4s // E w4567 fmla \tb\().4s, \o3\().4s, \t1\().4s // O w4567 fmls \t4\().4s, \o5\().4s, \t2\().4s // E j4567 fmls \t0\().4s, \o7\().4s, \t1\().4s // O j4567 fsub \t2\().4s, \t7\().4s, \t9\().4s fsub \t1\().4s, \t8\().4s, \t6\().4s fsub \t3\().4s, \t4\().4s, \ta\().4s fsub \t5\().4s, \t0\().4s, \tb\().4s fadd \t6\().4s, \t8\().4s, \t6\().4s fadd \t7\().4s, \t9\().4s, \t7\().4s fadd \t8\().4s, \t4\().4s, \ta\().4s fadd \t9\().4s, \t0\().4s, \tb\().4s fmul \t1\().4s, \t1\().4s, v31.4s fmul \t2\().4s, \t2\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s fmul \t5\().4s, \t5\().4s, v31.4s rev64 \t6\().4s, \t6\().4s rev64 \t8\().4s, \t8\().4s rev64 \t7\().4s, \t7\().4s rev64 \t9\().4s, \t9\().4s fsub \o0\().4s, \e0\().4s, \t6\().4s fsub \o1\().4s, \e1\().4s, \t8\().4s fsub \o2\().4s, \e2\().4s, \t1\().4s fsub \o3\().4s, \e3\().4s, \t3\().4s fsub \o4\().4s, \e4\().4s, \t7\().4s fsub \o5\().4s, \e6\().4s, \t9\().4s fadd \o6\().4s, \e5\().4s, \t2\().4s fsub \o7\().4s, \e7\().4s, \t5\().4s fadd \e0\().4s, \e0\().4s, \t6\().4s fadd \e1\().4s, \e1\().4s, \t8\().4s fadd \e2\().4s, \e2\().4s, \t1\().4s fadd \e3\().4s, \e3\().4s, \t3\().4s fadd \e4\().4s, \e4\().4s, \t7\().4s fsub \e5\().4s, \e5\().4s, \t2\().4s // swapped fadd \e6\().4s, \e6\().4s, \t9\().4s // swapped fadd \e7\().4s, \e7\().4s, \t5\().4s .endm .macro SR_COMBINE_HALF e0, e1, e2, e3, \ o0, o1, o2, o3, \ c0, c1, c2, c3, \ t0, t1, t2, t3, t4, t5, part .if \part == 0 trn1 \t4\().4s, \c0\().4s, \c0\().4s // cos0022 trn1 \c1\().4s, \c1\().4s, \c1\().4s // cos1133 .else trn2 \t4\().4s, \c0\().4s, \c0\().4s // cos0022 trn2 \c1\().4s, \c1\().4s, \c1\().4s // cos1133 .endif .if \part == 0 trn2 \t5\().4s, \c2\().4s, \c2\().4s // wim7755 trn2 \c3\().4s, \c3\().4s, \c3\().4s // wim3311 .else trn1 \t5\().4s, \c2\().4s, \c2\().4s // wim7755 trn1 \c3\().4s, \c3\().4s, \c3\().4s // wim3311 .endif fmul \t5\().4s, \t5\().4s, v31.4s fmul \c3\().4s, \c3\().4s, v31.4s rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre rev64 \t3\().4s, \o3\().4s // E m3[2,3].imre fmul \o0\().4s, \o0\().4s, \c3\().4s // E m2[0,1].imre*t1[0,2] fmul \o1\().4s, \o1\().4s, \t5\().4s // E m3[0,1].imre*t1[4,6] fmla \o0\().4s, \t0\().4s, \t4\().4s // E w0123 fmla \o1\().4s, \t2\().4s, \c1\().4s // E w4567 fmul \t1\().4s, \t1\().4s, \t4\().4s // E m2[2,3].imre*t1[0,2] fmul \t3\().4s, \t3\().4s, \c1\().4s // E m3[2,3].imre*t1[4,6] fmls \t1\().4s, \o2\().4s, \c3\().4s // E j0123 fmls \t3\().4s, \o3\().4s, \t5\().4s // E j4567 fsub \t0\().4s, \t1\().4s, \o0\().4s fadd \t1\().4s, \t1\().4s, \o0\().4s fadd \t2\().4s, \t3\().4s, \o1\().4s fsub \t3\().4s, \t3\().4s, \o1\().4s fmul \t0\().4s, \t0\().4s, v31.4s fmul \t3\().4s, \t3\().4s, v31.4s rev64 \t1\().4s, \t1\().4s rev64 \t2\().4s, \t2\().4s .if \part == 0 fsub \o0\().4s, \e0\().4s, \t1\().4s fsub \o1\().4s, \e1\().4s, \t2\().4s fsub \o2\().4s, \e2\().4s, \t0\().4s fsub \o3\().4s, \e3\().4s, \t3\().4s .else fsub \o0\().4s, \e0\().4s, \t1\().4s fadd \o2\().4s, \e1\().4s, \t2\().4s fsub \o1\().4s, \e2\().4s, \t0\().4s fadd \o3\().4s, \e3\().4s, \t3\().4s .endif .if \part == 0 fadd \e0\().4s, \e0\().4s, \t1\().4s fadd \e1\().4s, \e1\().4s, \t2\().4s fadd \e2\().4s, \e2\().4s, \t0\().4s fadd \e3\().4s, \e3\().4s, \t3\().4s .else fadd \e0\().4s, \e0\().4s, \t1\().4s fsub \e1\().4s, \e1\().4s, \t2\().4s // swapped fadd \e2\().4s, \e2\().4s, \t0\().4s // swapped fsub \e3\().4s, \e3\().4s, \t3\().4s .endif .endm /* Same as SR_COMBINE_HALF, but heroically tries to use 3 temporary registers * without touching the tables. */ .macro SR_COMBINE_LITE e0, e1, e2, e3, \ o0, o1, o2, o3, \ c0, c1, c2, c3, \ t0, t1, t2, part rev64 \t0\().4s, \o0\().4s // E m2[0,1].imre rev64 \t1\().4s, \o2\().4s // E m2[2,3].imre .if \part == 0 trn2 \t2\().4s, \c3\().4s, \c3\().4s // wim3311 .else trn1 \t2\().4s, \c3\().4s, \c3\().4s // wim3311 .endif fmul \t2\().4s, \t2\().4s, v31.4s fmul \o2\().4s, \o2\().4s, \t2\().4s fmul \o0\().4s, \o0\().4s, \t2\().4s // E m2[0,1].imre*t1[0,2] .if \part == 0 trn1 \t2\().4s, \c0\().4s, \c0\().4s // cos0022 .else trn2 \t2\().4s, \c0\().4s, \c0\().4s // cos0022 .endif fmul \t1\().4s, \t1\().4s, \t2\().4s // E m2[2,3].imre*t1[0,2] fmla \o0\().4s, \t0\().4s, \t2\().4s // E w0123 fsub \t1\().4s, \t1\().4s, \o2\().4s // E j0123 rev64 \t2\().4s, \o1\().4s // E m3[0,1].imre rev64 \o2\().4s, \o3\().4s // E m3[2,3].imre .if \part == 0 trn2 \t0\().4s, \c2\().4s, \c2\().4s // wim7755 .else trn1 \t0\().4s, \c2\().4s, \c2\().4s // wim7755 .endif fmul \t0\().4s, \t0\().4s, v31.4s fmul \o1\().4s, \o1\().4s, \t0\().4s // E m3[0,1].imre*t1[4,6] fmul \o3\().4s, \o3\().4s, \t0\().4s .if \part == 0 trn1 \t0\().4s, \c1\().4s, \c1\().4s // cos1133 .else trn2 \t0\().4s, \c1\().4s, \c1\().4s // cos1133 .endif fmul \o2\().4s, \o2\().4s, \t0\().4s // E m3[2,3].imre*t1[4,6] fmla \o1\().4s, \t2\().4s, \t0\().4s // E w4567 fsub \o2\().4s, \o2\().4s, \o3\().4s // E j4567 fsub \t0\().4s, \t1\().4s, \o0\().4s fadd \o0\().4s, \t1\().4s, \o0\().4s fadd \t2\().4s, \o2\().4s, \o1\().4s fsub \t1\().4s, \o2\().4s, \o1\().4s fmul \t0\().4s, \t0\().4s, v31.4s fmul \t1\().4s, \t1\().4s, v31.4s rev64 \t2\().4s, \t2\().4s rev64 \o0\().4s, \o0\().4s .if \part == 0 fsub \o1\().4s, \e1\().4s, \t2\().4s fsub \o2\().4s, \e2\().4s, \t0\().4s fsub \o3\().4s, \e3\().4s, \t1\().4s .else fadd \o2\().4s, \e1\().4s, \t0\().4s fsub \o1\().4s, \e2\().4s, \t2\().4s fadd \o3\().4s, \e3\().4s, \t1\().4s .endif .if \part == 0 fadd \e1\().4s, \e1\().4s, \t2\().4s fadd \e2\().4s, \e2\().4s, \t0\().4s fadd \e3\().4s, \e3\().4s, \t1\().4s .else fsub \e1\().4s, \e1\().4s, \t0\().4s // swapped fadd \e2\().4s, \e2\().4s, \t2\().4s // swapped fsub \e3\().4s, \e3\().4s, \t1\().4s .endif mov \t1\().16b, \o0\().16b fsub \o0\().4s, \e0\().4s, \t1\().4s fadd \e0\().4s, \e0\().4s, \t1\().4s .endm .macro SR_COMBINE_4 len, part, off add x10, x1, x21 add x11, x1, x21, lsl #1 add x12, x1, x22 ldp q0, q1, [x1, #((0 + \part)*32 + \off)] ldp q4, q5, [x1, #((2 + \part)*32 + \off)] ldp q2, q3, [x10, #((0 + \part)*32 + \off)] ldp q6, q7, [x10, #((2 + \part)*32 + \off)] ldp q8, q9, [x11, #((0 + \part)*32 + \off)] ldp q10, q11, [x11, #((2 + \part)*32 + \off)] ldp q12, q13, [x12, #((0 + \part)*32 + \off)] ldp q14, q15, [x12, #((2 + \part)*32 + \off)] SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 stp q0, q1, [x1, #((0 + \part)*32 + \off)] stp q4, q5, [x1, #((2 + \part)*32 + \off)] stp q2, q3, [x10, #((0 + \part)*32 + \off)] stp q6, q7, [x10, #((2 + \part)*32 + \off)] stp q8, q9, [x11, #((0 + \part)*32 + \off)] stp q12, q13, [x11, #((2 + \part)*32 + \off)] stp q10, q11, [x12, #((0 + \part)*32 + \off)] stp q14, q15, [x12, #((2 + \part)*32 + \off)] .endm .macro SR_COMBINE_FULL len, off=0 add x10, x1, x21 add x11, x1, x21, lsl #1 add x12, x1, x22 SR_COMBINE_4 \len, 0, \off SR_COMBINE_4 \len, 1, \off SR_COMBINE_4 \len, 4, \off SR_COMBINE_4 \len, 5, \off .endm .macro SR_COMBINE_D2 part, off add x10, x1, #((\part)*32 + \off) add x11, x14, #((\part)*32 + \off) add x12, x15, #((\part)*32 + \off) add x13, x16, #((\part)*32 + \off) ldp q0, q1, [x10] ldp q4, q5, [x10, #(2*32)] ldp q2, q3, [x11] ldp q6, q7, [x11, #(2*32)] ldp q8, q9, [x12] ldp q10, q11, [x12, #(2*32)] ldp q12, q13, [x13] ldp q14, q15, [x13, #(2*32)] SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0, \ v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v5.2d zip2 v19.2d, v1.2d, v5.2d zip1 v20.2d, v2.2d, v6.2d zip2 v21.2d, v2.2d, v6.2d zip1 v22.2d, v3.2d, v7.2d zip2 v23.2d, v3.2d, v7.2d ldp q0, q1, [x10, #(1*32)] ldp q4, q5, [x10, #(3*32)] ldp q2, q3, [x11, #(1*32)] ldp q6, q7, [x11, #(3*32)] st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10], #64 st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x11], #64 zip1 v20.2d, v8.2d, v12.2d zip2 v21.2d, v8.2d, v12.2d zip1 v22.2d, v9.2d, v13.2d zip2 v23.2d, v9.2d, v13.2d zip1 v24.2d, v10.2d, v14.2d zip2 v25.2d, v10.2d, v14.2d zip1 v26.2d, v11.2d, v15.2d zip2 v27.2d, v11.2d, v15.2d ldp q8, q9, [x12, #(1*32)] ldp q10, q11, [x12, #(3*32)] ldp q12, q13, [x13, #(1*32)] ldp q14, q15, [x13, #(3*32)] st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12], #64 st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13], #64 SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0, \ v16, v17, v18, v19, v20, v21, v22, v23, v24, v25, v26, v27 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v5.2d zip2 v19.2d, v1.2d, v5.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x10] zip1 v16.2d, v2.2d, v6.2d zip2 v17.2d, v2.2d, v6.2d zip1 v18.2d, v3.2d, v7.2d zip2 v19.2d, v3.2d, v7.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x11] zip1 v20.2d, v8.2d, v12.2d zip2 v21.2d, v8.2d, v12.2d zip1 v22.2d, v9.2d, v13.2d zip2 v23.2d, v9.2d, v13.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x12] zip1 v24.2d, v10.2d, v14.2d zip2 v25.2d, v10.2d, v14.2d zip1 v26.2d, v11.2d, v15.2d zip2 v27.2d, v11.2d, v15.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x13] .endm .macro SR_COMBINE_DINT off=0 add x14, x1, x21 add x15, x1, x21, lsl #1 add x16, x1, x22 SR_COMBINE_D2 0, \off SR_COMBINE_D2 4, \off .endm .macro FFT32_FN name, no_perm function ff_tx_fft32_\name\()_neon, export=1 stp d14, d15, [sp, #-16*4]! stp d8, d9, [sp, #16*3] stp d10, d11, [sp, #16*2] stp d12, d13, [sp, #16] LOAD_SUBADD SETUP_SR_RECOMB 32, x7, x8, x9 SETUP_LUT \no_perm LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 5, 6, 7, x2, \no_perm LOAD_INPUT 8, 9, 10, 11, x2, \no_perm LOAD_INPUT 12, 13, 14, 15, x2, \no_perm FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15 FFT16 v0, v1, v2, v3, v4, v5, v6, v7 SR_COMBINE v0, v1, v2, v3, v4, v5, v6, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 zip1 v16.2d, v0.2d, v4.2d zip2 v17.2d, v0.2d, v4.2d zip1 v18.2d, v1.2d, v6.2d zip2 v19.2d, v1.2d, v6.2d st1 { v16.4s, v17.4s, v18.4s, v19.4s }, [x1], #64 zip1 v20.2d, v2.2d, v5.2d zip2 v21.2d, v2.2d, v5.2d zip1 v22.2d, v3.2d, v7.2d zip2 v23.2d, v3.2d, v7.2d st1 { v20.4s, v21.4s, v22.4s, v23.4s }, [x1], #64 zip1 v24.2d, v8.2d, v12.2d zip2 v25.2d, v8.2d, v12.2d zip1 v26.2d, v9.2d, v13.2d zip2 v27.2d, v9.2d, v13.2d st1 { v24.4s, v25.4s, v26.4s, v27.4s }, [x1], #64 zip1 v28.2d, v10.2d, v14.2d zip2 v29.2d, v10.2d, v14.2d zip1 v30.2d, v11.2d, v15.2d zip2 v31.2d, v11.2d, v15.2d st1 { v28.4s, v29.4s, v30.4s, v31.4s }, [x1] ldp d12, d13, [sp, #16] ldp d10, d11, [sp, #16*2] ldp d8, d9, [sp, #16*3] ldp d14, d15, [sp], #16*4 ret endfunc .endm FFT32_FN float, 0 FFT32_FN ns_float, 1 .macro cmp_imm reg, imm .if \imm >= 4096 cmp \reg, #((\imm)/4096), lsl #12 .else cmp \reg, #(\imm) .endif .endm .macro SR_TRANSFORM_DEF len, next=0 \len: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #(\len/4) mov x5, #((\len*4) - (\len/1)) add x1, x1, x5 bl 32b mov x5, #((\len*2) - (\len/2)) add x1, x1, x5 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER ldr w5, =(\len*6 + \len/2) sub x1, x1, x5 SETUP_SR_RECOMB \len, x7, x8, x9 .if \next\() != 0 cmp_imm w19, \len b.eq 0f mov w5, #(\len/128) \len\()5: SR_COMBINE_FULL \len add x1, x1, 8*32 subs w5, w5, 1 b.gt \len\()5b cmp_imm w20, \len b.gt \next\()f ret .endif .endm .macro FFT_SPLIT_RADIX_FN name, no_perm function ff_tx_fft_sr_\name\()_neon, export=1 stp x21, x22, [sp, #-16*6]! stp d8, d9, [sp, #16*5] stp d10, d11, [sp, #16*4] stp d12, d13, [sp, #16*3] stp d14, d15, [sp, #16*2] stp x19, x20, [sp, #16] ldr w19, [x0, #0] // global target mov w20, w19 // local length LOAD_SUBADD SETUP_LUT \no_perm 32: SETUP_SR_RECOMB 32, x7, x8, x9 LOAD_INPUT 0, 1, 2, 3, x2, \no_perm LOAD_INPUT 4, 6, 5, 7, x2, \no_perm, 1 LOAD_INPUT 8, 9, 10, 11, x2, \no_perm LOAD_INPUT 12, 13, 14, 15, x2, \no_perm FFT8_X2 v8, v9, v10, v11, v12, v13, v14, v15 FFT16 v0, v1, v2, v3, v4, v6, v5, v7 SR_COMBINE v0, v1, v2, v3, v4, v6, v5, v7, \ v8, v9, v10, v11, v12, v13, v14, v15, \ x7, x8, x9, 0 stp q2, q3, [x1, #32*1] stp q6, q7, [x1, #32*3] stp q10, q11, [x1, #32*5] stp q14, q15, [x1, #32*7] cmp w20, #32 b.gt 64f stp q0, q1, [x1, #32*0] stp q4, q5, [x1, #32*2] stp q8, q9, [x1, #32*4] stp q12, q13, [x1, #32*6] ret 64: SETUP_SR_RECOMB 64, x7, x8, x9 LOAD_INPUT 2, 3, 10, 11, x2, \no_perm, 1 LOAD_INPUT 6, 14, 7, 15, x2, \no_perm, 1 FFT16 v2, v3, v10, v11, v6, v14, v7, v15 LOAD_INPUT 16, 17, 18, 19, x2, \no_perm LOAD_INPUT 20, 22, 21, 23, x2, \no_perm, 1 FFT16 v16, v17, v18, v19, v20, v22, v21, v23, \ v24, v25, v26, v27, v28, v29, v30 ld1 { v26.4s, v27.4s }, [x8], x9 ldp q24, q25, [x7], #32 ext v26.16b, v26.16b, v26.16b, #8 ext v27.16b, v27.16b, v27.16b, #8 cmp w19, #64 b.eq 2f // custom deinterleave // TODO: investigate doing the 2 combines like in deinterleave // TODO: experiment with spilling to gprs and converting to HALF or full SR_COMBINE_LITE v0, v1, v8, v9, \ v2, v3, v16, v17, \ v24, v25, v26, v27, \ v28, v29, v30, 0 stp q0, q1, [x1, #32* 0] stp q8, q9, [x1, #32* 4] stp q2, q3, [x1, #32* 8] stp q16, q17, [x1, #32*12] SR_COMBINE_HALF v4, v5, v12, v13, \ v6, v7, v20, v21, \ v24, v25, v26, v27, \ v28, v29, v30, v0, v1, v8, 1 stp q4, q20, [x1, #32* 2] stp q12, q21, [x1, #32* 6] stp q6, q5, [x1, #32*10] stp q7, q13, [x1, #32*14] ldp q2, q3, [x1, #32*1] ldp q6, q7, [x1, #32*3] ldp q12, q13, [x1, #32*5] ldp q16, q17, [x1, #32*7] SR_COMBINE v2, v3, v12, v13, v6, v16, v7, v17, \ v10, v11, v14, v15, v18, v19, v22, v23, \ x7, x8, x9, 0, \ v24, v25, v26, v27, v28, v29, v30, v8, v0, v1, v4, v5 stp q2, q3, [x1, #32* 1] stp q6, q7, [x1, #32* 3] stp q12, q13, [x1, #32* 5] stp q16, q17, [x1, #32* 7] stp q10, q11, [x1, #32* 9] stp q18, q19, [x1, #32*11] stp q14, q15, [x1, #32*13] stp q22, q23, [x1, #32*15] cmp w20, #64 b.gt 128f ret 128: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #32 add x1, x1, #16*32 bl 32b add x1, x1, #8*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #24*32 SETUP_SR_RECOMB 128, x7, x8, x9 cmp w19, #128 b.eq 0f SR_COMBINE_FULL 128 cmp w20, #128 b.gt 256f ret 256: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #64 add x1, x1, #32*32 bl 32b add x1, x1, #16*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #48*32 SETUP_SR_RECOMB 256, x7, x8, x9 cmp w19, #256 b.eq 0f SR_COMBINE_FULL 256 SR_COMBINE_FULL 256, 8*32 cmp w20, #256 b.gt 512f ret 512: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #128 add x1, x1, #64*32 bl 32b add x1, x1, #32*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER sub x1, x1, #96*32 SETUP_SR_RECOMB 512, x7, x8, x9 cmp w19, #512 b.eq 0f mov x5, 4 5125: SR_COMBINE_FULL 512 add x1, x1, 8*32 subs w5, w5, 1 b.gt 5125b cmp w20, #512 b.gt 1024f ret 1024: AARCH64_SIGN_LINK_REGISTER stp x20, x30, [sp, #-16]! mov w20, #256 add x1, x1, #96*32 bl 32b add x1, x1, #64*32 bl 32b ldp x20, x30, [sp], #16 AARCH64_VALIDATE_LINK_REGISTER mov x5, #192*32 sub x1, x1, x5 SETUP_SR_RECOMB 1024, x7, x8, x9 cmp w19, #1024 b.eq 0f mov w5, 8 10245: SR_COMBINE_FULL 1024 add x1, x1, 8*32 subs w5, w5, 1 b.gt 10245b cmp w20, #1024 b.gt 2048f ret SR_TRANSFORM_DEF 2048, 4096 SR_TRANSFORM_DEF 4096, 8192 SR_TRANSFORM_DEF 8192, 16384 SR_TRANSFORM_DEF 16384, 32768 SR_TRANSFORM_DEF 32768, 65536 SR_TRANSFORM_DEF 65536, 131072 SR_TRANSFORM_DEF 131072 0: // general deinterleave loop SR_COMBINE_DINT add x1, x1, #32*8 subs w19, w19, #32*4 b.gt 0b ldp x19, x20, [sp, #16] ldp d14, d15, [sp, #16*2] ldp d12, d13, [sp, #16*3] ldp d10, d11, [sp, #16*4] ldp d8, d9, [sp, #16*5] ldp x21, x22, [sp], #16*6 ret 2: // special case for 64 point deinterleave mov x10, v23.d[0] mov x11, v23.d[1] SR_COMBINE_LITE v0, v1, v8, v9, \ v2, v3, v16, v17, \ v24, v25, v26, v27, \ v28, v29, v30, 0 SR_COMBINE_HALF v4, v5, v12, v13, \ v6, v7, v20, v21, \ v24, v25, v26, v27, \ v28, v29, v30, v23, v24, v26, 1 zip1 v23.2d, v0.2d, v4.2d zip2 v24.2d, v0.2d, v4.2d zip1 v25.2d, v1.2d, v20.2d zip2 v26.2d, v1.2d, v20.2d zip1 v27.2d, v8.2d, v12.2d zip2 v28.2d, v8.2d, v12.2d zip1 v29.2d, v9.2d, v21.2d zip2 v30.2d, v9.2d, v21.2d mov v20.16b, v5.16b mov v21.16b, v7.16b mov x12, x1 add x13, x1, #32* 4 add x14, x1, #32* 8 add x15, x1, #32*12 zip1 v4.2d, v2.2d, v6.2d zip2 v5.2d, v2.2d, v6.2d zip1 v6.2d, v3.2d, v20.2d zip2 v7.2d, v3.2d, v20.2d zip1 v0.2d, v16.2d, v21.2d zip2 v1.2d, v16.2d, v21.2d zip1 v2.2d, v17.2d, v13.2d zip2 v3.2d, v17.2d, v13.2d // stp is faster by a little on A53, but this is faster on M1s (theory) ldp q8, q9, [x1, #32*1] ldp q12, q13, [x1, #32*5] st1 { v23.4s, v24.4s, v25.4s, v26.4s }, [x12], #64 // 32* 0...1 st1 { v27.4s, v28.4s, v29.4s, v30.4s }, [x13], #64 // 32* 4...5 st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x14], #64 // 32* 8...9 st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x15], #64 // 32*12..13 mov v23.d[0], x10 mov v23.d[1], x11 ldp q6, q7, [x1, #32*3] ldp q16, q17, [x1, #32*7] SR_COMBINE v8, v9, v12, v13, v6, v16, v7, v17, \ v10, v11, v14, v15, v18, v19, v22, v23, \ x7, x8, x9, 0, \ v24, v25, v26, v27, v28, v29, v30, v4, v0, v1, v5, v20 zip1 v0.2d, v8.2d, v6.2d zip2 v1.2d, v8.2d, v6.2d zip1 v2.2d, v9.2d, v7.2d zip2 v3.2d, v9.2d, v7.2d st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x12] zip1 v4.2d, v12.2d, v16.2d zip2 v5.2d, v12.2d, v16.2d zip1 v6.2d, v13.2d, v17.2d zip2 v7.2d, v13.2d, v17.2d st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x13] zip1 v0.2d, v10.2d, v18.2d zip2 v1.2d, v10.2d, v18.2d zip1 v2.2d, v11.2d, v19.2d zip2 v3.2d, v11.2d, v19.2d st1 { v0.4s, v1.4s, v2.4s, v3.4s }, [x14] zip1 v4.2d, v14.2d, v22.2d zip2 v5.2d, v14.2d, v22.2d zip1 v6.2d, v15.2d, v23.2d zip2 v7.2d, v15.2d, v23.2d st1 { v4.4s, v5.4s, v6.4s, v7.4s }, [x15] ldp x19, x20, [sp, #16] ldp d14, d15, [sp, #16*2] ldp d12, d13, [sp, #16*3] ldp d10, d11, [sp, #16*4] ldp d8, d9, [sp, #16*5] ldp x21, x22, [sp], #16*6 ret endfunc .endm FFT_SPLIT_RADIX_FN float, 0 FFT_SPLIT_RADIX_FN ns_float, 1