[5/5] aarch64: vp9itxfm: Do separate functions for half/quarter idct16 and idct32 (alternative 2)

Message ID 1480584422-24237-7-git-send-email-martin@martin.st
State Committed
Headers show

Commit Message

Martin Storsjö Dec. 1, 2016, 9:27 a.m.
This work is sponsored by, and copyright, Google.

This makes it easier to avoid filling the temp buffer with zeros for the
skipped slices, and leads to slightly more straightforward code for these
cases (for the 16x16 case, where the special case pass functions are
written out instead of templated from the same macro), instead of riddling
the common code with special case branches or macro .ifs.

The code size increases from 14740 bytes to 24472 bytes.

Before:
vp9_inv_dct_dct_16x16_sub1_add_neon:     235.3
vp9_inv_dct_dct_16x16_sub2_add_neon:    1051.0
vp9_inv_dct_dct_16x16_sub4_add_neon:    1051.0
vp9_inv_dct_dct_16x16_sub8_add_neon:    1051.0
vp9_inv_dct_dct_16x16_sub12_add_neon:   1390.3
vp9_inv_dct_dct_16x16_sub16_add_neon:   1390.1
vp9_inv_dct_dct_32x32_sub1_add_neon:     556.5
vp9_inv_dct_dct_32x32_sub2_add_neon:    5199.1
vp9_inv_dct_dct_32x32_sub4_add_neon:    5199.9
vp9_inv_dct_dct_32x32_sub8_add_neon:    5196.9
vp9_inv_dct_dct_32x32_sub12_add_neon:   6171.6
vp9_inv_dct_dct_32x32_sub16_add_neon:   6170.9
vp9_inv_dct_dct_32x32_sub20_add_neon:   7147.1
vp9_inv_dct_dct_32x32_sub24_add_neon:   7147.0
vp9_inv_dct_dct_32x32_sub28_add_neon:   8118.8
vp9_inv_dct_dct_32x32_sub32_add_neon:   8125.8

After:
vp9_inv_dct_dct_16x16_sub1_add_neon:     235.3
vp9_inv_dct_dct_16x16_sub2_add_neon:     639.0
vp9_inv_dct_dct_16x16_sub4_add_neon:     639.0
vp9_inv_dct_dct_16x16_sub8_add_neon:     845.0
vp9_inv_dct_dct_16x16_sub12_add_neon:   1389.4
vp9_inv_dct_dct_16x16_sub16_add_neon:   1389.3
vp9_inv_dct_dct_32x32_sub1_add_neon:     556.5
vp9_inv_dct_dct_32x32_sub2_add_neon:    3684.1
vp9_inv_dct_dct_32x32_sub4_add_neon:    3682.6
vp9_inv_dct_dct_32x32_sub8_add_neon:    3684.1
vp9_inv_dct_dct_32x32_sub12_add_neon:   5319.0
vp9_inv_dct_dct_32x32_sub16_add_neon:   5323.5
vp9_inv_dct_dct_32x32_sub20_add_neon:   7149.8
vp9_inv_dct_dct_32x32_sub24_add_neon:   7148.2
vp9_inv_dct_dct_32x32_sub28_add_neon:   8124.5
vp9_inv_dct_dct_32x32_sub32_add_neon:   8122.1

---
If we wouldn't have made the core transforms standalone functions,
the code size would end up at around 34 KB.

The binary output is 6 KB larger than in the other alternative,
but is more straightforward and gives better opportunities to
special case them further.

In the arm version, there was a significant speedup compared to the
other alternative (having cmps within the functions), skipping
zeroing of the temp buffer. Here there's much less difference.
---
 libavcodec/aarch64/vp9itxfm_neon.S | 628 +++++++++++++++++++++++++++++++++----
 1 file changed, 566 insertions(+), 62 deletions(-)

Comments

Janne Grunau Feb. 5, 2017, 10:12 a.m. | #1
On 2016-12-01 11:27:02 +0200, Martin Storsjö wrote:
> This work is sponsored by, and copyright, Google.
> 
> This makes it easier to avoid filling the temp buffer with zeros for the
> skipped slices, and leads to slightly more straightforward code for these
> cases (for the 16x16 case, where the special case pass functions are
> written out instead of templated from the same macro), instead of riddling
> the common code with special case branches or macro .ifs.
> 
> The code size increases from 14740 bytes to 24472 bytes.
> 
> Before:
> vp9_inv_dct_dct_16x16_sub1_add_neon:     235.3
> vp9_inv_dct_dct_16x16_sub2_add_neon:    1051.0
> vp9_inv_dct_dct_16x16_sub4_add_neon:    1051.0
> vp9_inv_dct_dct_16x16_sub8_add_neon:    1051.0
> vp9_inv_dct_dct_16x16_sub12_add_neon:   1390.3
> vp9_inv_dct_dct_16x16_sub16_add_neon:   1390.1
> vp9_inv_dct_dct_32x32_sub1_add_neon:     556.5
> vp9_inv_dct_dct_32x32_sub2_add_neon:    5199.1
> vp9_inv_dct_dct_32x32_sub4_add_neon:    5199.9
> vp9_inv_dct_dct_32x32_sub8_add_neon:    5196.9
> vp9_inv_dct_dct_32x32_sub12_add_neon:   6171.6
> vp9_inv_dct_dct_32x32_sub16_add_neon:   6170.9
> vp9_inv_dct_dct_32x32_sub20_add_neon:   7147.1
> vp9_inv_dct_dct_32x32_sub24_add_neon:   7147.0
> vp9_inv_dct_dct_32x32_sub28_add_neon:   8118.8
> vp9_inv_dct_dct_32x32_sub32_add_neon:   8125.8
> 
> After:
> vp9_inv_dct_dct_16x16_sub1_add_neon:     235.3
> vp9_inv_dct_dct_16x16_sub2_add_neon:     639.0
> vp9_inv_dct_dct_16x16_sub4_add_neon:     639.0
> vp9_inv_dct_dct_16x16_sub8_add_neon:     845.0
> vp9_inv_dct_dct_16x16_sub12_add_neon:   1389.4
> vp9_inv_dct_dct_16x16_sub16_add_neon:   1389.3
> vp9_inv_dct_dct_32x32_sub1_add_neon:     556.5
> vp9_inv_dct_dct_32x32_sub2_add_neon:    3684.1
> vp9_inv_dct_dct_32x32_sub4_add_neon:    3682.6
> vp9_inv_dct_dct_32x32_sub8_add_neon:    3684.1
> vp9_inv_dct_dct_32x32_sub12_add_neon:   5319.0
> vp9_inv_dct_dct_32x32_sub16_add_neon:   5323.5
> vp9_inv_dct_dct_32x32_sub20_add_neon:   7149.8
> vp9_inv_dct_dct_32x32_sub24_add_neon:   7148.2
> vp9_inv_dct_dct_32x32_sub28_add_neon:   8124.5
> vp9_inv_dct_dct_32x32_sub32_add_neon:   8122.1
> 
> ---
> If we wouldn't have made the core transforms standalone functions,
> the code size would end up at around 34 KB.
> 
> The binary output is 6 KB larger than in the other alternative,
> but is more straightforward and gives better opportunities to
> special case them further.
> 
> In the arm version, there was a significant speedup compared to the
> other alternative (having cmps within the functions), skipping
> zeroing of the temp buffer. Here there's much less difference.

And the relative binary size difference is even larger. It would a 
little strange to choose different alternatives for 32- and 64-bit but 
it sounds like alternative 1 might be better for arm64. Please run a 
full decoding benchmark for arm64 too.

> ---
>  libavcodec/aarch64/vp9itxfm_neon.S | 628 +++++++++++++++++++++++++++++++++----
>  1 file changed, 566 insertions(+), 62 deletions(-)
> 
> diff --git a/libavcodec/aarch64/vp9itxfm_neon.S b/libavcodec/aarch64/vp9itxfm_neon.S
> index be9643e..9910170 100644
> --- a/libavcodec/aarch64/vp9itxfm_neon.S
> +++ b/libavcodec/aarch64/vp9itxfm_neon.S
> @@ -75,6 +75,16 @@ endconst
>  .endif
>  .endm
>  
> +// Same as dmbutterfly0 above, but treating the input in in2 as zero,
> +// writing the same output into both out1 and out2.
> +.macro dmbutterfly0_h out1, out2, in1, in2, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6
> +        smull           \tmp1\().4s,  \in1\().4h,  v0.h[0]
> +        smull2          \tmp2\().4s,  \in1\().8h,  v0.h[0]
> +        rshrn           \out1\().4h,  \tmp1\().4s, #14
> +        rshrn2          \out1\().8h,  \tmp2\().4s, #14
> +        mov             \out2\().16b, \out1\().16b
> +.endm
> +
>  // out1,out2 = in1 * coef1 - in2 * coef2
>  // out3,out4 = in1 * coef2 + in2 * coef1
>  // out are 4 x .4s registers, in are 2 x .8h registers
> @@ -104,6 +114,43 @@ endconst
>          rshrn2          \inout2\().8h, \tmp4\().4s,  #14
>  .endm
>  
> +// Same as dmbutterfly above, but treating the input in inout2 as zero
> +.macro dmbutterfly_h1 inout1, inout2, coef1, coef2, tmp1, tmp2, tmp3, tmp4
> +        smull           \tmp1\().4s, \inout1\().4h, \coef1
> +        smull2          \tmp2\().4s, \inout1\().8h, \coef1
> +        smull           \tmp3\().4s, \inout1\().4h, \coef2
> +        smull2          \tmp4\().4s, \inout1\().8h, \coef2
> +        rshrn           \inout1\().4h, \tmp1\().4s, #14
> +        rshrn2          \inout1\().8h, \tmp2\().4s, #14
> +        rshrn           \inout2\().4h, \tmp3\().4s, #14
> +        rshrn2          \inout2\().8h, \tmp4\().4s, #14
> +.endm
> +
> +// Same as dmbutterfly above, but treating the input in inout1 as zero
> +.macro dmbutterfly_h2 inout1, inout2, coef1, coef2, tmp1, tmp2, tmp3, tmp4
> +        smull           \tmp1\().4s, \inout2\().4h, \coef2
> +        smull2          \tmp2\().4s, \inout2\().8h, \coef2
> +        smull           \tmp3\().4s, \inout2\().4h, \coef1
> +        smull2          \tmp4\().4s, \inout2\().8h, \coef1
> +        neg             \tmp1\().4s, \tmp1\().4s
> +        neg             \tmp2\().4s, \tmp2\().4s
> +        rshrn           \inout2\().4h, \tmp3\().4s, #14
> +        rshrn2          \inout2\().8h, \tmp4\().4s, #14
> +        rshrn           \inout1\().4h, \tmp1\().4s, #14
> +        rshrn2          \inout1\().8h, \tmp2\().4s, #14
> +.endm
> +
> +.macro dsmull_h out1, out2, in, coef
> +        smull           \out1\().4s, \in\().4h, \coef
> +        smull2          \out2\().4s, \in\().8h, \coef
> +.endm
> +
> +.macro drshrn_h out, in1, in2, shift
> +        rshrn           \out\().4h, \in1\().4s, \shift
> +        rshrn2          \out\().8h, \in2\().4s, \shift
> +.endm
> +
> +
>  // out1 = in1 + in2
>  // out2 = in1 - in2
>  .macro butterfly_8h out1, out2, in1, in2
> @@ -463,7 +510,7 @@ function idct16x16_dc_add_neon
>          ret
>  endfunc
>  
> -function idct16
> +.macro idct16_full
>          dmbutterfly0    v16, v24, v16, v24, v2, v3, v4, v5, v6, v7 // v16 = t0a,  v24 = t1a
>          dmbutterfly     v20, v28, v0.h[1], v0.h[2], v2, v3, v4, v5 // v20 = t2a,  v28 = t3a
>          dmbutterfly     v18, v30, v0.h[3], v0.h[4], v2, v3, v4, v5 // v18 = t4a,  v30 = t7a
> @@ -485,7 +532,10 @@ function idct16
>          dmbutterfly0    v22, v26, v22, v26, v2, v3, v18, v19, v30, v31        // v22 = t6a,  v26 = t5a
>          dmbutterfly     v23, v25, v0.h[1], v0.h[2], v18, v19, v30, v31        // v23 = t9a,  v25 = t14a
>          dmbutterfly     v27, v21, v0.h[1], v0.h[2], v18, v19, v30, v31, neg=1 // v27 = t13a, v21 = t10a
> +        idct16_end

see comment in alternative 1

> +.endm
>  
> +.macro idct16_end
>          butterfly_8h    v18, v7,  v4,  v7                // v18 = t0a,  v7  = t7a
>          butterfly_8h    v19, v22, v5,  v22               // v19 = t1a,  v22 = t6
>          butterfly_8h    v4,  v26, v20, v26               // v4  = t2a,  v26 = t5
> @@ -507,6 +557,68 @@ function idct16
>          butterfly_8h    v20, v27, v6,  v27               // v20 = out[4], v27 = out[11]
>          butterfly_8h    v21, v26, v26, v3                // v21 = out[5], v26 = out[10]
>          ret
> +.endm
> +
> +function idct16
> +        idct16_full
> +endfunc
> +
> +function idct16_half
> +        dmbutterfly0_h  v16, v24, v16, v24, v2, v3, v4, v5, v6, v7 // v16 = t0a,  v24 = t1a
> +        dmbutterfly_h1  v20, v28, v0.h[1], v0.h[2], v2, v3, v4, v5 // v20 = t2a,  v28 = t3a
> +        dmbutterfly_h1  v18, v30, v0.h[3], v0.h[4], v2, v3, v4, v5 // v18 = t4a,  v30 = t7a
> +        dmbutterfly_h2  v26, v22, v0.h[5], v0.h[6], v2, v3, v4, v5 // v26 = t5a,  v22 = t6a
> +        dmbutterfly_h1  v17, v31, v0.h[7], v1.h[0], v2, v3, v4, v5 // v17 = t8a,  v31 = t15a
> +        dmbutterfly_h2  v25, v23, v1.h[1], v1.h[2], v2, v3, v4, v5 // v25 = t9a,  v23 = t14a
> +        dmbutterfly_h1  v21, v27, v1.h[3], v1.h[4], v2, v3, v4, v5 // v21 = t10a, v27 = t13a
> +        dmbutterfly_h2  v29, v19, v1.h[5], v1.h[6], v2, v3, v4, v5 // v29 = t11a, v19 = t12a
> +
> +        butterfly_8h    v4,  v28, v16, v28               // v4  = t0,   v28 = t3
> +        butterfly_8h    v5,  v20, v24, v20               // v5  = t1,   v20 = t2
> +        butterfly_8h    v6,  v26, v18, v26               // v6  = t4,   v26 = t5
> +        butterfly_8h    v7,  v22, v30, v22               // v7  = t7,   v22 = t6
> +        butterfly_8h    v16, v25, v17, v25               // v16 = t8,   v25 = t9
> +        butterfly_8h    v24, v21, v29, v21               // v24 = t11,  v21 = t10
> +        butterfly_8h    v17, v27, v19, v27               // v17 = t12,  v27 = t13
> +        butterfly_8h    v29, v23, v31, v23               // v29 = t15,  v23 = t14
> +
> +        dmbutterfly0    v22, v26, v22, v26, v2, v3, v18, v19, v30, v31        // v22 = t6a,  v26 = t5a
> +        dmbutterfly     v23, v25, v0.h[1], v0.h[2], v18, v19, v30, v31        // v23 = t9a,  v25 = t14a
> +        dmbutterfly     v27, v21, v0.h[1], v0.h[2], v18, v19, v30, v31, neg=1 // v27 = t13a, v21 = t10a
> +        idct16_end
> +endfunc
> +
> +function idct16_quarter
> +        dsmull_h        v24, v25, v19, v1.h[6]
> +        dsmull_h        v4,  v5,  v17, v0.h[7]
> +        dsmull_h        v7,  v6,  v18, v0.h[4]
> +        dsmull_h        v30, v31, v18, v0.h[3]
> +        neg             v24.4s,  v24.4s
> +        neg             v25.4s,  v25.4s
> +        dsmull_h        v29, v28, v17, v1.h[0]
> +        dsmull_h        v26, v27, v19, v1.h[5]
> +        dsmull_h        v22, v23, v16, v0.h[0]
> +        drshrn_h        v24, v24, v25, #14
> +        drshrn_h        v16, v4,  v5,  #14
> +        drshrn_h        v7,  v7,  v6,  #14
> +        drshrn_h        v6,  v30, v31, #14
> +        drshrn_h        v29, v29, v28, #14
> +        drshrn_h        v17, v26, v27, #14
> +        drshrn_h        v28, v22, v23, #14
> +
> +        dmbutterfly_l   v20, v21, v22, v23, v17, v24, v0.h[1], v0.h[2]
> +        dmbutterfly_l   v18, v19, v30, v31, v29, v16, v0.h[1], v0.h[2]
> +        neg             v22.4s,  v22.4s
> +        neg             v23.4s,  v23.4s
> +        drshrn_h        v27, v20, v21, #14
> +        drshrn_h        v21, v22, v23, #14
> +        drshrn_h        v23, v18, v19, #14
> +        drshrn_h        v25, v30, v31, #14
> +        mov             v4.16b,  v28.16b
> +        mov             v5.16b,  v28.16b
> +        dmbutterfly0    v22, v26, v7,  v6,  v18, v19, v30, v31
> +        mov             v20.16b, v28.16b
> +        idct16_end
>  endfunc
>  
>  function iadst16
> @@ -598,6 +710,51 @@ endfunc
>          st1             {v2.8h},  [\src], \inc
>  .endm
>  
> +.macro load_add_store coef0, coef1, coef2, coef3, coef4, coef5, coef6, coef7, tmp1, tmp2
> +        srshr           \coef0, \coef0, #6
> +        ld1             {v2.8b},  [x0], x1
> +        srshr           \coef1, \coef1, #6
> +        ld1             {v3.8b},  [x3], x1
> +        srshr           \coef2, \coef2, #6
> +        ld1             {v4.8b},  [x0], x1
> +        srshr           \coef3, \coef3, #6
> +        uaddw           \coef0, \coef0, v2.8b
> +        ld1             {v5.8b},  [x3], x1
> +        uaddw           \coef1, \coef1, v3.8b
> +        srshr           \coef4, \coef4, #6
> +        ld1             {v6.8b},  [x0], x1
> +        srshr           \coef5, \coef5, #6
> +        ld1             {v7.8b},  [x3], x1
> +        sqxtun          v2.8b,  \coef0
> +        srshr           \coef6, \coef6, #6
> +        sqxtun          v3.8b,  \coef1
> +        srshr           \coef7, \coef7, #6
> +        uaddw           \coef2, \coef2, v4.8b
> +        ld1             {\tmp1},  [x0], x1
> +        uaddw           \coef3, \coef3, v5.8b
> +        ld1             {\tmp2},  [x3], x1
> +        sqxtun          v4.8b,  \coef2
> +        sub             x0,  x0,  x1, lsl #2
> +        sub             x3,  x3,  x1, lsl #2
> +        sqxtun          v5.8b,  \coef3
> +        uaddw           \coef4, \coef4, v6.8b
> +        st1             {v2.8b},  [x0], x1
> +        uaddw           \coef5, \coef5, v7.8b
> +        st1             {v3.8b},  [x3], x1
> +        sqxtun          v6.8b,  \coef4
> +        st1             {v4.8b},  [x0], x1
> +        sqxtun          v7.8b,  \coef5
> +        st1             {v5.8b},  [x3], x1
> +        uaddw           \coef6, \coef6, \tmp1
> +        st1             {v6.8b},  [x0], x1
> +        uaddw           \coef7, \coef7, \tmp2
> +        st1             {v7.8b},  [x3], x1
> +        sqxtun          \tmp1,  \coef6
> +        sqxtun          \tmp2,  \coef7
> +        st1             {\tmp1},  [x0], x1
> +        st1             {\tmp2},  [x3], x1
> +.endm
> +
>  // Read a vertical 8x16 slice out of a 16x16 matrix, do a transform on it,
>  // transpose into a horizontal 16x8 slice and store.
>  // x0 = dst (temp buffer)
> @@ -671,53 +828,8 @@ function \txfm\()16_1d_8x16_pass2_neon
>          lsl             x1,  x1,  #1
>          bl              \txfm\()16
>  
> -.macro load_add_store coef0, coef1, coef2, coef3, coef4, coef5, coef6, coef7, tmp1, tmp2
> -        srshr           \coef0, \coef0, #6
> -        ld1             {v2.8b},  [x0], x1
> -        srshr           \coef1, \coef1, #6
> -        ld1             {v3.8b},  [x3], x1
> -        srshr           \coef2, \coef2, #6
> -        ld1             {v4.8b},  [x0], x1
> -        srshr           \coef3, \coef3, #6
> -        uaddw           \coef0, \coef0, v2.8b
> -        ld1             {v5.8b},  [x3], x1
> -        uaddw           \coef1, \coef1, v3.8b
> -        srshr           \coef4, \coef4, #6
> -        ld1             {v6.8b},  [x0], x1
> -        srshr           \coef5, \coef5, #6
> -        ld1             {v7.8b},  [x3], x1
> -        sqxtun          v2.8b,  \coef0
> -        srshr           \coef6, \coef6, #6
> -        sqxtun          v3.8b,  \coef1
> -        srshr           \coef7, \coef7, #6
> -        uaddw           \coef2, \coef2, v4.8b
> -        ld1             {\tmp1},  [x0], x1
> -        uaddw           \coef3, \coef3, v5.8b
> -        ld1             {\tmp2},  [x3], x1
> -        sqxtun          v4.8b,  \coef2
> -        sub             x0,  x0,  x1, lsl #2
> -        sub             x3,  x3,  x1, lsl #2
> -        sqxtun          v5.8b,  \coef3
> -        uaddw           \coef4, \coef4, v6.8b
> -        st1             {v2.8b},  [x0], x1
> -        uaddw           \coef5, \coef5, v7.8b
> -        st1             {v3.8b},  [x3], x1
> -        sqxtun          v6.8b,  \coef4
> -        st1             {v4.8b},  [x0], x1
> -        sqxtun          v7.8b,  \coef5
> -        st1             {v5.8b},  [x3], x1
> -        uaddw           \coef6, \coef6, \tmp1
> -        st1             {v6.8b},  [x0], x1
> -        uaddw           \coef7, \coef7, \tmp2
> -        st1             {v7.8b},  [x3], x1
> -        sqxtun          \tmp1,  \coef6
> -        sqxtun          \tmp2,  \coef7
> -        st1             {\tmp1},  [x0], x1
> -        st1             {\tmp2},  [x3], x1
> -.endm
>          load_add_store  v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v16.8b, v17.8b
>          load_add_store  v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h, v16.8b, v17.8b
> -.purgem load_add_store
>  
>          br              x14
>  endfunc
> @@ -731,6 +843,10 @@ function ff_vp9_\txfm1\()_\txfm2\()_16x16_add_neon, export=1
>  .ifc \txfm1\()_\txfm2,idct_idct
>          cmp             w3,  #1
>          b.eq            idct16x16_dc_add_neon
> +        cmp             w3,  #10
> +        b.le            idct16x16_quarter_add_neon
> +        cmp             w3,  #38
> +        b.le            idct16x16_half_add_neon
>  .endif
>          mov             x15, x30
>          // iadst16 requires clobbering v8-v15, but idct16 doesn't need to.
> @@ -812,6 +928,155 @@ itxfm_func16x16 iadst, idct
>  itxfm_func16x16 idct,  iadst
>  itxfm_func16x16 iadst, iadst
>  
> +function idct16_1d_8x16_pass1_quarter_neon
> +        mov             x14, x30
> +        movi            v2.8h, #0
> +.irp i, 16, 17, 18, 19
> +        load_clear      \i,  x2,  x9
> +.endr
> +
> +        bl              idct16_quarter
> +
> +        // Do two 8x8 transposes. Originally, v16-v31 contain the
> +        // 16 rows. Afterwards, v16-v23 and v24-v31 contain the two
> +        // transposed 8x8 blocks.
> +        transpose_8x8H  v16, v17, v18, v19, v20, v21, v22, v23, v2, v3
> +        transpose_8x8H  v24, v25, v26, v27, v28, v29, v30, v31, v2, v3
> +
> +        // Store the transposed 8x8 blocks horizontally.
> +        // The first 8x8 block is kept in registers for the second pass,
> +        // store the rest in the temp buffer.
> +        // Since only a 4x4 part of the input was nonzero, this means that
> +        // only 4 rows are nonzero after transposing, and the second pass
> +        // only reads the topmost 4 rows. Therefore only store the topmost
> +        // 4 rows.
> +        add             x0,  x0,  #16
> +.irp i, 24, 25, 26, 27
> +        store           \i,  x0,  x9
> +.endr
> +        br              x14
> +endfunc
> +
> +function idct16_1d_8x16_pass2_quarter_neon
> +        mov             x14, x30
> +        cbz             x3,  1f
> +.irp i, 16, 17, 18, 19
> +        load            \i,  x2,  x9
> +.endr
> +1:
> +
> +        add             x3,  x0,  x1
> +        lsl             x1,  x1,  #1
> +        bl              idct16_quarter
> +
> +        load_add_store  v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v16.8b, v17.8b
> +        load_add_store  v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h, v16.8b, v17.8b
> +
> +        br              x14
> +endfunc
> +
> +function idct16_1d_8x16_pass1_half_neon
> +        mov             x14, x30
> +        movi            v2.8h, #0
> +.irp i, 16, 17, 18, 19, 20, 21, 22, 23
> +        load_clear      \i,  x2,  x9
> +.endr
> +
> +        bl              idct16_half
> +
> +        // Do two 8x8 transposes. Originally, v16-v31 contain the
> +        // 16 rows. Afterwards, v16-v23 and v24-v31 contain the two
> +        // transposed 8x8 blocks.
> +        transpose_8x8H  v16, v17, v18, v19, v20, v21, v22, v23, v2, v3
> +        transpose_8x8H  v24, v25, v26, v27, v28, v29, v30, v31, v2, v3
> +
> +        // Store the transposed 8x8 blocks horizontally.
> +        // The first 8x8 block is kept in registers for the second pass,
> +        // store the rest in the temp buffer.
> +        add             x0,  x0,  #16
> +.irp i, 24, 25, 26, 27, 28, 29, 30, 31
> +        store           \i,  x0,  x9
> +.endr
> +        br              x14
> +endfunc
> +
> +function idct16_1d_8x16_pass2_half_neon
> +        mov             x14, x30
> +        cbz             x3,  1f
> +.irp i, 16, 17, 18, 19, 20, 21, 22, 23
> +        load            \i,  x2,  x9
> +.endr
> +1:
> +
> +        add             x3,  x0,  x1
> +        lsl             x1,  x1,  #1
> +        bl              idct16_half
> +
> +        load_add_store  v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v16.8b, v17.8b
> +        load_add_store  v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h, v16.8b, v17.8b
> +
> +        br              x14
> +endfunc
> +
> +function idct16x16_quarter_add_neon, export=1
> +        mov             x15, x30
> +
> +        sub             sp,  sp,  #512
> +
> +        mov             x4,  x0
> +        mov             x5,  x1
> +        mov             x6,  x2
> +
> +        movrel          x10, idct_coeffs
> +        ld1             {v0.8h,v1.8h}, [x10]
> +        mov             x9, #32
> +
> +.irp i, 0
> +        add             x0,  sp,  #(\i*32)
> +        add             x2,  x6,  #(\i*2)
> +        bl              idct16_1d_8x16_pass1_quarter_neon
> +.endr
> +.irp i, 0, 8
> +        add             x0,  x4,  #(\i)
> +        mov             x1,  x5
> +        add             x2,  sp,  #(\i*2)
> +        mov             x3,  #\i
> +        bl              idct16_1d_8x16_pass2_quarter_neon
> +.endr
> +
> +        add             sp,  sp,  #512
> +        br              x15
> +endfunc
> +
> +function idct16x16_half_add_neon, export=1
> +        mov             x15, x30
> +
> +        sub             sp,  sp,  #512
> +
> +        mov             x4,  x0
> +        mov             x5,  x1
> +        mov             x6,  x2
> +
> +        movrel          x10, idct_coeffs
> +        ld1             {v0.8h,v1.8h}, [x10]
> +        mov             x9, #32
> +
> +.irp i, 0
> +        add             x0,  sp,  #(\i*32)
> +        add             x2,  x6,  #(\i*2)
> +        bl              idct16_1d_8x16_pass1_half_neon
> +.endr
> +.irp i, 0, 8
> +        add             x0,  x4,  #(\i)
> +        mov             x1,  x5
> +        add             x2,  sp,  #(\i*2)
> +        mov             x3,  #\i
> +        bl              idct16_1d_8x16_pass2_half_neon
> +.endr
> +
> +        add             sp,  sp,  #512
> +        br              x15
> +endfunc

this two should be templated

>  
>  function idct32x32_dc_add_neon
>          movrel          x4, idct_coeffs
> @@ -848,7 +1113,7 @@ function idct32x32_dc_add_neon
>          ret
>  endfunc
>  
> -function idct32_odd
> +.macro idct32_odd_full
>          ld1             {v0.8h,v1.8h}, [x11]
>  
>          dmbutterfly     v16, v31, v0.h[0], v0.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a
> @@ -875,7 +1140,10 @@ function idct32_odd
>          dmbutterfly     v27, v20, v0.h[3], v0.h[4], v16, v17, v18, v19, neg=1 // v27 = t29a, v20 = t18a
>          dmbutterfly     v21, v26, v0.h[5], v0.h[6], v16, v17, v18, v19        // v21 = t21a, v26 = t26a
>          dmbutterfly     v25, v22, v0.h[5], v0.h[6], v16, v17, v18, v19, neg=1 // v25 = t25a, v22 = t22a
> +        idct32_end
> +.endm
>  
> +.macro idct32_end
>          butterfly_8h    v16, v5,  v4,  v5  // v16 = t16a, v5  = t19a
>          butterfly_8h    v17, v20, v23, v20 // v17 = t17,  v20 = t18
>          butterfly_8h    v18, v6,  v7,  v6  // v18 = t23a, v6  = t20a
> @@ -904,8 +1172,91 @@ function idct32_odd
>          dmbutterfly0    v25, v22, v25, v22, v2, v3, v4, v5, v6, v7 // v25 = t25,  v22 = t22
>          dmbutterfly0    v24, v23, v24, v23, v2, v3, v4, v5, v6, v7 // v24 = t24a, v23 = t23a
>          ret
> +.endm
> +
> +function idct32_odd
> +        idct32_odd_full
> +endfunc
> +
> +function idct32_odd_half
> +        ld1             {v0.8h,v1.8h}, [x11]
> +
> +        dmbutterfly_h1  v16, v31, v0.h[0], v0.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a
> +        dmbutterfly_h2  v24, v23, v0.h[2], v0.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a
> +        dmbutterfly_h1  v20, v27, v0.h[4], v0.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a
> +        dmbutterfly_h2  v28, v19, v0.h[6], v0.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a
> +        dmbutterfly_h1  v18, v29, v1.h[0], v1.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a
> +        dmbutterfly_h2  v26, v21, v1.h[2], v1.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a
> +        dmbutterfly_h1  v22, v25, v1.h[4], v1.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a
> +        dmbutterfly_h2  v30, v17, v1.h[6], v1.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a
> +
> +        ld1             {v0.8h}, [x10]
> +
> +        butterfly_8h    v4,  v24, v16, v24 // v4  = t16, v24 = t17
> +        butterfly_8h    v5,  v20, v28, v20 // v5  = t19, v20 = t18
> +        butterfly_8h    v6,  v26, v18, v26 // v6  = t20, v26 = t21
> +        butterfly_8h    v7,  v22, v30, v22 // v7  = t23, v22 = t22
> +        butterfly_8h    v28, v25, v17, v25 // v28 = t24, v25 = t25
> +        butterfly_8h    v30, v21, v29, v21 // v30 = t27, v21 = t26
> +        butterfly_8h    v29, v23, v31, v23 // v29 = t31, v23 = t30
> +        butterfly_8h    v31, v27, v19, v27 // v31 = t28, v27 = t29
> +
> +        dmbutterfly     v23, v24, v0.h[3], v0.h[4], v16, v17, v18, v19        // v23 = t17a, v24 = t30a
> +        dmbutterfly     v27, v20, v0.h[3], v0.h[4], v16, v17, v18, v19, neg=1 // v27 = t29a, v20 = t18a
> +        dmbutterfly     v21, v26, v0.h[5], v0.h[6], v16, v17, v18, v19        // v21 = t21a, v26 = t26a
> +        dmbutterfly     v25, v22, v0.h[5], v0.h[6], v16, v17, v18, v19, neg=1 // v25 = t25a, v22 = t22a
> +        idct32_end
> +endfunc
> +
> +function idct32_odd_quarter
> +        ld1             {v0.8h,v1.8h}, [x11]
> +
> +        dsmull_h        v4,  v5,  v16, v0.h[0]
> +        dsmull_h        v28, v29, v19, v0.h[7]
> +        dsmull_h        v30, v31, v16, v0.h[1]
> +        dsmull_h        v22, v23, v17, v1.h[6]
> +        dsmull_h        v7,  v6,  v17, v1.h[7]
> +        dsmull_h        v26, v27, v19, v0.h[6]
> +        dsmull_h        v20, v21, v18, v1.h[0]
> +        dsmull_h        v24, v25, v18, v1.h[1]
> +
> +        ld1             {v0.8h}, [x10]
> +
> +        neg             v28.4s, v28.4s
> +        neg             v29.4s, v29.4s
> +        neg             v7.4s,  v7.4s
> +        neg             v6.4s,  v6.4s
> +
> +        drshrn_h        v4,  v4,  v5,  #14
> +        drshrn_h        v5,  v28, v29, #14
> +        drshrn_h        v29, v30, v31, #14
> +        drshrn_h        v28, v22, v23, #14
> +        drshrn_h        v7,  v7,  v6,  #14
> +        drshrn_h        v31, v26, v27, #14
> +        drshrn_h        v6,  v20, v21, #14
> +        drshrn_h        v30, v24, v25, #14
> +
> +        dmbutterfly_l   v16, v17, v18, v19, v29, v4,  v0.h[3], v0.h[4]
> +        dmbutterfly_l   v27, v26, v20, v21, v31, v5,  v0.h[3], v0.h[4]
> +        drshrn_h        v23, v16, v17, #14
> +        drshrn_h        v24, v18, v19, #14
> +        neg             v20.4s, v20.4s
> +        neg             v21.4s, v21.4s
> +        drshrn_h        v27, v27, v26, #14
> +        drshrn_h        v20, v20, v21, #14
> +        dmbutterfly_l   v16, v17, v18, v19, v30, v6,  v0.h[5], v0.h[6]
> +        drshrn_h        v21, v16, v17, #14
> +        drshrn_h        v26, v18, v19, #14
> +        dmbutterfly_l   v16, v17, v18, v19, v28, v7,  v0.h[5], v0.h[6]
> +        drshrn_h        v25, v16, v17, #14
> +        neg             v18.4s, v18.4s
> +        neg             v19.4s, v19.4s
> +        drshrn_h        v22, v18, v19, #14
> +
> +        idct32_end
>  endfunc
>  
> +.macro idct32_funcs suffix
>  // Do an 32-point IDCT of a 8x32 slice out of a 32x32 matrix.
>  // The 32-point IDCT can be decomposed into two 16-point IDCTs;
>  // a normal IDCT16 with every other input component (the even ones, with
> @@ -917,19 +1268,30 @@ endfunc
>  // x9 = double input stride
>  // x10 = idct_coeffs
>  // x11 = idct_coeffs + 32
> -function idct32_1d_8x32_pass1_neon
> +function idct32_1d_8x32_pass1\suffix\()_neon
>          mov             x14, x30
>          ld1             {v0.8h,v1.8h}, [x10]
>  
> -        movi            v4.8h, #0
> +        movi            v2.8h, #0
>  
>          // v16 = IN(0), v17 = IN(2) ... v31 = IN(30)
> +.ifb \suffix
>  .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
> -        ld1             {v\i\().8h}, [x2]
> -        st1             {v4.8h},  [x2], x9
> +        load_clear      \i, x2, x9
> +.endr
> +.endif
> +.ifc \suffix,_quarter
> +.irp i, 16, 17, 18, 19
> +        load_clear      \i, x2, x9
> +.endr
> +.endif
> +.ifc \suffix,_half
> +.irp i, 16, 17, 18, 19, 20, 21, 22, 23
> +        load_clear      \i, x2, x9
>  .endr
> +.endif
>  
> -        bl              idct16
> +        bl              idct16\suffix
>  
>          // Do two 8x8 transposes. Originally, v16-v31 contain the
>          // 16 rows. Afterwards, v16-v23 and v24-v31 contain the
> @@ -964,17 +1326,36 @@ function idct32_1d_8x32_pass1_neon
>  
>          // Move x2 back to the start of the input, and move
>          // to the first odd row
> +.ifb \suffix
>          sub             x2,  x2,  x9, lsl #4
> +.endif
> +.ifc \suffix,_quarter
> +        sub             x2,  x2,  x9, lsl #2
> +.endif
> +.ifc \suffix,_half
> +        sub             x2,  x2,  x9, lsl #3
> +.endif
>          add             x2,  x2,  #64
>  
> -        movi            v4.8h, #0
> +        movi            v2.8h, #0
>          // v16 = IN(1), v17 = IN(3) ... v31 = IN(31)
> +.ifb \suffix
>  .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
> -        ld1             {v\i\().8h}, [x2]
> -        st1             {v4.8h},  [x2], x9
> +        load_clear      \i, x2, x9
> +.endr
> +.endif
> +.ifc \suffix,_quarter
> +.irp i, 16, 17, 18, 19
> +        load_clear      \i, x2, x9
> +.endr
> +.endif
> +.ifc \suffix,_half
> +.irp i, 16, 17, 18, 19, 20, 21, 22, 23
> +        load_clear      \i, x2, x9
>  .endr
> +.endif
>  
> -        bl              idct32_odd
> +        bl              idct32_odd\suffix
>  
>          transpose_8x8H v31, v30, v29, v28, v27, v26, v25, v24, v2, v3
>          transpose_8x8H v23, v22, v21, v20, v19, v18, v17, v16, v2, v3
> @@ -1023,33 +1404,61 @@ endfunc
>  // x9 = double temp buffer stride
>  // x10 = idct_coeffs
>  // x11 = idct_coeffs + 32
> -function idct32_1d_8x32_pass2_neon
> +function idct32_1d_8x32_pass2\suffix\()_neon
>          mov             x14, x30
>          ld1             {v0.8h,v1.8h}, [x10]
>  
>          // v16 = IN(0), v17 = IN(2) ... v31 = IN(30)
> +.ifb \suffix
>  .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
> -        ld1             {v\i\().8h}, [x2], x9
> +        load            \i, x2, x9
>  .endr
>          sub             x2,  x2,  x9, lsl #4
> +.endif
> +.ifc \suffix,_quarter
> +.irp i, 16, 17, 18, 19
> +        load            \i, x2, x9
> +.endr
> +        sub             x2,  x2,  x9, lsl #2
> +.endif
> +.ifc \suffix,_half
> +.irp i, 16, 17, 18, 19, 20, 21, 22, 23
> +        load            \i, x2, x9
> +.endr
> +        sub             x2,  x2,  x9, lsl #3
> +.endif
>  
> -        bl              idct16
> +        bl              idct16\suffix
>  
>  .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
> -        st1             {v\i\().8h}, [x2], x9
> +        store           \i, x2, x9
>  .endr
>  
>          sub             x2,  x2,  x9, lsl #4
>          add             x2,  x2,  #64
>  
>          // v16 = IN(1), v17 = IN(3) ... v31 = IN(31)
> +.ifb \suffix
>  .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
> -        ld1             {v\i\().8h}, [x2], x9
> +        load            \i, x2, x9
>  .endr
>          sub             x2,  x2,  x9, lsl #4
> +.endif
> +.ifc \suffix,_quarter
> +.irp i, 16, 17, 18, 19
> +        load            \i, x2, x9
> +.endr
> +        sub             x2,  x2,  x9, lsl #2
> +.endif
> +.ifc \suffix,_half
> +.irp i, 16, 17, 18, 19, 20, 21, 22, 23
> +        load            \i, x2, x9
> +.endr
> +        sub             x2,  x2,  x9, lsl #3
> +.endif
>          sub             x2,  x2,  #64
>  
> -        bl              idct32_odd
> +        bl              idct32_odd\suffix
>  
>  .macro load_acc_store a, b, c, d, neg=0
>  .if \neg == 0
> @@ -1105,6 +1514,11 @@ function idct32_1d_8x32_pass2_neon
>  .purgem load_acc_store
>          br              x14
>  endfunc
> +.endm
> +
> +idct32_funcs
> +idct32_funcs _quarter
> +idct32_funcs _half
>  
>  const min_eob_idct_idct_32, align=4
>          .short  0, 34, 135, 336
> @@ -1113,6 +1527,10 @@ endconst
>  function ff_vp9_idct_idct_32x32_add_neon, export=1
>          cmp             w3,  #1
>          b.eq            idct32x32_dc_add_neon

saving d8-d15 should be done here, saves duplicating it in the 
quarter/half variants. same for the idct_coeffs and other stuff.

> +        cmp             w3,  #34
> +        b.le            idct32x32_quarter_add_neon
> +        cmp             w3,  #135
> +        b.le            idct32x32_half_add_neon
>  
>          movrel          x10, idct_coeffs
>          add             x11, x10, #32
> @@ -1177,3 +1595,89 @@ function ff_vp9_idct_idct_32x32_add_neon, export=1
>  
>          br              x15
>  endfunc
> +
> +function idct32x32_quarter_add_neon
> +        movrel          x10, idct_coeffs
> +        add             x11, x10, #32
> +
> +        mov             x15, x30
> +
> +        stp             d14, d15, [sp, #-0x10]!
> +        stp             d12, d13, [sp, #-0x10]!
> +        stp             d10, d11, [sp, #-0x10]!
> +        stp             d8,  d9,  [sp, #-0x10]!
> +
> +        sub             sp,  sp,  #2048
> +
> +        mov             x4,  x0
> +        mov             x5,  x1
> +        mov             x6,  x2
> +
> +        // Double stride of the input, since we only read every other line
> +        mov             x9,  #128
> +        neg             x7,  x9
> +
> +.irp i, 0
> +        add             x0,  sp,  #(\i*64)
> +        add             x2,  x6,  #(\i*2)
> +        bl              idct32_1d_8x32_pass1_quarter_neon
> +.endr
> +.irp i, 0, 8, 16, 24
> +        add             x0,  x4,  #(\i)
> +        mov             x1,  x5
> +        add             x2,  sp,  #(\i*2)
> +        bl              idct32_1d_8x32_pass2_quarter_neon
> +.endr
> +
> +        add             sp,  sp,  #2048
> +
> +        ldp             d8,  d9,  [sp], 0x10
> +        ldp             d10, d11, [sp], 0x10
> +        ldp             d12, d13, [sp], 0x10
> +        ldp             d14, d15, [sp], 0x10
> +
> +        br              x15
> +endfunc
> +
> +function idct32x32_half_add_neon
> +        movrel          x10, idct_coeffs
> +        add             x11, x10, #32
> +
> +        mov             x15, x30
> +
> +        stp             d14, d15, [sp, #-0x10]!
> +        stp             d12, d13, [sp, #-0x10]!
> +        stp             d10, d11, [sp, #-0x10]!
> +        stp             d8,  d9,  [sp, #-0x10]!
> +
> +        sub             sp,  sp,  #2048
> +
> +        mov             x4,  x0
> +        mov             x5,  x1
> +        mov             x6,  x2
> +
> +        // Double stride of the input, since we only read every other line
> +        mov             x9,  #128
> +        neg             x7,  x9
> +
> +.irp i, 0, 8
> +        add             x0,  sp,  #(\i*64)
> +        add             x2,  x6,  #(\i*2)
> +        bl              idct32_1d_8x32_pass1_half_neon
> +.endr
> +.irp i, 0, 8, 16, 24
> +        add             x0,  x4,  #(\i)
> +        mov             x1,  x5
> +        add             x2,  sp,  #(\i*2)
> +        bl              idct32_1d_8x32_pass2_half_neon
> +.endr
> +
> +        add             sp,  sp,  #2048
> +
> +        ldp             d8,  d9,  [sp], 0x10
> +        ldp             d10, d11, [sp], 0x10
> +        ldp             d12, d13, [sp], 0x10
> +        ldp             d14, d15, [sp], 0x10
> +
> +        br              x15
> +endfunc

otherwise ok

Janne
Martin Storsjö Feb. 5, 2017, 12:10 p.m. | #2
On Sun, 5 Feb 2017, Janne Grunau wrote:

> On 2016-12-01 11:27:02 +0200, Martin Storsjö wrote:
>> This work is sponsored by, and copyright, Google.
>> 
>> This makes it easier to avoid filling the temp buffer with zeros for the
>> skipped slices, and leads to slightly more straightforward code for these
>> cases (for the 16x16 case, where the special case pass functions are
>> written out instead of templated from the same macro), instead of riddling
>> the common code with special case branches or macro .ifs.
>> 
>> The code size increases from 14740 bytes to 24472 bytes.
>> 
>> Before:
>> vp9_inv_dct_dct_16x16_sub1_add_neon:     235.3
>> vp9_inv_dct_dct_16x16_sub2_add_neon:    1051.0
>> vp9_inv_dct_dct_16x16_sub4_add_neon:    1051.0
>> vp9_inv_dct_dct_16x16_sub8_add_neon:    1051.0
>> vp9_inv_dct_dct_16x16_sub12_add_neon:   1390.3
>> vp9_inv_dct_dct_16x16_sub16_add_neon:   1390.1
>> vp9_inv_dct_dct_32x32_sub1_add_neon:     556.5
>> vp9_inv_dct_dct_32x32_sub2_add_neon:    5199.1
>> vp9_inv_dct_dct_32x32_sub4_add_neon:    5199.9
>> vp9_inv_dct_dct_32x32_sub8_add_neon:    5196.9
>> vp9_inv_dct_dct_32x32_sub12_add_neon:   6171.6
>> vp9_inv_dct_dct_32x32_sub16_add_neon:   6170.9
>> vp9_inv_dct_dct_32x32_sub20_add_neon:   7147.1
>> vp9_inv_dct_dct_32x32_sub24_add_neon:   7147.0
>> vp9_inv_dct_dct_32x32_sub28_add_neon:   8118.8
>> vp9_inv_dct_dct_32x32_sub32_add_neon:   8125.8
>> 
>> After:
>> vp9_inv_dct_dct_16x16_sub1_add_neon:     235.3
>> vp9_inv_dct_dct_16x16_sub2_add_neon:     639.0
>> vp9_inv_dct_dct_16x16_sub4_add_neon:     639.0
>> vp9_inv_dct_dct_16x16_sub8_add_neon:     845.0
>> vp9_inv_dct_dct_16x16_sub12_add_neon:   1389.4
>> vp9_inv_dct_dct_16x16_sub16_add_neon:   1389.3
>> vp9_inv_dct_dct_32x32_sub1_add_neon:     556.5
>> vp9_inv_dct_dct_32x32_sub2_add_neon:    3684.1
>> vp9_inv_dct_dct_32x32_sub4_add_neon:    3682.6
>> vp9_inv_dct_dct_32x32_sub8_add_neon:    3684.1
>> vp9_inv_dct_dct_32x32_sub12_add_neon:   5319.0
>> vp9_inv_dct_dct_32x32_sub16_add_neon:   5323.5
>> vp9_inv_dct_dct_32x32_sub20_add_neon:   7149.8
>> vp9_inv_dct_dct_32x32_sub24_add_neon:   7148.2
>> vp9_inv_dct_dct_32x32_sub28_add_neon:   8124.5
>> vp9_inv_dct_dct_32x32_sub32_add_neon:   8122.1
>> 
>> ---
>> If we wouldn't have made the core transforms standalone functions,
>> the code size would end up at around 34 KB.
>> 
>> The binary output is 6 KB larger than in the other alternative,
>> but is more straightforward and gives better opportunities to
>> special case them further.
>> 
>> In the arm version, there was a significant speedup compared to the
>> other alternative (having cmps within the functions), skipping
>> zeroing of the temp buffer. Here there's much less difference.
>
> And the relative binary size difference is even larger. It would a 
> little strange to choose different alternatives for 32- and 64-bit but 
> it sounds like alternative 1 might be better for arm64. Please run a 
> full decoding benchmark for arm64 too.

Yeah, I need to do more extensive full benchmarks to know whether it's 
worth it or not. The difference in the arm case seemed bigger than it 
should be based on checkasm numbers as well, so perhaps I need to run a 
few more iterations to get more correct values.

>> +endfunc
>
> this two should be templated

Will do

>>  function ff_vp9_idct_idct_32x32_add_neon, export=1
>>          cmp             w3,  #1
>>          b.eq            idct32x32_dc_add_neon
>
> saving d8-d15 should be done here, saves duplicating it in the 
> quarter/half variants. same for the idct_coeffs and other stuff.

Ok, will try

// Martin
Martin Storsjö Feb. 5, 2017, 10:16 p.m. | #3
On Sun, 5 Feb 2017, Martin Storsjö wrote:

> On Sun, 5 Feb 2017, Janne Grunau wrote:
>
>> On 2016-12-01 11:27:02 +0200, Martin Storsjö wrote:
>>> This work is sponsored by, and copyright, Google.
>>> 
>>> This makes it easier to avoid filling the temp buffer with zeros for the
>>> skipped slices, and leads to slightly more straightforward code for these
>>> cases (for the 16x16 case, where the special case pass functions are
>>> written out instead of templated from the same macro), instead of riddling
>>> the common code with special case branches or macro .ifs.
>>> 
>>> The code size increases from 14740 bytes to 24472 bytes.
>>> 
>>> Before:
>>> vp9_inv_dct_dct_16x16_sub1_add_neon:     235.3
>>> vp9_inv_dct_dct_16x16_sub2_add_neon:    1051.0
>>> vp9_inv_dct_dct_16x16_sub4_add_neon:    1051.0
>>> vp9_inv_dct_dct_16x16_sub8_add_neon:    1051.0
>>> vp9_inv_dct_dct_16x16_sub12_add_neon:   1390.3
>>> vp9_inv_dct_dct_16x16_sub16_add_neon:   1390.1
>>> vp9_inv_dct_dct_32x32_sub1_add_neon:     556.5
>>> vp9_inv_dct_dct_32x32_sub2_add_neon:    5199.1
>>> vp9_inv_dct_dct_32x32_sub4_add_neon:    5199.9
>>> vp9_inv_dct_dct_32x32_sub8_add_neon:    5196.9
>>> vp9_inv_dct_dct_32x32_sub12_add_neon:   6171.6
>>> vp9_inv_dct_dct_32x32_sub16_add_neon:   6170.9
>>> vp9_inv_dct_dct_32x32_sub20_add_neon:   7147.1
>>> vp9_inv_dct_dct_32x32_sub24_add_neon:   7147.0
>>> vp9_inv_dct_dct_32x32_sub28_add_neon:   8118.8
>>> vp9_inv_dct_dct_32x32_sub32_add_neon:   8125.8
>>> 
>>> After:
>>> vp9_inv_dct_dct_16x16_sub1_add_neon:     235.3
>>> vp9_inv_dct_dct_16x16_sub2_add_neon:     639.0
>>> vp9_inv_dct_dct_16x16_sub4_add_neon:     639.0
>>> vp9_inv_dct_dct_16x16_sub8_add_neon:     845.0
>>> vp9_inv_dct_dct_16x16_sub12_add_neon:   1389.4
>>> vp9_inv_dct_dct_16x16_sub16_add_neon:   1389.3
>>> vp9_inv_dct_dct_32x32_sub1_add_neon:     556.5
>>> vp9_inv_dct_dct_32x32_sub2_add_neon:    3684.1
>>> vp9_inv_dct_dct_32x32_sub4_add_neon:    3682.6
>>> vp9_inv_dct_dct_32x32_sub8_add_neon:    3684.1
>>> vp9_inv_dct_dct_32x32_sub12_add_neon:   5319.0
>>> vp9_inv_dct_dct_32x32_sub16_add_neon:   5323.5
>>> vp9_inv_dct_dct_32x32_sub20_add_neon:   7149.8
>>> vp9_inv_dct_dct_32x32_sub24_add_neon:   7148.2
>>> vp9_inv_dct_dct_32x32_sub28_add_neon:   8124.5
>>> vp9_inv_dct_dct_32x32_sub32_add_neon:   8122.1
>>> 
>>> ---
>>> If we wouldn't have made the core transforms standalone functions,
>>> the code size would end up at around 34 KB.
>>> 
>>> The binary output is 6 KB larger than in the other alternative,
>>> but is more straightforward and gives better opportunities to
>>> special case them further.
>>> 
>>> In the arm version, there was a significant speedup compared to the
>>> other alternative (having cmps within the functions), skipping
>>> zeroing of the temp buffer. Here there's much less difference.
>>
>> And the relative binary size difference is even larger. It would a 
>> little strange to choose different alternatives for 32- and 64-bit but 
>> it sounds like alternative 1 might be better for arm64. Please run a 
>> full decoding benchmark for arm64 too.
>
> Yeah, I need to do more extensive full benchmarks to know whether it's 
> worth it or not. The difference in the arm case seemed bigger than it 
> should be based on checkasm numbers as well, so perhaps I need to run a 
> few more iterations to get more correct values.

Ok, so after running a slightly shorter clip (which seems to have about as 
large percentage of runtime doing IDCT as the previous one) with a bit 
more iterations, I've got the following results (the 'user' part from 
'time avconv -threads 1 -i foo -f null -'):

32 orig   32 alt1   32 alt2   64 orig   64 alt1   64 alt2
40.436s   40.148s   40.008s   37.428s   37.356s   37.192s
40.596s   40.140s   40.216s   37.572s   37.524s   37.384s
40.512s   40.228s   40.188s   37.740s   37.588s   37.368s
40.584s   40.136s   40.216s   37.880s   37.492s   37.348s
40.572s   40.292s   40.232s   37.756s   37.556s   37.676s
40.764s   40.312s   40.232s   37.876s   37.640s   37.468s
40.688s   40.284s   40.368s   37.972s   37.608s   37.460s

So while alt2 is faster in most runs, the margin is not quite as big as in 
the previous benchmark. (The benchmarks were done on a practically 
unloaded system so it shouldn't vary too much from run to run, but in 
practice, the first few runs seem to be slightly faster than the later 
ones.)

I.e. around 400 ms gain out of 40 s for alt1, and then another -50 - +150 
ms speedup on top of that for alt2.

What do you think?

// Martin
Janne Grunau Feb. 9, 2017, 7:57 a.m. | #4
On 2017-02-06 00:16:41 +0200, Martin Storsjö wrote:
> 
> Ok, so after running a slightly shorter clip (which seems to have about as
> large percentage of runtime doing IDCT as the previous one) with a bit more
> iterations, I've got the following results (the 'user' part from 'time
> avconv -threads 1 -i foo -f null -'):
> 
> 32 orig   32 alt1   32 alt2   64 orig   64 alt1   64 alt2
> 40.436s   40.148s   40.008s   37.428s   37.356s   37.192s
> 40.596s   40.140s   40.216s   37.572s   37.524s   37.384s
> 40.512s   40.228s   40.188s   37.740s   37.588s   37.368s
> 40.584s   40.136s   40.216s   37.880s   37.492s   37.348s
> 40.572s   40.292s   40.232s   37.756s   37.556s   37.676s
> 40.764s   40.312s   40.232s   37.876s   37.640s   37.468s
> 40.688s   40.284s   40.368s   37.972s   37.608s   37.460s
> 
> So while alt2 is faster in most runs, the margin is not quite as big as in
> the previous benchmark. (The benchmarks were done on a practically unloaded
> system so it shouldn't vary too much from run to run, but in practice, the
> first few runs seem to be slightly faster than the later ones.)
> 
> I.e. around 400 ms gain out of 40 s for alt1, and then another -50 - +150 ms
> speedup on top of that for alt2.
> 
> What do you think?

At least it looks like the difference between alt1 and alt2 are quite 
similar on 32- and 64-bit. So we should use the same variant on both 
archs. I favor alternate 2.

Janne
Martin Storsjö Feb. 9, 2017, 8:22 a.m. | #5
On Thu, 9 Feb 2017, Janne Grunau wrote:

> On 2017-02-06 00:16:41 +0200, Martin Storsjö wrote:
>> 
>> Ok, so after running a slightly shorter clip (which seems to have about as
>> large percentage of runtime doing IDCT as the previous one) with a bit more
>> iterations, I've got the following results (the 'user' part from 'time
>> avconv -threads 1 -i foo -f null -'):
>> 
>> 32 orig   32 alt1   32 alt2   64 orig   64 alt1   64 alt2
>> 40.436s   40.148s   40.008s   37.428s   37.356s   37.192s
>> 40.596s   40.140s   40.216s   37.572s   37.524s   37.384s
>> 40.512s   40.228s   40.188s   37.740s   37.588s   37.368s
>> 40.584s   40.136s   40.216s   37.880s   37.492s   37.348s
>> 40.572s   40.292s   40.232s   37.756s   37.556s   37.676s
>> 40.764s   40.312s   40.232s   37.876s   37.640s   37.468s
>> 40.688s   40.284s   40.368s   37.972s   37.608s   37.460s
>> 
>> So while alt2 is faster in most runs, the margin is not quite as big as in
>> the previous benchmark. (The benchmarks were done on a practically unloaded
>> system so it shouldn't vary too much from run to run, but in practice, the
>> first few runs seem to be slightly faster than the later ones.)
>> 
>> I.e. around 400 ms gain out of 40 s for alt1, and then another -50 - +150 ms
>> speedup on top of that for alt2.
>> 
>> What do you think?
>
> At least it looks like the difference between alt1 and alt2 are quite 
> similar on 32- and 64-bit. So we should use the same variant on both 
> archs. I favor alternate 2.

Ok then - I'll try to polish up and push alternative 2 based on the 
feedback I got.

// Martin

Patch

diff --git a/libavcodec/aarch64/vp9itxfm_neon.S b/libavcodec/aarch64/vp9itxfm_neon.S
index be9643e..9910170 100644
--- a/libavcodec/aarch64/vp9itxfm_neon.S
+++ b/libavcodec/aarch64/vp9itxfm_neon.S
@@ -75,6 +75,16 @@  endconst
 .endif
 .endm
 
+// Same as dmbutterfly0 above, but treating the input in in2 as zero,
+// writing the same output into both out1 and out2.
+.macro dmbutterfly0_h out1, out2, in1, in2, tmp1, tmp2, tmp3, tmp4, tmp5, tmp6
+        smull           \tmp1\().4s,  \in1\().4h,  v0.h[0]
+        smull2          \tmp2\().4s,  \in1\().8h,  v0.h[0]
+        rshrn           \out1\().4h,  \tmp1\().4s, #14
+        rshrn2          \out1\().8h,  \tmp2\().4s, #14
+        mov             \out2\().16b, \out1\().16b
+.endm
+
 // out1,out2 = in1 * coef1 - in2 * coef2
 // out3,out4 = in1 * coef2 + in2 * coef1
 // out are 4 x .4s registers, in are 2 x .8h registers
@@ -104,6 +114,43 @@  endconst
         rshrn2          \inout2\().8h, \tmp4\().4s,  #14
 .endm
 
+// Same as dmbutterfly above, but treating the input in inout2 as zero
+.macro dmbutterfly_h1 inout1, inout2, coef1, coef2, tmp1, tmp2, tmp3, tmp4
+        smull           \tmp1\().4s, \inout1\().4h, \coef1
+        smull2          \tmp2\().4s, \inout1\().8h, \coef1
+        smull           \tmp3\().4s, \inout1\().4h, \coef2
+        smull2          \tmp4\().4s, \inout1\().8h, \coef2
+        rshrn           \inout1\().4h, \tmp1\().4s, #14
+        rshrn2          \inout1\().8h, \tmp2\().4s, #14
+        rshrn           \inout2\().4h, \tmp3\().4s, #14
+        rshrn2          \inout2\().8h, \tmp4\().4s, #14
+.endm
+
+// Same as dmbutterfly above, but treating the input in inout1 as zero
+.macro dmbutterfly_h2 inout1, inout2, coef1, coef2, tmp1, tmp2, tmp3, tmp4
+        smull           \tmp1\().4s, \inout2\().4h, \coef2
+        smull2          \tmp2\().4s, \inout2\().8h, \coef2
+        smull           \tmp3\().4s, \inout2\().4h, \coef1
+        smull2          \tmp4\().4s, \inout2\().8h, \coef1
+        neg             \tmp1\().4s, \tmp1\().4s
+        neg             \tmp2\().4s, \tmp2\().4s
+        rshrn           \inout2\().4h, \tmp3\().4s, #14
+        rshrn2          \inout2\().8h, \tmp4\().4s, #14
+        rshrn           \inout1\().4h, \tmp1\().4s, #14
+        rshrn2          \inout1\().8h, \tmp2\().4s, #14
+.endm
+
+.macro dsmull_h out1, out2, in, coef
+        smull           \out1\().4s, \in\().4h, \coef
+        smull2          \out2\().4s, \in\().8h, \coef
+.endm
+
+.macro drshrn_h out, in1, in2, shift
+        rshrn           \out\().4h, \in1\().4s, \shift
+        rshrn2          \out\().8h, \in2\().4s, \shift
+.endm
+
+
 // out1 = in1 + in2
 // out2 = in1 - in2
 .macro butterfly_8h out1, out2, in1, in2
@@ -463,7 +510,7 @@  function idct16x16_dc_add_neon
         ret
 endfunc
 
-function idct16
+.macro idct16_full
         dmbutterfly0    v16, v24, v16, v24, v2, v3, v4, v5, v6, v7 // v16 = t0a,  v24 = t1a
         dmbutterfly     v20, v28, v0.h[1], v0.h[2], v2, v3, v4, v5 // v20 = t2a,  v28 = t3a
         dmbutterfly     v18, v30, v0.h[3], v0.h[4], v2, v3, v4, v5 // v18 = t4a,  v30 = t7a
@@ -485,7 +532,10 @@  function idct16
         dmbutterfly0    v22, v26, v22, v26, v2, v3, v18, v19, v30, v31        // v22 = t6a,  v26 = t5a
         dmbutterfly     v23, v25, v0.h[1], v0.h[2], v18, v19, v30, v31        // v23 = t9a,  v25 = t14a
         dmbutterfly     v27, v21, v0.h[1], v0.h[2], v18, v19, v30, v31, neg=1 // v27 = t13a, v21 = t10a
+        idct16_end
+.endm
 
+.macro idct16_end
         butterfly_8h    v18, v7,  v4,  v7                // v18 = t0a,  v7  = t7a
         butterfly_8h    v19, v22, v5,  v22               // v19 = t1a,  v22 = t6
         butterfly_8h    v4,  v26, v20, v26               // v4  = t2a,  v26 = t5
@@ -507,6 +557,68 @@  function idct16
         butterfly_8h    v20, v27, v6,  v27               // v20 = out[4], v27 = out[11]
         butterfly_8h    v21, v26, v26, v3                // v21 = out[5], v26 = out[10]
         ret
+.endm
+
+function idct16
+        idct16_full
+endfunc
+
+function idct16_half
+        dmbutterfly0_h  v16, v24, v16, v24, v2, v3, v4, v5, v6, v7 // v16 = t0a,  v24 = t1a
+        dmbutterfly_h1  v20, v28, v0.h[1], v0.h[2], v2, v3, v4, v5 // v20 = t2a,  v28 = t3a
+        dmbutterfly_h1  v18, v30, v0.h[3], v0.h[4], v2, v3, v4, v5 // v18 = t4a,  v30 = t7a
+        dmbutterfly_h2  v26, v22, v0.h[5], v0.h[6], v2, v3, v4, v5 // v26 = t5a,  v22 = t6a
+        dmbutterfly_h1  v17, v31, v0.h[7], v1.h[0], v2, v3, v4, v5 // v17 = t8a,  v31 = t15a
+        dmbutterfly_h2  v25, v23, v1.h[1], v1.h[2], v2, v3, v4, v5 // v25 = t9a,  v23 = t14a
+        dmbutterfly_h1  v21, v27, v1.h[3], v1.h[4], v2, v3, v4, v5 // v21 = t10a, v27 = t13a
+        dmbutterfly_h2  v29, v19, v1.h[5], v1.h[6], v2, v3, v4, v5 // v29 = t11a, v19 = t12a
+
+        butterfly_8h    v4,  v28, v16, v28               // v4  = t0,   v28 = t3
+        butterfly_8h    v5,  v20, v24, v20               // v5  = t1,   v20 = t2
+        butterfly_8h    v6,  v26, v18, v26               // v6  = t4,   v26 = t5
+        butterfly_8h    v7,  v22, v30, v22               // v7  = t7,   v22 = t6
+        butterfly_8h    v16, v25, v17, v25               // v16 = t8,   v25 = t9
+        butterfly_8h    v24, v21, v29, v21               // v24 = t11,  v21 = t10
+        butterfly_8h    v17, v27, v19, v27               // v17 = t12,  v27 = t13
+        butterfly_8h    v29, v23, v31, v23               // v29 = t15,  v23 = t14
+
+        dmbutterfly0    v22, v26, v22, v26, v2, v3, v18, v19, v30, v31        // v22 = t6a,  v26 = t5a
+        dmbutterfly     v23, v25, v0.h[1], v0.h[2], v18, v19, v30, v31        // v23 = t9a,  v25 = t14a
+        dmbutterfly     v27, v21, v0.h[1], v0.h[2], v18, v19, v30, v31, neg=1 // v27 = t13a, v21 = t10a
+        idct16_end
+endfunc
+
+function idct16_quarter
+        dsmull_h        v24, v25, v19, v1.h[6]
+        dsmull_h        v4,  v5,  v17, v0.h[7]
+        dsmull_h        v7,  v6,  v18, v0.h[4]
+        dsmull_h        v30, v31, v18, v0.h[3]
+        neg             v24.4s,  v24.4s
+        neg             v25.4s,  v25.4s
+        dsmull_h        v29, v28, v17, v1.h[0]
+        dsmull_h        v26, v27, v19, v1.h[5]
+        dsmull_h        v22, v23, v16, v0.h[0]
+        drshrn_h        v24, v24, v25, #14
+        drshrn_h        v16, v4,  v5,  #14
+        drshrn_h        v7,  v7,  v6,  #14
+        drshrn_h        v6,  v30, v31, #14
+        drshrn_h        v29, v29, v28, #14
+        drshrn_h        v17, v26, v27, #14
+        drshrn_h        v28, v22, v23, #14
+
+        dmbutterfly_l   v20, v21, v22, v23, v17, v24, v0.h[1], v0.h[2]
+        dmbutterfly_l   v18, v19, v30, v31, v29, v16, v0.h[1], v0.h[2]
+        neg             v22.4s,  v22.4s
+        neg             v23.4s,  v23.4s
+        drshrn_h        v27, v20, v21, #14
+        drshrn_h        v21, v22, v23, #14
+        drshrn_h        v23, v18, v19, #14
+        drshrn_h        v25, v30, v31, #14
+        mov             v4.16b,  v28.16b
+        mov             v5.16b,  v28.16b
+        dmbutterfly0    v22, v26, v7,  v6,  v18, v19, v30, v31
+        mov             v20.16b, v28.16b
+        idct16_end
 endfunc
 
 function iadst16
@@ -598,6 +710,51 @@  endfunc
         st1             {v2.8h},  [\src], \inc
 .endm
 
+.macro load_add_store coef0, coef1, coef2, coef3, coef4, coef5, coef6, coef7, tmp1, tmp2
+        srshr           \coef0, \coef0, #6
+        ld1             {v2.8b},  [x0], x1
+        srshr           \coef1, \coef1, #6
+        ld1             {v3.8b},  [x3], x1
+        srshr           \coef2, \coef2, #6
+        ld1             {v4.8b},  [x0], x1
+        srshr           \coef3, \coef3, #6
+        uaddw           \coef0, \coef0, v2.8b
+        ld1             {v5.8b},  [x3], x1
+        uaddw           \coef1, \coef1, v3.8b
+        srshr           \coef4, \coef4, #6
+        ld1             {v6.8b},  [x0], x1
+        srshr           \coef5, \coef5, #6
+        ld1             {v7.8b},  [x3], x1
+        sqxtun          v2.8b,  \coef0
+        srshr           \coef6, \coef6, #6
+        sqxtun          v3.8b,  \coef1
+        srshr           \coef7, \coef7, #6
+        uaddw           \coef2, \coef2, v4.8b
+        ld1             {\tmp1},  [x0], x1
+        uaddw           \coef3, \coef3, v5.8b
+        ld1             {\tmp2},  [x3], x1
+        sqxtun          v4.8b,  \coef2
+        sub             x0,  x0,  x1, lsl #2
+        sub             x3,  x3,  x1, lsl #2
+        sqxtun          v5.8b,  \coef3
+        uaddw           \coef4, \coef4, v6.8b
+        st1             {v2.8b},  [x0], x1
+        uaddw           \coef5, \coef5, v7.8b
+        st1             {v3.8b},  [x3], x1
+        sqxtun          v6.8b,  \coef4
+        st1             {v4.8b},  [x0], x1
+        sqxtun          v7.8b,  \coef5
+        st1             {v5.8b},  [x3], x1
+        uaddw           \coef6, \coef6, \tmp1
+        st1             {v6.8b},  [x0], x1
+        uaddw           \coef7, \coef7, \tmp2
+        st1             {v7.8b},  [x3], x1
+        sqxtun          \tmp1,  \coef6
+        sqxtun          \tmp2,  \coef7
+        st1             {\tmp1},  [x0], x1
+        st1             {\tmp2},  [x3], x1
+.endm
+
 // Read a vertical 8x16 slice out of a 16x16 matrix, do a transform on it,
 // transpose into a horizontal 16x8 slice and store.
 // x0 = dst (temp buffer)
@@ -671,53 +828,8 @@  function \txfm\()16_1d_8x16_pass2_neon
         lsl             x1,  x1,  #1
         bl              \txfm\()16
 
-.macro load_add_store coef0, coef1, coef2, coef3, coef4, coef5, coef6, coef7, tmp1, tmp2
-        srshr           \coef0, \coef0, #6
-        ld1             {v2.8b},  [x0], x1
-        srshr           \coef1, \coef1, #6
-        ld1             {v3.8b},  [x3], x1
-        srshr           \coef2, \coef2, #6
-        ld1             {v4.8b},  [x0], x1
-        srshr           \coef3, \coef3, #6
-        uaddw           \coef0, \coef0, v2.8b
-        ld1             {v5.8b},  [x3], x1
-        uaddw           \coef1, \coef1, v3.8b
-        srshr           \coef4, \coef4, #6
-        ld1             {v6.8b},  [x0], x1
-        srshr           \coef5, \coef5, #6
-        ld1             {v7.8b},  [x3], x1
-        sqxtun          v2.8b,  \coef0
-        srshr           \coef6, \coef6, #6
-        sqxtun          v3.8b,  \coef1
-        srshr           \coef7, \coef7, #6
-        uaddw           \coef2, \coef2, v4.8b
-        ld1             {\tmp1},  [x0], x1
-        uaddw           \coef3, \coef3, v5.8b
-        ld1             {\tmp2},  [x3], x1
-        sqxtun          v4.8b,  \coef2
-        sub             x0,  x0,  x1, lsl #2
-        sub             x3,  x3,  x1, lsl #2
-        sqxtun          v5.8b,  \coef3
-        uaddw           \coef4, \coef4, v6.8b
-        st1             {v2.8b},  [x0], x1
-        uaddw           \coef5, \coef5, v7.8b
-        st1             {v3.8b},  [x3], x1
-        sqxtun          v6.8b,  \coef4
-        st1             {v4.8b},  [x0], x1
-        sqxtun          v7.8b,  \coef5
-        st1             {v5.8b},  [x3], x1
-        uaddw           \coef6, \coef6, \tmp1
-        st1             {v6.8b},  [x0], x1
-        uaddw           \coef7, \coef7, \tmp2
-        st1             {v7.8b},  [x3], x1
-        sqxtun          \tmp1,  \coef6
-        sqxtun          \tmp2,  \coef7
-        st1             {\tmp1},  [x0], x1
-        st1             {\tmp2},  [x3], x1
-.endm
         load_add_store  v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v16.8b, v17.8b
         load_add_store  v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h, v16.8b, v17.8b
-.purgem load_add_store
 
         br              x14
 endfunc
@@ -731,6 +843,10 @@  function ff_vp9_\txfm1\()_\txfm2\()_16x16_add_neon, export=1
 .ifc \txfm1\()_\txfm2,idct_idct
         cmp             w3,  #1
         b.eq            idct16x16_dc_add_neon
+        cmp             w3,  #10
+        b.le            idct16x16_quarter_add_neon
+        cmp             w3,  #38
+        b.le            idct16x16_half_add_neon
 .endif
         mov             x15, x30
         // iadst16 requires clobbering v8-v15, but idct16 doesn't need to.
@@ -812,6 +928,155 @@  itxfm_func16x16 iadst, idct
 itxfm_func16x16 idct,  iadst
 itxfm_func16x16 iadst, iadst
 
+function idct16_1d_8x16_pass1_quarter_neon
+        mov             x14, x30
+        movi            v2.8h, #0
+.irp i, 16, 17, 18, 19
+        load_clear      \i,  x2,  x9
+.endr
+
+        bl              idct16_quarter
+
+        // Do two 8x8 transposes. Originally, v16-v31 contain the
+        // 16 rows. Afterwards, v16-v23 and v24-v31 contain the two
+        // transposed 8x8 blocks.
+        transpose_8x8H  v16, v17, v18, v19, v20, v21, v22, v23, v2, v3
+        transpose_8x8H  v24, v25, v26, v27, v28, v29, v30, v31, v2, v3
+
+        // Store the transposed 8x8 blocks horizontally.
+        // The first 8x8 block is kept in registers for the second pass,
+        // store the rest in the temp buffer.
+        // Since only a 4x4 part of the input was nonzero, this means that
+        // only 4 rows are nonzero after transposing, and the second pass
+        // only reads the topmost 4 rows. Therefore only store the topmost
+        // 4 rows.
+        add             x0,  x0,  #16
+.irp i, 24, 25, 26, 27
+        store           \i,  x0,  x9
+.endr
+        br              x14
+endfunc
+
+function idct16_1d_8x16_pass2_quarter_neon
+        mov             x14, x30
+        cbz             x3,  1f
+.irp i, 16, 17, 18, 19
+        load            \i,  x2,  x9
+.endr
+1:
+
+        add             x3,  x0,  x1
+        lsl             x1,  x1,  #1
+        bl              idct16_quarter
+
+        load_add_store  v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v16.8b, v17.8b
+        load_add_store  v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h, v16.8b, v17.8b
+
+        br              x14
+endfunc
+
+function idct16_1d_8x16_pass1_half_neon
+        mov             x14, x30
+        movi            v2.8h, #0
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        load_clear      \i,  x2,  x9
+.endr
+
+        bl              idct16_half
+
+        // Do two 8x8 transposes. Originally, v16-v31 contain the
+        // 16 rows. Afterwards, v16-v23 and v24-v31 contain the two
+        // transposed 8x8 blocks.
+        transpose_8x8H  v16, v17, v18, v19, v20, v21, v22, v23, v2, v3
+        transpose_8x8H  v24, v25, v26, v27, v28, v29, v30, v31, v2, v3
+
+        // Store the transposed 8x8 blocks horizontally.
+        // The first 8x8 block is kept in registers for the second pass,
+        // store the rest in the temp buffer.
+        add             x0,  x0,  #16
+.irp i, 24, 25, 26, 27, 28, 29, 30, 31
+        store           \i,  x0,  x9
+.endr
+        br              x14
+endfunc
+
+function idct16_1d_8x16_pass2_half_neon
+        mov             x14, x30
+        cbz             x3,  1f
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        load            \i,  x2,  x9
+.endr
+1:
+
+        add             x3,  x0,  x1
+        lsl             x1,  x1,  #1
+        bl              idct16_half
+
+        load_add_store  v16.8h, v17.8h, v18.8h, v19.8h, v20.8h, v21.8h, v22.8h, v23.8h, v16.8b, v17.8b
+        load_add_store  v24.8h, v25.8h, v26.8h, v27.8h, v28.8h, v29.8h, v30.8h, v31.8h, v16.8b, v17.8b
+
+        br              x14
+endfunc
+
+function idct16x16_quarter_add_neon, export=1
+        mov             x15, x30
+
+        sub             sp,  sp,  #512
+
+        mov             x4,  x0
+        mov             x5,  x1
+        mov             x6,  x2
+
+        movrel          x10, idct_coeffs
+        ld1             {v0.8h,v1.8h}, [x10]
+        mov             x9, #32
+
+.irp i, 0
+        add             x0,  sp,  #(\i*32)
+        add             x2,  x6,  #(\i*2)
+        bl              idct16_1d_8x16_pass1_quarter_neon
+.endr
+.irp i, 0, 8
+        add             x0,  x4,  #(\i)
+        mov             x1,  x5
+        add             x2,  sp,  #(\i*2)
+        mov             x3,  #\i
+        bl              idct16_1d_8x16_pass2_quarter_neon
+.endr
+
+        add             sp,  sp,  #512
+        br              x15
+endfunc
+
+function idct16x16_half_add_neon, export=1
+        mov             x15, x30
+
+        sub             sp,  sp,  #512
+
+        mov             x4,  x0
+        mov             x5,  x1
+        mov             x6,  x2
+
+        movrel          x10, idct_coeffs
+        ld1             {v0.8h,v1.8h}, [x10]
+        mov             x9, #32
+
+.irp i, 0
+        add             x0,  sp,  #(\i*32)
+        add             x2,  x6,  #(\i*2)
+        bl              idct16_1d_8x16_pass1_half_neon
+.endr
+.irp i, 0, 8
+        add             x0,  x4,  #(\i)
+        mov             x1,  x5
+        add             x2,  sp,  #(\i*2)
+        mov             x3,  #\i
+        bl              idct16_1d_8x16_pass2_half_neon
+.endr
+
+        add             sp,  sp,  #512
+        br              x15
+endfunc
 
 function idct32x32_dc_add_neon
         movrel          x4, idct_coeffs
@@ -848,7 +1113,7 @@  function idct32x32_dc_add_neon
         ret
 endfunc
 
-function idct32_odd
+.macro idct32_odd_full
         ld1             {v0.8h,v1.8h}, [x11]
 
         dmbutterfly     v16, v31, v0.h[0], v0.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a
@@ -875,7 +1140,10 @@  function idct32_odd
         dmbutterfly     v27, v20, v0.h[3], v0.h[4], v16, v17, v18, v19, neg=1 // v27 = t29a, v20 = t18a
         dmbutterfly     v21, v26, v0.h[5], v0.h[6], v16, v17, v18, v19        // v21 = t21a, v26 = t26a
         dmbutterfly     v25, v22, v0.h[5], v0.h[6], v16, v17, v18, v19, neg=1 // v25 = t25a, v22 = t22a
+        idct32_end
+.endm
 
+.macro idct32_end
         butterfly_8h    v16, v5,  v4,  v5  // v16 = t16a, v5  = t19a
         butterfly_8h    v17, v20, v23, v20 // v17 = t17,  v20 = t18
         butterfly_8h    v18, v6,  v7,  v6  // v18 = t23a, v6  = t20a
@@ -904,8 +1172,91 @@  function idct32_odd
         dmbutterfly0    v25, v22, v25, v22, v2, v3, v4, v5, v6, v7 // v25 = t25,  v22 = t22
         dmbutterfly0    v24, v23, v24, v23, v2, v3, v4, v5, v6, v7 // v24 = t24a, v23 = t23a
         ret
+.endm
+
+function idct32_odd
+        idct32_odd_full
+endfunc
+
+function idct32_odd_half
+        ld1             {v0.8h,v1.8h}, [x11]
+
+        dmbutterfly_h1  v16, v31, v0.h[0], v0.h[1], v4, v5, v6, v7 // v16 = t16a, v31 = t31a
+        dmbutterfly_h2  v24, v23, v0.h[2], v0.h[3], v4, v5, v6, v7 // v24 = t17a, v23 = t30a
+        dmbutterfly_h1  v20, v27, v0.h[4], v0.h[5], v4, v5, v6, v7 // v20 = t18a, v27 = t29a
+        dmbutterfly_h2  v28, v19, v0.h[6], v0.h[7], v4, v5, v6, v7 // v28 = t19a, v19 = t28a
+        dmbutterfly_h1  v18, v29, v1.h[0], v1.h[1], v4, v5, v6, v7 // v18 = t20a, v29 = t27a
+        dmbutterfly_h2  v26, v21, v1.h[2], v1.h[3], v4, v5, v6, v7 // v26 = t21a, v21 = t26a
+        dmbutterfly_h1  v22, v25, v1.h[4], v1.h[5], v4, v5, v6, v7 // v22 = t22a, v25 = t25a
+        dmbutterfly_h2  v30, v17, v1.h[6], v1.h[7], v4, v5, v6, v7 // v30 = t23a, v17 = t24a
+
+        ld1             {v0.8h}, [x10]
+
+        butterfly_8h    v4,  v24, v16, v24 // v4  = t16, v24 = t17
+        butterfly_8h    v5,  v20, v28, v20 // v5  = t19, v20 = t18
+        butterfly_8h    v6,  v26, v18, v26 // v6  = t20, v26 = t21
+        butterfly_8h    v7,  v22, v30, v22 // v7  = t23, v22 = t22
+        butterfly_8h    v28, v25, v17, v25 // v28 = t24, v25 = t25
+        butterfly_8h    v30, v21, v29, v21 // v30 = t27, v21 = t26
+        butterfly_8h    v29, v23, v31, v23 // v29 = t31, v23 = t30
+        butterfly_8h    v31, v27, v19, v27 // v31 = t28, v27 = t29
+
+        dmbutterfly     v23, v24, v0.h[3], v0.h[4], v16, v17, v18, v19        // v23 = t17a, v24 = t30a
+        dmbutterfly     v27, v20, v0.h[3], v0.h[4], v16, v17, v18, v19, neg=1 // v27 = t29a, v20 = t18a
+        dmbutterfly     v21, v26, v0.h[5], v0.h[6], v16, v17, v18, v19        // v21 = t21a, v26 = t26a
+        dmbutterfly     v25, v22, v0.h[5], v0.h[6], v16, v17, v18, v19, neg=1 // v25 = t25a, v22 = t22a
+        idct32_end
+endfunc
+
+function idct32_odd_quarter
+        ld1             {v0.8h,v1.8h}, [x11]
+
+        dsmull_h        v4,  v5,  v16, v0.h[0]
+        dsmull_h        v28, v29, v19, v0.h[7]
+        dsmull_h        v30, v31, v16, v0.h[1]
+        dsmull_h        v22, v23, v17, v1.h[6]
+        dsmull_h        v7,  v6,  v17, v1.h[7]
+        dsmull_h        v26, v27, v19, v0.h[6]
+        dsmull_h        v20, v21, v18, v1.h[0]
+        dsmull_h        v24, v25, v18, v1.h[1]
+
+        ld1             {v0.8h}, [x10]
+
+        neg             v28.4s, v28.4s
+        neg             v29.4s, v29.4s
+        neg             v7.4s,  v7.4s
+        neg             v6.4s,  v6.4s
+
+        drshrn_h        v4,  v4,  v5,  #14
+        drshrn_h        v5,  v28, v29, #14
+        drshrn_h        v29, v30, v31, #14
+        drshrn_h        v28, v22, v23, #14
+        drshrn_h        v7,  v7,  v6,  #14
+        drshrn_h        v31, v26, v27, #14
+        drshrn_h        v6,  v20, v21, #14
+        drshrn_h        v30, v24, v25, #14
+
+        dmbutterfly_l   v16, v17, v18, v19, v29, v4,  v0.h[3], v0.h[4]
+        dmbutterfly_l   v27, v26, v20, v21, v31, v5,  v0.h[3], v0.h[4]
+        drshrn_h        v23, v16, v17, #14
+        drshrn_h        v24, v18, v19, #14
+        neg             v20.4s, v20.4s
+        neg             v21.4s, v21.4s
+        drshrn_h        v27, v27, v26, #14
+        drshrn_h        v20, v20, v21, #14
+        dmbutterfly_l   v16, v17, v18, v19, v30, v6,  v0.h[5], v0.h[6]
+        drshrn_h        v21, v16, v17, #14
+        drshrn_h        v26, v18, v19, #14
+        dmbutterfly_l   v16, v17, v18, v19, v28, v7,  v0.h[5], v0.h[6]
+        drshrn_h        v25, v16, v17, #14
+        neg             v18.4s, v18.4s
+        neg             v19.4s, v19.4s
+        drshrn_h        v22, v18, v19, #14
+
+        idct32_end
 endfunc
 
+.macro idct32_funcs suffix
 // Do an 32-point IDCT of a 8x32 slice out of a 32x32 matrix.
 // The 32-point IDCT can be decomposed into two 16-point IDCTs;
 // a normal IDCT16 with every other input component (the even ones, with
@@ -917,19 +1268,30 @@  endfunc
 // x9 = double input stride
 // x10 = idct_coeffs
 // x11 = idct_coeffs + 32
-function idct32_1d_8x32_pass1_neon
+function idct32_1d_8x32_pass1\suffix\()_neon
         mov             x14, x30
         ld1             {v0.8h,v1.8h}, [x10]
 
-        movi            v4.8h, #0
+        movi            v2.8h, #0
 
         // v16 = IN(0), v17 = IN(2) ... v31 = IN(30)
+.ifb \suffix
 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
-        ld1             {v\i\().8h}, [x2]
-        st1             {v4.8h},  [x2], x9
+        load_clear      \i, x2, x9
+.endr
+.endif
+.ifc \suffix,_quarter
+.irp i, 16, 17, 18, 19
+        load_clear      \i, x2, x9
+.endr
+.endif
+.ifc \suffix,_half
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        load_clear      \i, x2, x9
 .endr
+.endif
 
-        bl              idct16
+        bl              idct16\suffix
 
         // Do two 8x8 transposes. Originally, v16-v31 contain the
         // 16 rows. Afterwards, v16-v23 and v24-v31 contain the
@@ -964,17 +1326,36 @@  function idct32_1d_8x32_pass1_neon
 
         // Move x2 back to the start of the input, and move
         // to the first odd row
+.ifb \suffix
         sub             x2,  x2,  x9, lsl #4
+.endif
+.ifc \suffix,_quarter
+        sub             x2,  x2,  x9, lsl #2
+.endif
+.ifc \suffix,_half
+        sub             x2,  x2,  x9, lsl #3
+.endif
         add             x2,  x2,  #64
 
-        movi            v4.8h, #0
+        movi            v2.8h, #0
         // v16 = IN(1), v17 = IN(3) ... v31 = IN(31)
+.ifb \suffix
 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
-        ld1             {v\i\().8h}, [x2]
-        st1             {v4.8h},  [x2], x9
+        load_clear      \i, x2, x9
+.endr
+.endif
+.ifc \suffix,_quarter
+.irp i, 16, 17, 18, 19
+        load_clear      \i, x2, x9
+.endr
+.endif
+.ifc \suffix,_half
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        load_clear      \i, x2, x9
 .endr
+.endif
 
-        bl              idct32_odd
+        bl              idct32_odd\suffix
 
         transpose_8x8H v31, v30, v29, v28, v27, v26, v25, v24, v2, v3
         transpose_8x8H v23, v22, v21, v20, v19, v18, v17, v16, v2, v3
@@ -1023,33 +1404,61 @@  endfunc
 // x9 = double temp buffer stride
 // x10 = idct_coeffs
 // x11 = idct_coeffs + 32
-function idct32_1d_8x32_pass2_neon
+function idct32_1d_8x32_pass2\suffix\()_neon
         mov             x14, x30
         ld1             {v0.8h,v1.8h}, [x10]
 
         // v16 = IN(0), v17 = IN(2) ... v31 = IN(30)
+.ifb \suffix
 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
-        ld1             {v\i\().8h}, [x2], x9
+        load            \i, x2, x9
 .endr
         sub             x2,  x2,  x9, lsl #4
+.endif
+.ifc \suffix,_quarter
+.irp i, 16, 17, 18, 19
+        load            \i, x2, x9
+.endr
+        sub             x2,  x2,  x9, lsl #2
+.endif
+.ifc \suffix,_half
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        load            \i, x2, x9
+.endr
+        sub             x2,  x2,  x9, lsl #3
+.endif
 
-        bl              idct16
+        bl              idct16\suffix
 
 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
-        st1             {v\i\().8h}, [x2], x9
+        store           \i, x2, x9
 .endr
 
         sub             x2,  x2,  x9, lsl #4
         add             x2,  x2,  #64
 
         // v16 = IN(1), v17 = IN(3) ... v31 = IN(31)
+.ifb \suffix
 .irp i, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31
-        ld1             {v\i\().8h}, [x2], x9
+        load            \i, x2, x9
 .endr
         sub             x2,  x2,  x9, lsl #4
+.endif
+.ifc \suffix,_quarter
+.irp i, 16, 17, 18, 19
+        load            \i, x2, x9
+.endr
+        sub             x2,  x2,  x9, lsl #2
+.endif
+.ifc \suffix,_half
+.irp i, 16, 17, 18, 19, 20, 21, 22, 23
+        load            \i, x2, x9
+.endr
+        sub             x2,  x2,  x9, lsl #3
+.endif
         sub             x2,  x2,  #64
 
-        bl              idct32_odd
+        bl              idct32_odd\suffix
 
 .macro load_acc_store a, b, c, d, neg=0
 .if \neg == 0
@@ -1105,6 +1514,11 @@  function idct32_1d_8x32_pass2_neon
 .purgem load_acc_store
         br              x14
 endfunc
+.endm
+
+idct32_funcs
+idct32_funcs _quarter
+idct32_funcs _half
 
 const min_eob_idct_idct_32, align=4
         .short  0, 34, 135, 336
@@ -1113,6 +1527,10 @@  endconst
 function ff_vp9_idct_idct_32x32_add_neon, export=1
         cmp             w3,  #1
         b.eq            idct32x32_dc_add_neon
+        cmp             w3,  #34
+        b.le            idct32x32_quarter_add_neon
+        cmp             w3,  #135
+        b.le            idct32x32_half_add_neon
 
         movrel          x10, idct_coeffs
         add             x11, x10, #32
@@ -1177,3 +1595,89 @@  function ff_vp9_idct_idct_32x32_add_neon, export=1
 
         br              x15
 endfunc
+
+function idct32x32_quarter_add_neon
+        movrel          x10, idct_coeffs
+        add             x11, x10, #32
+
+        mov             x15, x30
+
+        stp             d14, d15, [sp, #-0x10]!
+        stp             d12, d13, [sp, #-0x10]!
+        stp             d10, d11, [sp, #-0x10]!
+        stp             d8,  d9,  [sp, #-0x10]!
+
+        sub             sp,  sp,  #2048
+
+        mov             x4,  x0
+        mov             x5,  x1
+        mov             x6,  x2
+
+        // Double stride of the input, since we only read every other line
+        mov             x9,  #128
+        neg             x7,  x9
+
+.irp i, 0
+        add             x0,  sp,  #(\i*64)
+        add             x2,  x6,  #(\i*2)
+        bl              idct32_1d_8x32_pass1_quarter_neon
+.endr
+.irp i, 0, 8, 16, 24
+        add             x0,  x4,  #(\i)
+        mov             x1,  x5
+        add             x2,  sp,  #(\i*2)
+        bl              idct32_1d_8x32_pass2_quarter_neon
+.endr
+
+        add             sp,  sp,  #2048
+
+        ldp             d8,  d9,  [sp], 0x10
+        ldp             d10, d11, [sp], 0x10
+        ldp             d12, d13, [sp], 0x10
+        ldp             d14, d15, [sp], 0x10
+
+        br              x15
+endfunc
+
+function idct32x32_half_add_neon
+        movrel          x10, idct_coeffs
+        add             x11, x10, #32
+
+        mov             x15, x30
+
+        stp             d14, d15, [sp, #-0x10]!
+        stp             d12, d13, [sp, #-0x10]!
+        stp             d10, d11, [sp, #-0x10]!
+        stp             d8,  d9,  [sp, #-0x10]!
+
+        sub             sp,  sp,  #2048
+
+        mov             x4,  x0
+        mov             x5,  x1
+        mov             x6,  x2
+
+        // Double stride of the input, since we only read every other line
+        mov             x9,  #128
+        neg             x7,  x9
+
+.irp i, 0, 8
+        add             x0,  sp,  #(\i*64)
+        add             x2,  x6,  #(\i*2)
+        bl              idct32_1d_8x32_pass1_half_neon
+.endr
+.irp i, 0, 8, 16, 24
+        add             x0,  x4,  #(\i)
+        mov             x1,  x5
+        add             x2,  sp,  #(\i*2)
+        bl              idct32_1d_8x32_pass2_half_neon
+.endr
+
+        add             sp,  sp,  #2048
+
+        ldp             d8,  d9,  [sp], 0x10
+        ldp             d10, d11, [sp], 0x10
+        ldp             d12, d13, [sp], 0x10
+        ldp             d14, d15, [sp], 0x10
+
+        br              x15
+endfunc