[6/6] arm: vp9lpf: Implement the mix2_44 function with one single filter pass

Message ID 1484513752-7472-6-git-send-email-martin@martin.st
State Committed
Commit 575e31e931e4178e9f1e24407503c9b4ec0ef9ba
Headers show

Commit Message

Martin Storsjö Jan. 15, 2017, 8:55 p.m.
For this case, with 8 inputs but only changing 4 of them, we can fit
all 16 input pixels into a q register, and still have enough temporary
registers for doing the loop filter.

The wd=8 filters would require too many temporary registers for
processing all 16 pixels at once though.

Before:                          Cortex A7      A8     A9     A53
vp9_loop_filter_mix2_v_44_16_neon:   289.7   256.2  237.5   181.2
After:
vp9_loop_filter_mix2_v_44_16_neon:   221.2   150.5  177.7   138.0
---
 libavcodec/arm/vp9dsp_init_arm.c |   7 +-
 libavcodec/arm/vp9lpf_neon.S     | 191 +++++++++++++++++++++++++++++++++++++++
 2 files changed, 195 insertions(+), 3 deletions(-)

Comments

Janne Grunau Feb. 10, 2017, 9:02 p.m. | #1
On 2017-01-15 22:55:52 +0200, Martin Storsjö wrote:
> For this case, with 8 inputs but only changing 4 of them, we can fit
> all 16 input pixels into a q register, and still have enough temporary
> registers for doing the loop filter.
> 
> The wd=8 filters would require too many temporary registers for
> processing all 16 pixels at once though.
> 
> Before:                          Cortex A7      A8     A9     A53
> vp9_loop_filter_mix2_v_44_16_neon:   289.7   256.2  237.5   181.2
> After:
> vp9_loop_filter_mix2_v_44_16_neon:   221.2   150.5  177.7   138.0
> ---
>  libavcodec/arm/vp9dsp_init_arm.c |   7 +-
>  libavcodec/arm/vp9lpf_neon.S     | 191 +++++++++++++++++++++++++++++++++++++++
>  2 files changed, 195 insertions(+), 3 deletions(-)
> 
> diff --git a/libavcodec/arm/vp9dsp_init_arm.c b/libavcodec/arm/vp9dsp_init_arm.c
> index e99d931..1ede170 100644
> --- a/libavcodec/arm/vp9dsp_init_arm.c
> +++ b/libavcodec/arm/vp9dsp_init_arm.c
> @@ -194,6 +194,8 @@ define_loop_filters(8, 8);
>  define_loop_filters(16, 8);
>  define_loop_filters(16, 16);
>  
> +define_loop_filters(44, 16);
> +
>  #define lf_mix_fn(dir, wd1, wd2, stridea)                                                         \
>  static void loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,                                \
>                                                       ptrdiff_t stride,                            \
> @@ -207,7 +209,6 @@ static void loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,
>      lf_mix_fn(h, wd1, wd2, stride) \
>      lf_mix_fn(v, wd1, wd2, sizeof(uint8_t))
>  
> -lf_mix_fns(4, 4)
>  lf_mix_fns(4, 8)
>  lf_mix_fns(8, 4)
>  lf_mix_fns(8, 8)
> @@ -227,8 +228,8 @@ static av_cold void vp9dsp_loopfilter_init_arm(VP9DSPContext *dsp)
>          dsp->loop_filter_16[0] = ff_vp9_loop_filter_h_16_16_neon;
>          dsp->loop_filter_16[1] = ff_vp9_loop_filter_v_16_16_neon;
>  
> -        dsp->loop_filter_mix2[0][0][0] = loop_filter_h_44_16_neon;
> -        dsp->loop_filter_mix2[0][0][1] = loop_filter_v_44_16_neon;
> +        dsp->loop_filter_mix2[0][0][0] = ff_vp9_loop_filter_h_44_16_neon;
> +        dsp->loop_filter_mix2[0][0][1] = ff_vp9_loop_filter_v_44_16_neon;
>          dsp->loop_filter_mix2[0][1][0] = loop_filter_h_48_16_neon;
>          dsp->loop_filter_mix2[0][1][1] = loop_filter_v_48_16_neon;
>          dsp->loop_filter_mix2[1][0][0] = loop_filter_h_84_16_neon;
> diff --git a/libavcodec/arm/vp9lpf_neon.S b/libavcodec/arm/vp9lpf_neon.S
> index e31c807..12984a9 100644
> --- a/libavcodec/arm/vp9lpf_neon.S
> +++ b/libavcodec/arm/vp9lpf_neon.S
> @@ -44,6 +44,109 @@
>          vtrn.8          \r2,  \r3
>  .endm
>  
> +@ The input to and output from this macro is in the registers q8-q15,
> +@ and q0-q7 are used as scratch registers.
> +@ p3 = q8, p0 = q11, q0 = q12, q3 = q15
> +.macro loop_filter_q
> +        vdup.u8         d0,  r2          @ E
> +        lsr             r2,  r2,  #8
> +        vdup.u8         d2,  r3          @ I
> +        lsr             r3,  r3,  #8
> +        vdup.u8         d1,  r2          @ E
> +        vdup.u8         d3,  r3          @ I
> +
> +        vabd.u8         q2,  q8,  q9     @ abs(p3 - p2)
> +        vabd.u8         q3,  q9,  q10    @ abs(p2 - p1)
> +        vabd.u8         q4,  q10, q11    @ abs(p1 - p0)
> +        vabd.u8         q5,  q12, q13    @ abs(q0 - q1)
> +        vabd.u8         q6,  q13, q14    @ abs(q1 - q2)
> +        vabd.u8         q7,  q14, q15    @ abs(q2 - q3)
> +        vmax.u8         q2,  q2,  q3
> +        vmax.u8         q3,  q4,  q5
> +        vmax.u8         q4,  q6,  q7
> +        vabd.u8         q5,  q11, q12    @ abs(p0 - q0)
> +        vmax.u8         q2,  q2,  q3
> +        vqadd.u8        q5,  q5,  q5     @ abs(p0 - q0) * 2
> +        vabd.u8         q7,  q10, q13    @ abs(p1 - q1)
> +        vmax.u8         q2,  q2,  q4     @ max(abs(p3 - p2), ..., abs(q2 - q3))
> +        vshr.u8         q7,  q7,  #1
> +        vcle.u8         q2,  q2,  q1     @ max(abs()) <= I
> +        vqadd.u8        q5,  q5,  q7     @ abs(p0 - q0) * 2 + abs(p1 - q1) >> 1
> +        vcle.u8         q5,  q5,  q0
> +        vand            q2,  q2,  q5     @ fm
> +
> +        vshrn.u16       d10, q2,  #4
> +        vmov            r2,  r3,  d10
> +        orrs            r2,  r2,  r3
> +        @ If no pixels need filtering, just exit as soon as possible
> +        beq             9f
> +
> +        @ Calculate the normal inner loop filter for 2 or 4 pixels
> +        ldr             r3,  [sp, #64]
> +        vabd.u8         q3,  q10, q11    @ abs(p1 - p0)
> +        vabd.u8         q4,  q13, q12    @ abs(q1 - q0)
> +
> +        vsubl.u8        q5,  d20, d26    @ p1 - q1
> +        vsubl.u8        q6,  d21, d27    @ p1 - q1
> +        vmax.u8         q3,  q3,  q4     @ max(abs(p1 - p0), abs(q1 - q0))
> +        vqmovn.s16      d10, q5          @ av_clip_int8p(p1 - q1)
> +        vqmovn.s16      d11, q6          @ av_clip_int8p(p1 - q1)
> +        vdup.u8         d8,  r3          @ H
> +        lsr             r3,  r3,  #8
> +        vdup.u8         d9,  r3          @ H
> +        vsubl.u8        q6,  d24, d22    @ q0 - p0
> +        vsubl.u8        q7,  d25, d23    @ q0 - p0
> +        vcle.u8         q3,  q3,  q4     @ hev
> +        vmov.s16        q0,  #3
> +        vand            q3,  q3,  q2     @ !hev && fm && !flat8in
> +
> +        vmul.s16        q6,  q6,  q0     @ 3 * (q0 - p0)
> +        vmul.s16        q7,  q7,  q0     @ 3 * (q0 - p0)
> +        vbic            q5,  q5,  q3     @ if (!hev) av_clip_int8 = 0
> +        vaddw.s8        q6,  q6,  d10    @ 3 * (q0 - p0) [+ av_clip_int8(p1 - q1)]
> +        vaddw.s8        q7,  q7,  d11    @ 3 * (q0 - p0) [+ av_clip_int8(p1 - q1)]
> +        vmov.s8         q5,  #4
> +        vqmovn.s16      d12, q6
> +        vqmovn.s16      d13, q7          @ av_clip_int8(3 * (q0 - p0) [+ av_clip_int8(p1 - q1)], BIT_DEPTH - 1) = f
> +        vmov.s8         q0,  #3
> +
> +        vqadd.s8        q5,  q6,  q5     @ FFMIN(f + 4, 127)
> +        vqadd.s8        q0,  q6,  q0     @ FFMIN(f + 3, 127)
> +        vmovl.u8        q6,  d22         @ p0
> +        vmovl.u8        q7,  d23         @ p0
> +        vshr.s8         q5,  q5,  #3     @ f1
> +        vshr.s8         q0,  q0,  #3     @ f2
> +
> +        vaddw.s8        q6,  q6,  d0     @ p0 + f2
> +        vaddw.s8        q7,  q7,  d1     @ p0 + f2
> +        vqmovun.s16     d0,  q6          @ out p0
> +        vmovl.u8        q6,  d24         @ q0
> +        vqmovun.s16     d1,  q7          @ out p0
> +        vmovl.u8        q7,  d25         @ q0
> +        vsubw.s8        q6,  q6,  d10    @ q0 - f1
> +        vsubw.s8        q7,  q7,  d11    @ q0 - f1
> +        vqmovun.s16     d12, q6          @ out q0
> +        vqmovun.s16     d13, q7          @ out q0
> +        vrshr.s8        q5,  q5,  #1     @ f = (f1 + 1) >> 1
> +        vbit            q11, q0,  q2     @ if (fm && !flat8in)
> +        vbit            q12, q6,  q2
> +
> +        vmovl.u8        q0,  d20         @ p1
> +        vmovl.u8        q2,  d21         @ p1
> +        vmovl.u8        q6,  d26         @ q1
> +        vmovl.u8        q7,  d27         @ q1
> +        vaddw.s8        q0,  q0,  d10    @ p1 + f
> +        vaddw.s8        q2,  q2,  d11    @ p1 + f
> +        vsubw.s8        q6,  q6,  d10    @ q1 - f
> +        vsubw.s8        q7,  q7,  d11    @ q1 - f
> +        vqmovun.s16     d0,  q0          @ out p1
> +        vqmovun.s16     d1,  q2          @ out p1
> +        vqmovun.s16     d12, q6          @ out q1
> +        vqmovun.s16     d13, q7          @ out q1
> +        vbit            q10, q0,  q3     @ if (!hev && fm && !flat8in)
> +        vbit            q13, q6,  q3
> +.endm
> +
>  @ The input to and output from this macro is in the registers d16-d31,
>  @ and d0-d7 are used as scratch registers.
>  @ p7 = d16 .. p3 = d20, p0 = d23, q0 = d24, q3 = d27, q7 = d31
> @@ -455,6 +558,94 @@ function ff_vp9_loop_filter_h_4_8_neon, export=1
>          bx              lr
>  endfunc
>  
> +function ff_vp9_loop_filter_v_44_16_neon, export=1
> +        vpush           {q4-q7}
> +        sub             r12, r0,  r1, lsl #2
> +        vld1.8          {q8},  [r12,:128], r1 @ p3
> +        vld1.8          {q12}, [r0, :128], r1 @ q0
> +        vld1.8          {q9},  [r12,:128], r1 @ p2
> +        vld1.8          {q13}, [r0, :128], r1 @ q1
> +        vld1.8          {q10}, [r12,:128], r1 @ p1
> +        vld1.8          {q14}, [r0, :128], r1 @ q2
> +        vld1.8          {q11}, [r12,:128], r1 @ p0
> +        vld1.8          {q15}, [r0, :128], r1 @ q3
> +        sub             r0,  r0,  r1, lsl #2
> +        sub             r12, r12, r1, lsl #1
> +
> +        loop_filter_q
> +
> +        vst1.8          {q10}, [r12,:128], r1
> +        vst1.8          {q12}, [r0, :128], r1
> +        vst1.8          {q11}, [r12,:128], r1
> +        vst1.8          {q13}, [r0, :128], r1
> +9:
> +        vpop            {q4-q7}
> +        bx              lr
> +endfunc
> +
> +function ff_vp9_loop_filter_h_44_16_neon, export=1
> +        vpush           {q4-q7}
> +        sub             r12, r0,  #4
> +        add             r0,  r12, r1, lsl #2
> +        vld1.8          {d16}, [r12], r1
> +        vld1.8          {d24}, [r0],  r1
> +        vld1.8          {d18}, [r12], r1
> +        vld1.8          {d26}, [r0],  r1
> +        vld1.8          {d20}, [r12], r1
> +        vld1.8          {d28}, [r0],  r1
> +        vld1.8          {d22}, [r12], r1
> +        vld1.8          {d30}, [r0],  r1
> +        mov             r12, r0
> +        add             r0,  r0,  r1, lsl #2
> +        vld1.8          {d17}, [r12], r1
> +        vld1.8          {d25}, [r0],  r1
> +        vld1.8          {d19}, [r12], r1
> +        vld1.8          {d27}, [r0],  r1
> +        vld1.8          {d21}, [r12], r1
> +        vld1.8          {d29}, [r0],  r1
> +        vld1.8          {d23}, [r12], r1
> +        vld1.8          {d31}, [r0],  r1
> +
> +        @ Transpose the 16x8 pixels, as two 8x8 parts
> +        transpose_8x8   q8,  q9,  q10, q11, q12, q13, q14, q15
> +
> +        loop_filter_q
> +
> +        sub             r12, r0,  r1, lsl #4
> +        add             r0,  r12, r1, lsl #3
> +        @ Move r0/r12 forward by 2 pixels; we don't need to rewrite the
> +        @ outermost 2 pixels since they aren't changed.
> +        add             r12, r12, #2
> +        add             r0,  r0,  #2
> +
> +        @ We only will write the mid 4 pixels back; after the loop filter,
> +        @ these are in q10, q11, q12, q13, ordered as rows (16x4 pixels).
> +        @ We need to transpose them to columns, done with a 4x4 transpose
> +        @ (which in practice is four 4x4 transposes of the 4x4 blocks of
> +        @ the 16x4 pixels; into 4x16 pixels).
> +        transpose_4x4   q10, q11, q12, q13
> +
> +        vst1.32         {d20[0]}, [r12], r1
> +        vst1.32         {d21[0]}, [r0],  r1
> +        vst1.32         {d22[0]}, [r12], r1
> +        vst1.32         {d23[0]}, [r0],  r1
> +        vst1.32         {d24[0]}, [r12], r1
> +        vst1.32         {d25[0]}, [r0],  r1
> +        vst1.32         {d26[0]}, [r12], r1
> +        vst1.32         {d27[0]}, [r0],  r1
> +        vst1.32         {d20[1]}, [r12], r1
> +        vst1.32         {d21[1]}, [r0],  r1
> +        vst1.32         {d22[1]}, [r12], r1
> +        vst1.32         {d23[1]}, [r0],  r1
> +        vst1.32         {d24[1]}, [r12], r1
> +        vst1.32         {d25[1]}, [r0],  r1
> +        vst1.32         {d26[1]}, [r12], r1
> +        vst1.32         {d27[1]}, [r0],  r1
> +9:
> +        vpop            {q4-q7}
> +        bx              lr
> +endfunc
> +
>  function ff_vp9_loop_filter_v_8_8_neon, export=1
>          sub             r12, r0,  r1, lsl #2
>          vld1.8          {d20}, [r12,:64], r1 @ p3

ok

Janne
Martin Storsjö Feb. 11, 2017, 8:51 p.m. | #2
On Fri, 10 Feb 2017, Janne Grunau wrote:

> On 2017-01-15 22:55:52 +0200, Martin Storsjö wrote:
>> For this case, with 8 inputs but only changing 4 of them, we can fit
>> all 16 input pixels into a q register, and still have enough temporary
>> registers for doing the loop filter.
>> 
>> The wd=8 filters would require too many temporary registers for
>> processing all 16 pixels at once though.
>> 
>> Before:                          Cortex A7      A8     A9     A53
>> vp9_loop_filter_mix2_v_44_16_neon:   289.7   256.2  237.5   181.2
>> After:
>> vp9_loop_filter_mix2_v_44_16_neon:   221.2   150.5  177.7   138.0
>> ---
>>  libavcodec/arm/vp9dsp_init_arm.c |   7 +-
>>  libavcodec/arm/vp9lpf_neon.S     | 191 +++++++++++++++++++++++++++++++++++++++
>>  2 files changed, 195 insertions(+), 3 deletions(-)
>> 
>> diff --git a/libavcodec/arm/vp9dsp_init_arm.c b/libavcodec/arm/vp9dsp_init_arm.c
>> index e99d931..1ede170 100644
>> --- a/libavcodec/arm/vp9dsp_init_arm.c
>> +++ b/libavcodec/arm/vp9dsp_init_arm.c
>> @@ -194,6 +194,8 @@ define_loop_filters(8, 8);
>>  define_loop_filters(16, 8);
>>  define_loop_filters(16, 16);
>> 
>> +define_loop_filters(44, 16);
>> +
>>  #define lf_mix_fn(dir, wd1, wd2, stridea)                                                         \
>>  static void loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,                                \
>>                                                       ptrdiff_t stride,                            \
>> @@ -207,7 +209,6 @@ static void loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,
>>      lf_mix_fn(h, wd1, wd2, stride) \
>>      lf_mix_fn(v, wd1, wd2, sizeof(uint8_t))
>> 
>> -lf_mix_fns(4, 4)
>>  lf_mix_fns(4, 8)
>>  lf_mix_fns(8, 4)
>>  lf_mix_fns(8, 8)
>> @@ -227,8 +228,8 @@ static av_cold void vp9dsp_loopfilter_init_arm(VP9DSPContext *dsp)
>>          dsp->loop_filter_16[0] = ff_vp9_loop_filter_h_16_16_neon;
>>          dsp->loop_filter_16[1] = ff_vp9_loop_filter_v_16_16_neon;
>> 
>> -        dsp->loop_filter_mix2[0][0][0] = loop_filter_h_44_16_neon;
>> -        dsp->loop_filter_mix2[0][0][1] = loop_filter_v_44_16_neon;
>> +        dsp->loop_filter_mix2[0][0][0] = ff_vp9_loop_filter_h_44_16_neon;
>> +        dsp->loop_filter_mix2[0][0][1] = ff_vp9_loop_filter_v_44_16_neon;
>>          dsp->loop_filter_mix2[0][1][0] = loop_filter_h_48_16_neon;
>>          dsp->loop_filter_mix2[0][1][1] = loop_filter_v_48_16_neon;
>>          dsp->loop_filter_mix2[1][0][0] = loop_filter_h_84_16_neon;
>> diff --git a/libavcodec/arm/vp9lpf_neon.S b/libavcodec/arm/vp9lpf_neon.S
>> index e31c807..12984a9 100644
>> --- a/libavcodec/arm/vp9lpf_neon.S
>> +++ b/libavcodec/arm/vp9lpf_neon.S
>> @@ -44,6 +44,109 @@
>>          vtrn.8          \r2,  \r3
>>  .endm
>> 
>> +@ The input to and output from this macro is in the registers q8-q15,
>> +@ and q0-q7 are used as scratch registers.
>> +@ p3 = q8, p0 = q11, q0 = q12, q3 = q15
>> +.macro loop_filter_q
>> +        vdup.u8         d0,  r2          @ E
>> +        lsr             r2,  r2,  #8
>> +        vdup.u8         d2,  r3          @ I
>> +        lsr             r3,  r3,  #8
>> +        vdup.u8         d1,  r2          @ E
>> +        vdup.u8         d3,  r3          @ I

I tried implementing your suggestion with uzp here, but it ended up being 
slower actually. With the version of the patch I posted here:

vp9_loop_filter_mix2_v_44_16_neon:   221.2   150.5  185.0   139.0

With this block replaced with this:

         vdup.u16        q0,  r2          @ E
         vdup.u16        q1,  r3          @ I
         vuzp.u8         d0,  d1          @ E
         vuzp.u8         d2,  d3          @ I

I get the following:

vp9_loop_filter_mix2_v_44_16_neon:   223.2   150.5  186.1   142.0

I.e. 1-3 cycles slower on A7, A9 and A53, identical on A8.

// Martin
Martin Storsjö Feb. 11, 2017, 9:42 p.m. | #3
On Sat, 11 Feb 2017, Martin Storsjö wrote:

> On Fri, 10 Feb 2017, Janne Grunau wrote:
>
>> On 2017-01-15 22:55:52 +0200, Martin Storsjö wrote:
>>> For this case, with 8 inputs but only changing 4 of them, we can fit
>>> all 16 input pixels into a q register, and still have enough temporary
>>> registers for doing the loop filter.
>>> 
>>> The wd=8 filters would require too many temporary registers for
>>> processing all 16 pixels at once though.
>>> 
>>> Before:                          Cortex A7      A8     A9     A53
>>> vp9_loop_filter_mix2_v_44_16_neon:   289.7   256.2  237.5   181.2
>>> After:
>>> vp9_loop_filter_mix2_v_44_16_neon:   221.2   150.5  177.7   138.0
>>> ---
>>>  libavcodec/arm/vp9dsp_init_arm.c |   7 +-
>>>  libavcodec/arm/vp9lpf_neon.S     | 191 
> +++++++++++++++++++++++++++++++++++++++
>>>  2 files changed, 195 insertions(+), 3 deletions(-)
>>> 
>>> diff --git a/libavcodec/arm/vp9dsp_init_arm.c 
> b/libavcodec/arm/vp9dsp_init_arm.c
>>> index e99d931..1ede170 100644
>>> --- a/libavcodec/arm/vp9dsp_init_arm.c
>>> +++ b/libavcodec/arm/vp9dsp_init_arm.c
>>> @@ -194,6 +194,8 @@ define_loop_filters(8, 8);
>>>  define_loop_filters(16, 8);
>>>  define_loop_filters(16, 16);
>>> 
>>> +define_loop_filters(44, 16);
>>> +
>>>  #define lf_mix_fn(dir, wd1, wd2, stridea) 
> \
>>>  static void loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst, 
> \
>>>                                                       ptrdiff_t stride, 
> \
>>> @@ -207,7 +209,6 @@ static void 
> loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,
>>>      lf_mix_fn(h, wd1, wd2, stride) \
>>>      lf_mix_fn(v, wd1, wd2, sizeof(uint8_t))
>>> 
>>> -lf_mix_fns(4, 4)
>>>  lf_mix_fns(4, 8)
>>>  lf_mix_fns(8, 4)
>>>  lf_mix_fns(8, 8)
>>> @@ -227,8 +228,8 @@ static av_cold void 
> vp9dsp_loopfilter_init_arm(VP9DSPContext *dsp)
>>>          dsp->loop_filter_16[0] = ff_vp9_loop_filter_h_16_16_neon;
>>>          dsp->loop_filter_16[1] = ff_vp9_loop_filter_v_16_16_neon;
>>> 
>>> -        dsp->loop_filter_mix2[0][0][0] = loop_filter_h_44_16_neon;
>>> -        dsp->loop_filter_mix2[0][0][1] = loop_filter_v_44_16_neon;
>>> +        dsp->loop_filter_mix2[0][0][0] = ff_vp9_loop_filter_h_44_16_neon;
>>> +        dsp->loop_filter_mix2[0][0][1] = ff_vp9_loop_filter_v_44_16_neon;
>>>          dsp->loop_filter_mix2[0][1][0] = loop_filter_h_48_16_neon;
>>>          dsp->loop_filter_mix2[0][1][1] = loop_filter_v_48_16_neon;
>>>          dsp->loop_filter_mix2[1][0][0] = loop_filter_h_84_16_neon;
>>> diff --git a/libavcodec/arm/vp9lpf_neon.S b/libavcodec/arm/vp9lpf_neon.S
>>> index e31c807..12984a9 100644
>>> --- a/libavcodec/arm/vp9lpf_neon.S
>>> +++ b/libavcodec/arm/vp9lpf_neon.S
>>> @@ -44,6 +44,109 @@
>>>          vtrn.8          \r2,  \r3
>>>  .endm
>>> 
>>> +@ The input to and output from this macro is in the registers q8-q15,
>>> +@ and q0-q7 are used as scratch registers.
>>> +@ p3 = q8, p0 = q11, q0 = q12, q3 = q15
>>> +.macro loop_filter_q
>>> +        vdup.u8         d0,  r2          @ E
>>> +        lsr             r2,  r2,  #8
>>> +        vdup.u8         d2,  r3          @ I
>>> +        lsr             r3,  r3,  #8
>>> +        vdup.u8         d1,  r2          @ E
>>> +        vdup.u8         d3,  r3          @ I
>
> I tried implementing your suggestion with uzp here, but it ended up being 
> slower actually. With the version of the patch I posted here:
>
> vp9_loop_filter_mix2_v_44_16_neon:   221.2   150.5  185.0   139.0
>
> With this block replaced with this:
>
>         vdup.u16        q0,  r2          @ E
>         vdup.u16        q1,  r3          @ I
>         vuzp.u8         d0,  d1          @ E
>         vuzp.u8         d2,  d3          @ I
>
> I get the following:
>
> vp9_loop_filter_mix2_v_44_16_neon:   223.2   150.5  186.1   142.0
>
> I.e. 1-3 cycles slower on A7, A9 and A53, identical on A8.

If I move the two vuzp further down, I get the following:

vp9_loop_filter_mix2_v_44_16_neon:   223.2   148.5  185.1   141.0

I.e. +2 on A7, -2 on A8, 0 on A9, +2 on A53. So on average it's still 
worse, even though it codewise is neater.

// Martin
Janne Grunau Feb. 23, 2017, 8:30 p.m. | #4
On 2017-02-11 23:42:05 +0200, Martin Storsjö wrote:
> On Sat, 11 Feb 2017, Martin Storsjö wrote:
> 
> >On Fri, 10 Feb 2017, Janne Grunau wrote:
> >
> >>On 2017-01-15 22:55:52 +0200, Martin Storsjö wrote:
> >>>For this case, with 8 inputs but only changing 4 of them, we can fit
> >>>all 16 input pixels into a q register, and still have enough temporary
> >>>registers for doing the loop filter.
> >>>
> >>>The wd=8 filters would require too many temporary registers for
> >>>processing all 16 pixels at once though.
> >>>
> >>>Before:                          Cortex A7      A8     A9     A53
> >>>vp9_loop_filter_mix2_v_44_16_neon:   289.7   256.2  237.5   181.2
> >>>After:
> >>>vp9_loop_filter_mix2_v_44_16_neon:   221.2   150.5  177.7   138.0
> >>>---
> >>> libavcodec/arm/vp9dsp_init_arm.c |   7 +-
> >>> libavcodec/arm/vp9lpf_neon.S     | 191
> >+++++++++++++++++++++++++++++++++++++++
> >>> 2 files changed, 195 insertions(+), 3 deletions(-)
> >>>
> >>>diff --git a/libavcodec/arm/vp9dsp_init_arm.c
> >b/libavcodec/arm/vp9dsp_init_arm.c
> >>>index e99d931..1ede170 100644
> >>>--- a/libavcodec/arm/vp9dsp_init_arm.c
> >>>+++ b/libavcodec/arm/vp9dsp_init_arm.c
> >>>@@ -194,6 +194,8 @@ define_loop_filters(8, 8);
> >>> define_loop_filters(16, 8);
> >>> define_loop_filters(16, 16);
> >>>
> >>>+define_loop_filters(44, 16);
> >>>+
> >>> #define lf_mix_fn(dir, wd1, wd2, stridea)
> >\
> >>> static void loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,
> >\
> >>>                                                      ptrdiff_t
> >>>stride,
> >\
> >>>@@ -207,7 +209,6 @@ static void
> >loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,
> >>>     lf_mix_fn(h, wd1, wd2, stride) \
> >>>     lf_mix_fn(v, wd1, wd2, sizeof(uint8_t))
> >>>
> >>>-lf_mix_fns(4, 4)
> >>> lf_mix_fns(4, 8)
> >>> lf_mix_fns(8, 4)
> >>> lf_mix_fns(8, 8)
> >>>@@ -227,8 +228,8 @@ static av_cold void
> >vp9dsp_loopfilter_init_arm(VP9DSPContext *dsp)
> >>>         dsp->loop_filter_16[0] = ff_vp9_loop_filter_h_16_16_neon;
> >>>         dsp->loop_filter_16[1] = ff_vp9_loop_filter_v_16_16_neon;
> >>>
> >>>-        dsp->loop_filter_mix2[0][0][0] = loop_filter_h_44_16_neon;
> >>>-        dsp->loop_filter_mix2[0][0][1] = loop_filter_v_44_16_neon;
> >>>+        dsp->loop_filter_mix2[0][0][0] = ff_vp9_loop_filter_h_44_16_neon;
> >>>+        dsp->loop_filter_mix2[0][0][1] = ff_vp9_loop_filter_v_44_16_neon;
> >>>         dsp->loop_filter_mix2[0][1][0] = loop_filter_h_48_16_neon;
> >>>         dsp->loop_filter_mix2[0][1][1] = loop_filter_v_48_16_neon;
> >>>         dsp->loop_filter_mix2[1][0][0] = loop_filter_h_84_16_neon;
> >>>diff --git a/libavcodec/arm/vp9lpf_neon.S b/libavcodec/arm/vp9lpf_neon.S
> >>>index e31c807..12984a9 100644
> >>>--- a/libavcodec/arm/vp9lpf_neon.S
> >>>+++ b/libavcodec/arm/vp9lpf_neon.S
> >>>@@ -44,6 +44,109 @@
> >>>         vtrn.8          \r2,  \r3
> >>> .endm
> >>>
> >>>+@ The input to and output from this macro is in the registers q8-q15,
> >>>+@ and q0-q7 are used as scratch registers.
> >>>+@ p3 = q8, p0 = q11, q0 = q12, q3 = q15
> >>>+.macro loop_filter_q
> >>>+        vdup.u8         d0,  r2          @ E
> >>>+        lsr             r2,  r2,  #8
> >>>+        vdup.u8         d2,  r3          @ I
> >>>+        lsr             r3,  r3,  #8
> >>>+        vdup.u8         d1,  r2          @ E
> >>>+        vdup.u8         d3,  r3          @ I
> >
> >I tried implementing your suggestion with uzp here, but it ended up being
> >slower actually. With the version of the patch I posted here:
> >
> >vp9_loop_filter_mix2_v_44_16_neon:   221.2   150.5  185.0   139.0
> >
> >With this block replaced with this:
> >
> >        vdup.u16        q0,  r2          @ E
> >        vdup.u16        q1,  r3          @ I
> >        vuzp.u8         d0,  d1          @ E
> >        vuzp.u8         d2,  d3          @ I
> >
> >I get the following:
> >
> >vp9_loop_filter_mix2_v_44_16_neon:   223.2   150.5  186.1   142.0
> >
> >I.e. 1-3 cycles slower on A7, A9 and A53, identical on A8.
> 
> If I move the two vuzp further down, I get the following:
> 
> vp9_loop_filter_mix2_v_44_16_neon:   223.2   148.5  185.1   141.0
> 
> I.e. +2 on A7, -2 on A8, 0 on A9, +2 on A53. So on average it's still worse,
> even though it codewise is neater.

leave it as it was then

Janne

Patch

diff --git a/libavcodec/arm/vp9dsp_init_arm.c b/libavcodec/arm/vp9dsp_init_arm.c
index e99d931..1ede170 100644
--- a/libavcodec/arm/vp9dsp_init_arm.c
+++ b/libavcodec/arm/vp9dsp_init_arm.c
@@ -194,6 +194,8 @@  define_loop_filters(8, 8);
 define_loop_filters(16, 8);
 define_loop_filters(16, 16);
 
+define_loop_filters(44, 16);
+
 #define lf_mix_fn(dir, wd1, wd2, stridea)                                                         \
 static void loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,                                \
                                                      ptrdiff_t stride,                            \
@@ -207,7 +209,6 @@  static void loop_filter_##dir##_##wd1##wd2##_16_neon(uint8_t *dst,
     lf_mix_fn(h, wd1, wd2, stride) \
     lf_mix_fn(v, wd1, wd2, sizeof(uint8_t))
 
-lf_mix_fns(4, 4)
 lf_mix_fns(4, 8)
 lf_mix_fns(8, 4)
 lf_mix_fns(8, 8)
@@ -227,8 +228,8 @@  static av_cold void vp9dsp_loopfilter_init_arm(VP9DSPContext *dsp)
         dsp->loop_filter_16[0] = ff_vp9_loop_filter_h_16_16_neon;
         dsp->loop_filter_16[1] = ff_vp9_loop_filter_v_16_16_neon;
 
-        dsp->loop_filter_mix2[0][0][0] = loop_filter_h_44_16_neon;
-        dsp->loop_filter_mix2[0][0][1] = loop_filter_v_44_16_neon;
+        dsp->loop_filter_mix2[0][0][0] = ff_vp9_loop_filter_h_44_16_neon;
+        dsp->loop_filter_mix2[0][0][1] = ff_vp9_loop_filter_v_44_16_neon;
         dsp->loop_filter_mix2[0][1][0] = loop_filter_h_48_16_neon;
         dsp->loop_filter_mix2[0][1][1] = loop_filter_v_48_16_neon;
         dsp->loop_filter_mix2[1][0][0] = loop_filter_h_84_16_neon;
diff --git a/libavcodec/arm/vp9lpf_neon.S b/libavcodec/arm/vp9lpf_neon.S
index e31c807..12984a9 100644
--- a/libavcodec/arm/vp9lpf_neon.S
+++ b/libavcodec/arm/vp9lpf_neon.S
@@ -44,6 +44,109 @@ 
         vtrn.8          \r2,  \r3
 .endm
 
+@ The input to and output from this macro is in the registers q8-q15,
+@ and q0-q7 are used as scratch registers.
+@ p3 = q8, p0 = q11, q0 = q12, q3 = q15
+.macro loop_filter_q
+        vdup.u8         d0,  r2          @ E
+        lsr             r2,  r2,  #8
+        vdup.u8         d2,  r3          @ I
+        lsr             r3,  r3,  #8
+        vdup.u8         d1,  r2          @ E
+        vdup.u8         d3,  r3          @ I
+
+        vabd.u8         q2,  q8,  q9     @ abs(p3 - p2)
+        vabd.u8         q3,  q9,  q10    @ abs(p2 - p1)
+        vabd.u8         q4,  q10, q11    @ abs(p1 - p0)
+        vabd.u8         q5,  q12, q13    @ abs(q0 - q1)
+        vabd.u8         q6,  q13, q14    @ abs(q1 - q2)
+        vabd.u8         q7,  q14, q15    @ abs(q2 - q3)
+        vmax.u8         q2,  q2,  q3
+        vmax.u8         q3,  q4,  q5
+        vmax.u8         q4,  q6,  q7
+        vabd.u8         q5,  q11, q12    @ abs(p0 - q0)
+        vmax.u8         q2,  q2,  q3
+        vqadd.u8        q5,  q5,  q5     @ abs(p0 - q0) * 2
+        vabd.u8         q7,  q10, q13    @ abs(p1 - q1)
+        vmax.u8         q2,  q2,  q4     @ max(abs(p3 - p2), ..., abs(q2 - q3))
+        vshr.u8         q7,  q7,  #1
+        vcle.u8         q2,  q2,  q1     @ max(abs()) <= I
+        vqadd.u8        q5,  q5,  q7     @ abs(p0 - q0) * 2 + abs(p1 - q1) >> 1
+        vcle.u8         q5,  q5,  q0
+        vand            q2,  q2,  q5     @ fm
+
+        vshrn.u16       d10, q2,  #4
+        vmov            r2,  r3,  d10
+        orrs            r2,  r2,  r3
+        @ If no pixels need filtering, just exit as soon as possible
+        beq             9f
+
+        @ Calculate the normal inner loop filter for 2 or 4 pixels
+        ldr             r3,  [sp, #64]
+        vabd.u8         q3,  q10, q11    @ abs(p1 - p0)
+        vabd.u8         q4,  q13, q12    @ abs(q1 - q0)
+
+        vsubl.u8        q5,  d20, d26    @ p1 - q1
+        vsubl.u8        q6,  d21, d27    @ p1 - q1
+        vmax.u8         q3,  q3,  q4     @ max(abs(p1 - p0), abs(q1 - q0))
+        vqmovn.s16      d10, q5          @ av_clip_int8p(p1 - q1)
+        vqmovn.s16      d11, q6          @ av_clip_int8p(p1 - q1)
+        vdup.u8         d8,  r3          @ H
+        lsr             r3,  r3,  #8
+        vdup.u8         d9,  r3          @ H
+        vsubl.u8        q6,  d24, d22    @ q0 - p0
+        vsubl.u8        q7,  d25, d23    @ q0 - p0
+        vcle.u8         q3,  q3,  q4     @ hev
+        vmov.s16        q0,  #3
+        vand            q3,  q3,  q2     @ !hev && fm && !flat8in
+
+        vmul.s16        q6,  q6,  q0     @ 3 * (q0 - p0)
+        vmul.s16        q7,  q7,  q0     @ 3 * (q0 - p0)
+        vbic            q5,  q5,  q3     @ if (!hev) av_clip_int8 = 0
+        vaddw.s8        q6,  q6,  d10    @ 3 * (q0 - p0) [+ av_clip_int8(p1 - q1)]
+        vaddw.s8        q7,  q7,  d11    @ 3 * (q0 - p0) [+ av_clip_int8(p1 - q1)]
+        vmov.s8         q5,  #4
+        vqmovn.s16      d12, q6
+        vqmovn.s16      d13, q7          @ av_clip_int8(3 * (q0 - p0) [+ av_clip_int8(p1 - q1)], BIT_DEPTH - 1) = f
+        vmov.s8         q0,  #3
+
+        vqadd.s8        q5,  q6,  q5     @ FFMIN(f + 4, 127)
+        vqadd.s8        q0,  q6,  q0     @ FFMIN(f + 3, 127)
+        vmovl.u8        q6,  d22         @ p0
+        vmovl.u8        q7,  d23         @ p0
+        vshr.s8         q5,  q5,  #3     @ f1
+        vshr.s8         q0,  q0,  #3     @ f2
+
+        vaddw.s8        q6,  q6,  d0     @ p0 + f2
+        vaddw.s8        q7,  q7,  d1     @ p0 + f2
+        vqmovun.s16     d0,  q6          @ out p0
+        vmovl.u8        q6,  d24         @ q0
+        vqmovun.s16     d1,  q7          @ out p0
+        vmovl.u8        q7,  d25         @ q0
+        vsubw.s8        q6,  q6,  d10    @ q0 - f1
+        vsubw.s8        q7,  q7,  d11    @ q0 - f1
+        vqmovun.s16     d12, q6          @ out q0
+        vqmovun.s16     d13, q7          @ out q0
+        vrshr.s8        q5,  q5,  #1     @ f = (f1 + 1) >> 1
+        vbit            q11, q0,  q2     @ if (fm && !flat8in)
+        vbit            q12, q6,  q2
+
+        vmovl.u8        q0,  d20         @ p1
+        vmovl.u8        q2,  d21         @ p1
+        vmovl.u8        q6,  d26         @ q1
+        vmovl.u8        q7,  d27         @ q1
+        vaddw.s8        q0,  q0,  d10    @ p1 + f
+        vaddw.s8        q2,  q2,  d11    @ p1 + f
+        vsubw.s8        q6,  q6,  d10    @ q1 - f
+        vsubw.s8        q7,  q7,  d11    @ q1 - f
+        vqmovun.s16     d0,  q0          @ out p1
+        vqmovun.s16     d1,  q2          @ out p1
+        vqmovun.s16     d12, q6          @ out q1
+        vqmovun.s16     d13, q7          @ out q1
+        vbit            q10, q0,  q3     @ if (!hev && fm && !flat8in)
+        vbit            q13, q6,  q3
+.endm
+
 @ The input to and output from this macro is in the registers d16-d31,
 @ and d0-d7 are used as scratch registers.
 @ p7 = d16 .. p3 = d20, p0 = d23, q0 = d24, q3 = d27, q7 = d31
@@ -455,6 +558,94 @@  function ff_vp9_loop_filter_h_4_8_neon, export=1
         bx              lr
 endfunc
 
+function ff_vp9_loop_filter_v_44_16_neon, export=1
+        vpush           {q4-q7}
+        sub             r12, r0,  r1, lsl #2
+        vld1.8          {q8},  [r12,:128], r1 @ p3
+        vld1.8          {q12}, [r0, :128], r1 @ q0
+        vld1.8          {q9},  [r12,:128], r1 @ p2
+        vld1.8          {q13}, [r0, :128], r1 @ q1
+        vld1.8          {q10}, [r12,:128], r1 @ p1
+        vld1.8          {q14}, [r0, :128], r1 @ q2
+        vld1.8          {q11}, [r12,:128], r1 @ p0
+        vld1.8          {q15}, [r0, :128], r1 @ q3
+        sub             r0,  r0,  r1, lsl #2
+        sub             r12, r12, r1, lsl #1
+
+        loop_filter_q
+
+        vst1.8          {q10}, [r12,:128], r1
+        vst1.8          {q12}, [r0, :128], r1
+        vst1.8          {q11}, [r12,:128], r1
+        vst1.8          {q13}, [r0, :128], r1
+9:
+        vpop            {q4-q7}
+        bx              lr
+endfunc
+
+function ff_vp9_loop_filter_h_44_16_neon, export=1
+        vpush           {q4-q7}
+        sub             r12, r0,  #4
+        add             r0,  r12, r1, lsl #2
+        vld1.8          {d16}, [r12], r1
+        vld1.8          {d24}, [r0],  r1
+        vld1.8          {d18}, [r12], r1
+        vld1.8          {d26}, [r0],  r1
+        vld1.8          {d20}, [r12], r1
+        vld1.8          {d28}, [r0],  r1
+        vld1.8          {d22}, [r12], r1
+        vld1.8          {d30}, [r0],  r1
+        mov             r12, r0
+        add             r0,  r0,  r1, lsl #2
+        vld1.8          {d17}, [r12], r1
+        vld1.8          {d25}, [r0],  r1
+        vld1.8          {d19}, [r12], r1
+        vld1.8          {d27}, [r0],  r1
+        vld1.8          {d21}, [r12], r1
+        vld1.8          {d29}, [r0],  r1
+        vld1.8          {d23}, [r12], r1
+        vld1.8          {d31}, [r0],  r1
+
+        @ Transpose the 16x8 pixels, as two 8x8 parts
+        transpose_8x8   q8,  q9,  q10, q11, q12, q13, q14, q15
+
+        loop_filter_q
+
+        sub             r12, r0,  r1, lsl #4
+        add             r0,  r12, r1, lsl #3
+        @ Move r0/r12 forward by 2 pixels; we don't need to rewrite the
+        @ outermost 2 pixels since they aren't changed.
+        add             r12, r12, #2
+        add             r0,  r0,  #2
+
+        @ We only will write the mid 4 pixels back; after the loop filter,
+        @ these are in q10, q11, q12, q13, ordered as rows (16x4 pixels).
+        @ We need to transpose them to columns, done with a 4x4 transpose
+        @ (which in practice is four 4x4 transposes of the 4x4 blocks of
+        @ the 16x4 pixels; into 4x16 pixels).
+        transpose_4x4   q10, q11, q12, q13
+
+        vst1.32         {d20[0]}, [r12], r1
+        vst1.32         {d21[0]}, [r0],  r1
+        vst1.32         {d22[0]}, [r12], r1
+        vst1.32         {d23[0]}, [r0],  r1
+        vst1.32         {d24[0]}, [r12], r1
+        vst1.32         {d25[0]}, [r0],  r1
+        vst1.32         {d26[0]}, [r12], r1
+        vst1.32         {d27[0]}, [r0],  r1
+        vst1.32         {d20[1]}, [r12], r1
+        vst1.32         {d21[1]}, [r0],  r1
+        vst1.32         {d22[1]}, [r12], r1
+        vst1.32         {d23[1]}, [r0],  r1
+        vst1.32         {d24[1]}, [r12], r1
+        vst1.32         {d25[1]}, [r0],  r1
+        vst1.32         {d26[1]}, [r12], r1
+        vst1.32         {d27[1]}, [r0],  r1
+9:
+        vpop            {q4-q7}
+        bx              lr
+endfunc
+
 function ff_vp9_loop_filter_v_8_8_neon, export=1
         sub             r12, r0,  r1, lsl #2
         vld1.8          {d20}, [r12,:64], r1 @ p3