h264: Fix invalid interlaced/ progressive MB combinations for direct mode prediction.

Message ID 20120314210240.1E6195E088@aruru.libav.org
State New
Headers show

Commit Message

Janne Grunau March 14, 2012, 9:02 p.m.
Module: libav
Branch: release/0.8
Commit: de0ff4ce69c311a2879e10143f1cc2c4945f3ef0

Author:    Michael Niedermayer <michaelni@gmx.at>
Committer: Reinhard Tartler <siretart@tauware.de>
Date:      Mon Mar 12 18:26:50 2012 -0700

h264: Fix invalid interlaced/progressive MB combinations for direct mode prediction.

Found-by: Mateusz "j00ru" Jurczyk and Gynvael Coldwind
CC: libav-stable@libav.org
Signed-off-by: Ronald S. Bultje <rsbultje@gmail.com>
(cherry picked from commit 758ec111538ccd487686e8677aa754ee4d82beaa)

Signed-off-by: Reinhard Tartler <siretart@tauware.de>

---

 libavcodec/h264_direct.c |    8 ++++++++
 1 files changed, 8 insertions(+), 0 deletions(-)

Patch

diff --git a/libavcodec/h264_direct.c b/libavcodec/h264_direct.c
index a953728..4f70ff8 100644
--- a/libavcodec/h264_direct.c
+++ b/libavcodec/h264_direct.c
@@ -252,6 +252,10 @@  static void pred_spatial_direct_motion(H264Context * const h, int *mb_type){
             mb_type_col[1] = h->ref_list[1][0].f.mb_type[mb_xy + s->mb_stride];
             b8_stride = 2+4*s->mb_stride;
             b4_stride *= 6;
+            if (IS_INTERLACED(mb_type_col[0]) != IS_INTERLACED(mb_type_col[1])) {
+                mb_type_col[0] &= ~MB_TYPE_INTERLACED;
+                mb_type_col[1] &= ~MB_TYPE_INTERLACED;
+            }
 
             sub_mb_type |= MB_TYPE_16x16|MB_TYPE_DIRECT2; /* B_SUB_8x8 */
             if(    (mb_type_col[0] & MB_TYPE_16x16_OR_INTRA)
@@ -438,6 +442,10 @@  static void pred_temp_direct_motion(H264Context * const h, int *mb_type){
             mb_type_col[1] = h->ref_list[1][0].f.mb_type[mb_xy + s->mb_stride];
             b8_stride = 2+4*s->mb_stride;
             b4_stride *= 6;
+            if (IS_INTERLACED(mb_type_col[0]) != IS_INTERLACED(mb_type_col[1])) {
+                mb_type_col[0] &= ~MB_TYPE_INTERLACED;
+                mb_type_col[1] &= ~MB_TYPE_INTERLACED;
+            }
 
             sub_mb_type = MB_TYPE_16x16|MB_TYPE_P0L0|MB_TYPE_P0L1|MB_TYPE_DIRECT2; /* B_SUB_8x8 */