[1/3] put_bits: bounds check buffer

Message ID 20170228191155.13636-1-stebbins@jetheaddev.com
State New
Headers show

Commit Message

John Stebbins Feb. 28, 2017, 7:11 p.m.
This prevents invalid writes outside put_bits' buffer.

It also has the side effect of allowing measurement of the required
size of a buffer without the need to pre-allocate an over-sized buffer.

This fixes a crash in aacenc.c where it could write past the end of the
allocated packet, which is allocated to be the max size allowed by the
aac spec.  aacenc.c uses the above feature to check the size
of encoded data and try again when the size is too large.
---
 libavcodec/aacenc.c   |  2 +-
 libavcodec/put_bits.h | 25 ++++++++++++++++++++-----
 2 files changed, 21 insertions(+), 6 deletions(-)

Patch

diff --git a/libavcodec/aacenc.c b/libavcodec/aacenc.c
index 9b0e99b..a83b2a8 100644
--- a/libavcodec/aacenc.c
+++ b/libavcodec/aacenc.c
@@ -638,7 +638,7 @@  static int aac_encode_frame(AVCodecContext *avctx, AVPacket *avpkt,
             start_ch += chans;
         }
 
-        frame_bits = put_bits_count(&s->pb);
+        frame_bits = put_bits_attempted(&s->pb);
         if (frame_bits <= 6144 * s->channels - 3) {
             s->psy.bitres.bits = frame_bits / s->channels;
             break;
diff --git a/libavcodec/put_bits.h b/libavcodec/put_bits.h
index 17666fa..677fecd 100644
--- a/libavcodec/put_bits.h
+++ b/libavcodec/put_bits.h
@@ -30,6 +30,7 @@ 
 #include <stddef.h>
 #include <assert.h>
 
+#include "libavutil/common.h"
 #include "libavutil/intreadwrite.h"
 
 typedef struct PutBitContext {
@@ -62,11 +63,19 @@  static inline void init_put_bits(PutBitContext *s, uint8_t *buffer,
 }
 
 /**
+ * @return the total number of bits attempted to be written to the bitstream.
+ */
+static inline int put_bits_attempted(PutBitContext *s)
+{
+    return (s->buf_ptr - s->buf) * 8 + 32 - s->bit_left;
+}
+
+/**
  * @return the total number of bits written to the bitstream.
  */
 static inline int put_bits_count(PutBitContext *s)
 {
-    return (s->buf_ptr - s->buf) * 8 + 32 - s->bit_left;
+    return FFMIN(s->size_in_bits, put_bits_attempted(s));
 }
 
 /**
@@ -89,10 +98,14 @@  static inline void flush_put_bits(PutBitContext *s)
     while (s->bit_left < 32) {
         /* XXX: should test end of buffer */
 #ifdef BITSTREAM_WRITER_LE
-        *s->buf_ptr++ = s->bit_buf;
+        if (s->buf_ptr < s->buf_end)
+            *s->buf_ptr = s->bit_buf;
+        s->buf_ptr++;
         s->bit_buf  >>= 8;
 #else
-        *s->buf_ptr++ = s->bit_buf >> 24;
+        if (s->buf_ptr < s->buf_end)
+            *s->buf_ptr = s->bit_buf >> 24;
+        s->buf_ptr++;
         s->bit_buf  <<= 8;
 #endif
         s->bit_left  += 8;
@@ -145,7 +158,8 @@  static inline void put_bits(PutBitContext *s, int n, unsigned int value)
 #ifdef BITSTREAM_WRITER_LE
     bit_buf |= value << (32 - bit_left);
     if (n >= bit_left) {
-        AV_WL32(s->buf_ptr, bit_buf);
+        if (s->buf_ptr < s->buf_end)
+            AV_WL32(s->buf_ptr, bit_buf);
         s->buf_ptr += 4;
         bit_buf     = (bit_left == 32) ? 0 : value >> bit_left;
         bit_left   += 32;
@@ -158,7 +172,8 @@  static inline void put_bits(PutBitContext *s, int n, unsigned int value)
     } else {
         bit_buf   <<= bit_left;
         bit_buf    |= value >> (n - bit_left);
-        AV_WB32(s->buf_ptr, bit_buf);
+        if (s->buf_ptr < s->buf_end)
+            AV_WB32(s->buf_ptr, bit_buf);
         s->buf_ptr += 4;
         bit_left   += 32 - n;
         bit_buf     = value;