[6/7] asfdec: use the new rc4/des alloc functions

Message ID 1438032992-4680-7-git-send-email-jamrial@gmail.com
State New
Headers show

Commit Message

James Almer July 27, 2015, 9:36 p.m.
Signed-off-by: James Almer <jamrial@gmail.com>
---
 libavformat/asfcrypt.c | 35 ++++++++++++++++++++++++-----------
 libavformat/asfcrypt.h |  2 +-
 libavformat/asfdec.c   | 12 ++++++++----
 3 files changed, 33 insertions(+), 16 deletions(-)

Patch

diff --git a/libavformat/asfcrypt.c b/libavformat/asfcrypt.c
index c261475..8292eae 100644
--- a/libavformat/asfcrypt.c
+++ b/libavformat/asfcrypt.c
@@ -144,35 +144,42 @@  static uint64_t multiswap_dec(const uint32_t keys[12],
     return ((uint64_t)b << 32) | a;
 }
 
-void ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len)
+int ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len)
 {
-    struct AVDES des;
-    struct AVRC4 rc4;
+    struct AVDES *des;
+    struct AVRC4 *rc4;
     int num_qwords      = len >> 3;
     uint8_t *qwords     = data;
     uint64_t rc4buff[8] = { 0 };
     uint64_t packetkey;
     uint32_t ms_keys[12];
     uint64_t ms_state;
-    int i;
+    int i, ret = 0;
     if (len < 16) {
         for (i = 0; i < len; i++)
             data[i] ^= key[i];
-        return;
+        return ret;
     }
 
-    av_rc4_init(&rc4, key, 12 * 8, 1);
-    av_rc4_crypt(&rc4, (uint8_t *)rc4buff, NULL, sizeof(rc4buff), NULL, 1);
+    if (!(rc4 = av_rc4_alloc()))
+        return AVERROR(ENOMEM);
+    av_rc4_init(rc4, key, 12 * 8, 1);
+    av_rc4_crypt(rc4, (uint8_t *)rc4buff, NULL, sizeof(rc4buff), NULL, 1);
     multiswap_init((uint8_t *)rc4buff, ms_keys);
 
     packetkey  = AV_RN64(&qwords[num_qwords * 8 - 8]);
     packetkey ^= rc4buff[7];
-    av_des_init(&des, key + 12, 64, 1);
-    av_des_crypt(&des, (uint8_t *)&packetkey, (uint8_t *)&packetkey, 1, NULL, 1);
+
+    if (!(des = av_des_alloc())) {
+        ret = AVERROR(ENOMEM);
+        goto fail;
+    }
+    av_des_init(des, key + 12, 64, 1);
+    av_des_crypt(des, (uint8_t *)&packetkey, (uint8_t *)&packetkey, 1, NULL, 1);
     packetkey ^= rc4buff[6];
 
-    av_rc4_init(&rc4, (uint8_t *)&packetkey, 64, 1);
-    av_rc4_crypt(&rc4, data, data, len, NULL, 1);
+    av_rc4_init(rc4, (uint8_t *)&packetkey, 64, 1);
+    av_rc4_crypt(rc4, data, data, len, NULL, 1);
 
     ms_state = 0;
     for (i = 0; i < num_qwords - 1; i++, qwords += 8)
@@ -182,4 +189,10 @@  void ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len)
     packetkey = av_le2ne64(packetkey);
     packetkey = multiswap_dec(ms_keys, ms_state, packetkey);
     AV_WL64(qwords, packetkey);
+
+fail:
+    av_freep(&rc4);
+    av_freep(&des);
+
+    return ret;
 }
diff --git a/libavformat/asfcrypt.h b/libavformat/asfcrypt.h
index 53388b4..d4f311c 100644
--- a/libavformat/asfcrypt.h
+++ b/libavformat/asfcrypt.h
@@ -24,6 +24,6 @@ 
 
 #include <inttypes.h>
 
-void ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len);
+int ff_asfcrypt_dec(const uint8_t key[20], uint8_t *data, int len);
 
 #endif /* AVFORMAT_ASFCRYPT_H */
diff --git a/libavformat/asfdec.c b/libavformat/asfdec.c
index 37d91e0..058bfa2 100644
--- a/libavformat/asfdec.c
+++ b/libavformat/asfdec.c
@@ -1167,8 +1167,10 @@  static int asf_read_multiple_payload(AVFormatContext *s, AVPacket *pkt,
         }
         if ((ret = avio_read(pb, p, pay_len)) < 0)
             return ret;
-        if (s->key && s->keylen == 20)
-            ff_asfcrypt_dec(s->key, p, ret);
+        if (s->key && s->keylen == 20) {
+            if ((ret = ff_asfcrypt_dec(s->key, p, ret)) < 0)
+                return ret;
+        }
         avio_skip(pb, skip);
         asf_pkt->size_left -= pay_len;
         asf->nb_mult_left--;
@@ -1220,8 +1222,10 @@  static int asf_read_single_payload(AVFormatContext *s, AVPacket *pkt,
         asf_pkt->size_left = 0;
     if ((ret = avio_read(pb, p, size)) < 0)
         return ret;
-    if (s->key && s->keylen == 20)
-            ff_asfcrypt_dec(s->key, p, ret);
+    if (s->key && s->keylen == 20) {
+        if ((ret = ff_asfcrypt_dec(s->key, p, ret)) < 0)
+            return ret;
+    }
     if (asf->packet_size_internal)
         avio_skip(pb, asf->packet_size - asf->packet_size_internal);
     avio_skip(pb, asf->pad_len); // skip padding