From 5bb861d45b86803ec39295cfc04889d2a7138361 Mon Sep 17 00:00:00 2001
From: Michael Niedermayer <michael@niedermayer.cc>
Date: Sun, 16 Jul 2017 14:57:20 +0200
Subject: [PATCH] avcodec/apedec: Fix integer overflow

Fixes: out of array access
Fixes: PoC.ape and others

Found-by: Bingchang, Liu@VARAS of IIE
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
(cherry picked from commit ba4beaf6149f7241c8bd85fe853318c2f6837ad0)
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
---
 libavcodec/apedec.c | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git libavcodec/apedec.c libavcodec/apedec.c
index b99598b4ee7..072e3b42cff 100644
--- external/FFmpeg/libavcodec/apedec.c
+++ external/FFmpeg/libavcodec/apedec.c
@@ -1412,6 +1412,7 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
     int32_t *sample24;
     int i, ch, ret;
     int blockstodecode;
+    uint64_t decoded_buffer_size;
 
     /* this should never be negative, but bad things will happen if it is, so
        check it just to make sure. */
@@ -1467,7 +1468,7 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
                 skip_bits_long(&s->gb, offset);
         }
 
-        if (!nblocks || nblocks > INT_MAX) {
+        if (!nblocks || nblocks > INT_MAX / 2 / sizeof(*s->decoded_buffer) - 8) {
             av_log(avctx, AV_LOG_ERROR, "Invalid sample count: %"PRIu32".\n",
                    nblocks);
             return AVERROR_INVALIDDATA;
@@ -1493,8 +1494,9 @@ static int ape_decode_frame(AVCodecContext *avctx, void *data,
         blockstodecode = s->samples;
 
     /* reallocate decoded sample buffer if needed */
-    av_fast_malloc(&s->decoded_buffer, &s->decoded_size,
-                   2 * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer));
+    decoded_buffer_size = 2LL * FFALIGN(blockstodecode, 8) * sizeof(*s->decoded_buffer);
+    av_assert0(decoded_buffer_size <= INT_MAX);
+    av_fast_malloc(&s->decoded_buffer, &s->decoded_size, decoded_buffer_size);
     if (!s->decoded_buffer)
         return AVERROR(ENOMEM);
     memset(s->decoded_buffer, 0, s->decoded_size);
