commit b71a431eabce5983ccd3efadeb024b8ed08bb89d
author Pawin Vongmasa <pawin@google.com> Fri Aug 19 02:43:37 2016 +0000
committer android-build-merger <android-build-merger@google.com> Fri Aug 19 02:43:37 2016 +0000
tree 4d014fb84dffd894ca5b8f86439de50388f6cc6d
parent e40e452a3a7ab31e408d023ab7b4ad1fa65d01c5
parent 920b0f7887c5b8f8b1b7670c91f9ceedd9ad9cf2

DO NOT MERGE - SoftMPEG4: Check the buffer size before writing the reference frame. am: 940829f69b
am: 920b0f7887

Change-Id: I603ab931f82d8a83187be989dccb8686de8bf95e

media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp

diff --git a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
index 1a53c72..469e516 100644
--- a/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
+++ b/media/libstagefright/codecs/m4v_h263/dec/SoftMPEG4.cpp
@@ -190,8 +190,17 @@
             PortInfo *port = editPortInfo(1);
             OMX_BUFFERHEADERTYPE *outHeader = port->mBuffers.editItemAt(1).mHeader;
 
+            OMX_U32 yFrameSize = sizeof(uint8) * mHandle->size;
+            if ((outHeader->nAllocLen < yFrameSize) ||
+                    (outHeader->nAllocLen - yFrameSize < yFrameSize / 2)) {
+                ALOGE("Too small output buffer for reference frame: %zu bytes",
+                        outHeader->nAllocLen);
+                android_errorWriteLog(0x534e4554, "30033990");
+                notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+                mSignalledError = true;
+                return;
+            }
             PVSetReferenceYUV(mHandle, outHeader->pBuffer);
-
             mFramesConfigured = true;
         }
 
@@ -209,7 +218,16 @@
         int32_t bufferSize = inHeader->nFilledLen;
         int32_t tmp = bufferSize;
 
-        OMX_U32 frameSize = (mWidth * mHeight * 3) / 2;
+        OMX_U32 frameSize;
+        OMX_U64 yFrameSize = (OMX_U64)mWidth * (OMX_U64)mHeight;
+        if (yFrameSize > ((OMX_U64)UINT32_MAX / 3) * 2) {
+            ALOGE("Frame size too large");
+            notify(OMX_EventError, OMX_ErrorUndefined, 0, NULL);
+            mSignalledError = true;
+            return;
+        }
+        frameSize = (OMX_U32)(yFrameSize + (yFrameSize / 2));
+
         if (outHeader->nAllocLen < frameSize) {
             android_errorWriteLog(0x534e4554, "27833616");
             ALOGE("Insufficient output buffer size");

media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp

diff --git a/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp b/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
index 9d41351..787c46e 100644
--- a/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
+++ b/media/libstagefright/codecs/m4v_h263/enc/SoftMPEG4Encoder.cpp
@@ -129,6 +129,10 @@
         ALOGE("Failed to get default encoding parameters");
         return OMX_ErrorUndefined;
     }
+    if (mVideoFrameRate == 0) {
+        ALOGE("Framerate should not be 0");
+        return OMX_ErrorUndefined;
+    }
     mEncParams->encMode = mEncodeMode;
     mEncParams->encWidth[0] = mVideoWidth;
     mEncParams->encHeight[0] = mVideoHeight;