libeffects: validate channel mask in downmixer configure

Validate channel mask in downmixer configure, this returns
early warning to calling module which can fallback to another
downmixing module or return error instead of playing improper
data.

authored-by: Dhananjay Kumar <dhakumar@codeaurora.org>

CRs-Fixed: 862105
Change-Id: Ic19c4913608468d5c5197f65bb5d6a640c9d2d74
diff --git a/media/libeffects/downmix/EffectDownmix.c b/media/libeffects/downmix/EffectDownmix.c
index 4a41037..9823c55 100644
--- a/media/libeffects/downmix/EffectDownmix.c
+++ b/media/libeffects/downmix/EffectDownmix.c
@@ -141,6 +141,37 @@
 }
 #endif
 
+static bool Downmix_validChannelMask(uint32_t mask)
+{
+    if (!mask) {
+        return false;
+    }
+    // check against unsupported channels
+    if (mask & kUnsupported) {
+        ALOGE("Unsupported channels (top or front left/right of center)");
+        return false;
+    }
+    // verify has FL/FR
+    if ((mask & AUDIO_CHANNEL_OUT_STEREO) != AUDIO_CHANNEL_OUT_STEREO) {
+        ALOGE("Front channels must be present");
+        return false;
+    }
+    // verify uses SIDE as a pair (ok if not using SIDE at all)
+    if ((mask & kSides) != 0) {
+        if ((mask & kSides) != kSides) {
+            ALOGE("Side channels must be used as a pair");
+            return false;
+        }
+    }
+    // verify uses BACK as a pair (ok if not using BACK at all)
+    if ((mask & kBacks) != 0) {
+        if ((mask & kBacks) != kBacks) {
+            ALOGE("Back channels must be used as a pair");
+            return false;
+        }
+    }
+    return true;
+}
 
 /*----------------------------------------------------------------------------
  * Effect API implementation
@@ -624,9 +655,10 @@
         pDownmixer->apply_volume_correction = false;
         pDownmixer->input_channel_count = 8; // matches default input of AUDIO_CHANNEL_OUT_7POINT1
     } else {
-        // when configuring the effect, do not allow a blank channel mask
-        if (pConfig->inputCfg.channels == 0) {
-            ALOGE("Downmix_Configure error: input channel mask can't be 0");
+        // when configuring the effect, do not allow a blank or unsupported channel mask
+        if (!Downmix_validChannelMask(pConfig->inputCfg.channels)) {
+            ALOGE("Downmix_Configure error: input channel mask(0x%x) not supported",
+                                                        pConfig->inputCfg.channels);
             return -EINVAL;
         }
         pDownmixer->input_channel_count =
@@ -969,34 +1001,13 @@
  */
 bool Downmix_foldGeneric(
         uint32_t mask, int16_t *pSrc, int16_t*pDst, size_t numFrames, bool accumulate) {
-    // check against unsupported channels
-    if (mask & kUnsupported) {
-        ALOGE("Unsupported channels (top or front left/right of center)");
+
+    if (!Downmix_validChannelMask(mask)) {
         return false;
     }
-    // verify has FL/FR
-    if ((mask & AUDIO_CHANNEL_OUT_STEREO) != AUDIO_CHANNEL_OUT_STEREO) {
-        ALOGE("Front channels must be present");
-        return false;
-    }
-    // verify uses SIDE as a pair (ok if not using SIDE at all)
-    bool hasSides = false;
-    if ((mask & kSides) != 0) {
-        if ((mask & kSides) != kSides) {
-            ALOGE("Side channels must be used as a pair");
-            return false;
-        }
-        hasSides = true;
-    }
-    // verify uses BACK as a pair (ok if not using BACK at all)
-    bool hasBacks = false;
-    if ((mask & kBacks) != 0) {
-        if ((mask & kBacks) != kBacks) {
-            ALOGE("Back channels must be used as a pair");
-            return false;
-        }
-        hasBacks = true;
-    }
+
+    const bool hasSides = (mask & kSides) != 0;
+    const bool hasBacks = (mask & kBacks) != 0;
 
     const int numChan = audio_channel_count_from_out_mask(mask);
     const bool hasFC = ((mask & AUDIO_CHANNEL_OUT_FRONT_CENTER) == AUDIO_CHANNEL_OUT_FRONT_CENTER);