Initial libmediaformatshaper
mediaformatshaper library to amend MediaFormat parameters before
encoding. This is the initial cut of the library.
Disabled by default, enable with "setprop debug.stagefright.enableshaping 1"
Bug: 182827840
Test: build, boot, encode video
Change-Id: I8cefb6ed6ad286ae2192796bf15760e873e0d2f3
diff --git a/media/libmediaformatshaper/Android.bp b/media/libmediaformatshaper/Android.bp
new file mode 100644
index 0000000..731ff4c
--- /dev/null
+++ b/media/libmediaformatshaper/Android.bp
@@ -0,0 +1,91 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+// these headers include the structure of needed function pointers
+cc_library_headers {
+ name: "libmediaformatshaper_headers",
+ export_include_dirs: ["include"],
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.media",
+ ],
+ min_sdk_version: "29",
+ host_supported: true,
+ target: {
+ darwin: {
+ enabled: false,
+ },
+ },
+}
+
+cc_defaults {
+ name: "libmediaformatshaper_defaults",
+ srcs: [
+ "CodecProperties.cpp",
+ "FormatShaper.cpp",
+ "ManageShapingCodecs.cpp",
+ "VideoShaper.cpp",
+ "VQApply.cpp",
+ ],
+
+ local_include_dirs: [
+ "include",
+ ],
+
+ shared_libs: [
+ "liblog",
+ "libutils",
+ ],
+
+ cflags: [
+ "-Werror",
+ "-Wall",
+ "-fvisibility=hidden",
+ "-Wthread-safety", // enables GUARDED_BY()
+ ],
+
+ target: {
+ android: {
+ shared_libs: [
+ "libmediandk#29",
+ ],
+ },
+ },
+
+ sanitize: {
+ cfi: true,
+ misc_undefined: [
+ "unsigned-integer-overflow",
+ "signed-integer-overflow",
+ ],
+ },
+}
+
+cc_library {
+ name: "libmediaformatshaper",
+ defaults: ["libmediaformatshaper_defaults"],
+
+ min_sdk_version: "29",
+
+ apex_available: [
+ "//apex_available:platform",
+ "com.android.media",
+ ],
+
+ version_script: "exports.lds",
+
+}
diff --git a/media/libmediaformatshaper/CodecProperties.cpp b/media/libmediaformatshaper/CodecProperties.cpp
new file mode 100644
index 0000000..dccfd95
--- /dev/null
+++ b/media/libmediaformatshaper/CodecProperties.cpp
@@ -0,0 +1,143 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "CodecProperties"
+#include <utils/Log.h>
+
+#include <string>
+
+#include <media/formatshaper/CodecProperties.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+CodecProperties::CodecProperties(std::string name, std::string mediaType) {
+ mName = name;
+ mMediaType = mediaType;
+}
+
+std::string CodecProperties::getName(){
+ return mName;
+}
+
+std::string CodecProperties::getMediaType(){
+ return mMediaType;
+}
+
+int CodecProperties::supportedMinimumQuality() {
+ return mMinimumQuality;
+}
+void CodecProperties::setSupportedMinimumQuality(int vmaf) {
+ mMinimumQuality = vmaf;
+}
+
+int CodecProperties::targetQpMax() {
+ return mTargetQpMax;
+}
+void CodecProperties::setTargetQpMax(int qpMax) {
+ mTargetQpMax = qpMax;
+}
+
+// what API is this codec set up for (e.g. API of the associated partition)
+// vendor-side (OEM) codecs may be older, due to 'vendor freeze' and treble
+int CodecProperties::supportedApi() {
+ return mApi;
+}
+
+std::string CodecProperties::getMapping(std::string key, std::string kind) {
+ ALOGV("getMapping(key %s, kind %s )", key.c_str(), kind.c_str());
+ //play with mMappings
+ auto mapped = mMappings.find(kind + "-" + key);
+ if (mapped != mMappings.end()) {
+ std::string result = mapped->second;
+ ALOGV("getMapping(%s, %s) -> %s", key.c_str(), kind.c_str(), result.c_str());
+ return result;
+ }
+ ALOGV("nope, return unchanged key");
+ return key;
+}
+
+
+// really a bit of debugging code here.
+void CodecProperties::showMappings() {
+ ALOGD("Mappings:");
+ int count = 0;
+ for (const auto& [key, value] : mMappings) {
+ count++;
+ ALOGD("'%s' -> '%s'", key.c_str(), value.c_str());
+ }
+ ALOGD("total %d mappings", count);
+}
+
+void CodecProperties::setMapping(std::string kind, std::string key, std::string value) {
+ ALOGV("setMapping(%s,%s,%s)", kind.c_str(), key.c_str(), value.c_str());
+ std::string metaKey = kind + "-" + key;
+ mMappings.insert({metaKey, value});
+}
+
+const char **CodecProperties::getMappings(std::string kind, bool reverse) {
+ ALOGV("getMappings(kind %s, reverse %d", kind.c_str(), reverse);
+ // how many do we need?
+ int count = mMappings.size();
+ if (count == 0) {
+ ALOGV("empty mappings");
+ return nullptr;
+ }
+ size_t size = sizeof(char *) * (2 * count + 2);
+ const char **result = (const char **)malloc(size);
+ if (result == nullptr) {
+ ALOGW("no memory to return mappings");
+ return nullptr;
+ }
+ memset(result, '\0', size);
+
+ const char **pp = result;
+ for (const auto& [key, value] : mMappings) {
+ // split out the kind/key
+ size_t pos = key.find('-');
+ if (pos == std::string::npos) {
+ ALOGD("ignoring malformed key: %s", key.c_str());
+ continue;
+ }
+ std::string actualKind = key.substr(0,pos);
+ if (kind.length() != 0 && kind != actualKind) {
+ ALOGD("kinds don't match: want '%s' got '%s'", kind.c_str(), actualKind.c_str());
+ continue;
+ }
+ if (reverse) {
+ // codec specific -> std aka 'unmapping'
+ pp[0] = strdup( value.c_str());
+ pp[1] = strdup( key.substr(pos+1).c_str());
+ } else {
+ // std -> codec specific
+ pp[0] = strdup( key.substr(pos+1).c_str());
+ pp[1] = strdup( value.c_str());
+ }
+ ALOGV(" %s -> %s", pp[0], pp[1]);
+ pp += 2;
+ }
+
+ pp[0] = nullptr;
+ pp[1] = nullptr;
+
+ return result;
+}
+
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/FormatShaper.cpp b/media/libmediaformatshaper/FormatShaper.cpp
new file mode 100644
index 0000000..ca4dc72
--- /dev/null
+++ b/media/libmediaformatshaper/FormatShaper.cpp
@@ -0,0 +1,184 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "FormatShaper"
+#include <utils/Log.h>
+
+#include <string>
+#include <inttypes.h>
+
+#include <media/NdkMediaFormat.h>
+
+#include <media/formatshaper/VQops.h>
+#include <media/formatshaper/CodecProperties.h>
+#include <media/formatshaper/FormatShaper.h>
+#include <media/formatshaper/VideoShaper.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+//
+// Caller retains ownership of and responsibility for inFormat
+//
+
+//
+// the interface to the outside
+//
+
+int shapeFormat(shaperHandle_t shaper, AMediaFormat* inFormat, int flags) {
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr) {
+ return -1;
+ }
+ if (!codec->isRegistered()) {
+ return -1;
+ }
+
+ // run through the list of possible transformations
+ //
+
+ std::string mediaType = codec->getMediaType();
+ if (strncmp(mediaType.c_str(), "video/", 6) == 0) {
+ // video specific shaping
+ (void) videoShaper(codec, inFormat, flags);
+
+ } else if (strncmp(mediaType.c_str(), "audio/", 6) == 0) {
+ // audio specific shaping
+
+ } else {
+ ALOGV("unknown mediatype '%s', left untouched", mediaType.c_str());
+
+ }
+
+ return 0;
+}
+
+int setMap(shaperHandle_t shaper, const char *kind, const char *key, const char *value) {
+ ALOGV("setMap: kind %s key %s -> value %s", kind, key, value);
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr) {
+ return -1;
+ }
+ // must not yet be registered
+ if (codec->isRegistered()) {
+ return -1;
+ }
+
+ codec->setMapping(kind, key, value);
+ return 0;
+}
+
+int setFeature(shaperHandle_t shaper, const char *feature, int value) {
+ ALOGV("set_feature: feature %s value %d", feature, value);
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr) {
+ return -1;
+ }
+ // must not yet be registered
+ if (codec->isRegistered()) {
+ return -1;
+ }
+
+ if (!strcmp(feature, "vq-minimum-quality")) {
+ codec->setSupportedMinimumQuality(value);
+ } else if (!strcmp(feature, "vq-supports-qp")) {
+ codec->setSupportsQp(value != 0);
+ } else if (!strcmp(feature, "vq-target-qpmax")) {
+ codec->setTargetQpMax(value);
+ } else if (!strcmp(feature, "vq-target-bppx100")) {
+ double bpp = value / 100.0;
+ codec->setBpp(bpp);
+ } else {
+ // changed nothing, don't mark as configured
+ return 0;
+ }
+ return 0;
+}
+
+/*
+ * The routines that manage finding, creating, and registering the shapers.
+ */
+
+shaperHandle_t findShaper(const char *codecName, const char *mediaType) {
+ CodecProperties *codec = findCodec(codecName, mediaType);
+ return (shaperHandle_t) codec;
+}
+
+shaperHandle_t createShaper(const char *codecName, const char *mediaType) {
+ CodecProperties *codec = new CodecProperties(codecName, mediaType);
+ return (shaperHandle_t) codec;
+}
+
+shaperHandle_t registerShaper(shaperHandle_t shaper, const char *codecName, const char *mediaType) {
+ ALOGV("registerShaper(handle, codecName %s, mediaType %s", codecName, mediaType);
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr) {
+ return nullptr;
+ }
+ // must not yet be registered
+ if (codec->isRegistered()) {
+ return nullptr;
+ }
+
+ codec = registerCodec(codec, codecName, mediaType);
+ return (shaperHandle_t) codec;
+}
+
+// mapping & unmapping
+// give me the mappings for 'kind'.
+// kind==null (or empty string), means *all* mappings
+
+const char **getMappings(shaperHandle_t shaper, const char *kind) {
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr)
+ return nullptr;
+ if (kind == nullptr)
+ kind = "";
+
+ return codec->getMappings(kind, /* reverse */ false);
+}
+
+const char **getReverseMappings(shaperHandle_t shaper, const char *kind) {
+ CodecProperties *codec = (CodecProperties*) shaper;
+ if (codec == nullptr)
+ return nullptr;
+ if (kind == nullptr)
+ kind = "";
+
+ return codec->getMappings(kind, /* reverse */ true);
+}
+
+
+// the system grabs this structure
+__attribute__ ((visibility ("default")))
+extern "C" FormatShaperOps_t shaper_ops = {
+ .version = SHAPER_VERSION_V1,
+
+ .findShaper = findShaper,
+ .createShaper = createShaper,
+ .setMap = setMap,
+ .setFeature = setFeature,
+ .registerShaper = registerShaper,
+
+ .shapeFormat = shapeFormat,
+ .getMappings = getMappings,
+ .getReverseMappings = getReverseMappings,
+};
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/ManageShapingCodecs.cpp b/media/libmediaformatshaper/ManageShapingCodecs.cpp
new file mode 100644
index 0000000..bdc395f
--- /dev/null
+++ b/media/libmediaformatshaper/ManageShapingCodecs.cpp
@@ -0,0 +1,103 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "ManageShapingCodecs"
+#include <utils/Log.h>
+
+#include <mutex>
+#include <string>
+#include <inttypes.h>
+
+#include <media/NdkMediaFormat.h>
+#include <media/formatshaper/CodecProperties.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+// manage the list of codec information.
+//
+// XXX: the mutex here is too heavy; rework that.
+//
+
+static std::mutex sCodecMutex;
+static std::map<std::string, CodecProperties*> sCodecTraits;
+
+CodecProperties *findCodec(const char *codecName, const char *mediaType) {
+ CodecProperties *codec = nullptr;
+
+ // synthesize a name from both codecName + mediaType
+ // some codecs support multiple media types and may have different capabilities
+ // for each media type
+ //
+ std::string codecKey = codecName;
+ codecKey += "-";
+ codecKey += mediaType;
+
+ std::lock_guard _l(sCodecMutex);
+
+ auto it = sCodecTraits.find(codecKey);
+ if (it != sCodecTraits.end()) {
+ codec = it->second;
+ }
+
+ return codec;
+}
+
+CodecProperties *registerCodec(CodecProperties *codec, const char *codecName,
+ const char *mediaType) {
+
+ CodecProperties *registeredCodec = nullptr;
+
+ if (codec->isRegistered()) {
+ return nullptr;
+ }
+
+ // synthesize a name from both codecName + mediaType
+ // some codecs support multiple media types and may have different capabilities
+ // for each media type
+ //
+ std::string codecKey = codecName;
+ codecKey += "-";
+ codecKey += mediaType;
+
+ std::lock_guard _l(sCodecMutex);
+
+ auto it = sCodecTraits.find(codecKey);
+ if (it != sCodecTraits.end()) {
+ registeredCodec = it->second;
+ }
+
+ if (registeredCodec == nullptr) {
+ // register the one that was passed to us
+ ALOGV("Creating entry for codec %s, mediaType %s, key %s", codecName, mediaType,
+ codecKey.c_str());
+ sCodecTraits.insert({codecKey, codec});
+ registeredCodec = codec;
+ codec->setRegistered(true);
+ } else {
+ // one has already been registered, use that
+ // and discard the candidate
+ delete codec;
+ codec = nullptr;
+ }
+
+ return registeredCodec;
+}
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/VQApply.cpp b/media/libmediaformatshaper/VQApply.cpp
new file mode 100644
index 0000000..6f6f33c
--- /dev/null
+++ b/media/libmediaformatshaper/VQApply.cpp
@@ -0,0 +1,202 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "VQApply"
+#include <utils/Log.h>
+
+#include <string>
+#include <inttypes.h>
+
+#include <media/NdkMediaFormat.h>
+
+#include <media/formatshaper/VQops.h>
+#include <media/formatshaper/CodecProperties.h>
+#include <media/formatshaper/VideoShaper.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+
+// these are all NDK#31 and we run as NDK#29 (to be within the module)
+// the __builtin_available(android 31, *) constructs didn't work for me.
+//
+#define AMEDIAFORMAT_VIDEO_QP_MAX "video-qp-max"
+#define AMEDIAFORMAT_VIDEO_QP_MIN "video-qp-min"
+
+#define AMEDIAFORMAT_VIDEO_QP_B_MAX "video-qp-b-max"
+#define AMEDIAFORMAT_VIDEO_QP_B_MIN "video-qp-b-min"
+#define AMEDIAFORMAT_VIDEO_QP_I_MAX "video-qp-i-max"
+#define AMEDIAFORMAT_VIDEO_QP_I_MIN "video-qp-i-min"
+#define AMEDIAFORMAT_VIDEO_QP_P_MAX "video-qp-p-max"
+#define AMEDIAFORMAT_VIDEO_QP_P_MIN "video-qp-p-min"
+
+//
+// Caller retains ownership of and responsibility for inFormat
+//
+int VQApply(CodecProperties *codec, vqOps_t *info, AMediaFormat* inFormat, int flags) {
+ ALOGV("codecName %s inFormat %p flags x%x", codec->getName().c_str(), inFormat, flags);
+
+ if (codec->supportedMinimumQuality() > 0) {
+ // allow the codec provided minimum quality behavior to work at it
+ ALOGD("minquality(codec): codec says %d", codec->supportedMinimumQuality());
+ return 0;
+ }
+
+ ALOGD("considering other ways to improve quality...");
+
+ //
+ // apply any and all tools that we have.
+ // -- qp
+ // -- minimum bits-per-pixel
+ //
+ if (codec->supportsQp()) {
+ // use a (configurable) QP value to force better quality
+ //
+ // XXX: augment this so that we don't lower an existing QP setting
+ // (e.g. if user set it to 40, we don't want to set it back to 45)
+ int qpmax = codec->targetQpMax();
+ if (qpmax <= 0) {
+ qpmax = 45;
+ ALOGD("use default substitute QpMax == %d", qpmax);
+ }
+ ALOGD("minquality by QP: inject %s=%d", AMEDIAFORMAT_VIDEO_QP_MAX, qpmax);
+ AMediaFormat_setInt32(inFormat, AMEDIAFORMAT_VIDEO_QP_MAX, qpmax);
+
+ // force spreading the QP across frame types, since we imposing a value
+ qpSpreadMaxPerFrameType(inFormat, info->qpDelta, info->qpMax, /* override */ true);
+ } else {
+ ALOGD("codec %s: no qp bounding", codec->getName().c_str());
+ }
+
+ double bpp = codec->getBpp();
+ if (bpp > 0.0) {
+ // if we've decided to use bits-per-pixel (per second) to drive the quality
+ //
+ // (properly phrased as 'bits per second per pixel' so that it's resolution
+ // and framerate agnostic
+ //
+ // all of these is structured so that a missing value cleanly gets us to a
+ // non-faulting value of '0' for the minimum bits-per-pixel.
+ //
+ int32_t width = 0;
+ (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_WIDTH, &width);
+ int32_t height = 0;
+ (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_HEIGHT, &height);
+ int32_t bitrateConfigured = 0;
+ (void) AMediaFormat_getInt32(inFormat, AMEDIAFORMAT_KEY_BIT_RATE, &bitrateConfigured);
+
+ int64_t pixels = ((int64_t)width) * height;
+ int64_t bitrateFloor = pixels * bpp;
+
+ if (bitrateFloor > INT32_MAX) bitrateFloor = INT32_MAX;
+
+ ALOGD("minquality/bitrate: target %d floor %" PRId64 "(%.3f bpp * (%d w * %d h)",
+ bitrateConfigured, bitrateFloor, codec->getBpp(), height, width);
+
+ if (bitrateConfigured < bitrateFloor) {
+ ALOGD("minquality/target bitrate raised from %d to %" PRId64 " to maintain quality",
+ bitrateConfigured, bitrateFloor);
+ AMediaFormat_setInt32(inFormat, AMEDIAFORMAT_KEY_BIT_RATE, (int32_t)bitrateFloor);
+ }
+ }
+
+ return 0;
+}
+
+
+bool hasQpPerFrameType(AMediaFormat *format) {
+ int32_t value;
+
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, &value)
+ || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, &value)) {
+ return true;
+ }
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, &value)
+ || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, &value)) {
+ return true;
+ }
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, &value)
+ || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, &value)) {
+ return true;
+ }
+ return false;
+}
+
+bool hasQp(AMediaFormat *format) {
+ int32_t value;
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MAX, &value)
+ || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MIN, &value)) {
+ return true;
+ }
+ return hasQpPerFrameType(format);
+}
+
+void qpSpreadPerFrameType(AMediaFormat *format, int delta,
+ int qplow, int qphigh, bool override) {
+ qpSpreadMaxPerFrameType(format, delta, qphigh, override);
+ qpSpreadMinPerFrameType(format, qplow, override);
+}
+
+void qpSpreadMaxPerFrameType(AMediaFormat *format, int delta, int qphigh, bool override) {
+ ALOGV("format %p delta %d hi %d override %d", format, delta, qphigh, override);
+
+ int32_t qpOffered = 0;
+ if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MAX, &qpOffered)) {
+ // propagate to otherwise unspecified frame-specific keys
+ int32_t maxI;
+ if (override || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, &maxI)) {
+ int32_t value = std::min(qphigh, qpOffered);
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MAX, value);
+ }
+ int32_t maxP;
+ if (override || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, &maxP)) {
+ int32_t value = std::min(qphigh, (std::min(qpOffered, INT32_MAX-delta) + delta));
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MAX, value);
+ }
+ int32_t maxB;
+ if (override || !AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, &maxB)) {
+ int32_t value = std::min(qphigh, (std::min(qpOffered, INT32_MAX-2*delta) + 2*delta));
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MAX, value);
+ }
+ }
+}
+
+void qpSpreadMinPerFrameType(AMediaFormat *format, int qplow, bool override) {
+ ALOGV("format %p lo %d override %d", format, qplow, override);
+
+ int32_t qpOffered = 0;
+ if (AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_MIN, &qpOffered)) {
+ int value = std::max(qplow, qpOffered);
+ // propagate to otherwise unspecified frame-specific keys
+ int32_t minI;
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, &minI)) {
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_I_MIN, value);
+ }
+ int32_t minP;
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, &minP)) {
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_P_MIN, value);
+ }
+ int32_t minB;
+ if (!AMediaFormat_getInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, &minB)) {
+ AMediaFormat_setInt32(format, AMEDIAFORMAT_VIDEO_QP_B_MIN, value);
+ }
+ }
+}
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/VideoShaper.cpp b/media/libmediaformatshaper/VideoShaper.cpp
new file mode 100644
index 0000000..fecd3a1
--- /dev/null
+++ b/media/libmediaformatshaper/VideoShaper.cpp
@@ -0,0 +1,98 @@
+/*
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "VideoShaper"
+#include <utils/Log.h>
+
+#include <string>
+#include <inttypes.h>
+
+#include <media/NdkMediaFormat.h>
+
+#include <media/formatshaper/VQops.h>
+#include <media/formatshaper/CodecProperties.h>
+#include <media/formatshaper/VideoShaper.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+// mediatype-specific operations
+
+vqOps_t mediaInfo[] = {
+ {
+ .mediaType = "video/avc",
+ .qpMin = 0,
+ .qpMax = 51,
+ .qpDelta = 3,
+ },
+ {
+ .mediaType = "video/hevc",
+ .qpMin = 0,
+ .qpMax = 51,
+ .qpDelta = 3,
+ },
+ {
+ .mediaType = NULL, // matches everything, it must come last
+ .qpMin = INT32_MIN,
+ .qpMax = INT32_MAX,
+ .qpDelta = 3,
+ }
+};
+int nMediaInfos = sizeof(mediaInfo) / sizeof(mediaInfo[0]);
+
+//
+// Caller retains ownership of and responsibility for inFormat
+//
+
+int videoShaper(CodecProperties *codec, AMediaFormat* inFormat, int flags) {
+ if (codec == nullptr) {
+ return -1;
+ }
+ ALOGV("codec %s inFormat %p flags x%x", codec->getName().c_str(), inFormat, flags);
+
+ int ix;
+
+ std::string mediaType = codec->getMediaType();
+ // we should always come out of this with a selection, because the final entry
+ // is deliberaly a NULL -- so that it will act as a default
+ for(ix = 0; mediaInfo[ix].mediaType != NULL; ix++) {
+ if (strcmp(mediaType.c_str(), mediaInfo[ix].mediaType) == 0) {
+ break;
+ }
+ }
+ if (ix >= nMediaInfos) {
+ // shouldn't happen, but if it does .....
+ }
+
+ vqOps_t *info = &mediaInfo[ix];
+
+ // apply any quality transforms in here..
+ (void) VQApply(codec, info, inFormat, flags);
+
+ // We must always spread and map any QP parameters.
+ // Sometimes it's something we inserted here, sometimes it's a value that the user injected.
+ //
+ qpSpreadPerFrameType(inFormat, info->qpDelta, info->qpMin, info->qpMax, /* override */ false);
+
+ //
+ return 0;
+
+}
+
+} // namespace mediaformatshaper
+} // namespace android
+
diff --git a/media/libmediaformatshaper/exports.lds b/media/libmediaformatshaper/exports.lds
new file mode 100644
index 0000000..a29cadb
--- /dev/null
+++ b/media/libmediaformatshaper/exports.lds
@@ -0,0 +1,6 @@
+{
+ global:
+ shaper_ops;
+ local:
+ *;
+};
diff --git a/media/libmediaformatshaper/include/media/formatshaper/CodecProperties.h b/media/libmediaformatshaper/include/media/formatshaper/CodecProperties.h
new file mode 100644
index 0000000..f7177a4
--- /dev/null
+++ b/media/libmediaformatshaper/include/media/formatshaper/CodecProperties.h
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _LIBMEDIAFORMATSHAPER_CODECPROPERTIES_H_
+#define _LIBMEDIAFORMATSHAPER_CODECPROPERTIES_H_
+
+#include <map>
+#include <mutex>
+#include <string>
+
+#include <utils/RefBase.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+class CodecProperties {
+
+ public:
+ CodecProperties(std::string name, std::string mediaType);
+
+ std::string getName();
+ std::string getMediaType();
+
+ // establish a mapping from standard 'key' to non-standard 'value' in the namespace 'kind'
+ void setMapping(std::string kind, std::string key, std::string value);
+
+ // translate from from standard key to non-standard key
+ // return original standard key if there is no mapping
+ std::string getMapping(std::string key, std::string kind);
+
+ // returns an array of char *, which are paired "from" and "to" values
+ // for mapping (or unmapping). it's always expressed as from->to
+ // and 'reverse' describes which strings are to be on which side.
+ const char **getMappings(std::string kind, bool reverse);
+
+ // debugging of what's in the mapping dictionary
+ void showMappings();
+
+ // does the codec support the Android S minimum quality rules
+ void setSupportedMinimumQuality(int vmaf);
+ int supportedMinimumQuality();
+
+ // qp max bound used to compensate when SupportedMinimumQuality == 0
+ // 0 == let a system default handle it
+ void setTargetQpMax(int qpmax);
+ int targetQpMax();
+
+ // target bits-per-pixel (per second) for encoding operations.
+ // This is used to calculate a minimum bitrate for any particular resolution.
+ // A 1080p (1920*1080 = 2073600 pixels) to be encoded at 5Mbps has a bpp == 2.41
+ void setBpp(double bpp) { mBpp = bpp;}
+ double getBpp() {return mBpp;}
+
+ // Does this codec support QP bounding
+ // The getMapping() methods provide any needed mapping to non-standard keys.
+ void setSupportsQp(bool supported) { mSupportsQp = supported;}
+ bool supportsQp() { return mSupportsQp;}
+
+ int supportedApi();
+
+ // a codec is not usable until it has been registered with its
+ // name/mediaType.
+ bool isRegistered() { return mIsRegistered;}
+ void setRegistered(bool registered) { mIsRegistered = registered;}
+
+ private:
+ std::string mName;
+ std::string mMediaType;
+ int mApi = 0;
+ int mMinimumQuality = 0;
+ int mTargetQpMax = 0;
+ bool mSupportsQp = false;
+ double mBpp = 0.0;
+
+ std::mutex mMappingLock;
+ // XXX figure out why I'm having problems getting compiler to like GUARDED_BY
+ std::map<std::string, std::string> mMappings /*GUARDED_BY(mMappingLock)*/ ;
+ std::map<std::string, std::string> mUnMappings /*GUARDED_BY(mMappingLock)*/ ;
+
+ bool mIsRegistered = false;
+
+ // DISALLOW_EVIL_CONSTRUCTORS(CodecProperties);
+};
+
+extern CodecProperties *findCodec(const char *codecName, const char *mediaType);
+extern CodecProperties *registerCodec(CodecProperties *codec, const char *codecName,
+ const char *mediaType);
+
+
+} // namespace mediaformatshaper
+} // namespace android
+
+#endif // _LIBMEDIAFORMATSHAPER_CODECPROPERTIES_H_
diff --git a/media/libmediaformatshaper/include/media/formatshaper/FormatShaper.h b/media/libmediaformatshaper/include/media/formatshaper/FormatShaper.h
new file mode 100644
index 0000000..8ad81cd
--- /dev/null
+++ b/media/libmediaformatshaper/include/media/formatshaper/FormatShaper.h
@@ -0,0 +1,130 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+ * structure defining the function pointers that system-side folks
+ * use to invoke operations within the MediaFormat shaping library
+ *
+ * This is the include file the outside world uses.
+ */
+
+#ifndef LIBMEDIAFORMATSHAPER_FORMATSHAPER_H_
+#define LIBMEDIAFORMATSHAPER_FORMATSHAPER_H_
+
+namespace android {
+namespace mediaformatshaper {
+
+/*
+ * An opaque handle clients use to refer to codec+mediatype being shaped.
+ */
+typedef void (*shaperHandle_t);
+
+/*
+ * shapeFormat applies any re-shaping on the passed AMediaFormat.
+ * The updated format is returned in-place.
+ */
+typedef int (*shapeFormat_t)(shaperHandle_t shaperHandle,
+ AMediaFormat* inFormat, int flags);
+
+/*
+ * getMapping returns any mappings from standard keys to codec-specific keys.
+ * The return is a vector of const char* which are set up in pairs
+ * of "from", and "to".
+ * This array is always finished with a pair of nulls (to indicate a null from
+ * and a null to)
+ */
+
+typedef const char **(*getMappings_t)(shaperHandle_t shaperHandle, const char *kind);
+
+/*
+ * Returns a handle to the shaperHandle for the specified codec and mediatype.
+ * If none exists, it returns null.
+ */
+typedef shaperHandle_t (*findShaper_t)(const char *codecName, const char *mediaType);
+
+/*
+ * Creates and returns an empty shaperHandle that the client can populate using the
+ * setFeature() and setMap() operations.
+ */
+typedef shaperHandle_t (*createShaper_t)(const char *codecName, const char *mediaType);
+
+/*
+ * Registers the indicated shaperHandle for the indicated codec and mediatype.
+ * This call returns the shaperHandle that is to be used for further shaper operations.
+ * The returned value may be different than the one passed as an argument if another
+ * shaperinfo was registered while the passed one was being configured.
+ */
+typedef shaperHandle_t (*registerShaper_t)(shaperHandle_t shaper, const char *codecName,
+ const char *mediaType);
+
+/*
+ * establishes a mapping between the standard key "from" and the codec-specific key "to"
+ * in the "kind" namespace. This mapping is specific to the indicated codecName when
+ * encoding for the indicated mediaType.
+ */
+typedef int (*setMap_t)(shaperHandle_t shaper, const char *kind, const char *from, const char *to);
+
+/*
+ * establishes that codec "codecName" encoding for "mediaType" supports the indicated
+ * feature at the indicated value
+ */
+typedef int (*setFeature_t)(shaperHandle_t shaper, const char *feature, int value);
+
+/*
+ * The expectation is that the client will implement a flow similar to the following when
+ * setting up an encoding.
+ *
+ * if ((shaper=formatShaperops->findShaper(codecName, mediaType)) == NULL) {
+ * for (all codec features) {
+ * get feature name, feature value
+ * formatShaperops->setFeature(shaper,, featurename, featurevalue)
+ * }
+ * for (all codec mappings) {
+ * get mapping 'kind', mapping 'from', mapping 'to'
+ * formatShaperops->setMap(shaper, kind, from, to)
+ * }
+ * }
+ *
+ */
+
+typedef struct FormatShaperOps {
+ const uint32_t version;
+
+ /*
+ * find, create, setup, and register the shaper info
+ */
+ findShaper_t findShaper;
+ createShaper_t createShaper;
+ setMap_t setMap;
+ setFeature_t setFeature;
+ registerShaper_t registerShaper;
+
+ /*
+ * use the shaper info
+ */
+ shapeFormat_t shapeFormat;
+ getMappings_t getMappings;
+ getMappings_t getReverseMappings;
+} FormatShaperOps_t;
+
+// versioninf information
+const uint32_t SHAPER_VERSION_UNKNOWN = 0;
+const uint32_t SHAPER_VERSION_V1 = 1;
+
+} // namespace mediaformatshaper
+} // namespace android
+
+#endif // LIBMEDIAFORMATSHAPER_FORMATSHAPER_H_
diff --git a/media/libmediaformatshaper/include/media/formatshaper/VQops.h b/media/libmediaformatshaper/include/media/formatshaper/VQops.h
new file mode 100644
index 0000000..807e8af
--- /dev/null
+++ b/media/libmediaformatshaper/include/media/formatshaper/VQops.h
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBMEDIAFORMATSHAPER_VQOPS_H_
+#define LIBMEDIAFORMATSHAPER_VQOPS_H_
+
+#include <media/formatshaper/CodecProperties.h>
+#include <media/NdkMediaFormat.h>
+
+namespace android {
+namespace mediaformatshaper {
+
+// parameterized info for the different mediaType types
+typedef struct {
+ const char *mediaType;
+
+ int32_t qpMin; // codec type limit (e.g. h264, not c2.android.avc.encoder)
+ int32_t qpMax;
+ int32_t qpDelta; // from I to P to B
+
+} vqOps_t;
+
+int VQApply(CodecProperties *codec, vqOps_t *info, AMediaFormat* inFormat, int flags);
+
+// spread the overall QP setting to any un-set per-frame-type settings
+void qpSpreadPerFrameType(AMediaFormat *format, int delta, int qplow, int qphigh, bool override);
+void qpSpreadMaxPerFrameType(AMediaFormat *format, int delta, int qphigh, bool override);
+void qpSpreadMinPerFrameType(AMediaFormat *format, int qplow, bool override);
+
+// does the format have QP bounding entries
+bool hasQp(AMediaFormat *format);
+bool hasQpPerFrameType(AMediaFormat *format);
+
+} // namespace mediaformatshaper
+} // namespace android
+
+#endif // LIBMEDIAFORMATSHAPER_VQOPS_H_
diff --git a/media/libmediaformatshaper/include/media/formatshaper/VideoShaper.h b/media/libmediaformatshaper/include/media/formatshaper/VideoShaper.h
new file mode 100644
index 0000000..53f1b13
--- /dev/null
+++ b/media/libmediaformatshaper/include/media/formatshaper/VideoShaper.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2021, The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef LIBMEDIAFORMATSHAPER_VIDEOSHAPER_H_
+#define LIBMEDIAFORMATSHAPER_VIDEOSHAPER_H_
+
+namespace android {
+namespace mediaformatshaper {
+
+/*
+ * runs through video-specific shaping operations for the codec/format combination.
+ * updates inFormat in place.
+ */
+int videoShaper(CodecProperties *codec, AMediaFormat* inFormat, int flags);
+
+} // namespace mediaformatshaper
+} // namespace android
+
+#endif // LIBMEDIAFORMATSHAPER_VIDEOSHAPER_H_