RELAND CCodec: handle dataspace update from GraphicBufferSource
In addition, add ColorUtils::getColorConfigFromDataSpace and use it to
get platform color configs out of dataspace.
Bug: 178675686
Test: atest ccodec_unit_test
Change-Id: I657156093e9d9cdcc12e6ed55644a3f6648ed7c9
diff --git a/media/codec2/sfplugin/C2OMXNode.cpp b/media/codec2/sfplugin/C2OMXNode.cpp
index ab73245..2460490 100644
--- a/media/codec2/sfplugin/C2OMXNode.cpp
+++ b/media/codec2/sfplugin/C2OMXNode.cpp
@@ -33,12 +33,14 @@
#include <OMX_IndexExt.h>
#include <android/fdsan.h>
+#include <media/stagefright/foundation/ColorUtils.h>
#include <media/stagefright/omx/OMXUtils.h>
#include <media/stagefright/MediaErrors.h>
#include <ui/Fence.h>
#include <ui/GraphicBuffer.h>
#include <utils/Thread.h>
+#include "utils/Codec2Mapper.h"
#include "C2OMXNode.h"
namespace android {
@@ -71,6 +73,23 @@
jobs->cond.broadcast();
}
+ void setDataspace(android_dataspace dataspace) {
+ Mutexed<Jobs>::Locked jobs(mJobs);
+ ColorUtils::convertDataSpaceToV0(dataspace);
+ jobs->configUpdate.emplace_back(new C2StreamDataSpaceInfo::input(0u, dataspace));
+ int32_t standard;
+ int32_t transfer;
+ int32_t range;
+ ColorUtils::getColorConfigFromDataSpace(dataspace, &range, &standard, &transfer);
+ std::unique_ptr<C2StreamColorAspectsInfo::input> colorAspects =
+ std::make_unique<C2StreamColorAspectsInfo::input>(0u);
+ if (C2Mapper::map(standard, &colorAspects->primaries, &colorAspects->matrix)
+ && C2Mapper::map(transfer, &colorAspects->transfer)
+ && C2Mapper::map(range, &colorAspects->range)) {
+ jobs->configUpdate.push_back(std::move(colorAspects));
+ }
+ }
+
protected:
bool threadLoop() override {
constexpr nsecs_t kIntervalNs = nsecs_t(10) * 1000 * 1000; // 10ms
@@ -102,6 +121,9 @@
uniqueFds.push_back(std::move(queue.workList.front().fd1));
queue.workList.pop_front();
}
+ for (const std::unique_ptr<C2Param> ¶m : jobs->configUpdate) {
+ items.front()->input.configUpdate.emplace_back(C2Param::Copy(*param));
+ }
jobs.unlock();
for (int fenceFd : fenceFds) {
@@ -119,6 +141,7 @@
queued = true;
}
if (queued) {
+ jobs->configUpdate.clear();
return true;
}
if (i == 0) {
@@ -161,6 +184,7 @@
std::map<std::weak_ptr<Codec2Client::Component>,
Queue,
std::owner_less<std::weak_ptr<Codec2Client::Component>>> queues;
+ std::vector<std::unique_ptr<C2Param>> configUpdate;
Condition cond;
};
Mutexed<Jobs> mJobs;
@@ -172,6 +196,9 @@
mQueueThread(new QueueThread) {
android_fdsan_set_error_level(ANDROID_FDSAN_ERROR_LEVEL_WARN_ALWAYS);
mQueueThread->run("C2OMXNode", PRIORITY_AUDIO);
+
+ Mutexed<android_dataspace>::Locked ds(mDataspace);
+ *ds = HAL_DATASPACE_UNKNOWN;
}
status_t C2OMXNode::freeNode() {
@@ -459,8 +486,11 @@
android_dataspace dataSpace = (android_dataspace)msg.u.event_data.data1;
uint32_t pixelFormat = msg.u.event_data.data3;
- // TODO: set dataspace on component to see if it impacts color aspects
ALOGD("dataspace changed to %#x pixel format: %#x", dataSpace, pixelFormat);
+ mQueueThread->setDataspace(dataSpace);
+
+ Mutexed<android_dataspace>::Locked ds(mDataspace);
+ *ds = dataSpace;
return OK;
}
@@ -493,4 +523,8 @@
(void)mBufferSource->onInputBufferEmptied(bufferId, -1);
}
+android_dataspace C2OMXNode::getDataspace() {
+ return *mDataspace.lock();
+}
+
} // namespace android
diff --git a/media/codec2/sfplugin/C2OMXNode.h b/media/codec2/sfplugin/C2OMXNode.h
index 1717c96..9c04969 100644
--- a/media/codec2/sfplugin/C2OMXNode.h
+++ b/media/codec2/sfplugin/C2OMXNode.h
@@ -93,6 +93,11 @@
*/
void onInputBufferDone(c2_cntr64_t index);
+ /**
+ * Returns dataspace information from GraphicBufferSource.
+ */
+ android_dataspace getDataspace();
+
private:
std::weak_ptr<Codec2Client::Component> mComp;
sp<IOMXBufferSource> mBufferSource;
@@ -101,6 +106,7 @@
uint32_t mWidth;
uint32_t mHeight;
uint64_t mUsage;
+ Mutexed<android_dataspace> mDataspace;
// WORKAROUND: timestamp adjustment
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index f7564f6..12b3b8b 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -211,8 +211,6 @@
(OMX_INDEXTYPE)OMX_IndexParamConsumerUsageBits,
&usage, sizeof(usage));
- // NOTE: we do not use/pass through color aspects from GraphicBufferSource as we
- // communicate that directly to the component.
mSource->configure(
mOmxNode, static_cast<hardware::graphics::common::V1_0::Dataspace>(mDataSpace));
return OK;
@@ -411,6 +409,10 @@
mNode->onInputBufferDone(index);
}
+ android_dataspace getDataspace() override {
+ return mNode->getDataspace();
+ }
+
private:
sp<HGraphicBufferSource> mSource;
sp<C2OMXNode> mNode;
@@ -1571,6 +1573,7 @@
outputFormat = config->mOutputFormat = config->mOutputFormat->dup();
if (config->mInputSurface) {
err2 = config->mInputSurface->start();
+ config->mInputSurfaceDataspace = config->mInputSurface->getDataspace();
}
buffersBoundToCodec = config->mBuffersBoundToCodec;
}
@@ -1658,6 +1661,7 @@
if (config->mInputSurface) {
config->mInputSurface->disconnect();
config->mInputSurface = nullptr;
+ config->mInputSurfaceDataspace = HAL_DATASPACE_UNKNOWN;
}
}
{
@@ -1707,6 +1711,7 @@
if (config->mInputSurface) {
config->mInputSurface->disconnect();
config->mInputSurface = nullptr;
+ config->mInputSurfaceDataspace = HAL_DATASPACE_UNKNOWN;
}
}
diff --git a/media/codec2/sfplugin/CCodecConfig.cpp b/media/codec2/sfplugin/CCodecConfig.cpp
index f5cc98e..0d3ad9e 100644
--- a/media/codec2/sfplugin/CCodecConfig.cpp
+++ b/media/codec2/sfplugin/CCodecConfig.cpp
@@ -1128,6 +1128,12 @@
insertion.first->second = std::move(p);
}
}
+ if (mInputSurface
+ && (domain & mOutputDomain)
+ && mInputSurfaceDataspace != mInputSurface->getDataspace()) {
+ changed = true;
+ mInputSurfaceDataspace = mInputSurface->getDataspace();
+ }
ALOGV("updated configuration has %zu params (%s)", mCurrentConfig.size(),
changed ? "CHANGED" : "no change");
@@ -1356,7 +1362,6 @@
msg->removeEntryAt(msg->findEntryByName("color-matrix"));
}
-
// calculate dataspace for raw graphic buffers if not specified by component, or if
// using surface with unspecified aspects (as those must be defaulted which may change
// the dataspace)
@@ -1394,6 +1399,23 @@
}
}
+ if (mInputSurface) {
+ android_dataspace dataspace = mInputSurface->getDataspace();
+ ColorUtils::convertDataSpaceToV0(dataspace);
+ int32_t standard;
+ ColorUtils::getColorConfigFromDataSpace(dataspace, &range, &standard, &transfer);
+ if (range != 0) {
+ msg->setInt32(KEY_COLOR_RANGE, range);
+ }
+ if (standard != 0) {
+ msg->setInt32(KEY_COLOR_STANDARD, standard);
+ }
+ if (transfer != 0) {
+ msg->setInt32(KEY_COLOR_TRANSFER, transfer);
+ }
+ msg->setInt32("android._dataspace", dataspace);
+ }
+
// HDR static info
C2HdrStaticMetadataStruct hdr;
diff --git a/media/codec2/sfplugin/CCodecConfig.h b/media/codec2/sfplugin/CCodecConfig.h
index 7e060f6..d93ff8b 100644
--- a/media/codec2/sfplugin/CCodecConfig.h
+++ b/media/codec2/sfplugin/CCodecConfig.h
@@ -124,6 +124,7 @@
std::shared_ptr<InputSurfaceWrapper> mInputSurface;
std::unique_ptr<InputSurfaceWrapper::Config> mISConfig;
+ android_dataspace mInputSurfaceDataspace;
/// the current configuration. Updated after configure() and based on configUpdate in
/// onWorkDone
diff --git a/media/codec2/sfplugin/InputSurfaceWrapper.h b/media/codec2/sfplugin/InputSurfaceWrapper.h
index bb35763..3ddae01 100644
--- a/media/codec2/sfplugin/InputSurfaceWrapper.h
+++ b/media/codec2/sfplugin/InputSurfaceWrapper.h
@@ -106,6 +106,11 @@
*/
virtual void onInputBufferDone(c2_cntr64_t /* index */) {}
+ /**
+ * Returns dataspace information from GraphicBufferSource.
+ */
+ virtual android_dataspace getDataspace() { return mDataSpace; }
+
protected:
android_dataspace mDataSpace;
};
diff --git a/media/codec2/sfplugin/tests/CCodecConfig_test.cpp b/media/codec2/sfplugin/tests/CCodecConfig_test.cpp
index c9caa01..7c660dc 100644
--- a/media/codec2/sfplugin/tests/CCodecConfig_test.cpp
+++ b/media/codec2/sfplugin/tests/CCodecConfig_test.cpp
@@ -208,6 +208,24 @@
.withSetter(Setter<C2StreamPixelAspectRatioInfo::output>)
.build());
+ if (isEncoder) {
+ addParameter(
+ DefineParam(mInputBitrate, C2_PARAMKEY_BITRATE)
+ .withDefault(new C2StreamBitrateInfo::input(0u))
+ .withFields({C2F(mInputBitrate, value).any()})
+ .withSetter(Setter<C2StreamBitrateInfo::input>)
+ .build());
+
+ addParameter(
+ DefineParam(mOutputBitrate, C2_PARAMKEY_BITRATE)
+ .withDefault(new C2StreamBitrateInfo::output(0u))
+ .withFields({C2F(mOutputBitrate, value).any()})
+ .calculatedAs(
+ Copy<C2StreamBitrateInfo::output, C2StreamBitrateInfo::input>,
+ mInputBitrate)
+ .build());
+ }
+
// TODO: more SDK params
}
private:
@@ -221,11 +239,19 @@
std::shared_ptr<C2StreamVendorInt64Info::output> mInt64Output;
std::shared_ptr<C2PortVendorStringInfo::input> mStringInput;
std::shared_ptr<C2StreamPixelAspectRatioInfo::output> mPixelAspectRatio;
+ std::shared_ptr<C2StreamBitrateInfo::input> mInputBitrate;
+ std::shared_ptr<C2StreamBitrateInfo::output> mOutputBitrate;
template<typename T>
static C2R Setter(bool, C2P<T> &) {
return C2R::Ok();
}
+
+ template<typename ME, typename DEP>
+ static C2R Copy(bool, C2P<ME> &me, const C2P<DEP> &dep) {
+ me.set().value = dep.v.value;
+ return C2R::Ok();
+ }
};
Impl mImpl;
@@ -457,4 +483,97 @@
<< "mInputFormat = " << mConfig.mInputFormat->debugString().c_str();
}
+TEST_F(CCodecConfigTest, DataspaceUpdate) {
+ init(C2Component::DOMAIN_VIDEO, C2Component::KIND_ENCODER, MIMETYPE_VIDEO_AVC);
+
+ ASSERT_EQ(OK, mConfig.initialize(mReflector, mConfigurable));
+ class InputSurfaceStub : public InputSurfaceWrapper {
+ public:
+ ~InputSurfaceStub() override = default;
+ status_t connect(const std::shared_ptr<Codec2Client::Component> &) override {
+ return OK;
+ }
+ void disconnect() override {}
+ status_t start() override { return OK; }
+ status_t signalEndOfInputStream() override { return OK; }
+ status_t configure(Config &) override { return OK; }
+ };
+ mConfig.mInputSurface = std::make_shared<InputSurfaceStub>();
+
+ sp<AMessage> format{new AMessage};
+ format->setInt32(KEY_COLOR_RANGE, COLOR_RANGE_LIMITED);
+ format->setInt32(KEY_COLOR_STANDARD, COLOR_STANDARD_BT709);
+ format->setInt32(KEY_COLOR_TRANSFER, COLOR_TRANSFER_SDR_VIDEO);
+ format->setInt32(KEY_BIT_RATE, 100);
+
+ std::vector<std::unique_ptr<C2Param>> configUpdate;
+ ASSERT_EQ(OK, mConfig.getConfigUpdateFromSdkParams(
+ mConfigurable, format, D::ALL, C2_MAY_BLOCK, &configUpdate));
+ ASSERT_TRUE(mConfig.updateConfiguration(configUpdate, D::ALL));
+
+ int32_t range{0};
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_RANGE, &range))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_RANGE_LIMITED, range)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+
+ int32_t standard{0};
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_STANDARD, &standard))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_STANDARD_BT709, standard)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+
+ int32_t transfer{0};
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_TRANSFER, &transfer))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_TRANSFER_SDR_VIDEO, transfer)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+
+ mConfig.mInputSurface->setDataSpace(HAL_DATASPACE_BT2020_PQ);
+
+ // Dataspace from input surface should override the configured setting
+ mConfig.updateFormats(D::ALL);
+
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_RANGE, &range))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_RANGE_FULL, range)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_STANDARD, &standard))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_STANDARD_BT2020, standard)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_TRANSFER, &transfer))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_TRANSFER_ST2084, transfer)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+
+ // Simulate bitrate update
+ format = new AMessage;
+ format->setInt32(KEY_BIT_RATE, 200);
+ configUpdate.clear();
+ ASSERT_EQ(OK, mConfig.getConfigUpdateFromSdkParams(
+ mConfigurable, format, D::ALL, C2_MAY_BLOCK, &configUpdate));
+ ASSERT_EQ(OK, mConfig.setParameters(mConfigurable, configUpdate, C2_MAY_BLOCK));
+
+ // Color information should remain the same
+ mConfig.updateFormats(D::ALL);
+
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_RANGE, &range))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_RANGE_FULL, range)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_STANDARD, &standard))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_STANDARD_BT2020, standard)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+
+ ASSERT_TRUE(mConfig.mOutputFormat->findInt32(KEY_COLOR_TRANSFER, &transfer))
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+ EXPECT_EQ(COLOR_TRANSFER_ST2084, transfer)
+ << "mOutputFormat = " << mConfig.mOutputFormat->debugString().c_str();
+}
+
} // namespace android
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 71a4ad8..224ec8b 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -5683,15 +5683,18 @@
int32_t range, standard, transfer;
convertCodecColorAspectsToPlatformAspects(aspects, &range, &standard, &transfer);
+ int32_t dsRange, dsStandard, dsTransfer;
+ getColorConfigFromDataSpace(dataSpace, &dsRange, &dsStandard, &dsTransfer);
+
// if some aspects are unspecified, use dataspace fields
if (range == 0) {
- range = (dataSpace & HAL_DATASPACE_RANGE_MASK) >> HAL_DATASPACE_RANGE_SHIFT;
+ range = dsRange;
}
if (standard == 0) {
- standard = (dataSpace & HAL_DATASPACE_STANDARD_MASK) >> HAL_DATASPACE_STANDARD_SHIFT;
+ standard = dsStandard;
}
if (transfer == 0) {
- transfer = (dataSpace & HAL_DATASPACE_TRANSFER_MASK) >> HAL_DATASPACE_TRANSFER_SHIFT;
+ transfer = dsTransfer;
}
mOutputFormat = mOutputFormat->dup(); // trigger an output format changed event
diff --git a/media/libstagefright/MediaCodecSource.cpp b/media/libstagefright/MediaCodecSource.cpp
index 0f7df24..876d06c 100644
--- a/media/libstagefright/MediaCodecSource.cpp
+++ b/media/libstagefright/MediaCodecSource.cpp
@@ -775,12 +775,9 @@
android_dataspace dataspace = static_cast<android_dataspace>(ds);
ColorUtils::convertDataSpaceToV0(dataspace);
ALOGD("Updating dataspace to %x", dataspace);
- int32_t standard = (int32_t(dataspace) & HAL_DATASPACE_STANDARD_MASK)
- >> HAL_DATASPACE_STANDARD_SHIFT;
- int32_t transfer = (int32_t(dataspace) & HAL_DATASPACE_TRANSFER_MASK)
- >> HAL_DATASPACE_TRANSFER_SHIFT;
- int32_t range = (int32_t(dataspace) & HAL_DATASPACE_RANGE_MASK)
- >> HAL_DATASPACE_RANGE_SHIFT;
+ int32_t standard, transfer, range;
+ ColorUtils::getColorConfigFromDataSpace(
+ dataspace, &range, &standard, &transfer);
sp<AMessage> msg = new AMessage;
msg->setInt32(KEY_COLOR_STANDARD, standard);
msg->setInt32(KEY_COLOR_TRANSFER, transfer);
diff --git a/media/libstagefright/foundation/ColorUtils.cpp b/media/libstagefright/foundation/ColorUtils.cpp
index 070e325..3812afe 100644
--- a/media/libstagefright/foundation/ColorUtils.cpp
+++ b/media/libstagefright/foundation/ColorUtils.cpp
@@ -613,6 +613,35 @@
}
// static
+void ColorUtils::getColorConfigFromDataSpace(
+ const android_dataspace &dataspace, int32_t *range, int32_t *standard, int32_t *transfer) {
+ uint32_t gfxRange =
+ (dataspace & HAL_DATASPACE_RANGE_MASK) >> HAL_DATASPACE_RANGE_SHIFT;
+ uint32_t gfxStandard =
+ (dataspace & HAL_DATASPACE_STANDARD_MASK) >> HAL_DATASPACE_STANDARD_SHIFT;
+ uint32_t gfxTransfer =
+ (dataspace & HAL_DATASPACE_TRANSFER_MASK) >> HAL_DATASPACE_TRANSFER_SHIFT;
+
+ // assume 1-to-1 mapping to HAL values (to deal with potential vendor extensions)
+ CU::ColorRange cuRange = CU::kColorRangeUnspecified;
+ CU::ColorStandard cuStandard = CU::kColorStandardUnspecified;
+ CU::ColorTransfer cuTransfer = CU::kColorTransferUnspecified;
+ // TRICKY: use & to ensure all three mappings are completed
+ if (!(sGfxRanges.map(gfxRange, &cuRange) & sGfxStandards.map(gfxStandard, &cuStandard)
+ & sGfxTransfers.map(gfxTransfer, &cuTransfer))) {
+ ALOGW("could not safely map graphics dataspace (R:%u S:%u T:%u) to "
+ "platform color aspects (R:%u(%s) S:%u(%s) T:%u(%s)",
+ gfxRange, gfxStandard, gfxTransfer,
+ cuRange, asString(cuRange),
+ cuStandard, asString(cuStandard),
+ cuTransfer, asString(cuTransfer));
+ }
+ *range = cuRange;
+ *standard = cuStandard;
+ *transfer = cuTransfer;
+}
+
+// static
void ColorUtils::getColorConfigFromFormat(
const sp<AMessage> &format, int32_t *range, int32_t *standard, int32_t *transfer) {
if (!format->findInt32("color-range", range)) {
diff --git a/media/libstagefright/foundation/include/media/stagefright/foundation/ColorUtils.h b/media/libstagefright/foundation/include/media/stagefright/foundation/ColorUtils.h
index cd0af2b..9e3f718 100644
--- a/media/libstagefright/foundation/include/media/stagefright/foundation/ColorUtils.h
+++ b/media/libstagefright/foundation/include/media/stagefright/foundation/ColorUtils.h
@@ -156,6 +156,10 @@
// suited to blending. This requires implicit color space conversion on part of the device.
static android_dataspace getDataSpaceForColorAspects(ColorAspects &aspects, bool mayExpand);
+ // it returns the platform color configs from given |dataspace|.
+ static void getColorConfigFromDataSpace(
+ const android_dataspace &dataspace, int *range, int *standard, int *transfer);
+
// converts |dataSpace| to a V0 enum, and returns true if dataSpace is an aspect-only value
static bool convertDataSpaceToV0(android_dataspace &dataSpace);