MediaCodec: fix output format change event timing am: eb0f8a1136
Original change: https://android-review.googlesource.com/c/platform/frameworks/av/+/1478016
Change-Id: Ib5fd1ab66b78a8684c4e9413820f2361f23946d6
diff --git a/media/codec2/sfplugin/CCodecBuffers.cpp b/media/codec2/sfplugin/CCodecBuffers.cpp
index 689e3bb..692da58 100644
--- a/media/codec2/sfplugin/CCodecBuffers.cpp
+++ b/media/codec2/sfplugin/CCodecBuffers.cpp
@@ -274,8 +274,6 @@
// The output format can be processed without a registered slot.
if (outputFormat) {
- ALOGD("[%s] popFromStashAndRegister: output format changed to %s",
- mName, outputFormat->debugString().c_str());
updateSkipCutBuffer(outputFormat, entry.notify);
}
@@ -303,6 +301,10 @@
}
if (!entry.notify) {
+ if (outputFormat) {
+ ALOGD("[%s] popFromStashAndRegister: output format changed to %s",
+ mName, outputFormat->debugString().c_str());
+ }
mPending.pop_front();
return DISCARD;
}
@@ -319,6 +321,10 @@
// Append information from the front stash entry to outBuffer.
(*outBuffer)->meta()->setInt64("timeUs", entry.timestamp);
(*outBuffer)->meta()->setInt32("flags", entry.flags);
+ if (outputFormat) {
+ ALOGD("[%s] popFromStashAndRegister: output format changed to %s",
+ mName, outputFormat->debugString().c_str());
+ }
ALOGV("[%s] popFromStashAndRegister: "
"out buffer index = %zu [%p] => %p + %zu (%lld)",
mName, *index, outBuffer->get(),
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 69084bf..a3a4aba 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -2076,20 +2076,25 @@
} else if (mFlags & kFlagOutputBuffersChanged) {
PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);
mFlags &= ~kFlagOutputBuffersChanged;
- } else if (mFlags & kFlagOutputFormatChanged) {
- PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
- mFlags &= ~kFlagOutputFormatChanged;
} else {
sp<AMessage> response = new AMessage;
- ssize_t index = dequeuePortBuffer(kPortIndexOutput);
-
- if (index < 0) {
- CHECK_EQ(index, -EAGAIN);
+ BufferInfo *info = peekNextPortBuffer(kPortIndexOutput);
+ if (!info) {
return false;
}
- const sp<MediaCodecBuffer> &buffer =
- mPortBuffers[kPortIndexOutput][index].mData;
+ // In synchronous mode, output format change should be handled
+ // at dequeue to put the event at the correct order.
+
+ const sp<MediaCodecBuffer> &buffer = info->mData;
+ handleOutputFormatChangeIfNeeded(buffer);
+ if (mFlags & kFlagOutputFormatChanged) {
+ PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
+ mFlags &= ~kFlagOutputFormatChanged;
+ return true;
+ }
+
+ ssize_t index = dequeuePortBuffer(kPortIndexOutput);
response->setSize("index", index);
response->setSize("offset", buffer->offset());
@@ -2601,107 +2606,13 @@
break;
}
- sp<RefBase> obj;
- CHECK(msg->findObject("buffer", &obj));
- sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
-
- if (mOutputFormat != buffer->format()) {
- if (mFlags & kFlagUseBlockModel) {
- sp<AMessage> diff1 = mOutputFormat->changesFrom(buffer->format());
- sp<AMessage> diff2 = buffer->format()->changesFrom(mOutputFormat);
- std::set<std::string> keys;
- size_t numEntries = diff1->countEntries();
- AMessage::Type type;
- for (size_t i = 0; i < numEntries; ++i) {
- keys.emplace(diff1->getEntryNameAt(i, &type));
- }
- numEntries = diff2->countEntries();
- for (size_t i = 0; i < numEntries; ++i) {
- keys.emplace(diff2->getEntryNameAt(i, &type));
- }
- sp<WrapperObject<std::set<std::string>>> changedKeys{
- new WrapperObject<std::set<std::string>>{std::move(keys)}};
- buffer->meta()->setObject("changedKeys", changedKeys);
- }
- mOutputFormat = buffer->format();
- ALOGV("[%s] output format changed to: %s",
- mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
-
- if (mSoftRenderer == NULL &&
- mSurface != NULL &&
- (mFlags & kFlagUsesSoftwareRenderer)) {
- AString mime;
- CHECK(mOutputFormat->findString("mime", &mime));
-
- // TODO: propagate color aspects to software renderer to allow better
- // color conversion to RGB. For now, just mark dataspace for YUV
- // rendering.
- int32_t dataSpace;
- if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {
- ALOGD("[%s] setting dataspace on output surface to #%x",
- mComponentName.c_str(), dataSpace);
- int err = native_window_set_buffers_data_space(
- mSurface.get(), (android_dataspace)dataSpace);
- ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);
- }
- if (mOutputFormat->contains("hdr-static-info")) {
- HDRStaticInfo info;
- if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
- setNativeWindowHdrMetadata(mSurface.get(), &info);
- }
- }
-
- sp<ABuffer> hdr10PlusInfo;
- if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
- && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
- native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),
- hdr10PlusInfo->size(), hdr10PlusInfo->data());
- }
-
- if (mime.startsWithIgnoreCase("video/")) {
- mSurface->setDequeueTimeout(-1);
- mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
- }
- }
-
- requestCpuBoostIfNeeded();
-
- if (mFlags & kFlagIsEncoder) {
- // Before we announce the format change we should
- // collect codec specific data and amend the output
- // format as necessary.
- int32_t flags = 0;
- (void) buffer->meta()->findInt32("flags", &flags);
- if ((flags & BUFFER_FLAG_CODECCONFIG) && !(mFlags & kFlagIsSecure)) {
- status_t err =
- amendOutputFormatWithCodecSpecificData(buffer);
-
- if (err != OK) {
- ALOGE("Codec spit out malformed codec "
- "specific data!");
- }
- }
- }
- if (mFlags & kFlagIsAsync) {
- onOutputFormatChanged();
- } else {
- mFlags |= kFlagOutputFormatChanged;
- postActivityNotificationIfPossible();
- }
-
- // Notify mCrypto of video resolution changes
- if (mCrypto != NULL) {
- int32_t left, top, right, bottom, width, height;
- if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
- mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
- } else if (mOutputFormat->findInt32("width", &width)
- && mOutputFormat->findInt32("height", &height)) {
- mCrypto->notifyResolution(width, height);
- }
- }
- }
-
if (mFlags & kFlagIsAsync) {
+ sp<RefBase> obj;
+ CHECK(msg->findObject("buffer", &obj));
+ sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
+
+ // In asynchronous mode, output format change is processed immediately.
+ handleOutputFormatChangeIfNeeded(buffer);
onOutputBufferAvailable();
} else if (mFlags & kFlagDequeueOutputPending) {
CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
@@ -3604,6 +3515,106 @@
}
}
+void MediaCodec::handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer) {
+ sp<AMessage> format = buffer->format();
+ if (mOutputFormat == format) {
+ return;
+ }
+ if (mFlags & kFlagUseBlockModel) {
+ sp<AMessage> diff1 = mOutputFormat->changesFrom(format);
+ sp<AMessage> diff2 = format->changesFrom(mOutputFormat);
+ std::set<std::string> keys;
+ size_t numEntries = diff1->countEntries();
+ AMessage::Type type;
+ for (size_t i = 0; i < numEntries; ++i) {
+ keys.emplace(diff1->getEntryNameAt(i, &type));
+ }
+ numEntries = diff2->countEntries();
+ for (size_t i = 0; i < numEntries; ++i) {
+ keys.emplace(diff2->getEntryNameAt(i, &type));
+ }
+ sp<WrapperObject<std::set<std::string>>> changedKeys{
+ new WrapperObject<std::set<std::string>>{std::move(keys)}};
+ buffer->meta()->setObject("changedKeys", changedKeys);
+ }
+ mOutputFormat = format;
+ ALOGV("[%s] output format changed to: %s",
+ mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
+
+ if (mSoftRenderer == NULL &&
+ mSurface != NULL &&
+ (mFlags & kFlagUsesSoftwareRenderer)) {
+ AString mime;
+ CHECK(mOutputFormat->findString("mime", &mime));
+
+ // TODO: propagate color aspects to software renderer to allow better
+ // color conversion to RGB. For now, just mark dataspace for YUV
+ // rendering.
+ int32_t dataSpace;
+ if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {
+ ALOGD("[%s] setting dataspace on output surface to #%x",
+ mComponentName.c_str(), dataSpace);
+ int err = native_window_set_buffers_data_space(
+ mSurface.get(), (android_dataspace)dataSpace);
+ ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);
+ }
+ if (mOutputFormat->contains("hdr-static-info")) {
+ HDRStaticInfo info;
+ if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
+ setNativeWindowHdrMetadata(mSurface.get(), &info);
+ }
+ }
+
+ sp<ABuffer> hdr10PlusInfo;
+ if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
+ && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
+ native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),
+ hdr10PlusInfo->size(), hdr10PlusInfo->data());
+ }
+
+ if (mime.startsWithIgnoreCase("video/")) {
+ mSurface->setDequeueTimeout(-1);
+ mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
+ }
+ }
+
+ requestCpuBoostIfNeeded();
+
+ if (mFlags & kFlagIsEncoder) {
+ // Before we announce the format change we should
+ // collect codec specific data and amend the output
+ // format as necessary.
+ int32_t flags = 0;
+ (void) buffer->meta()->findInt32("flags", &flags);
+ if ((flags & BUFFER_FLAG_CODECCONFIG) && !(mFlags & kFlagIsSecure)) {
+ status_t err =
+ amendOutputFormatWithCodecSpecificData(buffer);
+
+ if (err != OK) {
+ ALOGE("Codec spit out malformed codec "
+ "specific data!");
+ }
+ }
+ }
+ if (mFlags & kFlagIsAsync) {
+ onOutputFormatChanged();
+ } else {
+ mFlags |= kFlagOutputFormatChanged;
+ postActivityNotificationIfPossible();
+ }
+
+ // Notify mCrypto of video resolution changes
+ if (mCrypto != NULL) {
+ int32_t left, top, right, bottom, width, height;
+ if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
+ mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
+ } else if (mOutputFormat->findInt32("width", &width)
+ && mOutputFormat->findInt32("height", &height)) {
+ mCrypto->notifyResolution(width, height);
+ }
+ }
+}
+
void MediaCodec::extractCSD(const sp<AMessage> &format) {
mCSD.clear();
@@ -4080,19 +4091,31 @@
return OK;
}
-ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
+MediaCodec::BufferInfo *MediaCodec::peekNextPortBuffer(int32_t portIndex) {
CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
if (availBuffers->empty()) {
+ return nullptr;
+ }
+
+ return &mPortBuffers[portIndex][*availBuffers->begin()];
+}
+
+ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
+ CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
+
+ BufferInfo *info = peekNextPortBuffer(portIndex);
+ if (!info) {
return -EAGAIN;
}
+ List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
size_t index = *availBuffers->begin();
+ CHECK_EQ(info, &mPortBuffers[portIndex][index]);
availBuffers->erase(availBuffers->begin());
- BufferInfo *info = &mPortBuffers[portIndex][index];
CHECK(!info->mOwnedByClient);
{
Mutex::Autolock al(mBufferLock);
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 4705e4a..5ff4e12 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -455,6 +455,7 @@
size_t updateBuffers(int32_t portIndex, const sp<AMessage> &msg);
status_t onQueueInputBuffer(const sp<AMessage> &msg);
status_t onReleaseOutputBuffer(const sp<AMessage> &msg);
+ BufferInfo *peekNextPortBuffer(int32_t portIndex);
ssize_t dequeuePortBuffer(int32_t portIndex);
status_t getBufferAndFormat(
@@ -486,6 +487,7 @@
status_t onSetParameters(const sp<AMessage> ¶ms);
status_t amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> &buffer);
+ void handleOutputFormatChangeIfNeeded(const sp<MediaCodecBuffer> &buffer);
bool isExecuting() const;
uint64_t getGraphicBufferSize();