In order to recover from video lagging behind audio, drop avc frames
that are not referenced by other frames before feeding them into the decoder.
Change-Id: I822190af8f8329567bff8da1ea23136d0a765481
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index 6f34ba7..7218faf 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -34,17 +34,21 @@
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
#include <media/stagefright/ACodec.h>
+#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/MediaErrors.h>
#include <media/stagefright/MetaData.h>
#include <surfaceflinger/Surface.h>
#include <gui/ISurfaceTexture.h>
+#include "avc_utils.h"
+
namespace android {
////////////////////////////////////////////////////////////////////////////////
NuPlayer::NuPlayer()
: mUIDValid(false),
+ mVideoIsAVC(false),
mAudioEOS(false),
mVideoEOS(false),
mScanSourcesPending(false),
@@ -52,7 +56,12 @@
mFlushingAudio(NONE),
mFlushingVideo(NONE),
mResetInProgress(false),
- mResetPostponed(false) {
+ mResetPostponed(false),
+ mSkipRenderingAudioUntilMediaTimeUs(-1ll),
+ mSkipRenderingVideoUntilMediaTimeUs(-1ll),
+ mVideoLateByUs(0ll),
+ mNumFramesTotal(0ll),
+ mNumFramesDropped(0ll) {
}
NuPlayer::~NuPlayer() {
@@ -185,10 +194,14 @@
{
LOGV("kWhatStart");
+ mVideoIsAVC = false;
mAudioEOS = false;
mVideoEOS = false;
mSkipRenderingAudioUntilMediaTimeUs = -1;
mSkipRenderingVideoUntilMediaTimeUs = -1;
+ mVideoLateByUs = 0;
+ mNumFramesTotal = 0;
+ mNumFramesDropped = 0;
mSource->start();
@@ -269,6 +282,8 @@
} else {
CHECK(IsFlushingState(mFlushingVideo, &needShutdown));
mFlushingVideo = FLUSHED;
+
+ mVideoLateByUs = 0;
}
LOGV("decoder %s flush completed", audio ? "audio" : "video");
@@ -397,13 +412,18 @@
int64_t positionUs;
CHECK(msg->findInt64("positionUs", &positionUs));
+ CHECK(msg->findInt64("videoLateByUs", &mVideoLateByUs));
+
if (mDriver != NULL) {
sp<NuPlayerDriver> driver = mDriver.promote();
if (driver != NULL) {
driver->notifyPosition(positionUs);
+
+ driver->notifyFrameStats(
+ mNumFramesTotal, mNumFramesDropped);
}
}
- } else {
+ } else if (what == Renderer::kWhatFlushComplete) {
CHECK_EQ(what, (int32_t)Renderer::kWhatFlushComplete);
int32_t audio;
@@ -565,6 +585,12 @@
return -EWOULDBLOCK;
}
+ if (!audio) {
+ const char *mime;
+ CHECK(meta->findCString(kKeyMIMEType, &mime));
+ mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime);
+ }
+
sp<AMessage> notify =
new AMessage(audio ? kWhatAudioNotify : kWhatVideoNotify,
id());
@@ -598,53 +624,70 @@
}
sp<ABuffer> accessUnit;
- status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
- if (err == -EWOULDBLOCK) {
- return err;
- } else if (err != OK) {
- if (err == INFO_DISCONTINUITY) {
- int32_t type;
- CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
+ bool dropAccessUnit;
+ do {
+ status_t err = mSource->dequeueAccessUnit(audio, &accessUnit);
- bool formatChange =
- type == ATSParser::DISCONTINUITY_FORMATCHANGE;
+ if (err == -EWOULDBLOCK) {
+ return err;
+ } else if (err != OK) {
+ if (err == INFO_DISCONTINUITY) {
+ int32_t type;
+ CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
- LOGV("%s discontinuity (formatChange=%d)",
- audio ? "audio" : "video", formatChange);
+ bool formatChange =
+ type == ATSParser::DISCONTINUITY_FORMATCHANGE;
- if (audio) {
- mSkipRenderingAudioUntilMediaTimeUs = -1;
- } else {
- mSkipRenderingVideoUntilMediaTimeUs = -1;
- }
+ LOGV("%s discontinuity (formatChange=%d)",
+ audio ? "audio" : "video", formatChange);
- sp<AMessage> extra;
- if (accessUnit->meta()->findMessage("extra", &extra)
- && extra != NULL) {
- int64_t resumeAtMediaTimeUs;
- if (extra->findInt64(
- "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
- LOGI("suppressing rendering of %s until %lld us",
- audio ? "audio" : "video", resumeAtMediaTimeUs);
+ if (audio) {
+ mSkipRenderingAudioUntilMediaTimeUs = -1;
+ } else {
+ mSkipRenderingVideoUntilMediaTimeUs = -1;
+ }
- if (audio) {
- mSkipRenderingAudioUntilMediaTimeUs =
- resumeAtMediaTimeUs;
- } else {
- mSkipRenderingVideoUntilMediaTimeUs =
- resumeAtMediaTimeUs;
+ sp<AMessage> extra;
+ if (accessUnit->meta()->findMessage("extra", &extra)
+ && extra != NULL) {
+ int64_t resumeAtMediaTimeUs;
+ if (extra->findInt64(
+ "resume-at-mediatimeUs", &resumeAtMediaTimeUs)) {
+ LOGI("suppressing rendering of %s until %lld us",
+ audio ? "audio" : "video", resumeAtMediaTimeUs);
+
+ if (audio) {
+ mSkipRenderingAudioUntilMediaTimeUs =
+ resumeAtMediaTimeUs;
+ } else {
+ mSkipRenderingVideoUntilMediaTimeUs =
+ resumeAtMediaTimeUs;
+ }
}
}
+
+ flushDecoder(audio, formatChange);
}
- flushDecoder(audio, formatChange);
+ reply->setInt32("err", err);
+ reply->post();
+ return OK;
}
- reply->setInt32("err", err);
- reply->post();
- return OK;
- }
+ if (!audio) {
+ ++mNumFramesTotal;
+ }
+
+ dropAccessUnit = false;
+ if (!audio
+ && mVideoLateByUs > 100000ll
+ && mVideoIsAVC
+ && !IsAVCReferenceFrame(accessUnit)) {
+ dropAccessUnit = true;
+ ++mNumFramesDropped;
+ }
+ } while (dropAccessUnit);
// LOGV("returned a valid buffer of %s data", audio ? "audio" : "video");
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h
index 41e015b..a5382b4 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h
@@ -92,6 +92,7 @@
sp<NativeWindowWrapper> mNativeWindow;
sp<MediaPlayerBase::AudioSink> mAudioSink;
sp<Decoder> mVideoDecoder;
+ bool mVideoIsAVC;
sp<Decoder> mAudioDecoder;
sp<Renderer> mRenderer;
@@ -119,6 +120,9 @@
int64_t mSkipRenderingAudioUntilMediaTimeUs;
int64_t mSkipRenderingVideoUntilMediaTimeUs;
+ int64_t mVideoLateByUs;
+ int64_t mNumFramesTotal, mNumFramesDropped;
+
status_t instantiateDecoder(bool audio, sp<Decoder> *decoder);
status_t feedDecoderInputData(bool audio, const sp<AMessage> &msg);
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
index c6fca2c..b1e917d 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
@@ -31,6 +31,8 @@
: mResetInProgress(false),
mDurationUs(-1),
mPositionUs(-1),
+ mNumFramesTotal(0),
+ mNumFramesDropped(0),
mLooper(new ALooper),
mState(UNINITIALIZED),
mStartupSeekTimeUs(-1) {
@@ -292,4 +294,30 @@
sendEvent(MEDIA_SEEK_COMPLETE);
}
+void NuPlayerDriver::notifyFrameStats(
+ int64_t numFramesTotal, int64_t numFramesDropped) {
+ Mutex::Autolock autoLock(mLock);
+ mNumFramesTotal = numFramesTotal;
+ mNumFramesDropped = numFramesDropped;
+}
+
+status_t NuPlayerDriver::dump(int fd, const Vector<String16> &args) const {
+ Mutex::Autolock autoLock(mLock);
+
+ FILE *out = fdopen(dup(fd), "w");
+
+ fprintf(out, " NuPlayer\n");
+ fprintf(out, " numFramesTotal(%lld), numFramesDropped(%lld), "
+ "percentageDropped(%.2f)\n",
+ mNumFramesTotal,
+ mNumFramesDropped,
+ mNumFramesTotal == 0
+ ? 0.0 : (double)mNumFramesDropped / mNumFramesTotal);
+
+ fclose(out);
+ out = NULL;
+
+ return OK;
+}
+
} // namespace android
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h
index 1bb7ca2..181c37d 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.h
@@ -60,16 +60,19 @@
virtual status_t getMetadata(
const media::Metadata::Filter& ids, Parcel *records);
+ virtual status_t dump(int fd, const Vector<String16> &args) const;
+
void notifyResetComplete();
void notifyDuration(int64_t durationUs);
void notifyPosition(int64_t positionUs);
void notifySeekComplete();
+ void notifyFrameStats(int64_t numFramesTotal, int64_t numFramesDropped);
protected:
virtual ~NuPlayerDriver();
private:
- Mutex mLock;
+ mutable Mutex mLock;
Condition mCondition;
// The following are protected through "mLock"
@@ -77,6 +80,8 @@
bool mResetInProgress;
int64_t mDurationUs;
int64_t mPositionUs;
+ int64_t mNumFramesTotal;
+ int64_t mNumFramesDropped;
// <<<
sp<ALooper> mLooper;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
index 3b3fca2..07e347e 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.cpp
@@ -47,7 +47,8 @@
mHasVideo(false),
mSyncQueues(false),
mPaused(false),
- mLastPositionUpdateUs(-1ll) {
+ mLastPositionUpdateUs(-1ll),
+ mVideoLateByUs(0ll) {
}
NuPlayer::Renderer::~Renderer() {
@@ -357,22 +358,26 @@
mVideoQueue.erase(mVideoQueue.begin());
entry = NULL;
+
+ mVideoLateByUs = 0ll;
+
+ notifyPosition();
return;
}
-#if 0
int64_t mediaTimeUs;
CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
int64_t realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
- int64_t lateByUs = ALooper::GetNowUs() - realTimeUs;
+ mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
- if (lateByUs > 40000) {
- LOGI("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
+ bool tooLate = (mVideoLateByUs > 40000);
+
+ if (tooLate) {
+ LOGV("video late by %lld us (%.2f secs)", lateByUs, lateByUs / 1E6);
} else {
LOGV("rendering video at media time %.2f secs", mediaTimeUs / 1E6);
}
-#endif
entry->mNotifyConsumed->setInt32("render", true);
entry->mNotifyConsumed->post();
@@ -604,6 +609,7 @@
sp<AMessage> notify = mNotify->dup();
notify->setInt32("what", kWhatPosition);
notify->setInt64("positionUs", positionUs);
+ notify->setInt64("videoLateByUs", mVideoLateByUs);
notify->post();
}
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
index 44c5d44..268628b 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerRenderer.h
@@ -101,6 +101,7 @@
bool mPaused;
int64_t mLastPositionUpdateUs;
+ int64_t mVideoLateByUs;
bool onDrainAudioQueue();
void postDrainAudioQueue(int64_t delayUs = 0);
@@ -118,6 +119,7 @@
void notifyEOS(bool audio, status_t finalResult);
void notifyFlushComplete(bool audio);
void notifyPosition();
+ void notifyVideoLateBy(int64_t lateByUs);
void flushQueue(List<QueueEntry> *queue);
bool dropBufferWhileFlushing(bool audio, const sp<AMessage> &msg);
diff --git a/media/libstagefright/avc_utils.cpp b/media/libstagefright/avc_utils.cpp
index 8a42e8b..07aa140 100644
--- a/media/libstagefright/avc_utils.cpp
+++ b/media/libstagefright/avc_utils.cpp
@@ -329,6 +329,28 @@
return foundIDR;
}
+bool IsAVCReferenceFrame(const sp<ABuffer> &accessUnit) {
+ const uint8_t *data = accessUnit->data();
+ size_t size = accessUnit->size();
+
+ const uint8_t *nalStart;
+ size_t nalSize;
+ while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
+ CHECK_GT(nalSize, 0u);
+
+ unsigned nalType = nalStart[0] & 0x1f;
+
+ if (nalType == 5) {
+ return true;
+ } else if (nalType == 1) {
+ unsigned nal_ref_idc = (nalStart[0] >> 5) & 3;
+ return nal_ref_idc != 0;
+ }
+ }
+
+ return true;
+}
+
sp<MetaData> MakeAACCodecSpecificData(
unsigned profile, unsigned sampling_freq_index,
unsigned channel_configuration) {
diff --git a/media/libstagefright/include/avc_utils.h b/media/libstagefright/include/avc_utils.h
index 15cd4d4..e418822 100644
--- a/media/libstagefright/include/avc_utils.h
+++ b/media/libstagefright/include/avc_utils.h
@@ -50,6 +50,7 @@
sp<MetaData> MakeAVCCodecSpecificData(const sp<ABuffer> &accessUnit);
bool IsIDR(const sp<ABuffer> &accessUnit);
+bool IsAVCReferenceFrame(const sp<ABuffer> &accessUnit);
const char *AVCProfileToString(uint8_t profile);