VT: Enhancements on RTP depacketizer (2)
- Send FIR if there was no I-frame within a sec in packetLost
- Fixed an issue dropped frame even it is completed. caused by
ABuffer marked as 'damaged'.
- Provides completed I-Frame only to prevent codec stuck.
- Change time mapping as realtime since rtp time is already covered
by jitter buffer.
- Removed periodic RTCP:FIR requests if video call. Cause i-frame
insertion should happened only if video stream is broken.
- Give default value for uninitialize TrackInfo Value because
video stucked at the first frame all the times by Uninitialized
trash TimeScale & NormalPlaytime values make AccessUnit's ntp time
wrong.
Bug: 165061754
Merged-in: Ied41ea57d4fa96d4c05d7f8eba3d96bcb8f3fc69
Change-Id: Ied41ea57d4fa96d4c05d7f8eba3d96bcb8f3fc69
Signed-off-by: Kim Sungyeon <sy85.kim@samsung.com>
Signed-off-by: Byeongjo Park <bjo.park@samsung.com>
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.cpp b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
index f2bb896..5388bfd 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.cpp
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
@@ -135,11 +135,16 @@
if (info->mIsAudio) {
mAudioTrack = source;
+ info->mTimeScale = 16000;
} else {
mVideoTrack = source;
+ info->mTimeScale = 90000;
}
info->mSource = source;
+ info->mRTPTime = 0;
+ info->mNormalPlaytimeUs = 0;
+ info->mNPTMappingValid = false;
}
if (mInPreparationPhase) {
@@ -346,6 +351,11 @@
schedulePollBuffering();
}
+bool NuPlayer::RTPSource::isRealTime() const {
+ ALOGD("RTPSource::isRealTime=%d", true);
+ return true;
+}
+
void NuPlayer::RTPSource::onMessageReceived(const sp<AMessage> &msg) {
ALOGV("onMessageReceived =%d", msg->what());
@@ -428,7 +438,6 @@
source->queueAccessUnit(accessUnit);
break;
}
- */
int64_t nptUs =
((double)rtpTime - (double)info->mRTPTime)
@@ -436,7 +445,8 @@
* 1000000ll
+ info->mNormalPlaytimeUs;
- accessUnit->meta()->setInt64("timeUs", nptUs);
+ */
+ accessUnit->meta()->setInt64("timeUs", ALooper::GetNowUs());
source->queueAccessUnit(accessUnit);
}
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.h b/media/libmediaplayerservice/nuplayer/RTPSource.h
index 288104c..5d70142 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.h
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.h
@@ -80,6 +80,8 @@
int64_t seekTimeUs,
MediaPlayerSeekMode mode = MediaPlayerSeekMode::SEEK_PREVIOUS_SYNC) override;
+ virtual bool isRealTime() const;
+
void onMessageReceived(const sp<AMessage> &msg);
virtual void setTargetBitrate(int32_t bitrate) override;
diff --git a/media/libstagefright/rtsp/AAVCAssembler.cpp b/media/libstagefright/rtsp/AAVCAssembler.cpp
index 9d75828..efbde54 100644
--- a/media/libstagefright/rtsp/AAVCAssembler.cpp
+++ b/media/libstagefright/rtsp/AAVCAssembler.cpp
@@ -37,7 +37,9 @@
mAccessUnitRTPTime(0),
mNextExpectedSeqNoValid(false),
mNextExpectedSeqNo(0),
- mAccessUnitDamaged(false) {
+ mAccessUnitDamaged(false),
+ mFirstIFrameProvided(false),
+ mLastIFrameProvidedAt(0) {
}
AAVCAssembler::~AAVCAssembler() {
@@ -218,12 +220,27 @@
}
}
+void AAVCAssembler::checkIFrameProvided(const sp<ABuffer> &buffer) {
+ const uint8_t *data = buffer->data();
+ unsigned nalType = data[0] & 0x1f;
+ if (nalType == 0x5) {
+ mFirstIFrameProvided = true;
+ mLastIFrameProvidedAt = ALooper::GetNowUs() / 1000;
+
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ ALOGD("got First I-frame to be decoded. rtpTime=%d, size=%zu", rtpTime, buffer->size());
+ }
+}
+
void AAVCAssembler::addSingleNALUnit(const sp<ABuffer> &buffer) {
ALOGV("addSingleNALUnit of size %zu", buffer->size());
#if !LOG_NDEBUG
hexdump(buffer->data(), buffer->size());
#endif
+ checkIFrameProvided(buffer);
+
uint32_t rtpTime;
CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
@@ -324,6 +341,8 @@
complete = true;
} else {
List<sp<ABuffer> >::iterator it = ++queue->begin();
+ int32_t connected = 1;
+ bool snapped = false;
while (it != queue->end()) {
ALOGV("sequence length %zu", totalCount);
@@ -333,13 +352,17 @@
size_t size = buffer->size();
if ((uint32_t)buffer->int32Data() != expectedSeqNo) {
- ALOGV("sequence not complete, expected seqNo %d, got %d, pFrame %d",
- expectedSeqNo, (uint32_t)buffer->int32Data(), pFrame);
+ ALOGV("sequence not complete, expected seqNo %d, got %d, nalType %d",
+ expectedSeqNo, (uint32_t)buffer->int32Data(), nalType);
+ snapped = true;
if (!pFrame)
return WRONG_SEQUENCE_NUMBER;
}
+ if (!snapped)
+ connected++;
+
uint32_t rtpTime;
CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
if (size < 2
@@ -363,12 +386,14 @@
expectedSeqNo = buffer->int32Data() + 1;
if (data[1] & 0x40) {
- if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo, totalCount, 0.5f)) {
- mNextExpectedSeqNo = expectedSeqNo;
- deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
+ if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo,
+ connected, totalCount, 0.5f)) {
+ mNextExpectedSeqNo = expectedSeqNo;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
- return MALFORMED_PACKET;
+ return MALFORMED_PACKET;
}
+
// This is the last fragment.
complete = true;
break;
@@ -493,13 +518,15 @@
return nextSeqNo;
}
-bool AAVCAssembler::recycleUnit(uint32_t start, uint32_t end, size_t avail, float goodRatio) {
+bool AAVCAssembler::recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
+ size_t avail, float goodRatio) {
float total = end - start;
+ float valid = connected;
float exist = avail;
- bool isRecycle = (exist / total) >= goodRatio;
+ bool isRecycle = (valid / total) >= goodRatio;
- ALOGV("checking p-frame losses.. recvBufs %f diff %f recycle? %d",
- exist, total, isRecycle);
+ ALOGV("checking p-frame losses.. recvBufs %f valid %f diff %f recycle? %d",
+ exist, valid, total, isRecycle);
return isRecycle;
}
@@ -531,18 +558,20 @@
const sp<ARTPSource> &source) {
AssemblyStatus status = addNALUnit(source);
if (status == MALFORMED_PACKET) {
- mAccessUnitDamaged = true;
+ uint64_t timeAfterLastIFrame = (ALooper::GetNowUs() / 1000) - mLastIFrameProvidedAt;
+ if (timeAfterLastIFrame > 1000) {
+ ALOGV("request FIR to get a new I-Frame, time after "
+ "last I-Frame in miils %llu", (unsigned long long)timeAfterLastIFrame);
+ source->onIssueFIRByAssembler();
+ }
}
return status;
}
void AAVCAssembler::packetLost() {
CHECK(mNextExpectedSeqNoValid);
- ALOGV("packetLost (expected %d)", mNextExpectedSeqNo);
-
+ ALOGD("packetLost (expected %d)", mNextExpectedSeqNo);
++mNextExpectedSeqNo;
-
- mAccessUnitDamaged = true;
}
void AAVCAssembler::onByeReceived() {
diff --git a/media/libstagefright/rtsp/AAVCAssembler.h b/media/libstagefright/rtsp/AAVCAssembler.h
index b9add64..ac2c9a0 100644
--- a/media/libstagefright/rtsp/AAVCAssembler.h
+++ b/media/libstagefright/rtsp/AAVCAssembler.h
@@ -47,9 +47,12 @@
bool mNextExpectedSeqNoValid;
uint32_t mNextExpectedSeqNo;
bool mAccessUnitDamaged;
+ bool mFirstIFrameProvided;
+ uint64_t mLastIFrameProvidedAt;
List<sp<ABuffer> > mNALUnits;
int32_t addNack(const sp<ARTPSource> &source);
+ void checkIFrameProvided(const sp<ABuffer> &buffer);
AssemblyStatus addNALUnit(const sp<ARTPSource> &source);
void addSingleNALUnit(const sp<ABuffer> &buffer);
AssemblyStatus addFragmentedNALUnit(List<sp<ABuffer> > *queue);
@@ -58,7 +61,8 @@
void submitAccessUnit();
int32_t pickProperSeq(const Q *q, uint32_t jit, int64_t play);
- bool recycleUnit(uint32_t start, uint32_t end, size_t avail, float goodRatio);
+ bool recycleUnit(uint32_t start, uint32_t end, uint32_t conneceted,
+ size_t avail, float goodRatio);
int32_t deleteUnitUnderSeq(Q *q, uint32_t seq);
void printNowTimeUs(int64_t start, int64_t now, int64_t play);
void printRTPTime(uint32_t rtp, int64_t play, uint32_t exp, bool isExp);
diff --git a/media/libstagefright/rtsp/ARTPConnection.cpp b/media/libstagefright/rtsp/ARTPConnection.cpp
index a8f68dd..9c374ac 100644
--- a/media/libstagefright/rtsp/ARTPConnection.cpp
+++ b/media/libstagefright/rtsp/ARTPConnection.cpp
@@ -448,7 +448,7 @@
continue;
}
- // addNACK
+ // add NACK and FIR that needs to be sent immediately.
sp<ABuffer> buffer = new ABuffer(kMaxUDPSize);
for (size_t i = 0; i < it->mSources.size(); ++i) {
buffer->setRange(0, 0);
@@ -457,6 +457,13 @@
ALOGV("Send NACK for lost %d Packets", cnt);
send(&*it, buffer);
}
+
+ buffer->setRange(0, 0);
+ it->mSources.valueAt(i)->addFIR(buffer);
+ if (buffer->size() > 0) {
+ ALOGD("Send FIR immediately for lost Packets");
+ send(&*it, buffer);
+ }
}
++it;
@@ -1032,6 +1039,9 @@
source = new ARTPSource(
srcId, info->mSessionDesc, info->mIndex, info->mNotifyMsg);
+ if (mFlags & kViLTEConnection)
+ source->setPeriodicFIR(false);
+
source->setSelfID(mSelfID);
source->setJbTime(mJbTime > 0 ? mJbTime : 300);
info->mSources.add(srcId, source);
diff --git a/media/libstagefright/rtsp/ARTPSource.cpp b/media/libstagefright/rtsp/ARTPSource.cpp
index 5c8c71e..361773e 100644
--- a/media/libstagefright/rtsp/ARTPSource.cpp
+++ b/media/libstagefright/rtsp/ARTPSource.cpp
@@ -60,6 +60,7 @@
mLastNTPTime(0),
mLastNTPTimeUpdateUs(0),
mIssueFIRRequests(false),
+ mIssueFIRByAssembler(false),
mLastFIRRequestUs(-1),
mNextFIRSeqNo((rand() * 256.0) / RAND_MAX),
mNotify(notify) {
@@ -210,16 +211,29 @@
}
void ARTPSource::addFIR(const sp<ABuffer> &buffer) {
- if (!mIssueFIRRequests) {
+ if (!mIssueFIRRequests && !mIssueFIRByAssembler) {
return;
}
+ bool send = false;
int64_t nowUs = ALooper::GetNowUs();
- if (mLastFIRRequestUs >= 0 && mLastFIRRequestUs + 5000000LL > nowUs) {
- // Send FIR requests at most every 5 secs.
- return;
+ int64_t timeAfterLastFIR = nowUs - mLastFIRRequestUs;
+ if (mLastFIRRequestUs < 0) {
+ // A first FIR, just send it.
+ send = true;
+ } else if (mIssueFIRByAssembler && (timeAfterLastFIR > 1000000)) {
+ // A FIR issued by Assembler.
+ // Send it if last FIR is not sent within a sec.
+ send = true;
+ } else if (mIssueFIRRequests && (timeAfterLastFIR > 5000000)) {
+ // A FIR issued periodically reagardless packet loss.
+ // Send it if last FIR is not sent within 5 secs.
+ send = true;
}
+ if (!send)
+ return;
+
mLastFIRRequestUs = nowUs;
if (buffer->size() + 20 > buffer->capacity()) {
@@ -256,6 +270,9 @@
buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
+ if (mIssueFIRByAssembler)
+ mIssueFIRByAssembler = false;
+
ALOGV("Added FIR request.");
}
@@ -490,6 +507,11 @@
mJbTime = jbTime;
}
+void ARTPSource::setPeriodicFIR(bool enable) {
+ ALOGD("setPeriodicFIR %d", enable);
+ mIssueFIRRequests = enable;
+}
+
void ARTPSource::notifyPktInfo(int32_t bitrate, int64_t /*time*/) {
sp<AMessage> notify = mNotify->dup();
notify->setInt32("rtcp-event", 1);
@@ -509,6 +531,10 @@
mPrevNumBuffersReceived = mNumBuffersReceived;
}
+void ARTPSource::onIssueFIRByAssembler() {
+ mIssueFIRByAssembler = true;
+}
+
void ARTPSource::noticeAbandonBuffer(int cnt) {
mNumBuffersReceived -= cnt;
}
diff --git a/media/libstagefright/rtsp/ARTPSource.h b/media/libstagefright/rtsp/ARTPSource.h
index a68740e..9d4ccf2 100644
--- a/media/libstagefright/rtsp/ARTPSource.h
+++ b/media/libstagefright/rtsp/ARTPSource.h
@@ -54,7 +54,10 @@
uint32_t getSelfID();
void setSelfID(const uint32_t selfID);
void setJbTime(const uint32_t jbTime);
+ void setPeriodicFIR(bool enable);
void notifyPktInfo(int32_t bitrate, int64_t time);
+ // FIR needs to be sent by missing packet or broken video image.
+ void onIssueFIRByAssembler();
void noticeAbandonBuffer(int cnt=1);
@@ -96,6 +99,7 @@
int64_t mLastNTPTimeUpdateUs;
bool mIssueFIRRequests;
+ bool mIssueFIRByAssembler;
int64_t mLastFIRRequestUs;
uint8_t mNextFIRSeqNo;