VT: SFP: H264: CVO implementation for Rx side
CVO(Coordination of Video Orientation) implemented for H264.
Support Coordination of Video Orientation (CVO) feature that carried in RTP Header extension.
it parses a device degrees(orientation) in RTP extension that remote sent.
(6.2.3 of 3GPP Release 12 TS 26.114)
Once AAVCAssembler parsed a video degress from RTP(CVO) extension,
A native window will be rotated as per parsed video degress.
Bug: 121230209
Change-Id: I802a9e43d9ee54b7970d2541f18fb4af8022336a
Signed-off-by: Byeongjo Park <bjo.park@samsung.com>
Signed-off-by: Kim Sungyeon <sy85.kim@samsung.com>
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index 3c4a96b..4d33b4e 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -2838,28 +2838,34 @@
}
void NuPlayer::sendIMSRxNotice(const sp<AMessage> &msg) {
- int32_t notice, payloadType, feedbackType, id, bitrate;
+ int32_t payloadType;
- CHECK(msg->findInt32("IMS-Rx-notice", ¬ice));
CHECK(msg->findInt32("payload-type", &payloadType));
- CHECK(msg->findInt32("feedback-type", &feedbackType));
- CHECK(msg->findInt32("sender", &id));
Parcel in;
in.writeInt32(payloadType);
- in.writeInt32(feedbackType);
- in.writeInt32(id);
switch (payloadType) {
- case 205: // TSFB
+ case NuPlayer::RTPSource::RTCP_TSFB: // RTCP TSFB
+ case NuPlayer::RTPSource::RTCP_PSFB: // RTCP PSFB
{
- CHECK(msg->findInt32("bit-rate", &bitrate));
- in.writeInt32(bitrate);
+ int32_t feedbackType, id;
+ CHECK(msg->findInt32("feedback-type", &feedbackType));
+ CHECK(msg->findInt32("sender", &id));
+ in.writeInt32(feedbackType);
+ in.writeInt32(id);
+ if (payloadType == NuPlayer::RTPSource::RTCP_TSFB) {
+ int32_t bitrate;
+ CHECK(msg->findInt32("bit-rate", &bitrate));
+ in.writeInt32(bitrate);
+ }
break;
}
- case 206: // PSFB
+ case NuPlayer::RTPSource::RTP_CVO:
{
- // nothing to do yet
+ int32_t cvo;
+ CHECK(msg->findInt32("cvo", &cvo));
+ in.writeInt32(cvo);
break;
}
default:
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
index f734439..8628edc 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
@@ -1050,7 +1050,7 @@
uint32_t flags = 0;
CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
- int32_t eos, csd;
+ int32_t eos, csd, cvo;
// we do not expect SYNCFRAME for decoder
if (buffer->meta()->findInt32("eos", &eos) && eos) {
flags |= MediaCodec::BUFFER_FLAG_EOS;
@@ -1058,6 +1058,24 @@
flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
}
+ if (buffer->meta()->findInt32("cvo", (int32_t*)&cvo)) {
+ ALOGV("[%s] cvo(%d) found at %lld us", mComponentName.c_str(), cvo, (long long)timeUs);
+ switch (cvo) {
+ case 0:
+ codecBuffer->meta()->setInt32("cvo", MediaCodec::CVO_DEGREE_0);
+ break;
+ case 1:
+ codecBuffer->meta()->setInt32("cvo", MediaCodec::CVO_DEGREE_90);
+ break;
+ case 2:
+ codecBuffer->meta()->setInt32("cvo", MediaCodec::CVO_DEGREE_180);
+ break;
+ case 3:
+ codecBuffer->meta()->setInt32("cvo", MediaCodec::CVO_DEGREE_270);
+ break;
+ }
+ }
+
// Modular DRM
MediaBufferBase *mediaBuf = NULL;
NuPlayerDrm::CryptoInfo *cryptInfo = NULL;
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.cpp b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
index 6f4933c..e9e7d06 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.cpp
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
@@ -44,7 +44,8 @@
mInPreparationPhase(true),
mRTPConn(new ARTPConnection),
mEOSTimeoutAudio(0),
- mEOSTimeoutVideo(0) {
+ mEOSTimeoutVideo(0),
+ mLastCVOUpdated(-1) {
ALOGD("RTPSource initialized with rtpParams=%s", rtpParams.string());
}
@@ -92,7 +93,7 @@
AString sdp;
ASessionDescription::SDPStringFactory(sdp, info->mLocalIp,
info->mIsAudio, info->mLocalPort, info->mPayloadType, info->mAS, info->mCodecName,
- NULL, info->mWidth, info->mHeight);
+ NULL, info->mWidth, info->mHeight, info->mCVOExtMap);
ALOGD("RTPSource SDP =>\n%s", sdp.c_str());
sp<ASessionDescription> desc = new ASessionDescription;
@@ -273,7 +274,29 @@
setEOSTimeout(audio, 0);
- return source->dequeueAccessUnit(accessUnit);
+ finalResult = source->dequeueAccessUnit(accessUnit);
+ if (finalResult != OK) {
+ return finalResult;
+ }
+
+ int32_t cvo;
+ if ((*accessUnit) != NULL && (*accessUnit)->meta()->findInt32("cvo", &cvo)) {
+ if (cvo != mLastCVOUpdated) {
+ sp<AMessage> msg = new AMessage();
+ msg->setInt32("payload-type", NuPlayer::RTPSource::RTP_CVO);
+ msg->setInt32("cvo", cvo);
+
+ sp<AMessage> notify = dupNotify();
+ notify->setInt32("what", kWhatIMSRxNotice);
+ notify->setMessage("message", msg);
+ notify->post();
+
+ ALOGV("notify cvo updated (%d)->(%d) to upper layer", mLastCVOUpdated, cvo);
+ mLastCVOUpdated = cvo;
+ }
+ }
+
+ return finalResult;
}
sp<AnotherPacketSource> NuPlayer::RTPSource::getSource(bool audio) {
@@ -666,6 +689,8 @@
} else if (key == "rtp-param-time-scale") {
} else if (key == "rtp-param-self-id") {
info->mSelfID = atoi(value);
+ } else if (key == "rtp-param-ext-cvo-extmap") {
+ info->mCVOExtMap = atoi(value);
}
return OK;
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.h b/media/libmediaplayerservice/nuplayer/RTPSource.h
index 5b26478..85e77c6 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.h
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.h
@@ -53,6 +53,12 @@
const sp<AMessage> ¬ify,
const String8& rtpParams);
+ enum {
+ RTCP_TSFB = 205,
+ RTCP_PSFB = 206,
+ RTP_CVO = 300,
+ };
+
virtual status_t getBufferingSettings(
BufferingSettings* buffering /* nonnull */) override;
virtual status_t setBufferingSettings(const BufferingSettings& buffering) override;
@@ -116,6 +122,8 @@
/* Unique ID indicates itself */
uint32_t mSelfID;
+ /* extmap:<value> for CVO will be set to here */
+ int32_t mCVOExtMap;
/* a copy of TrackInfo in RTSPSource */
sp<AnotherPacketSource> mSource;
@@ -168,6 +176,7 @@
int64_t mMediaAnchorUs;
int64_t mLastMediaTimeUs;
int64_t mNumAccessUnitsReceived;
+ int32_t mLastCVOUpdated;
bool mReceivedFirstRTCPPacket;
bool mReceivedFirstRTPPacket;
bool mPausing;
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index 73d3a0b..d966df2 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -6064,6 +6064,13 @@
return;
}
+ int32_t cvo;
+ if (mCodec->mNativeWindow != NULL && buffer != NULL &&
+ buffer->meta()->findInt32("cvo", &cvo)) {
+ ALOGV("cvo(%d) found in buffer #%u", cvo, bufferID);
+ setNativeWindowRotation(mCodec->mNativeWindow.get(), cvo);
+ }
+
info->mStatus = BufferInfo::OWNED_BY_US;
info->mData = buffer;
diff --git a/media/libstagefright/SurfaceUtils.cpp b/media/libstagefright/SurfaceUtils.cpp
index 4c94baa..85ff474 100644
--- a/media/libstagefright/SurfaceUtils.cpp
+++ b/media/libstagefright/SurfaceUtils.cpp
@@ -165,6 +165,22 @@
ALOGW_IF(err != 0, "failed to set cta861_3 metadata on surface (%d)", err);
}
+status_t setNativeWindowRotation(
+ ANativeWindow *nativeWindow /* nonnull */, int rotation) {
+
+ int transform = 0;
+ if ((rotation % 90) == 0) {
+ switch ((rotation / 90) & 3) {
+ case 1: transform = HAL_TRANSFORM_ROT_90; break;
+ case 2: transform = HAL_TRANSFORM_ROT_180; break;
+ case 3: transform = HAL_TRANSFORM_ROT_270; break;
+ default: transform = 0; break;
+ }
+ }
+
+ return native_window_set_buffers_transform(nativeWindow, transform);
+}
+
status_t pushBlankBuffersToNativeWindow(ANativeWindow *nativeWindow /* nonnull */) {
status_t err = NO_ERROR;
ANativeWindowBuffer* anb = NULL;
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 022c48e..736f626 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -81,6 +81,13 @@
BUFFER_FLAG_MUXER_DATA = 16,
};
+ enum CVODegree {
+ CVO_DEGREE_0 = 0,
+ CVO_DEGREE_90 = 90,
+ CVO_DEGREE_180 = 180,
+ CVO_DEGREE_270 = 270,
+ };
+
enum {
CB_INPUT_AVAILABLE = 1,
CB_OUTPUT_AVAILABLE = 2,
diff --git a/media/libstagefright/include/media/stagefright/SurfaceUtils.h b/media/libstagefright/include/media/stagefright/SurfaceUtils.h
index ae55c65..35b3fa2 100644
--- a/media/libstagefright/include/media/stagefright/SurfaceUtils.h
+++ b/media/libstagefright/include/media/stagefright/SurfaceUtils.h
@@ -38,6 +38,8 @@
int width, int height, int format, int rotation, int usage, bool reconnect);
void setNativeWindowHdrMetadata(
ANativeWindow *nativeWindow /* nonnull */, HDRStaticInfo *info /* nonnull */);
+status_t setNativeWindowRotation(
+ ANativeWindow *nativeWindow /* nonnull */, int rotation);
status_t pushBlankBuffersToNativeWindow(ANativeWindow *nativeWindow /* nonnull */);
status_t nativeWindowConnect(ANativeWindow *surface, const char *reason);
status_t nativeWindowDisconnect(ANativeWindow *surface, const char *reason);
diff --git a/media/libstagefright/rtsp/AAVCAssembler.cpp b/media/libstagefright/rtsp/AAVCAssembler.cpp
index 13d74e4..a02c99a 100644
--- a/media/libstagefright/rtsp/AAVCAssembler.cpp
+++ b/media/libstagefright/rtsp/AAVCAssembler.cpp
@@ -338,6 +338,7 @@
unit->data()[0] = (nri << 5) | nalType;
size_t offset = 1;
+ int32_t cvo = -1;
List<sp<ABuffer> >::iterator it = queue->begin();
for (size_t i = 0; i < totalCount; ++i) {
const sp<ABuffer> &buffer = *it;
@@ -348,6 +349,8 @@
#endif
memcpy(unit->data() + offset, buffer->data() + 2, buffer->size() - 2);
+
+ buffer->meta()->findInt32("cvo", &cvo);
offset += buffer->size() - 2;
it = queue->erase(it);
@@ -355,6 +358,10 @@
unit->setRange(0, totalSize);
+ if (cvo >= 0) {
+ unit->meta()->setInt32("cvo", cvo);
+ }
+
addSingleNALUnit(unit);
ALOGV("successfully assembled a NAL unit from fragments.");
@@ -375,6 +382,7 @@
sp<ABuffer> accessUnit = new ABuffer(totalSize);
size_t offset = 0;
+ int32_t cvo = -1;
for (List<sp<ABuffer> >::iterator it = mNALUnits.begin();
it != mNALUnits.end(); ++it) {
memcpy(accessUnit->data() + offset, "\x00\x00\x00\x01", 4);
@@ -383,6 +391,8 @@
sp<ABuffer> nal = *it;
memcpy(accessUnit->data() + offset, nal->data(), nal->size());
offset += nal->size();
+
+ nal->meta()->findInt32("cvo", &cvo);
}
CopyTimes(accessUnit, *mNALUnits.begin());
@@ -391,6 +401,9 @@
printf(mAccessUnitDamaged ? "X" : ".");
fflush(stdout);
#endif
+ if (cvo >= 0) {
+ accessUnit->meta()->setInt32("cvo", cvo);
+ }
if (mAccessUnitDamaged) {
accessUnit->meta()->setInt32("damaged", true);
diff --git a/media/libstagefright/rtsp/ARTPConnection.cpp b/media/libstagefright/rtsp/ARTPConnection.cpp
index 53bf045..769e3f4 100644
--- a/media/libstagefright/rtsp/ARTPConnection.cpp
+++ b/media/libstagefright/rtsp/ARTPConnection.cpp
@@ -67,6 +67,9 @@
struct sockaddr_in6 mRemoteRTCPAddr6;
bool mIsInjected;
+
+ // RTCP Extension for CVO
+ int mCVOExtMap; // will be set to 0 if cvo is not negotiated in sdp
};
ARTPConnection::ARTPConnection(uint32_t flags)
@@ -304,6 +307,18 @@
memset(&info->mRemoteRTCPAddr, 0, sizeof(info->mRemoteRTCPAddr));
memset(&info->mRemoteRTCPAddr6, 0, sizeof(info->mRemoteRTCPAddr6));
+ sp<ASessionDescription> sessionDesc = info->mSessionDesc;
+ info->mCVOExtMap = 0;
+ for (size_t i = 1; i < sessionDesc->countTracks(); ++i) {
+ int32_t cvoExtMap;
+ if (sessionDesc->getCvoExtMap(i, &cvoExtMap)) {
+ info->mCVOExtMap = cvoExtMap;
+ ALOGI("urn:3gpp:video-orientation(cvo) found as extmap:%d", info->mCVOExtMap);
+ } else {
+ ALOGI("urn:3gpp:video-orientation(cvo) not found :%d", info->mCVOExtMap);
+ }
+ }
+
if (!injected) {
postPollEvent();
}
@@ -576,6 +591,7 @@
return -1;
}
+ int32_t cvoDegrees = -1;
if (data[0] & 0x10) {
// Header eXtension present.
@@ -595,6 +611,7 @@
return -1;
}
+ parseRTPExt(s, (const uint8_t *)extensionData, extensionLength, &cvoDegrees);
payloadOffset += 4 + extensionLength;
}
@@ -609,6 +626,8 @@
meta->setInt32("rtp-time", rtpTime);
meta->setInt32("PT", data[1] & 0x7f);
meta->setInt32("M", data[1] >> 7);
+ if (cvoDegrees >= 0)
+ meta->setInt32("cvo", cvoDegrees);
buffer->setInt32Data(u16at(&data[2]));
buffer->setRange(payloadOffset, size - payloadOffset);
@@ -618,6 +637,51 @@
return OK;
}
+status_t ARTPConnection::parseRTPExt(StreamInfo *s,
+ const uint8_t *extHeader, size_t extLen, int32_t *cvoDegrees) {
+ if (extLen < 4)
+ return -1;
+
+ uint16_t header = (extHeader[0] << 8) | (extHeader[1]);
+ bool isOnebyteHeader = false;
+
+ if (header == 0xBEDE) {
+ isOnebyteHeader = true;
+ } else if (header == 0x1000) {
+ ALOGW("parseRTPExt: two-byte header is not implemented yet");
+ return -1;
+ } else {
+ ALOGW("parseRTPExt: can not recognize header");
+ return -1;
+ }
+
+ const uint8_t *extPayload = extHeader + 4;
+ size_t offset = 0; //start from first payload of rtp extension.
+ // one-byte header parser
+ while (isOnebyteHeader && offset < extLen) {
+ uint8_t extmapId = extPayload[offset] >> 4;
+ uint8_t length = (extPayload[offset] & 0xF) + 1;
+ offset++;
+
+ // padding case
+ if(extmapId == 0)
+ continue;
+
+ uint8_t data[length];
+ for (uint8_t j = 0; j < length; j++)
+ data[j] = extPayload[offset + j];
+
+ offset += length;
+
+ if (extmapId == s->mCVOExtMap) {
+ *cvoDegrees = (int32_t)data[0];
+ return OK;
+ }
+ }
+
+ return BAD_VALUE;
+}
+
status_t ARTPConnection::parseRTCP(StreamInfo *s, const sp<ABuffer> &buffer) {
if (s->mNumRTCPPacketsReceived++ == 0) {
sp<AMessage> notify = s->mNotifyMsg->dup();
diff --git a/media/libstagefright/rtsp/ARTPConnection.h b/media/libstagefright/rtsp/ARTPConnection.h
index ae638e4..4e690b2 100644
--- a/media/libstagefright/rtsp/ARTPConnection.h
+++ b/media/libstagefright/rtsp/ARTPConnection.h
@@ -99,6 +99,7 @@
status_t receive(StreamInfo *info, bool receiveRTP);
status_t parseRTP(StreamInfo *info, const sp<ABuffer> &buffer);
+ status_t parseRTPExt(StreamInfo *s, const uint8_t *extData, size_t extLen, int32_t *cvoDegrees);
status_t parseRTCP(StreamInfo *info, const sp<ABuffer> &buffer);
status_t parseSR(StreamInfo *info, const uint8_t *data, size_t size);
status_t parseTSFB(StreamInfo *info, const uint8_t *data, size_t size);
diff --git a/media/libstagefright/rtsp/ASessionDescription.cpp b/media/libstagefright/rtsp/ASessionDescription.cpp
index d8fde76..63f39f4 100644
--- a/media/libstagefright/rtsp/ASessionDescription.cpp
+++ b/media/libstagefright/rtsp/ASessionDescription.cpp
@@ -103,7 +103,7 @@
key.setTo(line, 0, colonPos);
if (key == "a=fmtp" || key == "a=rtpmap"
- || key == "a=framesize") {
+ || key == "a=framesize" || key == "a=extmap") {
ssize_t spacePos = line.find(" ", colonPos + 1);
if (spacePos < 0) {
return false;
@@ -201,6 +201,33 @@
return true;
}
+bool ASessionDescription::getCvoExtMap(
+ size_t index, int32_t *cvoExtMap) const {
+ CHECK_GE(index, 0u);
+ CHECK_LT(index, mTracks.size());
+
+ AString key, value;
+ *cvoExtMap = 0;
+
+ const Attribs &track = mTracks.itemAt(index);
+ for (size_t i = 0; i < track.size(); i++) {
+ value = track.valueAt(i);
+ if (value.size() > 0 && strcmp(value.c_str(), "urn:3gpp:video-orientation") == 0) {
+ key = track.keyAt(i);
+ break;
+ }
+ }
+
+ if (key.size() > 0) {
+ const char *colonPos = strrchr(key.c_str(), ':');
+ colonPos++;
+ *cvoExtMap = atoi(colonPos);
+ return true;
+ }
+
+ return false;
+}
+
void ASessionDescription::getFormatType(
size_t index, unsigned long *PT,
AString *desc, AString *params) const {
@@ -347,8 +374,9 @@
// static
void ASessionDescription::SDPStringFactory(AString &sdp,
- const char *ip, bool isAudio, unsigned port, unsigned payloadType,
- unsigned as, const char *codec, const char *fmtp, int32_t width, int32_t height)
+ const char *ip, bool isAudio, unsigned port, unsigned payloadType,
+ unsigned as, const char *codec, const char *fmtp,
+ int32_t width, int32_t height, int32_t cvoExtMap)
{
bool isIPv4 = (AString(ip).find("::") == -1) ? true : false;
sdp.clear();
@@ -401,6 +429,14 @@
sdp.append("\r\n");
}
+ if(cvoExtMap > 0) {
+ sdp.append("a=extmap:");
+ sdp.append(cvoExtMap);
+ sdp.append(" ");
+ sdp.append("urn:3gpp:video-orientation");
+ sdp.append("\r\n");
+ }
+
ALOGV("SDPStringFactory => %s", sdp.c_str());
}
diff --git a/media/libstagefright/rtsp/ASessionDescription.h b/media/libstagefright/rtsp/ASessionDescription.h
index bd92916..91f5442 100644
--- a/media/libstagefright/rtsp/ASessionDescription.h
+++ b/media/libstagefright/rtsp/ASessionDescription.h
@@ -40,6 +40,8 @@
size_t countTracks() const;
void getFormat(size_t index, AString *value) const;
+ bool getCvoExtMap(size_t index, int32_t *cvoExtMap) const;
+
void getFormatType(
size_t index, unsigned long *PT,
AString *desc, AString *params) const;
@@ -65,7 +67,7 @@
static void SDPStringFactory(AString &sdp, const char *ip, bool isAudio, unsigned port,
unsigned payloadType, unsigned as, const char *codec, const char *fmtp = NULL,
- int32_t width = 0, int32_t height = 0);
+ int32_t width = 0, int32_t height = 0, int32_t cvoExtMap = 0);
protected:
virtual ~ASessionDescription();