support for CEA-608 closed caption
Bug: 15470448
Change-Id: Ic6a527f5c35a8ee0a08a5b043336e4d193216083
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index dc69f73..b333043 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -375,14 +375,24 @@
inbandTracks = mSource->getTrackCount();
}
+ size_t ccTracks = 0;
+ if (mCCDecoder != NULL) {
+ ccTracks = mCCDecoder->getTrackCount();
+ }
+
// total track count
- reply->writeInt32(inbandTracks);
+ reply->writeInt32(inbandTracks + ccTracks);
// write inband tracks
for (size_t i = 0; i < inbandTracks; ++i) {
writeTrackInfo(reply, mSource->getTrackInfo(i));
}
+ // write CC track
+ for (size_t i = 0; i < ccTracks; ++i) {
+ writeTrackInfo(reply, mCCDecoder->getTrackInfo(i));
+ }
+
sp<AMessage> response = new AMessage;
response->postReply(replyID);
break;
@@ -404,9 +414,19 @@
if (mSource != NULL) {
inbandTracks = mSource->getTrackCount();
}
+ size_t ccTracks = 0;
+ if (mCCDecoder != NULL) {
+ ccTracks = mCCDecoder->getTrackCount();
+ }
if (trackIndex < inbandTracks) {
err = mSource->selectTrack(trackIndex, select);
+ } else {
+ trackIndex -= inbandTracks;
+
+ if (trackIndex < ccTracks) {
+ err = mCCDecoder->selectTrack(trackIndex, select);
+ }
}
sp<AMessage> response = new AMessage;
@@ -870,6 +890,12 @@
break;
}
+ case kWhatClosedCaptionNotify:
+ {
+ onClosedCaptionNotify(msg);
+ break;
+ }
+
default:
TRESPASS();
break;
@@ -933,6 +959,9 @@
AString mime;
CHECK(format->findString("mime", &mime));
mVideoIsAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
+
+ sp<AMessage> ccNotify = new AMessage(kWhatClosedCaptionNotify, id());
+ mCCDecoder = new CCDecoder(ccNotify);
}
sp<AMessage> notify =
@@ -1073,6 +1102,10 @@
mediaTimeUs / 1E6);
#endif
+ if (!audio) {
+ mCCDecoder->decode(accessUnit);
+ }
+
reply->setBuffer("buffer", accessUnit);
reply->post();
@@ -1101,14 +1134,15 @@
sp<ABuffer> buffer;
CHECK(msg->findBuffer("buffer", &buffer));
+ int64_t mediaTimeUs;
+ CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
+
int64_t &skipUntilMediaTimeUs =
audio
? mSkipRenderingAudioUntilMediaTimeUs
: mSkipRenderingVideoUntilMediaTimeUs;
if (skipUntilMediaTimeUs >= 0) {
- int64_t mediaTimeUs;
- CHECK(buffer->meta()->findInt64("timeUs", &mediaTimeUs));
if (mediaTimeUs < skipUntilMediaTimeUs) {
ALOGV("dropping %s buffer at time %lld as requested.",
@@ -1122,6 +1156,10 @@
skipUntilMediaTimeUs = -1;
}
+ if (!audio && mCCDecoder->isSelected()) {
+ mCCDecoder->display(mediaTimeUs);
+ }
+
mRenderer->queueBuffer(audio, buffer, reply);
}
@@ -1510,6 +1548,39 @@
}
}
+void NuPlayer::onClosedCaptionNotify(const sp<AMessage> &msg) {
+ int32_t what;
+ CHECK(msg->findInt32("what", &what));
+
+ switch (what) {
+ case NuPlayer::CCDecoder::kWhatClosedCaptionData:
+ {
+ sp<ABuffer> buffer;
+ CHECK(msg->findBuffer("buffer", &buffer));
+
+ size_t inbandTracks = 0;
+ if (mSource != NULL) {
+ inbandTracks = mSource->getTrackCount();
+ }
+
+ sendSubtitleData(buffer, inbandTracks);
+ break;
+ }
+
+ case NuPlayer::CCDecoder::kWhatTrackAdded:
+ {
+ notifyListener(MEDIA_INFO, MEDIA_INFO_METADATA_UPDATE, 0);
+
+ break;
+ }
+
+ default:
+ TRESPASS();
+ }
+
+
+}
+
void NuPlayer::sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex) {
int32_t trackIndex;
int64_t timeUs, durationUs;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h
index f95cc11..5be71fb 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h
@@ -76,6 +76,7 @@
private:
struct Decoder;
+ struct CCDecoder;
struct GenericSource;
struct HTTPLiveSource;
struct Renderer;
@@ -98,6 +99,7 @@
kWhatScanSources = 'scan',
kWhatVideoNotify = 'vidN',
kWhatAudioNotify = 'audN',
+ kWhatClosedCaptionNotify = 'capN',
kWhatRendererNotify = 'renN',
kWhatReset = 'rset',
kWhatSeek = 'seek',
@@ -119,6 +121,7 @@
sp<Decoder> mVideoDecoder;
bool mVideoIsAVC;
sp<Decoder> mAudioDecoder;
+ sp<CCDecoder> mCCDecoder;
sp<Renderer> mRenderer;
List<sp<Action> > mDeferredActions;
@@ -186,6 +189,7 @@
void performSetSurface(const sp<NativeWindowWrapper> &wrapper);
void onSourceNotify(const sp<AMessage> &msg);
+ void onClosedCaptionNotify(const sp<AMessage> &msg);
void queueDecoderShutdown(
bool audio, bool video, const sp<AMessage> &reply);
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
index cfbf282..5abfb71 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
@@ -22,6 +22,7 @@
#include "NuPlayerDecoder.h"
#include <media/ICrypto.h>
+#include <media/stagefright/foundation/ABitReader.h>
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
@@ -535,5 +536,272 @@
return seamless;
}
+struct NuPlayer::CCDecoder::CCData {
+ CCData(uint8_t type, uint8_t data1, uint8_t data2)
+ : mType(type), mData1(data1), mData2(data2) {
+ }
+
+ uint8_t mType;
+ uint8_t mData1;
+ uint8_t mData2;
+};
+
+NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> ¬ify)
+ : mNotify(notify),
+ mTrackCount(0),
+ mSelectedTrack(-1) {
+}
+
+size_t NuPlayer::CCDecoder::getTrackCount() const {
+ return mTrackCount;
+}
+
+sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
+ CHECK(index == 0);
+
+ sp<AMessage> format = new AMessage();
+
+ format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
+ format->setString("language", "und");
+ format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
+ format->setInt32("auto", 1);
+ format->setInt32("default", 1);
+ format->setInt32("forced", 0);
+
+ return format;
+}
+
+status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
+ CHECK(index < mTrackCount);
+
+ if (select) {
+ if (mSelectedTrack == (ssize_t)index) {
+ ALOGE("track %zu already selected", index);
+ return BAD_VALUE;
+ }
+ ALOGV("selected track %zu", index);
+ mSelectedTrack = index;
+ } else {
+ if (mSelectedTrack != (ssize_t)index) {
+ ALOGE("track %zu is not selected", index);
+ return BAD_VALUE;
+ }
+ ALOGV("unselected track %zu", index);
+ mSelectedTrack = -1;
+ }
+
+ return OK;
+}
+
+bool NuPlayer::CCDecoder::isSelected() const {
+ return mSelectedTrack >= 0 && mSelectedTrack < (int32_t)mTrackCount;
+}
+
+bool NuPlayer::CCDecoder::isNullPad(CCData *cc) const {
+ return cc->mData1 < 0x10 && cc->mData2 < 0x10;
+}
+
+void NuPlayer::CCDecoder::dumpBytePair(const sp<ABuffer> &ccBuf) const {
+ size_t offset = 0;
+ AString out;
+
+ while (offset < ccBuf->size()) {
+ char tmp[128];
+
+ CCData *cc = (CCData *) (ccBuf->data() + offset);
+
+ if (isNullPad(cc)) {
+ // 1 null pad or XDS metadata, ignore
+ offset += sizeof(CCData);
+ continue;
+ }
+
+ if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
+ // 2 basic chars
+ sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
+ } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
+ && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
+ // 1 special char
+ sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+ } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
+ && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
+ // 1 Spanish/French char
+ sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+ } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
+ && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
+ // 1 Portuguese/German/Danish char
+ sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+ } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
+ && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
+ // Mid-Row Codes (Table 69)
+ sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+ } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
+ && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
+ ||
+ ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
+ && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
+ // Misc Control Codes (Table 70)
+ sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+ } else if ((cc->mData1 & 0x70) == 0x10
+ && (cc->mData2 & 0x40) == 0x40
+ && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
+ // Preamble Address Codes (Table 71)
+ sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+ } else {
+ sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
+ }
+
+ if (out.size() > 0) {
+ out.append(", ");
+ }
+
+ out.append(tmp);
+
+ offset += sizeof(CCData);
+ }
+
+ ALOGI("%s", out.c_str());
+}
+
+bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
+ int64_t timeUs;
+ CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
+
+ sp<ABuffer> sei;
+ if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
+ return false;
+ }
+
+ bool hasCC = false;
+
+ ABitReader br(sei->data() + 1, sei->size() - 1);
+ // sei_message()
+ while (br.numBitsLeft() >= 16) { // at least 16-bit for sei_message()
+ uint32_t payload_type = 0;
+ size_t payload_size = 0;
+ uint8_t last_byte;
+
+ do {
+ last_byte = br.getBits(8);
+ payload_type += last_byte;
+ } while (last_byte == 0xFF);
+
+ do {
+ last_byte = br.getBits(8);
+ payload_size += last_byte;
+ } while (last_byte == 0xFF);
+
+ // sei_payload()
+ if (payload_type == 4) {
+ // user_data_registered_itu_t_t35()
+
+ // ATSC A/72: 6.4.2
+ uint8_t itu_t_t35_country_code = br.getBits(8);
+ uint16_t itu_t_t35_provider_code = br.getBits(16);
+ uint32_t user_identifier = br.getBits(32);
+ uint8_t user_data_type_code = br.getBits(8);
+
+ payload_size -= 1 + 2 + 4 + 1;
+
+ if (itu_t_t35_country_code == 0xB5
+ && itu_t_t35_provider_code == 0x0031
+ && user_identifier == 'GA94'
+ && user_data_type_code == 0x3) {
+ hasCC = true;
+
+ // MPEG_cc_data()
+ // ATSC A/53 Part 4: 6.2.3.1
+ br.skipBits(1); //process_em_data_flag
+ bool process_cc_data_flag = br.getBits(1);
+ br.skipBits(1); //additional_data_flag
+ size_t cc_count = br.getBits(5);
+ br.skipBits(8); // em_data;
+ payload_size -= 2;
+
+ if (process_cc_data_flag) {
+ AString out;
+
+ sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData));
+ ccBuf->setRange(0, 0);
+
+ for (size_t i = 0; i < cc_count; i++) {
+ uint8_t marker = br.getBits(5);
+ CHECK_EQ(marker, 0x1f);
+
+ bool cc_valid = br.getBits(1);
+ uint8_t cc_type = br.getBits(2);
+ // remove odd parity bit
+ uint8_t cc_data_1 = br.getBits(8) & 0x7f;
+ uint8_t cc_data_2 = br.getBits(8) & 0x7f;
+
+ if (cc_valid
+ && (cc_type == 0 || cc_type == 1)) {
+ CCData cc(cc_type, cc_data_1, cc_data_2);
+ if (!isNullPad(&cc)) {
+ memcpy(ccBuf->data() + ccBuf->size(),
+ (void *)&cc, sizeof(cc));
+ ccBuf->setRange(0, ccBuf->size() + sizeof(CCData));
+ }
+ }
+ }
+ payload_size -= cc_count * 3;
+
+ mCCMap.add(timeUs, ccBuf);
+ break;
+ }
+ } else {
+ ALOGV("Malformed SEI payload type 4");
+ }
+ } else {
+ ALOGV("Unsupported SEI payload type %d", payload_type);
+ }
+
+ // skipping remaining bits of this payload
+ br.skipBits(payload_size * 8);
+ }
+
+ return hasCC;
+}
+
+void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
+ if (extractFromSEI(accessUnit) && mTrackCount == 0) {
+ mTrackCount++;
+
+ ALOGI("Found CEA-608 track");
+ sp<AMessage> msg = mNotify->dup();
+ msg->setInt32("what", kWhatTrackAdded);
+ msg->post();
+ }
+ // TODO: extract CC from other sources
+}
+
+void NuPlayer::CCDecoder::display(int64_t timeUs) {
+ ssize_t index = mCCMap.indexOfKey(timeUs);
+ if (index < 0) {
+ ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
+ return;
+ }
+
+ sp<ABuffer> &ccBuf = mCCMap.editValueAt(index);
+
+ if (ccBuf->size() > 0) {
+#if 0
+ dumpBytePair(ccBuf);
+#endif
+
+ ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
+ ccBuf->meta()->setInt64("timeUs", timeUs);
+ ccBuf->meta()->setInt64("durationUs", 0ll);
+
+ sp<AMessage> msg = mNotify->dup();
+ msg->setInt32("what", kWhatClosedCaptionData);
+ msg->setBuffer("buffer", ccBuf);
+ msg->post();
+ }
+
+ // remove all entries before timeUs
+ mCCMap.removeItemsAt(0, index + 1);
+}
+
} // namespace android
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
index 2892584..1a4f4ab 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.h
@@ -101,6 +101,36 @@
DISALLOW_EVIL_CONSTRUCTORS(Decoder);
};
+struct NuPlayer::CCDecoder : public RefBase {
+ enum {
+ kWhatClosedCaptionData,
+ kWhatTrackAdded,
+ };
+
+ CCDecoder(const sp<AMessage> ¬ify);
+
+ size_t getTrackCount() const;
+ sp<AMessage> getTrackInfo(size_t index) const;
+ status_t selectTrack(size_t index, bool select);
+ bool isSelected() const;
+ void decode(const sp<ABuffer> &accessUnit);
+ void display(int64_t timeUs);
+
+private:
+ struct CCData;
+
+ sp<AMessage> mNotify;
+ KeyedVector<int64_t, sp<ABuffer> > mCCMap;
+ size_t mTrackCount;
+ int32_t mSelectedTrack;
+
+ bool isNullPad(CCData *cc) const;
+ void dumpBytePair(const sp<ABuffer> &ccBuf) const;
+ bool extractFromSEI(const sp<ABuffer> &accessUnit);
+
+ DISALLOW_EVIL_CONSTRUCTORS(CCDecoder);
+};
+
} // namespace android
#endif // NUPLAYER_DECODER_H_
diff --git a/media/libstagefright/MediaDefs.cpp b/media/libstagefright/MediaDefs.cpp
index f38729e..d48dd84 100644
--- a/media/libstagefright/MediaDefs.cpp
+++ b/media/libstagefright/MediaDefs.cpp
@@ -59,5 +59,6 @@
const char *MEDIA_MIMETYPE_TEXT_3GPP = "text/3gpp-tt";
const char *MEDIA_MIMETYPE_TEXT_SUBRIP = "application/x-subrip";
const char *MEDIA_MIMETYPE_TEXT_VTT = "text/vtt";
+const char *MEDIA_MIMETYPE_TEXT_CEA_608 = "text/cea-608";
} // namespace android
diff --git a/media/libstagefright/mpeg2ts/ESQueue.cpp b/media/libstagefright/mpeg2ts/ESQueue.cpp
index f7abf01..3c8f03e 100644
--- a/media/libstagefright/mpeg2ts/ESQueue.cpp
+++ b/media/libstagefright/mpeg2ts/ESQueue.cpp
@@ -777,6 +777,12 @@
unsigned nalType = mBuffer->data()[pos.nalOffset] & 0x1f;
+ if (nalType == 6) {
+ sp<ABuffer> sei = new ABuffer(pos.nalSize);
+ memcpy(sei->data(), mBuffer->data() + pos.nalOffset, pos.nalSize);
+ accessUnit->meta()->setBuffer("sei", sei);
+ }
+
#if !LOG_NDEBUG
char tmp[128];
sprintf(tmp, "0x%02x", nalType);