tunnel NuPlayer source and decoder input
Bug: 18342383
Change-Id: Ieff1cd3bad2b39d46f127ddd5d5139b919992461
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
index e695c43..0439a9a 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
@@ -1,5 +1,5 @@
/*
- * Copyright (C) 2010 The Android Open Source Project
+ * Copyright 2014 The Android Open Source Project
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -19,13 +19,12 @@
#include <utils/Log.h>
#include <inttypes.h>
+#include "NuPlayerCCDecoder.h"
#include "NuPlayerDecoder.h"
-
#include "NuPlayerRenderer.h"
#include "NuPlayerSource.h"
#include <media/ICrypto.h>
-#include <media/stagefright/foundation/ABitReader.h>
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
@@ -34,76 +33,103 @@
#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/MediaErrors.h>
+#include "avc_utils.h"
+#include "ATSParser.h"
+
namespace android {
NuPlayer::Decoder::Decoder(
const sp<AMessage> ¬ify,
const sp<Source> &source,
const sp<Renderer> &renderer,
- const sp<NativeWindowWrapper> &nativeWindow)
+ const sp<NativeWindowWrapper> &nativeWindow,
+ const sp<CCDecoder> &ccDecoder)
: mNotify(notify),
mNativeWindow(nativeWindow),
mSource(source),
mRenderer(renderer),
+ mCCDecoder(ccDecoder),
mSkipRenderingUntilMediaTimeUs(-1ll),
+ mNumFramesTotal(0ll),
+ mNumFramesDropped(0ll),
+ mIsAudio(true),
+ mIsVideoAVC(false),
+ mIsSecure(false),
+ mFormatChangePending(false),
mBufferGeneration(0),
mPaused(true),
mComponentName("decoder") {
- // Every decoder has its own looper because MediaCodec operations
- // are blocking, but NuPlayer needs asynchronous operations.
- mDecoderLooper = new ALooper;
- mDecoderLooper->setName("NPDecoder");
- mDecoderLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
-
mCodecLooper = new ALooper;
mCodecLooper->setName("NPDecoder-CL");
mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
}
NuPlayer::Decoder::~Decoder() {
- mDecoderLooper->unregisterHandler(id());
- mDecoderLooper->stop();
-
releaseAndResetMediaBuffers();
}
-static
-status_t PostAndAwaitResponse(
- const sp<AMessage> &msg, sp<AMessage> *response) {
- status_t err = msg->postAndAwaitResponse(response);
-
- if (err != OK) {
- return err;
- }
-
- if (!(*response)->findInt32("err", &err)) {
- err = OK;
- }
-
- return err;
+void NuPlayer::Decoder::getStats(
+ int64_t *numFramesTotal,
+ int64_t *numFramesDropped) const {
+ *numFramesTotal = mNumFramesTotal;
+ *numFramesDropped = mNumFramesDropped;
}
-void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
- mCSDsForCurrentFormat.clear();
- for (int32_t i = 0; ; ++i) {
- AString tag = "csd-";
- tag.append(i);
- sp<ABuffer> buffer;
- if (!format->findBuffer(tag.c_str(), &buffer)) {
+void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
+ ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
+
+ switch (msg->what()) {
+ case kWhatCodecNotify:
+ {
+ if (!isStaleReply(msg)) {
+ int32_t numInput, numOutput;
+
+ if (!msg->findInt32("input-buffers", &numInput)) {
+ numInput = INT32_MAX;
+ }
+
+ if (!msg->findInt32("output-buffers", &numOutput)) {
+ numOutput = INT32_MAX;
+ }
+
+ if (!mPaused) {
+ while (numInput-- > 0 && handleAnInputBuffer()) {}
+ }
+
+ while (numOutput-- > 0 && handleAnOutputBuffer()) {}
+ }
+
+ requestCodecNotification();
break;
}
- mCSDsForCurrentFormat.push(buffer);
+
+ case kWhatRenderBuffer:
+ {
+ if (!isStaleReply(msg)) {
+ onRenderBuffer(msg);
+ }
+ break;
+ }
+
+ default:
+ DecoderBase::onMessageReceived(msg);
+ break;
}
}
void NuPlayer::Decoder::onConfigure(const sp<AMessage> &format) {
CHECK(mCodec == NULL);
+ mFormatChangePending = false;
+
++mBufferGeneration;
AString mime;
CHECK(format->findString("mime", &mime));
+ mIsAudio = !strncasecmp("audio/", mime.c_str(), 6);
+ mIsVideoAVC = !strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime.c_str());
+
sp<Surface> surface = NULL;
if (mNativeWindow != NULL) {
surface = mNativeWindow->getSurfaceTextureClient();
@@ -131,6 +157,7 @@
handleError(UNKNOWN_ERROR);
return;
}
+ mIsSecure = secure;
mCodec->getName(&mComponentName);
@@ -183,69 +210,120 @@
mPaused = false;
}
-void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
- for (size_t i = 0; i < mMediaBuffers.size(); i++) {
- if (mMediaBuffers[i] != NULL) {
- mMediaBuffers[i]->release();
- mMediaBuffers.editItemAt(i) = NULL;
+void NuPlayer::Decoder::onSetRenderer(const sp<Renderer> &renderer) {
+ bool hadNoRenderer = (mRenderer == NULL);
+ mRenderer = renderer;
+ if (hadNoRenderer && mRenderer != NULL) {
+ requestCodecNotification();
+ }
+}
+
+void NuPlayer::Decoder::onGetInputBuffers(
+ Vector<sp<ABuffer> > *dstBuffers) {
+ dstBuffers->clear();
+ for (size_t i = 0; i < mInputBuffers.size(); i++) {
+ dstBuffers->push(mInputBuffers[i]);
+ }
+}
+
+void NuPlayer::Decoder::onResume() {
+ mPaused = false;
+}
+
+void NuPlayer::Decoder::onFlush(bool notifyComplete) {
+ if (mCCDecoder != NULL) {
+ mCCDecoder->flush();
+ }
+
+ if (mRenderer != NULL) {
+ mRenderer->flush(mIsAudio, notifyComplete);
+ mRenderer->signalTimeDiscontinuity();
+ }
+
+ status_t err = OK;
+ if (mCodec != NULL) {
+ err = mCodec->flush();
+ mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
+ ++mBufferGeneration;
+ }
+
+ if (err != OK) {
+ ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
+ handleError(err);
+ // finish with posting kWhatFlushCompleted.
+ // we attempt to release the buffers even if flush fails.
+ }
+ releaseAndResetMediaBuffers();
+
+ if (notifyComplete) {
+ sp<AMessage> notify = mNotify->dup();
+ notify->setInt32("what", kWhatFlushCompleted);
+ notify->post();
+ mPaused = true;
+ }
+}
+
+void NuPlayer::Decoder::onShutdown(bool notifyComplete) {
+ status_t err = OK;
+ if (mCodec != NULL) {
+ err = mCodec->release();
+ mCodec = NULL;
+ ++mBufferGeneration;
+
+ if (mNativeWindow != NULL) {
+ // reconnect to surface as MediaCodec disconnected from it
+ status_t error =
+ native_window_api_connect(
+ mNativeWindow->getNativeWindow().get(),
+ NATIVE_WINDOW_API_MEDIA);
+ ALOGW_IF(error != NO_ERROR,
+ "[%s] failed to connect to native window, error=%d",
+ mComponentName.c_str(), error);
+ }
+ mComponentName = "decoder";
+ }
+
+ releaseAndResetMediaBuffers();
+
+ if (err != OK) {
+ ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
+ handleError(err);
+ // finish with posting kWhatShutdownCompleted.
+ }
+
+ if (notifyComplete) {
+ sp<AMessage> notify = mNotify->dup();
+ notify->setInt32("what", kWhatShutdownCompleted);
+ notify->post();
+ mPaused = true;
+ }
+}
+
+void NuPlayer::Decoder::doRequestBuffers() {
+ if (mFormatChangePending) {
+ return;
+ }
+ status_t err = OK;
+ while (!mDequeuedInputBuffers.empty()) {
+ size_t bufferIx = *mDequeuedInputBuffers.begin();
+ sp<AMessage> msg = new AMessage();
+ msg->setSize("buffer-ix", bufferIx);
+ err = fetchInputData(msg);
+ if (err != OK) {
+ break;
+ }
+ mDequeuedInputBuffers.erase(mDequeuedInputBuffers.begin());
+
+ if (!mPendingInputMessages.empty()
+ || !onInputBufferFetched(msg)) {
+ mPendingInputMessages.push_back(msg);
}
}
- mMediaBuffers.resize(mInputBuffers.size());
- for (size_t i = 0; i < mMediaBuffers.size(); i++) {
- mMediaBuffers.editItemAt(i) = NULL;
+
+ if (err == -EWOULDBLOCK
+ && mSource->feedMoreTSData() == OK) {
+ scheduleRequestBuffers();
}
- mInputBufferIsDequeued.clear();
- mInputBufferIsDequeued.resize(mInputBuffers.size());
- for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
- mInputBufferIsDequeued.editItemAt(i) = false;
- }
-
- mPendingInputMessages.clear();
- mSkipRenderingUntilMediaTimeUs = -1;
-}
-
-void NuPlayer::Decoder::requestCodecNotification() {
- if (mCodec != NULL) {
- sp<AMessage> reply = new AMessage(kWhatCodecNotify, id());
- reply->setInt32("generation", mBufferGeneration);
- mCodec->requestActivityNotification(reply);
- }
-}
-
-bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
- int32_t generation;
- CHECK(msg->findInt32("generation", &generation));
- return generation != mBufferGeneration;
-}
-
-void NuPlayer::Decoder::init() {
- mDecoderLooper->registerHandler(this);
-}
-
-void NuPlayer::Decoder::configure(const sp<AMessage> &format) {
- sp<AMessage> msg = new AMessage(kWhatConfigure, id());
- msg->setMessage("format", format);
- msg->post();
-}
-
-void NuPlayer::Decoder::setRenderer(const sp<Renderer> &renderer) {
- sp<AMessage> msg = new AMessage(kWhatSetRenderer, id());
- msg->setObject("renderer", renderer);
- msg->post();
-}
-
-void NuPlayer::Decoder::signalUpdateFormat(const sp<AMessage> &format) {
- sp<AMessage> msg = new AMessage(kWhatUpdateFormat, id());
- msg->setMessage("format", format);
- msg->post();
-}
-
-status_t NuPlayer::Decoder::getInputBuffers(Vector<sp<ABuffer> > *buffers) const {
- sp<AMessage> msg = new AMessage(kWhatGetInputBuffers, id());
- msg->setPointer("buffers", buffers);
-
- sp<AMessage> response;
- return PostAndAwaitResponse(msg, &response);
}
void NuPlayer::Decoder::handleError(int32_t err)
@@ -263,6 +341,9 @@
}
bool NuPlayer::Decoder::handleAnInputBuffer() {
+ if (mFormatChangePending) {
+ return false;
+ }
size_t bufferIx = -1;
status_t res = mCodec->dequeueInputBuffer(&bufferIx);
ALOGV("[%s] dequeued input: %d",
@@ -284,22 +365,21 @@
}
mInputBufferIsDequeued.editItemAt(bufferIx) = true;
- sp<AMessage> reply = new AMessage(kWhatInputBufferFilled, id());
- reply->setSize("buffer-ix", bufferIx);
- reply->setInt32("generation", mBufferGeneration);
-
if (!mCSDsToSubmit.isEmpty()) {
+ sp<AMessage> msg = new AMessage();
+ msg->setSize("buffer-ix", bufferIx);
+
sp<ABuffer> buffer = mCSDsToSubmit.itemAt(0);
ALOGI("[%s] resubmitting CSD", mComponentName.c_str());
- reply->setBuffer("buffer", buffer);
+ msg->setBuffer("buffer", buffer);
mCSDsToSubmit.removeAt(0);
- CHECK(onInputBufferFilled(reply));
+ CHECK(onInputBufferFetched(msg));
return true;
}
while (!mPendingInputMessages.empty()) {
sp<AMessage> msg = *mPendingInputMessages.begin();
- if (!onInputBufferFilled(msg)) {
+ if (!onInputBufferFetched(msg)) {
break;
}
mPendingInputMessages.erase(mPendingInputMessages.begin());
@@ -309,15 +389,263 @@
return true;
}
- sp<AMessage> notify = mNotify->dup();
- notify->setInt32("what", kWhatFillThisBuffer);
- notify->setBuffer("buffer", mInputBuffers[bufferIx]);
- notify->setMessage("reply", reply);
- notify->post();
+ mDequeuedInputBuffers.push_back(bufferIx);
+
+ onRequestInputBuffers();
return true;
}
-bool android::NuPlayer::Decoder::onInputBufferFilled(const sp<AMessage> &msg) {
+bool NuPlayer::Decoder::handleAnOutputBuffer() {
+ if (mFormatChangePending) {
+ return false;
+ }
+ size_t bufferIx = -1;
+ size_t offset;
+ size_t size;
+ int64_t timeUs;
+ uint32_t flags;
+ status_t res = mCodec->dequeueOutputBuffer(
+ &bufferIx, &offset, &size, &timeUs, &flags);
+
+ if (res != OK) {
+ ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res);
+ } else {
+ ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")",
+ mComponentName.c_str(), (int)bufferIx, timeUs, flags);
+ }
+
+ if (res == INFO_OUTPUT_BUFFERS_CHANGED) {
+ res = mCodec->getOutputBuffers(&mOutputBuffers);
+ if (res != OK) {
+ ALOGE("Failed to get output buffers for %s after INFO event (err=%d)",
+ mComponentName.c_str(), res);
+ handleError(res);
+ return false;
+ }
+ // NuPlayer ignores this
+ return true;
+ } else if (res == INFO_FORMAT_CHANGED) {
+ sp<AMessage> format = new AMessage();
+ res = mCodec->getOutputFormat(&format);
+ if (res != OK) {
+ ALOGE("Failed to get output format for %s after INFO event (err=%d)",
+ mComponentName.c_str(), res);
+ handleError(res);
+ return false;
+ }
+
+ if (!mIsAudio) {
+ sp<AMessage> notify = mNotify->dup();
+ notify->setInt32("what", kWhatVideoSizeChanged);
+ notify->setMessage("format", format);
+ notify->post();
+ } else if (mRenderer != NULL) {
+ uint32_t flags;
+ int64_t durationUs;
+ bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
+ if (!hasVideo &&
+ mSource->getDuration(&durationUs) == OK &&
+ durationUs
+ > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
+ flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
+ } else {
+ flags = AUDIO_OUTPUT_FLAG_NONE;
+ }
+
+ mRenderer->openAudioSink(
+ format, false /* offloadOnly */, hasVideo, flags);
+ }
+ return true;
+ } else if (res == INFO_DISCONTINUITY) {
+ // nothing to do
+ return true;
+ } else if (res != OK) {
+ if (res != -EAGAIN) {
+ ALOGE("Failed to dequeue output buffer for %s (err=%d)",
+ mComponentName.c_str(), res);
+ handleError(res);
+ }
+ return false;
+ }
+
+ CHECK_LT(bufferIx, mOutputBuffers.size());
+ sp<ABuffer> buffer = mOutputBuffers[bufferIx];
+ buffer->setRange(offset, size);
+ buffer->meta()->clear();
+ buffer->meta()->setInt64("timeUs", timeUs);
+ if (flags & MediaCodec::BUFFER_FLAG_EOS) {
+ buffer->meta()->setInt32("eos", true);
+ }
+ // we do not expect CODECCONFIG or SYNCFRAME for decoder
+
+ sp<AMessage> reply = new AMessage(kWhatRenderBuffer, id());
+ reply->setSize("buffer-ix", bufferIx);
+ reply->setInt32("generation", mBufferGeneration);
+
+ if (mSkipRenderingUntilMediaTimeUs >= 0) {
+ if (timeUs < mSkipRenderingUntilMediaTimeUs) {
+ ALOGV("[%s] dropping buffer at time %lld as requested.",
+ mComponentName.c_str(), (long long)timeUs);
+
+ reply->post();
+ return true;
+ }
+
+ mSkipRenderingUntilMediaTimeUs = -1;
+ }
+
+ if (mRenderer != NULL) {
+ // send the buffer to renderer.
+ mRenderer->queueBuffer(mIsAudio, buffer, reply);
+ if (flags & MediaCodec::BUFFER_FLAG_EOS) {
+ mRenderer->queueEOS(mIsAudio, ERROR_END_OF_STREAM);
+ }
+ }
+
+ return true;
+}
+
+void NuPlayer::Decoder::releaseAndResetMediaBuffers() {
+ for (size_t i = 0; i < mMediaBuffers.size(); i++) {
+ if (mMediaBuffers[i] != NULL) {
+ mMediaBuffers[i]->release();
+ mMediaBuffers.editItemAt(i) = NULL;
+ }
+ }
+ mMediaBuffers.resize(mInputBuffers.size());
+ for (size_t i = 0; i < mMediaBuffers.size(); i++) {
+ mMediaBuffers.editItemAt(i) = NULL;
+ }
+ mInputBufferIsDequeued.clear();
+ mInputBufferIsDequeued.resize(mInputBuffers.size());
+ for (size_t i = 0; i < mInputBufferIsDequeued.size(); i++) {
+ mInputBufferIsDequeued.editItemAt(i) = false;
+ }
+
+ mPendingInputMessages.clear();
+ mDequeuedInputBuffers.clear();
+ mSkipRenderingUntilMediaTimeUs = -1;
+}
+
+void NuPlayer::Decoder::requestCodecNotification() {
+ if (mFormatChangePending) {
+ return;
+ }
+ if (mCodec != NULL) {
+ sp<AMessage> reply = new AMessage(kWhatCodecNotify, id());
+ reply->setInt32("generation", mBufferGeneration);
+ mCodec->requestActivityNotification(reply);
+ }
+}
+
+bool NuPlayer::Decoder::isStaleReply(const sp<AMessage> &msg) {
+ int32_t generation;
+ CHECK(msg->findInt32("generation", &generation));
+ return generation != mBufferGeneration;
+}
+
+status_t NuPlayer::Decoder::fetchInputData(sp<AMessage> &reply) {
+ sp<ABuffer> accessUnit;
+ bool dropAccessUnit;
+ do {
+ status_t err = mSource->dequeueAccessUnit(mIsAudio, &accessUnit);
+
+ if (err == -EWOULDBLOCK) {
+ return err;
+ } else if (err != OK) {
+ if (err == INFO_DISCONTINUITY) {
+ int32_t type;
+ CHECK(accessUnit->meta()->findInt32("discontinuity", &type));
+
+ bool formatChange =
+ (mIsAudio &&
+ (type & ATSParser::DISCONTINUITY_AUDIO_FORMAT))
+ || (!mIsAudio &&
+ (type & ATSParser::DISCONTINUITY_VIDEO_FORMAT));
+
+ bool timeChange = (type & ATSParser::DISCONTINUITY_TIME) != 0;
+
+ ALOGI("%s discontinuity (format=%d, time=%d)",
+ mIsAudio ? "audio" : "video", formatChange, timeChange);
+
+ bool seamlessFormatChange = false;
+ sp<AMessage> newFormat = mSource->getFormat(mIsAudio);
+ if (formatChange) {
+ seamlessFormatChange =
+ supportsSeamlessFormatChange(newFormat);
+ // treat seamless format change separately
+ formatChange = !seamlessFormatChange;
+ }
+
+ if (formatChange || timeChange) {
+ sp<AMessage> msg = mNotify->dup();
+ msg->setInt32("what", kWhatInputDiscontinuity);
+ msg->setInt32("formatChange", formatChange);
+ msg->post();
+ }
+
+ if (formatChange /* not seamless */) {
+ // must change decoder
+ // return EOS and wait to be killed
+ mFormatChangePending = true;
+ return ERROR_END_OF_STREAM;
+ } else if (timeChange) {
+ // need to flush
+ // TODO: Ideally we shouldn't need a flush upon time
+ // discontinuity, flushing will cause loss of frames.
+ // We probably should queue a time change marker to the
+ // output queue, and handles it in renderer instead.
+ rememberCodecSpecificData(newFormat);
+ onFlush(false /* notifyComplete */);
+ err = OK;
+ } else if (seamlessFormatChange) {
+ // reuse existing decoder and don't flush
+ rememberCodecSpecificData(newFormat);
+ err = OK;
+ } else {
+ // This stream is unaffected by the discontinuity
+ return -EWOULDBLOCK;
+ }
+ }
+
+ reply->setInt32("err", err);
+ return OK;
+ }
+
+ if (!mIsAudio) {
+ ++mNumFramesTotal;
+ }
+
+ dropAccessUnit = false;
+ if (!mIsAudio
+ && !mIsSecure
+ && mRenderer->getVideoLateByUs() > 100000ll
+ && mIsVideoAVC
+ && !IsAVCReferenceFrame(accessUnit)) {
+ dropAccessUnit = true;
+ ++mNumFramesDropped;
+ }
+ } while (dropAccessUnit);
+
+ // ALOGV("returned a valid buffer of %s data", mIsAudio ? "mIsAudio" : "video");
+#if 0
+ int64_t mediaTimeUs;
+ CHECK(accessUnit->meta()->findInt64("timeUs", &mediaTimeUs));
+ ALOGV("feeding %s input buffer at media time %.2f secs",
+ mIsAudio ? "audio" : "video",
+ mediaTimeUs / 1E6);
+#endif
+
+ if (mCCDecoder != NULL) {
+ mCCDecoder->decode(accessUnit);
+ }
+
+ reply->setBuffer("buffer", accessUnit);
+
+ return OK;
+}
+
+bool NuPlayer::Decoder::onInputBufferFetched(const sp<AMessage> &msg) {
size_t bufferIx;
CHECK(msg->findSize("buffer-ix", &bufferIx));
CHECK_LT(bufferIx, mInputBuffers.size());
@@ -444,127 +772,20 @@
return true;
}
-bool NuPlayer::Decoder::handleAnOutputBuffer() {
- size_t bufferIx = -1;
- size_t offset;
- size_t size;
- int64_t timeUs;
- uint32_t flags;
- status_t res = mCodec->dequeueOutputBuffer(
- &bufferIx, &offset, &size, &timeUs, &flags);
-
- if (res != OK) {
- ALOGV("[%s] dequeued output: %d", mComponentName.c_str(), res);
- } else {
- ALOGV("[%s] dequeued output: %d (time=%lld flags=%" PRIu32 ")",
- mComponentName.c_str(), (int)bufferIx, timeUs, flags);
- }
-
- if (res == INFO_OUTPUT_BUFFERS_CHANGED) {
- res = mCodec->getOutputBuffers(&mOutputBuffers);
- if (res != OK) {
- ALOGE("Failed to get output buffers for %s after INFO event (err=%d)",
- mComponentName.c_str(), res);
- handleError(res);
- return false;
- }
- // NuPlayer ignores this
- return true;
- } else if (res == INFO_FORMAT_CHANGED) {
- sp<AMessage> format = new AMessage();
- res = mCodec->getOutputFormat(&format);
- if (res != OK) {
- ALOGE("Failed to get output format for %s after INFO event (err=%d)",
- mComponentName.c_str(), res);
- handleError(res);
- return false;
- }
-
- if (isVideo()) {
- sp<AMessage> notify = mNotify->dup();
- notify->setInt32("what", kWhatVideoSizeChanged);
- notify->setMessage("format", format);
- notify->post();
- } else if (mRenderer != NULL) {
- uint32_t flags;
- int64_t durationUs;
- bool hasVideo = (mSource->getFormat(false /* audio */) != NULL);
- if (!hasVideo &&
- mSource->getDuration(&durationUs) == OK &&
- durationUs
- > AUDIO_SINK_MIN_DEEP_BUFFER_DURATION_US) {
- flags = AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
- } else {
- flags = AUDIO_OUTPUT_FLAG_NONE;
- }
-
- mRenderer->openAudioSink(
- format, false /* offloadOnly */, hasVideo, flags);
- }
- return true;
- } else if (res == INFO_DISCONTINUITY) {
- // nothing to do
- return true;
- } else if (res != OK) {
- if (res != -EAGAIN) {
- ALOGE("Failed to dequeue output buffer for %s (err=%d)",
- mComponentName.c_str(), res);
- handleError(res);
- }
- return false;
- }
-
- CHECK_LT(bufferIx, mOutputBuffers.size());
- sp<ABuffer> buffer = mOutputBuffers[bufferIx];
- buffer->setRange(offset, size);
- buffer->meta()->clear();
- buffer->meta()->setInt64("timeUs", timeUs);
- if (flags & MediaCodec::BUFFER_FLAG_EOS) {
- buffer->meta()->setInt32("eos", true);
- }
- // we do not expect CODECCONFIG or SYNCFRAME for decoder
-
- sp<AMessage> reply = new AMessage(kWhatRenderBuffer, id());
- reply->setSize("buffer-ix", bufferIx);
- reply->setInt32("generation", mBufferGeneration);
-
- if (mSkipRenderingUntilMediaTimeUs >= 0) {
- if (timeUs < mSkipRenderingUntilMediaTimeUs) {
- ALOGV("[%s] dropping buffer at time %lld as requested.",
- mComponentName.c_str(), (long long)timeUs);
-
- reply->post();
- return true;
- }
-
- mSkipRenderingUntilMediaTimeUs = -1;
- }
-
- if (mRenderer != NULL) {
- // send the buffer to renderer.
- mRenderer->queueBuffer(!isVideo(), buffer, reply);
- if (flags & MediaCodec::BUFFER_FLAG_EOS) {
- mRenderer->queueEOS(!isVideo(), ERROR_END_OF_STREAM);
- }
- }
-
- return true;
-}
-
void NuPlayer::Decoder::onRenderBuffer(const sp<AMessage> &msg) {
status_t err;
int32_t render;
size_t bufferIx;
CHECK(msg->findSize("buffer-ix", &bufferIx));
- if (isVideo()) {
+ if (!mIsAudio) {
int64_t timeUs;
sp<ABuffer> buffer = mOutputBuffers[bufferIx];
buffer->meta()->findInt64("timeUs", &timeUs);
- sp<AMessage> notify = mNotify->dup();
- notify->setInt32("what", kWhatRenderBufferTime);
- notify->setInt64("timeUs", timeUs);
- notify->post();
+
+ if (mCCDecoder != NULL && mCCDecoder->isSelected()) {
+ mCCDecoder->display(timeUs);
+ }
}
if (msg->findInt32("render", &render) && render) {
@@ -581,208 +802,8 @@
}
}
-void NuPlayer::Decoder::onFlush() {
- if (mRenderer != NULL) {
- mRenderer->flush(!isVideo());
- }
-
- status_t err = OK;
- if (mCodec != NULL) {
- err = mCodec->flush();
- mCSDsToSubmit = mCSDsForCurrentFormat; // copy operator
- ++mBufferGeneration;
- }
-
- if (err != OK) {
- ALOGE("failed to flush %s (err=%d)", mComponentName.c_str(), err);
- handleError(err);
- // finish with posting kWhatFlushCompleted.
- // we attempt to release the buffers even if flush fails.
- }
- releaseAndResetMediaBuffers();
-
- sp<AMessage> notify = mNotify->dup();
- notify->setInt32("what", kWhatFlushCompleted);
- notify->post();
- mPaused = true;
-}
-
-void NuPlayer::Decoder::onResume() {
- mPaused = false;
-}
-
-void NuPlayer::Decoder::onShutdown() {
- status_t err = OK;
- if (mCodec != NULL) {
- err = mCodec->release();
- mCodec = NULL;
- ++mBufferGeneration;
-
- if (mNativeWindow != NULL) {
- // reconnect to surface as MediaCodec disconnected from it
- status_t error =
- native_window_api_connect(
- mNativeWindow->getNativeWindow().get(),
- NATIVE_WINDOW_API_MEDIA);
- ALOGW_IF(error != NO_ERROR,
- "[%s] failed to connect to native window, error=%d",
- mComponentName.c_str(), error);
- }
- mComponentName = "decoder";
- }
-
- releaseAndResetMediaBuffers();
-
- if (err != OK) {
- ALOGE("failed to release %s (err=%d)", mComponentName.c_str(), err);
- handleError(err);
- // finish with posting kWhatShutdownCompleted.
- }
-
- sp<AMessage> notify = mNotify->dup();
- notify->setInt32("what", kWhatShutdownCompleted);
- notify->post();
- mPaused = true;
-}
-
-void NuPlayer::Decoder::onMessageReceived(const sp<AMessage> &msg) {
- ALOGV("[%s] onMessage: %s", mComponentName.c_str(), msg->debugString().c_str());
-
- switch (msg->what()) {
- case kWhatConfigure:
- {
- sp<AMessage> format;
- CHECK(msg->findMessage("format", &format));
- onConfigure(format);
- break;
- }
-
- case kWhatSetRenderer:
- {
- bool hadNoRenderer = (mRenderer == NULL);
- sp<RefBase> obj;
- CHECK(msg->findObject("renderer", &obj));
- mRenderer = static_cast<Renderer *>(obj.get());
- if (hadNoRenderer && mRenderer != NULL) {
- requestCodecNotification();
- }
- break;
- }
-
- case kWhatUpdateFormat:
- {
- sp<AMessage> format;
- CHECK(msg->findMessage("format", &format));
- rememberCodecSpecificData(format);
- break;
- }
-
- case kWhatGetInputBuffers:
- {
- uint32_t replyID;
- CHECK(msg->senderAwaitsResponse(&replyID));
-
- Vector<sp<ABuffer> > *dstBuffers;
- CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
-
- dstBuffers->clear();
- for (size_t i = 0; i < mInputBuffers.size(); i++) {
- dstBuffers->push(mInputBuffers[i]);
- }
-
- (new AMessage)->postReply(replyID);
- break;
- }
-
- case kWhatCodecNotify:
- {
- if (!isStaleReply(msg)) {
- int32_t numInput, numOutput;
-
- if (!msg->findInt32("input-buffers", &numInput)) {
- numInput = INT32_MAX;
- }
-
- if (!msg->findInt32("output-buffers", &numOutput)) {
- numOutput = INT32_MAX;
- }
-
- if (!mPaused) {
- while (numInput-- > 0 && handleAnInputBuffer()) {}
- }
-
- while (numOutput-- > 0 && handleAnOutputBuffer()) {}
- }
-
- requestCodecNotification();
- break;
- }
-
- case kWhatInputBufferFilled:
- {
- if (!isStaleReply(msg)) {
- if (!mPendingInputMessages.empty()
- || !onInputBufferFilled(msg)) {
- mPendingInputMessages.push_back(msg);
- }
- }
-
- break;
- }
-
- case kWhatRenderBuffer:
- {
- if (!isStaleReply(msg)) {
- onRenderBuffer(msg);
- }
- break;
- }
-
- case kWhatFlush:
- {
- sp<AMessage> format;
- if (msg->findMessage("new-format", &format)) {
- rememberCodecSpecificData(format);
- }
- onFlush();
- break;
- }
-
- case kWhatResume:
- {
- onResume();
- break;
- }
-
- case kWhatShutdown:
- {
- onShutdown();
- break;
- }
-
- default:
- TRESPASS();
- break;
- }
-}
-
-void NuPlayer::Decoder::signalFlush(const sp<AMessage> &format) {
- sp<AMessage> msg = new AMessage(kWhatFlush, id());
- if (format != NULL) {
- msg->setMessage("new-format", format);
- }
- msg->post();
-}
-
-void NuPlayer::Decoder::signalResume() {
- (new AMessage(kWhatResume, id()))->post();
-}
-
-void NuPlayer::Decoder::initiateShutdown() {
- (new AMessage(kWhatShutdown, id()))->post();
-}
-
-bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(const sp<AMessage> &targetFormat) const {
+bool NuPlayer::Decoder::supportsSeamlessAudioFormatChange(
+ const sp<AMessage> &targetFormat) const {
if (targetFormat == NULL) {
return true;
}
@@ -847,336 +868,20 @@
return seamless;
}
-bool NuPlayer::Decoder::isVideo() {
- return mNativeWindow != NULL;
-}
-
-struct CCData {
- CCData(uint8_t type, uint8_t data1, uint8_t data2)
- : mType(type), mData1(data1), mData2(data2) {
- }
- bool getChannel(size_t *channel) const {
- if (mData1 >= 0x10 && mData1 <= 0x1f) {
- *channel = (mData1 >= 0x18 ? 1 : 0) + (mType ? 2 : 0);
- return true;
- }
- return false;
- }
-
- uint8_t mType;
- uint8_t mData1;
- uint8_t mData2;
-};
-
-static bool isNullPad(CCData *cc) {
- return cc->mData1 < 0x10 && cc->mData2 < 0x10;
-}
-
-static void dumpBytePair(const sp<ABuffer> &ccBuf) {
- size_t offset = 0;
- AString out;
-
- while (offset < ccBuf->size()) {
- char tmp[128];
-
- CCData *cc = (CCData *) (ccBuf->data() + offset);
-
- if (isNullPad(cc)) {
- // 1 null pad or XDS metadata, ignore
- offset += sizeof(CCData);
- continue;
- }
-
- if (cc->mData1 >= 0x20 && cc->mData1 <= 0x7f) {
- // 2 basic chars
- sprintf(tmp, "[%d]Basic: %c %c", cc->mType, cc->mData1, cc->mData2);
- } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
- && cc->mData2 >= 0x30 && cc->mData2 <= 0x3f) {
- // 1 special char
- sprintf(tmp, "[%d]Special: %02x %02x", cc->mType, cc->mData1, cc->mData2);
- } else if ((cc->mData1 == 0x12 || cc->mData1 == 0x1A)
- && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
- // 1 Spanish/French char
- sprintf(tmp, "[%d]Spanish: %02x %02x", cc->mType, cc->mData1, cc->mData2);
- } else if ((cc->mData1 == 0x13 || cc->mData1 == 0x1B)
- && cc->mData2 >= 0x20 && cc->mData2 <= 0x3f){
- // 1 Portuguese/German/Danish char
- sprintf(tmp, "[%d]German: %02x %02x", cc->mType, cc->mData1, cc->mData2);
- } else if ((cc->mData1 == 0x11 || cc->mData1 == 0x19)
- && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f){
- // Mid-Row Codes (Table 69)
- sprintf(tmp, "[%d]Mid-row: %02x %02x", cc->mType, cc->mData1, cc->mData2);
- } else if (((cc->mData1 == 0x14 || cc->mData1 == 0x1c)
- && cc->mData2 >= 0x20 && cc->mData2 <= 0x2f)
- ||
- ((cc->mData1 == 0x17 || cc->mData1 == 0x1f)
- && cc->mData2 >= 0x21 && cc->mData2 <= 0x23)){
- // Misc Control Codes (Table 70)
- sprintf(tmp, "[%d]Ctrl: %02x %02x", cc->mType, cc->mData1, cc->mData2);
- } else if ((cc->mData1 & 0x70) == 0x10
- && (cc->mData2 & 0x40) == 0x40
- && ((cc->mData1 & 0x07) || !(cc->mData2 & 0x20)) ) {
- // Preamble Address Codes (Table 71)
- sprintf(tmp, "[%d]PAC: %02x %02x", cc->mType, cc->mData1, cc->mData2);
- } else {
- sprintf(tmp, "[%d]Invalid: %02x %02x", cc->mType, cc->mData1, cc->mData2);
- }
-
- if (out.size() > 0) {
- out.append(", ");
- }
-
- out.append(tmp);
-
- offset += sizeof(CCData);
- }
-
- ALOGI("%s", out.c_str());
-}
-
-NuPlayer::CCDecoder::CCDecoder(const sp<AMessage> ¬ify)
- : mNotify(notify),
- mCurrentChannel(0),
- mSelectedTrack(-1) {
- for (size_t i = 0; i < sizeof(mTrackIndices)/sizeof(mTrackIndices[0]); ++i) {
- mTrackIndices[i] = -1;
- }
-}
-
-size_t NuPlayer::CCDecoder::getTrackCount() const {
- return mFoundChannels.size();
-}
-
-sp<AMessage> NuPlayer::CCDecoder::getTrackInfo(size_t index) const {
- if (!isTrackValid(index)) {
- return NULL;
- }
-
- sp<AMessage> format = new AMessage();
-
- format->setInt32("type", MEDIA_TRACK_TYPE_SUBTITLE);
- format->setString("language", "und");
- format->setString("mime", MEDIA_MIMETYPE_TEXT_CEA_608);
- //CC1, field 0 channel 0
- bool isDefaultAuto = (mFoundChannels[index] == 0);
- format->setInt32("auto", isDefaultAuto);
- format->setInt32("default", isDefaultAuto);
- format->setInt32("forced", 0);
-
- return format;
-}
-
-status_t NuPlayer::CCDecoder::selectTrack(size_t index, bool select) {
- if (!isTrackValid(index)) {
- return BAD_VALUE;
- }
-
- if (select) {
- if (mSelectedTrack == (ssize_t)index) {
- ALOGE("track %zu already selected", index);
- return BAD_VALUE;
- }
- ALOGV("selected track %zu", index);
- mSelectedTrack = index;
- } else {
- if (mSelectedTrack != (ssize_t)index) {
- ALOGE("track %zu is not selected", index);
- return BAD_VALUE;
- }
- ALOGV("unselected track %zu", index);
- mSelectedTrack = -1;
- }
-
- return OK;
-}
-
-bool NuPlayer::CCDecoder::isSelected() const {
- return mSelectedTrack >= 0 && mSelectedTrack < (int32_t) getTrackCount();
-}
-
-bool NuPlayer::CCDecoder::isTrackValid(size_t index) const {
- return index < getTrackCount();
-}
-
-int32_t NuPlayer::CCDecoder::getTrackIndex(size_t channel) const {
- if (channel < sizeof(mTrackIndices)/sizeof(mTrackIndices[0])) {
- return mTrackIndices[channel];
- }
- return -1;
-}
-
-// returns true if a new CC track is found
-bool NuPlayer::CCDecoder::extractFromSEI(const sp<ABuffer> &accessUnit) {
- int64_t timeUs;
- CHECK(accessUnit->meta()->findInt64("timeUs", &timeUs));
-
- sp<ABuffer> sei;
- if (!accessUnit->meta()->findBuffer("sei", &sei) || sei == NULL) {
- return false;
- }
-
- bool trackAdded = false;
-
- NALBitReader br(sei->data() + 1, sei->size() - 1);
- // sei_message()
- while (br.atLeastNumBitsLeft(16)) { // at least 16-bit for sei_message()
- uint32_t payload_type = 0;
- size_t payload_size = 0;
- uint8_t last_byte;
-
- do {
- last_byte = br.getBits(8);
- payload_type += last_byte;
- } while (last_byte == 0xFF);
-
- do {
- last_byte = br.getBits(8);
- payload_size += last_byte;
- } while (last_byte == 0xFF);
-
- // sei_payload()
- if (payload_type == 4) {
- // user_data_registered_itu_t_t35()
-
- // ATSC A/72: 6.4.2
- uint8_t itu_t_t35_country_code = br.getBits(8);
- uint16_t itu_t_t35_provider_code = br.getBits(16);
- uint32_t user_identifier = br.getBits(32);
- uint8_t user_data_type_code = br.getBits(8);
-
- payload_size -= 1 + 2 + 4 + 1;
-
- if (itu_t_t35_country_code == 0xB5
- && itu_t_t35_provider_code == 0x0031
- && user_identifier == 'GA94'
- && user_data_type_code == 0x3) {
- // MPEG_cc_data()
- // ATSC A/53 Part 4: 6.2.3.1
- br.skipBits(1); //process_em_data_flag
- bool process_cc_data_flag = br.getBits(1);
- br.skipBits(1); //additional_data_flag
- size_t cc_count = br.getBits(5);
- br.skipBits(8); // em_data;
- payload_size -= 2;
-
- if (process_cc_data_flag) {
- AString out;
-
- sp<ABuffer> ccBuf = new ABuffer(cc_count * sizeof(CCData));
- ccBuf->setRange(0, 0);
-
- for (size_t i = 0; i < cc_count; i++) {
- uint8_t marker = br.getBits(5);
- CHECK_EQ(marker, 0x1f);
-
- bool cc_valid = br.getBits(1);
- uint8_t cc_type = br.getBits(2);
- // remove odd parity bit
- uint8_t cc_data_1 = br.getBits(8) & 0x7f;
- uint8_t cc_data_2 = br.getBits(8) & 0x7f;
-
- if (cc_valid
- && (cc_type == 0 || cc_type == 1)) {
- CCData cc(cc_type, cc_data_1, cc_data_2);
- if (!isNullPad(&cc)) {
- size_t channel;
- if (cc.getChannel(&channel) && getTrackIndex(channel) < 0) {
- mTrackIndices[channel] = mFoundChannels.size();
- mFoundChannels.push_back(channel);
- trackAdded = true;
- }
- memcpy(ccBuf->data() + ccBuf->size(),
- (void *)&cc, sizeof(cc));
- ccBuf->setRange(0, ccBuf->size() + sizeof(CCData));
- }
- }
- }
- payload_size -= cc_count * 3;
-
- mCCMap.add(timeUs, ccBuf);
- break;
- }
- } else {
- ALOGV("Malformed SEI payload type 4");
- }
- } else {
- ALOGV("Unsupported SEI payload type %d", payload_type);
- }
-
- // skipping remaining bits of this payload
- br.skipBits(payload_size * 8);
- }
-
- return trackAdded;
-}
-
-sp<ABuffer> NuPlayer::CCDecoder::filterCCBuf(
- const sp<ABuffer> &ccBuf, size_t index) {
- sp<ABuffer> filteredCCBuf = new ABuffer(ccBuf->size());
- filteredCCBuf->setRange(0, 0);
-
- size_t cc_count = ccBuf->size() / sizeof(CCData);
- const CCData* cc_data = (const CCData*)ccBuf->data();
- for (size_t i = 0; i < cc_count; ++i) {
- size_t channel;
- if (cc_data[i].getChannel(&channel)) {
- mCurrentChannel = channel;
- }
- if (mCurrentChannel == mFoundChannels[index]) {
- memcpy(filteredCCBuf->data() + filteredCCBuf->size(),
- (void *)&cc_data[i], sizeof(CCData));
- filteredCCBuf->setRange(0, filteredCCBuf->size() + sizeof(CCData));
- }
- }
-
- return filteredCCBuf;
-}
-
-void NuPlayer::CCDecoder::decode(const sp<ABuffer> &accessUnit) {
- if (extractFromSEI(accessUnit)) {
- ALOGI("Found CEA-608 track");
- sp<AMessage> msg = mNotify->dup();
- msg->setInt32("what", kWhatTrackAdded);
- msg->post();
- }
- // TODO: extract CC from other sources
-}
-
-void NuPlayer::CCDecoder::display(int64_t timeUs) {
- if (!isTrackValid(mSelectedTrack)) {
- ALOGE("Could not find current track(index=%d)", mSelectedTrack);
+void NuPlayer::Decoder::rememberCodecSpecificData(const sp<AMessage> &format) {
+ if (format == NULL) {
return;
}
-
- ssize_t index = mCCMap.indexOfKey(timeUs);
- if (index < 0) {
- ALOGV("cc for timestamp %" PRId64 " not found", timeUs);
- return;
+ mCSDsForCurrentFormat.clear();
+ for (int32_t i = 0; ; ++i) {
+ AString tag = "csd-";
+ tag.append(i);
+ sp<ABuffer> buffer;
+ if (!format->findBuffer(tag.c_str(), &buffer)) {
+ break;
+ }
+ mCSDsForCurrentFormat.push(buffer);
}
-
- sp<ABuffer> ccBuf = filterCCBuf(mCCMap.valueAt(index), mSelectedTrack);
-
- if (ccBuf->size() > 0) {
-#if 0
- dumpBytePair(ccBuf);
-#endif
-
- ccBuf->meta()->setInt32("trackIndex", mSelectedTrack);
- ccBuf->meta()->setInt64("timeUs", timeUs);
- ccBuf->meta()->setInt64("durationUs", 0ll);
-
- sp<AMessage> msg = mNotify->dup();
- msg->setInt32("what", kWhatClosedCaptionData);
- msg->setBuffer("buffer", ccBuf);
- msg->post();
- }
-
- // remove all entries before timeUs
- mCCMap.removeItemsAt(0, index + 1);
-}
-
-void NuPlayer::CCDecoder::flush() {
- mCCMap.clear();
}
} // namespace android