transcoder: wait for actual encoder output format to start writer
bug: 154734285
bug: 158240484
test: unit tests. In particular:
run MediaTranscoderTests and MediaTranscodingServiceRealTests, and
check the output files. Should play with proper aspect ratio on
host and device.
Change-Id: I8254fd8bd42678a414cab5fb0f61c4ba56d6c114
diff --git a/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
index 3a281ba..b485826 100644
--- a/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/MediaTrackTranscoder.cpp
@@ -102,6 +102,12 @@
return false;
}
+void MediaTrackTranscoder::notifyTrackFormatAvailable() {
+ if (auto callbacks = mTranscoderCallback.lock()) {
+ callbacks->onTrackFormatAvailable(this);
+ }
+}
+
std::shared_ptr<MediaSampleQueue> MediaTrackTranscoder::getOutputQueue() const {
return mOutputQueue;
}
diff --git a/media/libmediatranscoding/transcoder/MediaTranscoder.cpp b/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
index 3db7455..69932f4 100644
--- a/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/MediaTranscoder.cpp
@@ -95,12 +95,41 @@
// Transcoding is done and the callback to the client has been sent, so tear down the
// pipeline but do it asynchronously to avoid deadlocks. If an error occurred, client
// should clean up the file.
- std::thread asyncCancelThread{
- [self = shared_from_this()] { self->cancel(); }};
+ std::thread asyncCancelThread{[self = shared_from_this()] { self->cancel(); }};
asyncCancelThread.detach();
}
}
+void MediaTranscoder::onTrackFormatAvailable(const MediaTrackTranscoder* transcoder) {
+ LOG(INFO) << "TrackTranscoder " << transcoder << " format available.";
+
+ std::scoped_lock lock{mTracksAddedMutex};
+
+ // Ignore duplicate format change.
+ if (mTracksAdded.count(transcoder) > 0) {
+ return;
+ }
+
+ // Add track to the writer.
+ const bool ok =
+ mSampleWriter->addTrack(transcoder->getOutputQueue(), transcoder->getOutputFormat());
+ if (!ok) {
+ LOG(ERROR) << "Unable to add track to sample writer.";
+ sendCallback(AMEDIA_ERROR_UNKNOWN);
+ return;
+ }
+
+ mTracksAdded.insert(transcoder);
+ if (mTracksAdded.size() == mTrackTranscoders.size()) {
+ LOG(INFO) << "Starting sample writer.";
+ bool started = mSampleWriter->start();
+ if (!started) {
+ LOG(ERROR) << "Unable to start sample writer.";
+ sendCallback(AMEDIA_ERROR_UNKNOWN);
+ }
+ }
+}
+
void MediaTranscoder::onTrackFinished(const MediaTrackTranscoder* transcoder) {
LOG(DEBUG) << "TrackTranscoder " << transcoder << " finished";
}
@@ -116,7 +145,7 @@
}
MediaTranscoder::MediaTranscoder(const std::shared_ptr<CallbackInterface>& callbacks)
- : mCallbacks(callbacks) {}
+ : mCallbacks(callbacks) {}
std::shared_ptr<MediaTranscoder> MediaTranscoder::create(
const std::shared_ptr<CallbackInterface>& callbacks,
@@ -268,25 +297,9 @@
return AMEDIA_ERROR_INVALID_OPERATION;
}
- // Add tracks to the writer.
- for (auto& transcoder : mTrackTranscoders) {
- const bool ok = mSampleWriter->addTrack(transcoder->getOutputQueue(),
- transcoder->getOutputFormat());
- if (!ok) {
- LOG(ERROR) << "Unable to add track to sample writer.";
- return AMEDIA_ERROR_UNKNOWN;
- }
- }
-
- bool started = mSampleWriter->start();
- if (!started) {
- LOG(ERROR) << "Unable to start sample writer.";
- return AMEDIA_ERROR_UNKNOWN;
- }
-
// Start transcoders
for (auto& transcoder : mTrackTranscoders) {
- started = transcoder->start();
+ bool started = transcoder->start();
if (!started) {
LOG(ERROR) << "Unable to start track transcoder.";
cancel();
diff --git a/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
index 7806208..a4cbf33 100644
--- a/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/PassthroughTrackTranscoder.cpp
@@ -96,6 +96,9 @@
MediaSampleInfo info;
std::shared_ptr<MediaSample> sample;
+ // Notify the track format as soon as we start. It's same as the source format.
+ notifyTrackFormatAvailable();
+
MediaSample::OnSampleReleasedCallback bufferReleaseCallback =
[bufferPool = mBufferPool](MediaSample* sample) {
bufferPool->returnBuffer(const_cast<uint8_t*>(sample->buffer));
diff --git a/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp b/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
index 3818545..96e2e61 100644
--- a/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
+++ b/media/libmediatranscoding/transcoder/VideoTrackTranscoder.cpp
@@ -86,6 +86,10 @@
VideoTrackTranscoder* transcoder = static_cast<VideoTrackTranscoder*>(userdata);
const char* kCodecName = (codec == transcoder->mDecoder ? "Decoder" : "Encoder");
LOG(DEBUG) << kCodecName << " format changed: " << AMediaFormat_toString(format);
+ if (codec == transcoder->mEncoder.get()) {
+ transcoder->mCodecMessageQueue.push(
+ [transcoder, format] { transcoder->updateTrackFormat(format); });
+ }
}
static void onAsyncError(AMediaCodec* codec, void* userdata, media_status_t error,
@@ -311,6 +315,56 @@
}
}
+void VideoTrackTranscoder::updateTrackFormat(AMediaFormat* outputFormat) {
+ if (mActualOutputFormat != nullptr) {
+ LOG(WARNING) << "Ignoring duplicate format change.";
+ return;
+ }
+
+ AMediaFormat* formatCopy = AMediaFormat_new();
+ if (!formatCopy || AMediaFormat_copy(formatCopy, outputFormat) != AMEDIA_OK) {
+ LOG(ERROR) << "Unable to copy outputFormat";
+ AMediaFormat_delete(formatCopy);
+ mStatus = AMEDIA_ERROR_INVALID_PARAMETER;
+ return;
+ }
+
+ // Generate the actual track format for muxer based on the encoder output format,
+ // since many vital information comes in the encoder format (eg. CSD).
+ // Transfer necessary fields from the user-configured track format (derived from
+ // source track format and user transcoding request) where needed.
+
+ // Transfer SAR settings:
+ // If mDestinationFormat has SAR set, it means the original source has SAR specified
+ // at container level. This is supposed to override any SAR settings in the bitstream,
+ // thus should always be transferred to the container of the transcoded file.
+ int32_t sarWidth, sarHeight;
+ if (AMediaFormat_getInt32(mDestinationFormat.get(), AMEDIAFORMAT_KEY_SAR_WIDTH, &sarWidth) &&
+ (sarWidth > 0) &&
+ AMediaFormat_getInt32(mDestinationFormat.get(), AMEDIAFORMAT_KEY_SAR_HEIGHT, &sarHeight) &&
+ (sarHeight > 0)) {
+ AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_SAR_WIDTH, sarWidth);
+ AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_SAR_HEIGHT, sarHeight);
+ }
+ // Transfer DAR settings.
+ int32_t displayWidth, displayHeight;
+ if (AMediaFormat_getInt32(mDestinationFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_WIDTH,
+ &displayWidth) &&
+ (displayWidth > 0) &&
+ AMediaFormat_getInt32(mDestinationFormat.get(), AMEDIAFORMAT_KEY_DISPLAY_HEIGHT,
+ &displayHeight) &&
+ (displayHeight > 0)) {
+ AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_DISPLAY_WIDTH, displayWidth);
+ AMediaFormat_setInt32(formatCopy, AMEDIAFORMAT_KEY_DISPLAY_HEIGHT, displayHeight);
+ }
+
+ // TODO: transfer other fields as required.
+
+ mActualOutputFormat = std::shared_ptr<AMediaFormat>(formatCopy, &AMediaFormat_delete);
+
+ notifyTrackFormatAvailable();
+}
+
media_status_t VideoTrackTranscoder::runTranscodeLoop() {
media_status_t status = AMEDIA_OK;
@@ -350,7 +404,7 @@
}
std::shared_ptr<AMediaFormat> VideoTrackTranscoder::getOutputFormat() const {
- return mDestinationFormat;
+ return mActualOutputFormat;
}
} // namespace android
diff --git a/media/libmediatranscoding/transcoder/include/media/MediaTrackTranscoder.h b/media/libmediatranscoding/transcoder/include/media/MediaTrackTranscoder.h
index 0a35a57..60a9139 100644
--- a/media/libmediatranscoding/transcoder/include/media/MediaTrackTranscoder.h
+++ b/media/libmediatranscoding/transcoder/include/media/MediaTrackTranscoder.h
@@ -94,6 +94,9 @@
: mOutputQueue(std::make_shared<MediaSampleQueue>()),
mTranscoderCallback(transcoderCallback){};
+ // Called by subclasses when the actual track format becomes available.
+ void notifyTrackFormatAvailable();
+
// configureDestinationFormat needs to be implemented by subclasses, and gets called on an
// external thread before start.
virtual media_status_t configureDestinationFormat(
diff --git a/media/libmediatranscoding/transcoder/include/media/MediaTrackTranscoderCallback.h b/media/libmediatranscoding/transcoder/include/media/MediaTrackTranscoderCallback.h
index d535ada..654171e 100644
--- a/media/libmediatranscoding/transcoder/include/media/MediaTrackTranscoderCallback.h
+++ b/media/libmediatranscoding/transcoder/include/media/MediaTrackTranscoderCallback.h
@@ -27,6 +27,11 @@
class MediaTrackTranscoderCallback {
public:
/**
+ * Called when the MediaTrackTranscoder's actual track format becomes available.
+ * @param transcoder The MediaTrackTranscoder whose track format becomes available.
+ */
+ virtual void onTrackFormatAvailable(const MediaTrackTranscoder* transcoder);
+ /**
* Called when the MediaTrackTranscoder instance have finished transcoding all media samples
* successfully.
* @param transcoder The MediaTrackTranscoder that finished the transcoding.
diff --git a/media/libmediatranscoding/transcoder/include/media/MediaTranscoder.h b/media/libmediatranscoding/transcoder/include/media/MediaTranscoder.h
index 72a5559..e5a5d64 100644
--- a/media/libmediatranscoding/transcoder/include/media/MediaTranscoder.h
+++ b/media/libmediatranscoding/transcoder/include/media/MediaTranscoder.h
@@ -22,9 +22,11 @@
#include <media/MediaTrackTranscoderCallback.h>
#include <media/NdkMediaError.h>
#include <media/NdkMediaFormat.h>
+#include <utils/Mutex.h>
#include <atomic>
#include <memory>
+#include <mutex>
#include <unordered_set>
namespace android {
@@ -116,6 +118,7 @@
MediaTranscoder(const std::shared_ptr<CallbackInterface>& callbacks);
// MediaTrackTranscoderCallback
+ virtual void onTrackFormatAvailable(const MediaTrackTranscoder* transcoder) override;
virtual void onTrackFinished(const MediaTrackTranscoder* transcoder) override;
virtual void onTrackError(const MediaTrackTranscoder* transcoder,
media_status_t status) override;
@@ -128,6 +131,8 @@
std::unique_ptr<MediaSampleWriter> mSampleWriter;
std::vector<std::shared_ptr<AMediaFormat>> mSourceTrackFormats;
std::vector<std::unique_ptr<MediaTrackTranscoder>> mTrackTranscoders;
+ std::mutex mTracksAddedMutex;
+ std::unordered_set<const MediaTrackTranscoder*> mTracksAdded GUARDED_BY(mTracksAddedMutex);
std::atomic_bool mCallbackSent = false;
std::atomic_bool mCancelled = false;
diff --git a/media/libmediatranscoding/transcoder/include/media/VideoTrackTranscoder.h b/media/libmediatranscoding/transcoder/include/media/VideoTrackTranscoder.h
index c47e4b7..1ba205b 100644
--- a/media/libmediatranscoding/transcoder/include/media/VideoTrackTranscoder.h
+++ b/media/libmediatranscoding/transcoder/include/media/VideoTrackTranscoder.h
@@ -73,6 +73,9 @@
// Dequeues an encoded buffer from the encoder and adds it to the output queue.
void dequeueOutputSample(int32_t bufferIndex, AMediaCodecBufferInfo bufferInfo);
+ // Updates the video track's actual format based on encoder output format.
+ void updateTrackFormat(AMediaFormat* outputFormat);
+
AMediaCodec* mDecoder = nullptr;
// Sample release callback holds a reference to the encoder, hence the shared_ptr.
std::shared_ptr<AMediaCodec> mEncoder;
@@ -84,6 +87,7 @@
MediaSampleInfo mSampleInfo;
BlockingQueue<std::function<void()>> mCodecMessageQueue;
std::shared_ptr<AMediaFormat> mDestinationFormat;
+ std::shared_ptr<AMediaFormat> mActualOutputFormat;
};
} // namespace android
diff --git a/media/libmediatranscoding/transcoder/tests/TrackTranscoderTestUtils.h b/media/libmediatranscoding/transcoder/tests/TrackTranscoderTestUtils.h
index 32a1a0a..54b42fe 100644
--- a/media/libmediatranscoding/transcoder/tests/TrackTranscoderTestUtils.h
+++ b/media/libmediatranscoding/transcoder/tests/TrackTranscoderTestUtils.h
@@ -57,6 +57,8 @@
~TestCallback() = default;
// MediaTrackTranscoderCallback
+ void onTrackFormatAvailable(const MediaTrackTranscoder* transcoder __unused) {}
+
void onTrackFinished(const MediaTrackTranscoder* transcoder __unused) {
std::unique_lock<std::mutex> lock(mMutex);
mTranscodingFinished = true;