transcoding: add watchdog to prevent transcoder hang
Add a watchdog to monitor transcoder progress. Make transcoder
report heart beat regularly as long as there is new progress.
If heartbeat stops, watchdog will initiate a timeout to
1) Abandon old TranscoderWrapper. We try to shut it down nicely,
however, if it's really stuck, we'll have to leave it there.
2) Instantiate a new TranscoderWrapper with new looper.
3) Report Watchdog timeout to client.
Tests:
- New unit tests to MediaTranscoder, TranscodingSessionController
and MediaTranscodingService's simulated test (for error code reporting).
- Manually tested that long recording works properly without timeout.
bug: 169453212
Change-Id: Iae89e49e8e12d6078dc49eef2960efd03e91c431
diff --git a/media/libmediatranscoding/transcoder/MediaSampleWriter.cpp b/media/libmediatranscoding/transcoder/MediaSampleWriter.cpp
index 389b941..0efe85d 100644
--- a/media/libmediatranscoding/transcoder/MediaSampleWriter.cpp
+++ b/media/libmediatranscoding/transcoder/MediaSampleWriter.cpp
@@ -83,12 +83,14 @@
}
}
-bool MediaSampleWriter::init(int fd, const std::weak_ptr<CallbackInterface>& callbacks) {
- return init(DefaultMuxer::create(fd), callbacks);
+bool MediaSampleWriter::init(int fd, const std::weak_ptr<CallbackInterface>& callbacks,
+ int64_t heartBeatIntervalUs) {
+ return init(DefaultMuxer::create(fd), callbacks, heartBeatIntervalUs);
}
bool MediaSampleWriter::init(const std::shared_ptr<MediaSampleWriterMuxerInterface>& muxer,
- const std::weak_ptr<CallbackInterface>& callbacks) {
+ const std::weak_ptr<CallbackInterface>& callbacks,
+ int64_t heartBeatIntervalUs) {
if (callbacks.lock() == nullptr) {
LOG(ERROR) << "Callback object cannot be null";
return false;
@@ -106,6 +108,7 @@
mState = INITIALIZED;
mMuxer = muxer;
mCallbacks = callbacks;
+ mHeartBeatIntervalUs = heartBeatIntervalUs;
return true;
}
@@ -219,6 +222,7 @@
media_status_t MediaSampleWriter::runWriterLoop(bool* wasStopped) NO_THREAD_SAFETY_ANALYSIS {
AMediaCodecBufferInfo bufferInfo;
int32_t lastProgressUpdate = 0;
+ bool progressSinceLastReport = false;
int trackEosCount = 0;
// Set the "primary" track that will be used to determine progress to the track with longest
@@ -232,6 +236,10 @@
}
}
+ std::chrono::microseconds updateInterval(mHeartBeatIntervalUs);
+ std::chrono::system_clock::time_point nextUpdateTime =
+ std::chrono::system_clock::now() + updateInterval;
+
while (true) {
if (trackEosCount >= mTracks.size()) {
break;
@@ -242,7 +250,21 @@
{
std::unique_lock lock(mMutex);
while (mSampleQueue.empty() && mState == STARTED) {
- mSampleSignal.wait(lock);
+ if (mHeartBeatIntervalUs <= 0) {
+ mSampleSignal.wait(lock);
+ continue;
+ }
+
+ if (mSampleSignal.wait_until(lock, nextUpdateTime) == std::cv_status::timeout) {
+ // Send heart-beat if there is any progress since last update time.
+ if (progressSinceLastReport) {
+ if (auto callbacks = mCallbacks.lock()) {
+ callbacks->onHeartBeat(this);
+ }
+ progressSinceLastReport = false;
+ }
+ nextUpdateTime += updateInterval;
+ }
}
if (mState == STOPPED) {
@@ -306,6 +328,7 @@
}
lastProgressUpdate = progress;
}
+ progressSinceLastReport = true;
}
}