blob: b47a4f1d1015a8ac35b7b0cd82df2c709e29f858 [file] [log] [blame]
Andreas Huberf9334412010-12-15 15:17:42 -08001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
Andy Hungf2c87b32016-04-07 19:49:29 -070022#include <algorithm>
Andy Hunga0b39712015-05-31 22:40:49 -070023#include <cutils/properties.h>
Andreas Huberf9334412010-12-15 15:17:42 -080024#include <media/stagefright/foundation/ABuffer.h>
25#include <media/stagefright/foundation/ADebug.h>
Andreas Huber5bc087c2010-12-23 10:27:40 -080026#include <media/stagefright/foundation/AMessage.h>
Ronghua Wua73d9e02014-10-08 15:13:29 -070027#include <media/stagefright/foundation/AUtils.h>
Weiyin Jiang35d5af12015-01-28 16:14:02 +080028#include <media/stagefright/foundation/AWakeLock.h>
Wei Jia5833b6a2015-03-09 17:01:47 -070029#include <media/stagefright/MediaClock.h>
Wei Jiabc2fb722014-07-08 16:37:57 -070030#include <media/stagefright/MediaErrors.h>
31#include <media/stagefright/MetaData.h>
Chong Zhang3b9eb1f2014-10-15 17:05:08 -070032#include <media/stagefright/Utils.h>
Lajos Molnara3725d72015-07-01 16:47:22 -070033#include <media/stagefright/VideoFrameScheduler.h>
Lajos Molnardc43dfa2014-05-07 15:33:04 -070034
Lajos Molnar09524832014-07-17 14:29:51 -070035#include <inttypes.h>
36
Andreas Huberf9334412010-12-15 15:17:42 -080037namespace android {
38
Andy Hunga0b39712015-05-31 22:40:49 -070039/*
40 * Example of common configuration settings in shell script form
41
42 #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
43 adb shell setprop audio.offload.disable 1
44
45 #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
46 adb shell setprop audio.offload.video 1
47
48 #Use audio callbacks for PCM data
49 adb shell setprop media.stagefright.audio.cbk 1
50
Andy Hung288da022015-05-31 22:55:59 -070051 #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
52 adb shell setprop media.stagefright.audio.deep 1
53
Andy Hung179652e2015-05-31 22:49:46 -070054 #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
55 adb shell setprop media.stagefright.audio.sink 1000
56
Andy Hunga0b39712015-05-31 22:40:49 -070057 * These configurations take effect for the next track played (not the current track).
58 */
59
60static inline bool getUseAudioCallbackSetting() {
61 return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
62}
63
Andy Hung179652e2015-05-31 22:49:46 -070064static inline int32_t getAudioSinkPcmMsSetting() {
65 return property_get_int32(
66 "media.stagefright.audio.sink", 500 /* default_value */);
67}
68
Ronghua Wuf5b1db12014-09-09 10:11:08 -070069// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
70// is closed to allow the audio DSP to power down.
Eric Laurenta5d316f2014-10-29 15:32:17 -070071static const int64_t kOffloadPauseMaxUs = 10000000ll;
Ronghua Wuf5b1db12014-09-09 10:11:08 -070072
Wei Jia12b62652016-02-09 17:49:31 -080073// Maximum allowed delay from AudioSink, 1.5 seconds.
74static const int64_t kMaxAllowedAudioSinkDelayUs = 1500000ll;
75
Andy Hung528c8402016-01-12 12:39:34 -080076static const int64_t kMinimumAudioClockUpdatePeriodUs = 20 /* msec */ * 1000;
77
Andreas Huber714aa7b2011-09-13 08:28:38 -070078// static
Andy Hungf0e83642014-12-19 17:55:56 -080079const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
80 AUDIO_CHANNEL_NONE,
81 AUDIO_OUTPUT_FLAG_NONE,
82 AUDIO_FORMAT_INVALID,
83 0, // mNumChannels
84 0 // mSampleRate
85};
86
87// static
Andreas Huber714aa7b2011-09-13 08:28:38 -070088const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
89
Andreas Huberf9334412010-12-15 15:17:42 -080090NuPlayer::Renderer::Renderer(
91 const sp<MediaPlayerBase::AudioSink> &sink,
Andreas Huberd5e56232013-03-12 11:01:43 -070092 const sp<AMessage> &notify,
93 uint32_t flags)
Andreas Huberf9334412010-12-15 15:17:42 -080094 : mAudioSink(sink),
Wei Jiad2f35de2016-02-17 16:48:06 -080095 mUseVirtualAudioSink(false),
Andreas Huberf9334412010-12-15 15:17:42 -080096 mNotify(notify),
Andreas Huberd5e56232013-03-12 11:01:43 -070097 mFlags(flags),
Andreas Huberf9334412010-12-15 15:17:42 -080098 mNumFramesWritten(0),
99 mDrainAudioQueuePending(false),
100 mDrainVideoQueuePending(false),
101 mAudioQueueGeneration(0),
102 mVideoQueueGeneration(0),
Wei Jia7b15cb32015-02-03 17:46:06 -0800103 mAudioDrainGeneration(0),
104 mVideoDrainGeneration(0),
Wei Jia7c8d0e02015-08-27 17:40:21 -0700105 mAudioEOSGeneration(0),
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700106 mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
Ronghua Wua73d9e02014-10-08 15:13:29 -0700107 mAudioFirstAnchorTimeMediaUs(-1),
Ronghua Wueecb7802014-10-19 23:12:50 -0700108 mAnchorTimeMediaUs(-1),
Lajos Molnarf5926712014-10-21 09:36:43 -0700109 mAnchorNumFramesWritten(-1),
Ronghua Wua73d9e02014-10-08 15:13:29 -0700110 mVideoLateByUs(0ll),
Andreas Huberbc7f5b22011-01-21 10:15:23 -0800111 mHasAudio(false),
112 mHasVideo(false),
Chong Zhang7137ec72014-11-12 16:41:05 -0800113 mNotifyCompleteAudio(false),
114 mNotifyCompleteVideo(false),
Andreas Huberbc7f5b22011-01-21 10:15:23 -0800115 mSyncQueues(false),
Andreas Huber714aa7b2011-09-13 08:28:38 -0700116 mPaused(false),
Andy Hungb03dcb32015-08-27 16:18:59 -0700117 mPauseDrainAudioAllowedUs(0),
Andy Hung09e0c362014-09-12 15:12:16 -0700118 mVideoSampleReceived(false),
James Dongf57b4ea2012-07-20 13:38:36 -0700119 mVideoRenderingStarted(false),
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700120 mVideoRenderingStartGeneration(0),
121 mAudioRenderingStartGeneration(0),
Andy Hungb12ea0b2015-09-03 15:13:01 -0700122 mRenderingDataDelivered(false),
Andy Hung528c8402016-01-12 12:39:34 -0800123 mNextAudioClockUpdateTimeUs(-1),
Wei Jia2995dc72015-07-24 16:00:33 -0700124 mLastAudioMediaTimeUs(-1),
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700125 mAudioOffloadPauseTimeoutGeneration(0),
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700126 mAudioTornDown(false),
Lajos Molnard5923402014-10-20 17:47:54 -0700127 mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
Andy Hungf0e83642014-12-19 17:55:56 -0800128 mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
Lajos Molnard5923402014-10-20 17:47:54 -0700129 mTotalBuffersQueued(0),
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800130 mLastAudioBufferDrained(0),
Andy Hunga0b39712015-05-31 22:40:49 -0700131 mUseAudioCallback(false),
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800132 mWakeLock(new AWakeLock()) {
Wei Jia7b15cb32015-02-03 17:46:06 -0800133 mMediaClock = new MediaClock;
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700134 mPlaybackRate = mPlaybackSettings.mSpeed;
135 mMediaClock->setPlaybackRate(mPlaybackRate);
Andreas Huberf9334412010-12-15 15:17:42 -0800136}
137
138NuPlayer::Renderer::~Renderer() {
Wei Jiabc2fb722014-07-08 16:37:57 -0700139 if (offloadingAudio()) {
140 mAudioSink->stop();
141 mAudioSink->flush();
142 mAudioSink->close();
143 }
Andreas Huberf9334412010-12-15 15:17:42 -0800144}
145
146void NuPlayer::Renderer::queueBuffer(
147 bool audio,
148 const sp<ABuffer> &buffer,
149 const sp<AMessage> &notifyConsumed) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800150 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800151 msg->setInt32("queueGeneration", getQueueGeneration(audio));
Andreas Huberf9334412010-12-15 15:17:42 -0800152 msg->setInt32("audio", static_cast<int32_t>(audio));
Andreas Huber2d8bedd2012-02-21 14:38:23 -0800153 msg->setBuffer("buffer", buffer);
Andreas Huberf9334412010-12-15 15:17:42 -0800154 msg->setMessage("notifyConsumed", notifyConsumed);
155 msg->post();
156}
157
158void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
159 CHECK_NE(finalResult, (status_t)OK);
160
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800161 sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800162 msg->setInt32("queueGeneration", getQueueGeneration(audio));
Andreas Huberf9334412010-12-15 15:17:42 -0800163 msg->setInt32("audio", static_cast<int32_t>(audio));
164 msg->setInt32("finalResult", finalResult);
165 msg->post();
166}
167
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700168status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
169 sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
170 writeToAMessage(msg, rate);
171 sp<AMessage> response;
172 status_t err = msg->postAndAwaitResponse(&response);
173 if (err == OK && response != NULL) {
174 CHECK(response->findInt32("err", &err));
175 }
176 return err;
177}
178
179status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
180 if (rate.mSpeed == 0.f) {
181 onPause();
182 // don't call audiosink's setPlaybackRate if pausing, as pitch does not
183 // have to correspond to the any non-0 speed (e.g old speed). Keep
184 // settings nonetheless, using the old speed, in case audiosink changes.
185 AudioPlaybackRate newRate = rate;
186 newRate.mSpeed = mPlaybackSettings.mSpeed;
187 mPlaybackSettings = newRate;
188 return OK;
189 }
190
Wei Jia27ea08e2015-05-12 14:50:35 -0700191 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700192 status_t err = mAudioSink->setPlaybackRate(rate);
193 if (err != OK) {
194 return err;
195 }
196 }
197 mPlaybackSettings = rate;
198 mPlaybackRate = rate.mSpeed;
199 mMediaClock->setPlaybackRate(mPlaybackRate);
200 return OK;
201}
202
203status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
204 sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
205 sp<AMessage> response;
206 status_t err = msg->postAndAwaitResponse(&response);
207 if (err == OK && response != NULL) {
208 CHECK(response->findInt32("err", &err));
209 if (err == OK) {
210 readFromAMessage(response, rate);
211 }
212 }
213 return err;
214}
215
216status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
Wei Jia27ea08e2015-05-12 14:50:35 -0700217 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700218 status_t err = mAudioSink->getPlaybackRate(rate);
219 if (err == OK) {
220 if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
221 ALOGW("correcting mismatch in internal/external playback rate");
222 }
223 // get playback settings used by audiosink, as it may be
224 // slightly off due to audiosink not taking small changes.
225 mPlaybackSettings = *rate;
226 if (mPaused) {
227 rate->mSpeed = 0.f;
228 }
229 }
230 return err;
231 }
232 *rate = mPlaybackSettings;
233 return OK;
234}
235
236status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
237 sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
238 writeToAMessage(msg, sync, videoFpsHint);
239 sp<AMessage> response;
240 status_t err = msg->postAndAwaitResponse(&response);
241 if (err == OK && response != NULL) {
242 CHECK(response->findInt32("err", &err));
243 }
244 return err;
245}
246
247status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
248 if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
249 return BAD_VALUE;
250 }
251 // TODO: support sync sources
252 return INVALID_OPERATION;
253}
254
255status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
256 sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
257 sp<AMessage> response;
258 status_t err = msg->postAndAwaitResponse(&response);
259 if (err == OK && response != NULL) {
260 CHECK(response->findInt32("err", &err));
261 if (err == OK) {
262 readFromAMessage(response, sync, videoFps);
263 }
264 }
265 return err;
266}
267
268status_t NuPlayer::Renderer::onGetSyncSettings(
269 AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
270 *sync = mSyncSettings;
271 *videoFps = -1.f;
272 return OK;
Wei Jia98160162015-02-04 17:01:11 -0800273}
274
Chong Zhang7137ec72014-11-12 16:41:05 -0800275void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
Andreas Huberf9334412010-12-15 15:17:42 -0800276 {
Wei Jia7b15cb32015-02-03 17:46:06 -0800277 Mutex::Autolock autoLock(mLock);
Andreas Huberf9334412010-12-15 15:17:42 -0800278 if (audio) {
Chong Zhang7137ec72014-11-12 16:41:05 -0800279 mNotifyCompleteAudio |= notifyComplete;
Wei Jia3b0cd262015-07-13 11:28:10 -0700280 clearAudioFirstAnchorTime_l();
Wei Jia7b15cb32015-02-03 17:46:06 -0800281 ++mAudioQueueGeneration;
282 ++mAudioDrainGeneration;
Andreas Huberf9334412010-12-15 15:17:42 -0800283 } else {
Chong Zhang7137ec72014-11-12 16:41:05 -0800284 mNotifyCompleteVideo |= notifyComplete;
Wei Jia7b15cb32015-02-03 17:46:06 -0800285 ++mVideoQueueGeneration;
286 ++mVideoDrainGeneration;
Andreas Huberf9334412010-12-15 15:17:42 -0800287 }
Wei Jia7b15cb32015-02-03 17:46:06 -0800288
289 clearAnchorTime_l();
Wei Jia7b15cb32015-02-03 17:46:06 -0800290 mVideoLateByUs = 0;
291 mSyncQueues = false;
Andreas Huberf9334412010-12-15 15:17:42 -0800292 }
293
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800294 sp<AMessage> msg = new AMessage(kWhatFlush, this);
Andreas Huberf9334412010-12-15 15:17:42 -0800295 msg->setInt32("audio", static_cast<int32_t>(audio));
296 msg->post();
297}
298
299void NuPlayer::Renderer::signalTimeDiscontinuity() {
Wei Jia28a8a9f2014-08-18 11:29:50 -0700300}
301
302void NuPlayer::Renderer::signalDisableOffloadAudio() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800303 (new AMessage(kWhatDisableOffloadAudio, this))->post();
Wei Jia28a8a9f2014-08-18 11:29:50 -0700304}
305
Ronghua Wua10fd232014-11-06 16:15:20 -0800306void NuPlayer::Renderer::signalEnableOffloadAudio() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800307 (new AMessage(kWhatEnableOffloadAudio, this))->post();
Ronghua Wua10fd232014-11-06 16:15:20 -0800308}
309
Andreas Huberb4082222011-01-20 15:23:04 -0800310void NuPlayer::Renderer::pause() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800311 (new AMessage(kWhatPause, this))->post();
Andreas Huberb4082222011-01-20 15:23:04 -0800312}
313
314void NuPlayer::Renderer::resume() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800315 (new AMessage(kWhatResume, this))->post();
Andreas Huberb4082222011-01-20 15:23:04 -0800316}
317
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700318void NuPlayer::Renderer::setVideoFrameRate(float fps) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800319 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700320 msg->setFloat("frame-rate", fps);
321 msg->post();
322}
323
Wei Jia4ecbea32015-10-20 18:09:57 -0700324// Called on any threads without mLock acquired.
Wei Jia7b15cb32015-02-03 17:46:06 -0800325status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
Wei Jia4ecbea32015-10-20 18:09:57 -0700326 status_t result = mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
327 if (result == OK) {
328 return result;
329 }
330
331 // MediaClock has not started yet. Try to start it if possible.
332 {
333 Mutex::Autolock autoLock(mLock);
334 if (mAudioFirstAnchorTimeMediaUs == -1) {
335 return result;
336 }
337
338 AudioTimestamp ts;
339 status_t res = mAudioSink->getTimestamp(ts);
340 if (res != OK) {
341 return result;
342 }
343
344 // AudioSink has rendered some frames.
345 int64_t nowUs = ALooper::GetNowUs();
Wei Jiad855a732015-10-21 13:31:15 -0700346 int64_t nowMediaUs = mAudioSink->getPlayedOutDurationUs(nowUs)
Wei Jia4ecbea32015-10-20 18:09:57 -0700347 + mAudioFirstAnchorTimeMediaUs;
348 mMediaClock->updateAnchor(nowMediaUs, nowUs, -1);
349 }
350
Wei Jia98160162015-02-04 17:01:11 -0800351 return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700352}
353
Wei Jia7b15cb32015-02-03 17:46:06 -0800354void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
355 mAudioFirstAnchorTimeMediaUs = -1;
356 mMediaClock->setStartingTimeMedia(-1);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700357}
358
Wei Jia7b15cb32015-02-03 17:46:06 -0800359void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
Ronghua Wua73d9e02014-10-08 15:13:29 -0700360 if (mAudioFirstAnchorTimeMediaUs == -1) {
361 mAudioFirstAnchorTimeMediaUs = mediaUs;
Wei Jia7b15cb32015-02-03 17:46:06 -0800362 mMediaClock->setStartingTimeMedia(mediaUs);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700363 }
364}
365
Wei Jia7b15cb32015-02-03 17:46:06 -0800366void NuPlayer::Renderer::clearAnchorTime_l() {
367 mMediaClock->clearAnchor();
368 mAnchorTimeMediaUs = -1;
369 mAnchorNumFramesWritten = -1;
Ronghua Wua73d9e02014-10-08 15:13:29 -0700370}
371
372void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
Wei Jia7b15cb32015-02-03 17:46:06 -0800373 Mutex::Autolock autoLock(mLock);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700374 mVideoLateByUs = lateUs;
375}
376
377int64_t NuPlayer::Renderer::getVideoLateByUs() {
Wei Jia7b15cb32015-02-03 17:46:06 -0800378 Mutex::Autolock autoLock(mLock);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700379 return mVideoLateByUs;
380}
381
Andy Hung202bce12014-12-03 11:47:36 -0800382status_t NuPlayer::Renderer::openAudioSink(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700383 const sp<AMessage> &format,
384 bool offloadOnly,
385 bool hasVideo,
Andy Hung202bce12014-12-03 11:47:36 -0800386 uint32_t flags,
387 bool *isOffloaded) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800388 sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700389 msg->setMessage("format", format);
390 msg->setInt32("offload-only", offloadOnly);
391 msg->setInt32("has-video", hasVideo);
392 msg->setInt32("flags", flags);
393
394 sp<AMessage> response;
395 msg->postAndAwaitResponse(&response);
396
Andy Hung202bce12014-12-03 11:47:36 -0800397 int32_t err;
398 if (!response->findInt32("err", &err)) {
399 err = INVALID_OPERATION;
400 } else if (err == OK && isOffloaded != NULL) {
401 int32_t offload;
402 CHECK(response->findInt32("offload", &offload));
403 *isOffloaded = (offload != 0);
404 }
405 return err;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700406}
407
408void NuPlayer::Renderer::closeAudioSink() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800409 sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700410
411 sp<AMessage> response;
412 msg->postAndAwaitResponse(&response);
413}
414
Andreas Huberf9334412010-12-15 15:17:42 -0800415void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
416 switch (msg->what()) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700417 case kWhatOpenAudioSink:
418 {
419 sp<AMessage> format;
420 CHECK(msg->findMessage("format", &format));
421
422 int32_t offloadOnly;
423 CHECK(msg->findInt32("offload-only", &offloadOnly));
424
425 int32_t hasVideo;
426 CHECK(msg->findInt32("has-video", &hasVideo));
427
428 uint32_t flags;
429 CHECK(msg->findInt32("flags", (int32_t *)&flags));
430
Andy Hung202bce12014-12-03 11:47:36 -0800431 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700432
433 sp<AMessage> response = new AMessage;
Andy Hung202bce12014-12-03 11:47:36 -0800434 response->setInt32("err", err);
435 response->setInt32("offload", offloadingAudio());
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700436
Lajos Molnar3f274362015-03-05 14:35:41 -0800437 sp<AReplyToken> replyID;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700438 CHECK(msg->senderAwaitsResponse(&replyID));
439 response->postReply(replyID);
440
441 break;
442 }
443
444 case kWhatCloseAudioSink:
445 {
Lajos Molnar3f274362015-03-05 14:35:41 -0800446 sp<AReplyToken> replyID;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700447 CHECK(msg->senderAwaitsResponse(&replyID));
448
449 onCloseAudioSink();
450
451 sp<AMessage> response = new AMessage;
452 response->postReply(replyID);
453 break;
454 }
455
Wei Jiabc2fb722014-07-08 16:37:57 -0700456 case kWhatStopAudioSink:
457 {
458 mAudioSink->stop();
459 break;
460 }
461
Andreas Huberf9334412010-12-15 15:17:42 -0800462 case kWhatDrainAudioQueue:
463 {
Wei Jia3ab25452015-06-10 09:37:47 -0700464 mDrainAudioQueuePending = false;
465
Andreas Huberf9334412010-12-15 15:17:42 -0800466 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800467 CHECK(msg->findInt32("drainGeneration", &generation));
468 if (generation != getDrainGeneration(true /* audio */)) {
Andreas Huberf9334412010-12-15 15:17:42 -0800469 break;
470 }
471
Andreas Huber078cfcf2011-09-15 12:25:04 -0700472 if (onDrainAudioQueue()) {
473 uint32_t numFramesPlayed;
474 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
475 (status_t)OK);
Andreas Huberf9334412010-12-15 15:17:42 -0800476
Andreas Huber078cfcf2011-09-15 12:25:04 -0700477 uint32_t numFramesPendingPlayout =
478 mNumFramesWritten - numFramesPlayed;
479
480 // This is how long the audio sink will have data to
481 // play back.
482 int64_t delayUs =
483 mAudioSink->msecsPerFrame()
484 * numFramesPendingPlayout * 1000ll;
Wei Jiad9c2e9c2015-04-10 19:04:09 -0700485 if (mPlaybackRate > 1.0f) {
486 delayUs /= mPlaybackRate;
487 }
Andreas Huber078cfcf2011-09-15 12:25:04 -0700488
489 // Let's give it more data after about half that time
490 // has elapsed.
Andy Hungf2c87b32016-04-07 19:49:29 -0700491 delayUs /= 2;
492 // check the buffer size to estimate maximum delay permitted.
493 const int64_t maxDrainDelayUs = std::max(
494 mAudioSink->getBufferDurationInUs(), (int64_t)500000 /* half second */);
495 ALOGD_IF(delayUs > maxDrainDelayUs, "postDrainAudioQueue long delay: %lld > %lld",
496 (long long)delayUs, (long long)maxDrainDelayUs);
Wei Jia7b15cb32015-02-03 17:46:06 -0800497 Mutex::Autolock autoLock(mLock);
Andy Hungf2c87b32016-04-07 19:49:29 -0700498 postDrainAudioQueue_l(delayUs);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700499 }
Andreas Huberf9334412010-12-15 15:17:42 -0800500 break;
501 }
502
503 case kWhatDrainVideoQueue:
504 {
505 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800506 CHECK(msg->findInt32("drainGeneration", &generation));
507 if (generation != getDrainGeneration(false /* audio */)) {
Andreas Huberf9334412010-12-15 15:17:42 -0800508 break;
509 }
510
511 mDrainVideoQueuePending = false;
512
513 onDrainVideoQueue();
514
Wei Jia7b15cb32015-02-03 17:46:06 -0800515 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -0800516 break;
517 }
518
Lajos Molnard5923402014-10-20 17:47:54 -0700519 case kWhatPostDrainVideoQueue:
520 {
521 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800522 CHECK(msg->findInt32("drainGeneration", &generation));
523 if (generation != getDrainGeneration(false /* audio */)) {
Lajos Molnard5923402014-10-20 17:47:54 -0700524 break;
525 }
526
527 mDrainVideoQueuePending = false;
Wei Jia7b15cb32015-02-03 17:46:06 -0800528 postDrainVideoQueue();
Lajos Molnard5923402014-10-20 17:47:54 -0700529 break;
530 }
531
Andreas Huberf9334412010-12-15 15:17:42 -0800532 case kWhatQueueBuffer:
533 {
534 onQueueBuffer(msg);
535 break;
536 }
537
538 case kWhatQueueEOS:
539 {
540 onQueueEOS(msg);
541 break;
542 }
543
Wei Jia7c8d0e02015-08-27 17:40:21 -0700544 case kWhatEOS:
545 {
546 int32_t generation;
547 CHECK(msg->findInt32("audioEOSGeneration", &generation));
548 if (generation != mAudioEOSGeneration) {
549 break;
550 }
551 status_t finalResult;
552 CHECK(msg->findInt32("finalResult", &finalResult));
553 notifyEOS(true /* audio */, finalResult);
554 break;
555 }
556
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700557 case kWhatConfigPlayback:
Wei Jia98160162015-02-04 17:01:11 -0800558 {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700559 sp<AReplyToken> replyID;
560 CHECK(msg->senderAwaitsResponse(&replyID));
561 AudioPlaybackRate rate;
562 readFromAMessage(msg, &rate);
563 status_t err = onConfigPlayback(rate);
564 sp<AMessage> response = new AMessage;
565 response->setInt32("err", err);
566 response->postReply(replyID);
567 break;
568 }
569
570 case kWhatGetPlaybackSettings:
571 {
572 sp<AReplyToken> replyID;
573 CHECK(msg->senderAwaitsResponse(&replyID));
574 AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
575 status_t err = onGetPlaybackSettings(&rate);
576 sp<AMessage> response = new AMessage;
577 if (err == OK) {
578 writeToAMessage(response, rate);
579 }
580 response->setInt32("err", err);
581 response->postReply(replyID);
582 break;
583 }
584
585 case kWhatConfigSync:
586 {
587 sp<AReplyToken> replyID;
588 CHECK(msg->senderAwaitsResponse(&replyID));
589 AVSyncSettings sync;
590 float videoFpsHint;
591 readFromAMessage(msg, &sync, &videoFpsHint);
592 status_t err = onConfigSync(sync, videoFpsHint);
593 sp<AMessage> response = new AMessage;
594 response->setInt32("err", err);
595 response->postReply(replyID);
596 break;
597 }
598
599 case kWhatGetSyncSettings:
600 {
601 sp<AReplyToken> replyID;
602 CHECK(msg->senderAwaitsResponse(&replyID));
603
604 ALOGV("kWhatGetSyncSettings");
605 AVSyncSettings sync;
606 float videoFps = -1.f;
607 status_t err = onGetSyncSettings(&sync, &videoFps);
608 sp<AMessage> response = new AMessage;
609 if (err == OK) {
610 writeToAMessage(response, sync, videoFps);
611 }
612 response->setInt32("err", err);
613 response->postReply(replyID);
Wei Jia98160162015-02-04 17:01:11 -0800614 break;
615 }
616
Andreas Huberf9334412010-12-15 15:17:42 -0800617 case kWhatFlush:
618 {
619 onFlush(msg);
620 break;
621 }
622
Wei Jiabc2fb722014-07-08 16:37:57 -0700623 case kWhatDisableOffloadAudio:
624 {
625 onDisableOffloadAudio();
626 break;
627 }
628
Ronghua Wua10fd232014-11-06 16:15:20 -0800629 case kWhatEnableOffloadAudio:
630 {
631 onEnableOffloadAudio();
632 break;
633 }
634
Andreas Huberb4082222011-01-20 15:23:04 -0800635 case kWhatPause:
636 {
637 onPause();
638 break;
639 }
640
641 case kWhatResume:
642 {
643 onResume();
644 break;
645 }
646
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700647 case kWhatSetVideoFrameRate:
648 {
649 float fps;
650 CHECK(msg->findFloat("frame-rate", &fps));
651 onSetVideoFrameRate(fps);
652 break;
653 }
654
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700655 case kWhatAudioTearDown:
Wei Jia3a2956d2014-07-22 16:01:33 -0700656 {
Wei Jiaa05f1e32016-03-25 16:31:22 -0700657 int32_t reason;
658 CHECK(msg->findInt32("reason", &reason));
659
660 onAudioTearDown((AudioTearDownReason)reason);
Wei Jia3a2956d2014-07-22 16:01:33 -0700661 break;
662 }
663
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700664 case kWhatAudioOffloadPauseTimeout:
665 {
666 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800667 CHECK(msg->findInt32("drainGeneration", &generation));
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700668 if (generation != mAudioOffloadPauseTimeoutGeneration) {
669 break;
670 }
Ronghua Wu08529172014-10-02 16:55:52 -0700671 ALOGV("Audio Offload tear down due to pause timeout.");
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700672 onAudioTearDown(kDueToTimeout);
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800673 mWakeLock->release();
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700674 break;
675 }
676
Andreas Huberf9334412010-12-15 15:17:42 -0800677 default:
678 TRESPASS();
679 break;
680 }
681}
682
Wei Jiabc2fb722014-07-08 16:37:57 -0700683void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
Andy Hung005e9d02015-05-31 23:16:07 -0700684 if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
Andreas Huberf9334412010-12-15 15:17:42 -0800685 return;
686 }
687
688 if (mAudioQueue.empty()) {
689 return;
690 }
691
Andy Hungb03dcb32015-08-27 16:18:59 -0700692 // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data.
693 if (mPaused) {
694 const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs();
695 if (diffUs > delayUs) {
696 delayUs = diffUs;
697 }
698 }
699
Andreas Huberf9334412010-12-15 15:17:42 -0800700 mDrainAudioQueuePending = true;
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800701 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800702 msg->setInt32("drainGeneration", mAudioDrainGeneration);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700703 msg->post(delayUs);
Andreas Huberf9334412010-12-15 15:17:42 -0800704}
705
Wei Jia7b15cb32015-02-03 17:46:06 -0800706void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
707 mAudioRenderingStartGeneration = mAudioDrainGeneration;
708 mVideoRenderingStartGeneration = mVideoDrainGeneration;
Andy Hungb12ea0b2015-09-03 15:13:01 -0700709 mRenderingDataDelivered = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700710}
711
Wei Jia7b15cb32015-02-03 17:46:06 -0800712void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
713 if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
714 mAudioRenderingStartGeneration == mAudioDrainGeneration) {
Andy Hungb12ea0b2015-09-03 15:13:01 -0700715 mRenderingDataDelivered = true;
716 if (mPaused) {
717 return;
718 }
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700719 mVideoRenderingStartGeneration = -1;
720 mAudioRenderingStartGeneration = -1;
721
722 sp<AMessage> notify = mNotify->dup();
723 notify->setInt32("what", kWhatMediaRenderingStart);
724 notify->post();
725 }
726}
727
Wei Jiabc2fb722014-07-08 16:37:57 -0700728// static
729size_t NuPlayer::Renderer::AudioSinkCallback(
730 MediaPlayerBase::AudioSink * /* audioSink */,
731 void *buffer,
732 size_t size,
733 void *cookie,
734 MediaPlayerBase::AudioSink::cb_event_t event) {
735 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
736
737 switch (event) {
738 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
739 {
740 return me->fillAudioBuffer(buffer, size);
741 break;
742 }
743
744 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
745 {
Andy Hunga0b39712015-05-31 22:40:49 -0700746 ALOGV("AudioSink::CB_EVENT_STREAM_END");
Wei Jiabc2fb722014-07-08 16:37:57 -0700747 me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
748 break;
749 }
750
751 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
752 {
Andy Hunga0b39712015-05-31 22:40:49 -0700753 ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
Wei Jiaa05f1e32016-03-25 16:31:22 -0700754 me->notifyAudioTearDown(kDueToError);
Wei Jiabc2fb722014-07-08 16:37:57 -0700755 break;
756 }
757 }
758
759 return 0;
760}
761
762size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
763 Mutex::Autolock autoLock(mLock);
764
Andy Hung85e48142015-05-31 23:04:15 -0700765 if (!mUseAudioCallback) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700766 return 0;
767 }
768
769 bool hasEOS = false;
770
771 size_t sizeCopied = 0;
Ronghua Wu3e5efb32014-08-18 16:27:08 -0700772 bool firstEntry = true;
Andy Hunga0b39712015-05-31 22:40:49 -0700773 QueueEntry *entry; // will be valid after while loop if hasEOS is set.
Wei Jiabc2fb722014-07-08 16:37:57 -0700774 while (sizeCopied < size && !mAudioQueue.empty()) {
Andy Hunga0b39712015-05-31 22:40:49 -0700775 entry = &*mAudioQueue.begin();
Wei Jiabc2fb722014-07-08 16:37:57 -0700776
777 if (entry->mBuffer == NULL) { // EOS
778 hasEOS = true;
779 mAudioQueue.erase(mAudioQueue.begin());
Wei Jiabc2fb722014-07-08 16:37:57 -0700780 break;
781 }
782
Ronghua Wu3e5efb32014-08-18 16:27:08 -0700783 if (firstEntry && entry->mOffset == 0) {
784 firstEntry = false;
Wei Jiabc2fb722014-07-08 16:37:57 -0700785 int64_t mediaTimeUs;
786 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andy Hunga0b39712015-05-31 22:40:49 -0700787 ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
Wei Jia7b15cb32015-02-03 17:46:06 -0800788 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
Wei Jiabc2fb722014-07-08 16:37:57 -0700789 }
790
791 size_t copy = entry->mBuffer->size() - entry->mOffset;
792 size_t sizeRemaining = size - sizeCopied;
793 if (copy > sizeRemaining) {
794 copy = sizeRemaining;
795 }
796
797 memcpy((char *)buffer + sizeCopied,
798 entry->mBuffer->data() + entry->mOffset,
799 copy);
800
801 entry->mOffset += copy;
802 if (entry->mOffset == entry->mBuffer->size()) {
803 entry->mNotifyConsumed->post();
804 mAudioQueue.erase(mAudioQueue.begin());
805 entry = NULL;
806 }
807 sizeCopied += copy;
Wei Jia7b15cb32015-02-03 17:46:06 -0800808
809 notifyIfMediaRenderingStarted_l();
Wei Jiabc2fb722014-07-08 16:37:57 -0700810 }
811
Lajos Molnarf5926712014-10-21 09:36:43 -0700812 if (mAudioFirstAnchorTimeMediaUs >= 0) {
813 int64_t nowUs = ALooper::GetNowUs();
Wei Jia98160162015-02-04 17:01:11 -0800814 int64_t nowMediaUs =
Wei Jiac4ac8172015-10-21 10:35:48 -0700815 mAudioFirstAnchorTimeMediaUs + mAudioSink->getPlayedOutDurationUs(nowUs);
Wei Jia7b15cb32015-02-03 17:46:06 -0800816 // we don't know how much data we are queueing for offloaded tracks.
Wei Jia98160162015-02-04 17:01:11 -0800817 mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
Lajos Molnarf5926712014-10-21 09:36:43 -0700818 }
819
Andy Hunga0b39712015-05-31 22:40:49 -0700820 // for non-offloaded audio, we need to compute the frames written because
821 // there is no EVENT_STREAM_END notification. The frames written gives
822 // an estimate on the pending played out duration.
823 if (!offloadingAudio()) {
824 mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
Wei Jiabc2fb722014-07-08 16:37:57 -0700825 }
826
Andy Hunga0b39712015-05-31 22:40:49 -0700827 if (hasEOS) {
828 (new AMessage(kWhatStopAudioSink, this))->post();
829 // As there is currently no EVENT_STREAM_END callback notification for
830 // non-offloaded audio tracks, we need to post the EOS ourselves.
831 if (!offloadingAudio()) {
832 int64_t postEOSDelayUs = 0;
833 if (mAudioSink->needsTrailingPadding()) {
834 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
835 }
836 ALOGV("fillAudioBuffer: notifyEOS "
837 "mNumFramesWritten:%u finalResult:%d postEOSDelay:%lld",
838 mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
839 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
840 }
841 }
Wei Jiabc2fb722014-07-08 16:37:57 -0700842 return sizeCopied;
843}
844
Chong Zhang9da0ce42015-06-11 17:18:58 -0700845void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
846 List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
847 bool foundEOS = false;
848 while (it != mAudioQueue.end()) {
849 int32_t eos;
850 QueueEntry *entry = &*it++;
851 if (entry->mBuffer == NULL
852 || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
853 itEOS = it;
854 foundEOS = true;
855 }
856 }
857
858 if (foundEOS) {
859 // post all replies before EOS and drop the samples
860 for (it = mAudioQueue.begin(); it != itEOS; it++) {
861 if (it->mBuffer == NULL) {
862 // delay doesn't matter as we don't even have an AudioTrack
863 notifyEOS(true /* audio */, it->mFinalResult);
864 } else {
865 it->mNotifyConsumed->post();
866 }
867 }
868 mAudioQueue.erase(mAudioQueue.begin(), itEOS);
869 }
870}
871
Andreas Huber078cfcf2011-09-15 12:25:04 -0700872bool NuPlayer::Renderer::onDrainAudioQueue() {
Andy Hung58d315c2015-09-11 18:44:56 -0700873 // do not drain audio during teardown as queued buffers may be invalid.
874 if (mAudioTornDown) {
875 return false;
876 }
Wei Jia230b1882015-03-19 15:13:51 -0700877 // TODO: This call to getPosition checks if AudioTrack has been created
878 // in AudioSink before draining audio. If AudioTrack doesn't exist, then
879 // CHECKs on getPosition will fail.
880 // We still need to figure out why AudioTrack is not created when
881 // this function is called. One possible reason could be leftover
882 // audio. Another possible place is to check whether decoder
883 // has received INFO_FORMAT_CHANGED as the first buffer since
884 // AudioSink is opened there, and possible interactions with flush
885 // immediately after start. Investigate error message
886 // "vorbis_dsp_synthesis returned -135", along with RTSP.
Andreas Huber078cfcf2011-09-15 12:25:04 -0700887 uint32_t numFramesPlayed;
Andreas Huber2bfdd422011-10-11 15:24:07 -0700888 if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
Chong Zhang9da0ce42015-06-11 17:18:58 -0700889 // When getPosition fails, renderer will not reschedule the draining
890 // unless new samples are queued.
891 // If we have pending EOS (or "eos" marker for discontinuities), we need
892 // to post these now as NuPlayerDecoder might be waiting for it.
893 drainAudioQueueUntilLastEOS();
894
895 ALOGW("onDrainAudioQueue(): audio sink is not ready");
Andreas Huber2bfdd422011-10-11 15:24:07 -0700896 return false;
897 }
Andreas Huberc92fd242011-08-16 13:48:44 -0700898
Wei Jia230b1882015-03-19 15:13:51 -0700899#if 0
Andreas Huber078cfcf2011-09-15 12:25:04 -0700900 ssize_t numFramesAvailableToWrite =
901 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
902
Andreas Huber078cfcf2011-09-15 12:25:04 -0700903 if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
Steve Blockdf64d152012-01-04 20:05:49 +0000904 ALOGI("audio sink underrun");
Andreas Huber078cfcf2011-09-15 12:25:04 -0700905 } else {
Steve Block3856b092011-10-20 11:56:00 +0100906 ALOGV("audio queue has %d frames left to play",
Andreas Huber078cfcf2011-09-15 12:25:04 -0700907 mAudioSink->frameCount() - numFramesAvailableToWrite);
908 }
909#endif
910
Andy Hung005e9d02015-05-31 23:16:07 -0700911 uint32_t prevFramesWritten = mNumFramesWritten;
Wei Jia7d3f4df2015-03-03 15:28:00 -0800912 while (!mAudioQueue.empty()) {
Andreas Huberc92fd242011-08-16 13:48:44 -0700913 QueueEntry *entry = &*mAudioQueue.begin();
914
Lajos Molnard5923402014-10-20 17:47:54 -0700915 mLastAudioBufferDrained = entry->mBufferOrdinal;
916
Andreas Huberc92fd242011-08-16 13:48:44 -0700917 if (entry->mBuffer == NULL) {
918 // EOS
Ronghua Wu5095d702014-08-27 12:05:48 -0700919 int64_t postEOSDelayUs = 0;
920 if (mAudioSink->needsTrailingPadding()) {
Lajos Molnar06ad1522014-08-28 07:27:44 -0700921 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
Ronghua Wu5095d702014-08-27 12:05:48 -0700922 }
923 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
Wei Jia2995dc72015-07-24 16:00:33 -0700924 mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
Andreas Huberc92fd242011-08-16 13:48:44 -0700925
926 mAudioQueue.erase(mAudioQueue.begin());
927 entry = NULL;
Andy Hung34912322014-12-18 18:48:39 -0800928 if (mAudioSink->needsTrailingPadding()) {
929 // If we're not in gapless playback (i.e. through setNextPlayer), we
930 // need to stop the track here, because that will play out the last
931 // little bit at the end of the file. Otherwise short files won't play.
932 mAudioSink->stop();
933 mNumFramesWritten = 0;
934 }
Andreas Huber078cfcf2011-09-15 12:25:04 -0700935 return false;
Eric Laurent9b7d9502011-03-21 11:49:00 -0700936 }
937
Chong Zhang81636762015-04-21 09:33:47 -0700938 // ignore 0-sized buffer which could be EOS marker with no data
939 if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
Andreas Huberf9334412010-12-15 15:17:42 -0800940 int64_t mediaTimeUs;
941 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andy Hunga0b39712015-05-31 22:40:49 -0700942 ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
943 mediaTimeUs / 1E6);
Ronghua Wueecb7802014-10-19 23:12:50 -0700944 onNewAudioMediaTime(mediaTimeUs);
Andreas Huberf9334412010-12-15 15:17:42 -0800945 }
946
947 size_t copy = entry->mBuffer->size() - entry->mOffset;
Andreas Huberf9334412010-12-15 15:17:42 -0800948
Wei Jia7d3f4df2015-03-03 15:28:00 -0800949 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
950 copy, false /* blocking */);
Andy Hunga31335a2014-08-20 17:37:59 -0700951 if (written < 0) {
Andy Hung202bce12014-12-03 11:47:36 -0800952 // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
Wei Jia7d3f4df2015-03-03 15:28:00 -0800953 if (written == WOULD_BLOCK) {
Wei Jiadf809472015-06-09 11:21:13 -0700954 ALOGV("AudioSink write would block when writing %zu bytes", copy);
Wei Jia7d3f4df2015-03-03 15:28:00 -0800955 } else {
956 ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
Ronghua Wu7665f582015-09-02 10:15:30 -0700957 // This can only happen when AudioSink was opened with doNotReconnect flag set to
958 // true, in which case the NuPlayer will handle the reconnect.
Wei Jiaa05f1e32016-03-25 16:31:22 -0700959 notifyAudioTearDown(kDueToError);
Wei Jia7d3f4df2015-03-03 15:28:00 -0800960 }
Andy Hung202bce12014-12-03 11:47:36 -0800961 break;
Andy Hunga31335a2014-08-20 17:37:59 -0700962 }
Andreas Huberf9334412010-12-15 15:17:42 -0800963
Andy Hunga31335a2014-08-20 17:37:59 -0700964 entry->mOffset += written;
Wei Jia03cee242016-05-26 15:22:59 -0700965 size_t remainder = entry->mBuffer->size() - entry->mOffset;
966 if ((ssize_t)remainder < mAudioSink->frameSize()) {
967 if (remainder > 0) {
968 ALOGW("Corrupted audio buffer has fractional frames, discarding %zu bytes.",
969 remainder);
970 entry->mOffset += remainder;
971 copy -= remainder;
972 }
973
Andreas Huberf9334412010-12-15 15:17:42 -0800974 entry->mNotifyConsumed->post();
975 mAudioQueue.erase(mAudioQueue.begin());
Eric Laurent9b7d9502011-03-21 11:49:00 -0700976
Andreas Huberf9334412010-12-15 15:17:42 -0800977 entry = NULL;
978 }
979
Andy Hunga31335a2014-08-20 17:37:59 -0700980 size_t copiedFrames = written / mAudioSink->frameSize();
Andreas Huber078cfcf2011-09-15 12:25:04 -0700981 mNumFramesWritten += copiedFrames;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700982
Wei Jia7b15cb32015-02-03 17:46:06 -0800983 {
984 Mutex::Autolock autoLock(mLock);
Wei Jiad005c5d2015-10-13 10:58:07 -0700985 int64_t maxTimeMedia;
986 maxTimeMedia =
987 mAnchorTimeMediaUs +
988 (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
989 * 1000LL * mAudioSink->msecsPerFrame());
990 mMediaClock->updateMaxTimeMedia(maxTimeMedia);
991
Wei Jia7b15cb32015-02-03 17:46:06 -0800992 notifyIfMediaRenderingStarted_l();
993 }
Andreas Huber43c3e6c2011-01-05 12:17:08 -0800994
Andy Hunga31335a2014-08-20 17:37:59 -0700995 if (written != (ssize_t)copy) {
996 // A short count was received from AudioSink::write()
997 //
Wei Jia7d3f4df2015-03-03 15:28:00 -0800998 // AudioSink write is called in non-blocking mode.
999 // It may return with a short count when:
Andy Hunga31335a2014-08-20 17:37:59 -07001000 //
Wei Jia03cee242016-05-26 15:22:59 -07001001 // 1) Size to be copied is not a multiple of the frame size. Fractional frames are
1002 // discarded.
Wei Jia7d3f4df2015-03-03 15:28:00 -08001003 // 2) The data to be copied exceeds the available buffer in AudioSink.
1004 // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
1005 // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
Andy Hunga31335a2014-08-20 17:37:59 -07001006
1007 // (Case 1)
1008 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it
1009 // needs to fail, as we should not carry over fractional frames between calls.
1010 CHECK_EQ(copy % mAudioSink->frameSize(), 0);
1011
Wei Jia7d3f4df2015-03-03 15:28:00 -08001012 // (Case 2, 3, 4)
Andy Hunga31335a2014-08-20 17:37:59 -07001013 // Return early to the caller.
1014 // Beware of calling immediately again as this may busy-loop if you are not careful.
Wei Jia7d3f4df2015-03-03 15:28:00 -08001015 ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
Andy Hunga31335a2014-08-20 17:37:59 -07001016 break;
1017 }
1018 }
Lajos Molnarf5926712014-10-21 09:36:43 -07001019
Andy Hung005e9d02015-05-31 23:16:07 -07001020 // calculate whether we need to reschedule another write.
1021 bool reschedule = !mAudioQueue.empty()
1022 && (!mPaused
1023 || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
1024 //ALOGD("reschedule:%d empty:%d mPaused:%d prevFramesWritten:%u mNumFramesWritten:%u",
1025 // reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
1026 return reschedule;
Andreas Huberf9334412010-12-15 15:17:42 -08001027}
1028
Wei Jia98160162015-02-04 17:01:11 -08001029int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
1030 int32_t sampleRate = offloadingAudio() ?
1031 mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
Wei Jia4d7ac852015-08-31 18:33:14 -07001032 if (sampleRate == 0) {
1033 ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
1034 return 0;
1035 }
Wei Jia98160162015-02-04 17:01:11 -08001036 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
1037 return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
1038}
1039
1040// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
Lajos Molnar06ad1522014-08-28 07:27:44 -07001041int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
Wei Jia98160162015-02-04 17:01:11 -08001042 int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
Wei Jiad2f35de2016-02-17 16:48:06 -08001043 if (mUseVirtualAudioSink) {
1044 int64_t nowUs = ALooper::GetNowUs();
1045 int64_t mediaUs;
1046 if (mMediaClock->getMediaTime(nowUs, &mediaUs) != OK) {
1047 return 0ll;
1048 } else {
1049 return writtenAudioDurationUs - (mediaUs - mAudioFirstAnchorTimeMediaUs);
1050 }
1051 }
Wei Jiac4ac8172015-10-21 10:35:48 -07001052 return writtenAudioDurationUs - mAudioSink->getPlayedOutDurationUs(nowUs);
Ronghua Wu5095d702014-08-27 12:05:48 -07001053}
1054
Ronghua Wua73d9e02014-10-08 15:13:29 -07001055int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
Wei Jia98160162015-02-04 17:01:11 -08001056 int64_t realUs;
1057 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001058 // If failed to get current position, e.g. due to audio clock is
1059 // not ready, then just play out video immediately without delay.
Ronghua Wueecb7802014-10-19 23:12:50 -07001060 return nowUs;
Ronghua Wua73d9e02014-10-08 15:13:29 -07001061 }
Wei Jia98160162015-02-04 17:01:11 -08001062 return realUs;
Ronghua Wua73d9e02014-10-08 15:13:29 -07001063}
1064
Ronghua Wueecb7802014-10-19 23:12:50 -07001065void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001066 Mutex::Autolock autoLock(mLock);
Ronghua Wueecb7802014-10-19 23:12:50 -07001067 // TRICKY: vorbis decoder generates multiple frames with the same
1068 // timestamp, so only update on the first frame with a given timestamp
1069 if (mediaTimeUs == mAnchorTimeMediaUs) {
1070 return;
1071 }
Wei Jia7b15cb32015-02-03 17:46:06 -08001072 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
Wei Jia4ecbea32015-10-20 18:09:57 -07001073
Andy Hung528c8402016-01-12 12:39:34 -08001074 // mNextAudioClockUpdateTimeUs is -1 if we're waiting for audio sink to start
1075 if (mNextAudioClockUpdateTimeUs == -1) {
1076 AudioTimestamp ts;
1077 if (mAudioSink->getTimestamp(ts) == OK && ts.mPosition > 0) {
1078 mNextAudioClockUpdateTimeUs = 0; // start our clock updates
1079 }
1080 }
Wei Jia12b62652016-02-09 17:49:31 -08001081 int64_t nowUs = ALooper::GetNowUs();
Andy Hung528c8402016-01-12 12:39:34 -08001082 if (mNextAudioClockUpdateTimeUs >= 0) {
1083 if (nowUs >= mNextAudioClockUpdateTimeUs) {
1084 int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
1085 mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
Wei Jiad2f35de2016-02-17 16:48:06 -08001086 mUseVirtualAudioSink = false;
Andy Hung528c8402016-01-12 12:39:34 -08001087 mNextAudioClockUpdateTimeUs = nowUs + kMinimumAudioClockUpdatePeriodUs;
1088 }
Wei Jia12b62652016-02-09 17:49:31 -08001089 } else {
1090 int64_t unused;
1091 if ((mMediaClock->getMediaTime(nowUs, &unused) != OK)
1092 && (getDurationUsIfPlayedAtSampleRate(mNumFramesWritten)
1093 > kMaxAllowedAudioSinkDelayUs)) {
1094 // Enough data has been sent to AudioSink, but AudioSink has not rendered
1095 // any data yet. Something is wrong with AudioSink, e.g., the device is not
1096 // connected to audio out.
1097 // Switch to system clock. This essentially creates a virtual AudioSink with
1098 // initial latenty of getDurationUsIfPlayedAtSampleRate(mNumFramesWritten).
1099 // This virtual AudioSink renders audio data starting from the very first sample
1100 // and it's paced by system clock.
Wei Jia07af6512016-02-10 10:45:47 -08001101 ALOGW("AudioSink stuck. ARE YOU CONNECTED TO AUDIO OUT? Switching to system clock.");
Wei Jia12b62652016-02-09 17:49:31 -08001102 mMediaClock->updateAnchor(mAudioFirstAnchorTimeMediaUs, nowUs, mediaTimeUs);
Wei Jiad2f35de2016-02-17 16:48:06 -08001103 mUseVirtualAudioSink = true;
Wei Jia12b62652016-02-09 17:49:31 -08001104 }
Wei Jia4ecbea32015-10-20 18:09:57 -07001105 }
Wei Jia8edb3f82015-03-11 11:07:45 -07001106 mAnchorNumFramesWritten = mNumFramesWritten;
Wei Jia7b15cb32015-02-03 17:46:06 -08001107 mAnchorTimeMediaUs = mediaTimeUs;
Ronghua Wueecb7802014-10-19 23:12:50 -07001108}
1109
Wei Jia7b15cb32015-02-03 17:46:06 -08001110// Called without mLock acquired.
1111void NuPlayer::Renderer::postDrainVideoQueue() {
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001112 if (mDrainVideoQueuePending
Wei Jia7b15cb32015-02-03 17:46:06 -08001113 || getSyncQueues()
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001114 || (mPaused && mVideoSampleReceived)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001115 return;
1116 }
1117
1118 if (mVideoQueue.empty()) {
1119 return;
1120 }
1121
1122 QueueEntry &entry = *mVideoQueue.begin();
1123
Lajos Molnar1d15ab52015-03-04 16:46:34 -08001124 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
Wei Jia7b15cb32015-02-03 17:46:06 -08001125 msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
Andreas Huberf9334412010-12-15 15:17:42 -08001126
Andreas Huberf9334412010-12-15 15:17:42 -08001127 if (entry.mBuffer == NULL) {
1128 // EOS doesn't carry a timestamp.
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001129 msg->post();
1130 mDrainVideoQueuePending = true;
1131 return;
1132 }
1133
Robert Shih00541e22015-12-18 17:04:00 -08001134 bool needRepostDrainVideoQueue = false;
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001135 int64_t delayUs;
1136 int64_t nowUs = ALooper::GetNowUs();
1137 int64_t realTimeUs;
1138 if (mFlags & FLAG_REAL_TIME) {
Andreas Huberd5e56232013-03-12 11:01:43 -07001139 int64_t mediaTimeUs;
1140 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001141 realTimeUs = mediaTimeUs;
Andreas Huberf9334412010-12-15 15:17:42 -08001142 } else {
1143 int64_t mediaTimeUs;
1144 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1145
Wei Jia7b15cb32015-02-03 17:46:06 -08001146 {
1147 Mutex::Autolock autoLock(mLock);
1148 if (mAnchorTimeMediaUs < 0) {
1149 mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1150 mAnchorTimeMediaUs = mediaTimeUs;
1151 realTimeUs = nowUs;
Wei Jia25d696f2015-11-19 10:57:15 -08001152 } else if (!mVideoSampleReceived) {
1153 // Always render the first video frame.
1154 realTimeUs = nowUs;
Robert Shih00541e22015-12-18 17:04:00 -08001155 } else if (mAudioFirstAnchorTimeMediaUs < 0
1156 || mMediaClock->getRealTimeFor(mediaTimeUs, &realTimeUs) == OK) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001157 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
Robert Shih00541e22015-12-18 17:04:00 -08001158 } else if (mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0) {
1159 needRepostDrainVideoQueue = true;
1160 realTimeUs = nowUs;
1161 } else {
1162 realTimeUs = nowUs;
Wei Jia7b15cb32015-02-03 17:46:06 -08001163 }
Andreas Huberf9334412010-12-15 15:17:42 -08001164 }
Lajos Molnarf5926712014-10-21 09:36:43 -07001165 if (!mHasAudio) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001166 // smooth out videos >= 10fps
1167 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
Lajos Molnarf5926712014-10-21 09:36:43 -07001168 }
Lajos Molnard5923402014-10-20 17:47:54 -07001169
1170 // Heuristics to handle situation when media time changed without a
1171 // discontinuity. If we have not drained an audio buffer that was
1172 // received after this buffer, repost in 10 msec. Otherwise repost
1173 // in 500 msec.
1174 delayUs = realTimeUs - nowUs;
Robert Shih00541e22015-12-18 17:04:00 -08001175 int64_t postDelayUs = -1;
Lajos Molnard5923402014-10-20 17:47:54 -07001176 if (delayUs > 500000) {
Robert Shih00541e22015-12-18 17:04:00 -08001177 postDelayUs = 500000;
Lajos Molnard5923402014-10-20 17:47:54 -07001178 if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
1179 postDelayUs = 10000;
1180 }
Robert Shih00541e22015-12-18 17:04:00 -08001181 } else if (needRepostDrainVideoQueue) {
1182 // CHECK(mPlaybackRate > 0);
1183 // CHECK(mAudioFirstAnchorTimeMediaUs >= 0);
1184 // CHECK(mediaTimeUs - mAudioFirstAnchorTimeMediaUs >= 0);
1185 postDelayUs = mediaTimeUs - mAudioFirstAnchorTimeMediaUs;
1186 postDelayUs /= mPlaybackRate;
1187 }
1188
1189 if (postDelayUs >= 0) {
Lajos Molnard5923402014-10-20 17:47:54 -07001190 msg->setWhat(kWhatPostDrainVideoQueue);
1191 msg->post(postDelayUs);
1192 mVideoScheduler->restart();
Robert Shih00541e22015-12-18 17:04:00 -08001193 ALOGI("possible video time jump of %dms or uninitialized media clock, retrying in %dms",
Lajos Molnard5923402014-10-20 17:47:54 -07001194 (int)(delayUs / 1000), (int)(postDelayUs / 1000));
1195 mDrainVideoQueuePending = true;
1196 return;
1197 }
Andreas Huberf9334412010-12-15 15:17:42 -08001198 }
1199
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001200 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1201 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1202
1203 delayUs = realTimeUs - nowUs;
1204
Lajos Molnar09524832014-07-17 14:29:51 -07001205 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001206 // post 2 display refreshes before rendering is due
1207 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
Andreas Huberf9334412010-12-15 15:17:42 -08001208
1209 mDrainVideoQueuePending = true;
1210}
1211
1212void NuPlayer::Renderer::onDrainVideoQueue() {
1213 if (mVideoQueue.empty()) {
1214 return;
1215 }
1216
1217 QueueEntry *entry = &*mVideoQueue.begin();
1218
1219 if (entry->mBuffer == NULL) {
1220 // EOS
1221
Andreas Huberc92fd242011-08-16 13:48:44 -07001222 notifyEOS(false /* audio */, entry->mFinalResult);
Andreas Huberf9334412010-12-15 15:17:42 -08001223
1224 mVideoQueue.erase(mVideoQueue.begin());
1225 entry = NULL;
Andreas Huber3fe62152011-09-16 15:09:22 -07001226
Ronghua Wua73d9e02014-10-08 15:13:29 -07001227 setVideoLateByUs(0);
Andreas Huberf9334412010-12-15 15:17:42 -08001228 return;
1229 }
1230
Wei Jia25d696f2015-11-19 10:57:15 -08001231 int64_t nowUs = ALooper::GetNowUs();
Andreas Huberd5e56232013-03-12 11:01:43 -07001232 int64_t realTimeUs;
Wei Jia2995dc72015-07-24 16:00:33 -07001233 int64_t mediaTimeUs = -1;
Andreas Huberd5e56232013-03-12 11:01:43 -07001234 if (mFlags & FLAG_REAL_TIME) {
1235 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1236 } else {
Andreas Huberd5e56232013-03-12 11:01:43 -07001237 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andreas Huberf9334412010-12-15 15:17:42 -08001238
Ronghua Wua73d9e02014-10-08 15:13:29 -07001239 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
Andreas Huberd5e56232013-03-12 11:01:43 -07001240 }
1241
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001242 bool tooLate = false;
Andreas Huber3fe62152011-09-16 15:09:22 -07001243
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001244 if (!mPaused) {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001245 setVideoLateByUs(nowUs - realTimeUs);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001246 tooLate = (mVideoLateByUs > 40000);
1247
1248 if (tooLate) {
1249 ALOGV("video late by %lld us (%.2f secs)",
Lajos Molnar6d339f12015-04-17 16:15:53 -07001250 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001251 } else {
Wei Jia98160162015-02-04 17:01:11 -08001252 int64_t mediaUs = 0;
1253 mMediaClock->getMediaTime(realTimeUs, &mediaUs);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001254 ALOGV("rendering video at media time %.2f secs",
1255 (mFlags & FLAG_REAL_TIME ? realTimeUs :
Wei Jia98160162015-02-04 17:01:11 -08001256 mediaUs) / 1E6);
Wei Jia2995dc72015-07-24 16:00:33 -07001257
1258 if (!(mFlags & FLAG_REAL_TIME)
1259 && mLastAudioMediaTimeUs != -1
1260 && mediaTimeUs > mLastAudioMediaTimeUs) {
1261 // If audio ends before video, video continues to drive media clock.
1262 // Also smooth out videos >= 10fps.
1263 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1264 }
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001265 }
Andreas Huber078cfcf2011-09-15 12:25:04 -07001266 } else {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001267 setVideoLateByUs(0);
Ronghua Wueecb7802014-10-19 23:12:50 -07001268 if (!mVideoSampleReceived && !mHasAudio) {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001269 // This will ensure that the first frame after a flush won't be used as anchor
1270 // when renderer is in paused state, because resume can happen any time after seek.
Wei Jia7b15cb32015-02-03 17:46:06 -08001271 Mutex::Autolock autoLock(mLock);
1272 clearAnchorTime_l();
Wei Jia49966ff2014-10-08 18:44:45 -07001273 }
Andreas Huber078cfcf2011-09-15 12:25:04 -07001274 }
Andreas Huberf9334412010-12-15 15:17:42 -08001275
Wei Jia25d696f2015-11-19 10:57:15 -08001276 // Always render the first video frame while keeping stats on A/V sync.
1277 if (!mVideoSampleReceived) {
1278 realTimeUs = nowUs;
1279 tooLate = false;
1280 }
1281
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001282 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
Glenn Kasten683525b2011-11-04 18:05:35 -07001283 entry->mNotifyConsumed->setInt32("render", !tooLate);
Andreas Huberf9334412010-12-15 15:17:42 -08001284 entry->mNotifyConsumed->post();
1285 mVideoQueue.erase(mVideoQueue.begin());
1286 entry = NULL;
Andreas Huber43c3e6c2011-01-05 12:17:08 -08001287
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001288 mVideoSampleReceived = true;
James Dongf57b4ea2012-07-20 13:38:36 -07001289
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001290 if (!mPaused) {
1291 if (!mVideoRenderingStarted) {
1292 mVideoRenderingStarted = true;
1293 notifyVideoRenderingStart();
1294 }
Wei Jia7b15cb32015-02-03 17:46:06 -08001295 Mutex::Autolock autoLock(mLock);
1296 notifyIfMediaRenderingStarted_l();
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001297 }
Andreas Huberf9334412010-12-15 15:17:42 -08001298}
1299
James Dongf57b4ea2012-07-20 13:38:36 -07001300void NuPlayer::Renderer::notifyVideoRenderingStart() {
1301 sp<AMessage> notify = mNotify->dup();
1302 notify->setInt32("what", kWhatVideoRenderingStart);
1303 notify->post();
1304}
1305
Ronghua Wu5095d702014-08-27 12:05:48 -07001306void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
Wei Jia7c8d0e02015-08-27 17:40:21 -07001307 if (audio && delayUs > 0) {
1308 sp<AMessage> msg = new AMessage(kWhatEOS, this);
1309 msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
1310 msg->setInt32("finalResult", finalResult);
1311 msg->post(delayUs);
1312 return;
1313 }
Andreas Huberf9334412010-12-15 15:17:42 -08001314 sp<AMessage> notify = mNotify->dup();
1315 notify->setInt32("what", kWhatEOS);
1316 notify->setInt32("audio", static_cast<int32_t>(audio));
Andreas Huberc92fd242011-08-16 13:48:44 -07001317 notify->setInt32("finalResult", finalResult);
Ronghua Wu5095d702014-08-27 12:05:48 -07001318 notify->post(delayUs);
Andreas Huberf9334412010-12-15 15:17:42 -08001319}
1320
Wei Jiaa05f1e32016-03-25 16:31:22 -07001321void NuPlayer::Renderer::notifyAudioTearDown(AudioTearDownReason reason) {
1322 sp<AMessage> msg = new AMessage(kWhatAudioTearDown, this);
1323 msg->setInt32("reason", reason);
1324 msg->post();
Wei Jia3a2956d2014-07-22 16:01:33 -07001325}
1326
Andreas Huberf9334412010-12-15 15:17:42 -08001327void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1328 int32_t audio;
1329 CHECK(msg->findInt32("audio", &audio));
1330
Wei Jia7b15cb32015-02-03 17:46:06 -08001331 if (dropBufferIfStale(audio, msg)) {
1332 return;
1333 }
1334
1335 if (audio) {
1336 mHasAudio = true;
1337 } else {
1338 mHasVideo = true;
1339 }
Ronghua Wua73d9e02014-10-08 15:13:29 -07001340
1341 if (mHasVideo) {
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001342 if (mVideoScheduler == NULL) {
1343 mVideoScheduler = new VideoFrameScheduler();
1344 mVideoScheduler->init();
1345 }
Andreas Huberbc7f5b22011-01-21 10:15:23 -08001346 }
1347
Andreas Huber2d8bedd2012-02-21 14:38:23 -08001348 sp<ABuffer> buffer;
1349 CHECK(msg->findBuffer("buffer", &buffer));
Andreas Huberf9334412010-12-15 15:17:42 -08001350
1351 sp<AMessage> notifyConsumed;
1352 CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1353
1354 QueueEntry entry;
1355 entry.mBuffer = buffer;
1356 entry.mNotifyConsumed = notifyConsumed;
1357 entry.mOffset = 0;
1358 entry.mFinalResult = OK;
Lajos Molnard5923402014-10-20 17:47:54 -07001359 entry.mBufferOrdinal = ++mTotalBuffersQueued;
Andreas Huberf9334412010-12-15 15:17:42 -08001360
1361 if (audio) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001362 Mutex::Autolock autoLock(mLock);
Andreas Huberf9334412010-12-15 15:17:42 -08001363 mAudioQueue.push_back(entry);
Wei Jiabc2fb722014-07-08 16:37:57 -07001364 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001365 } else {
1366 mVideoQueue.push_back(entry);
Wei Jia7b15cb32015-02-03 17:46:06 -08001367 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -08001368 }
1369
Wei Jia7b15cb32015-02-03 17:46:06 -08001370 Mutex::Autolock autoLock(mLock);
Andreas Hubercb67cd12011-08-26 16:02:19 -07001371 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1372 return;
Andreas Huberf9334412010-12-15 15:17:42 -08001373 }
Andreas Hubercb67cd12011-08-26 16:02:19 -07001374
1375 sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1376 sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1377
1378 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1379 // EOS signalled on either queue.
Wei Jiabc2fb722014-07-08 16:37:57 -07001380 syncQueuesDone_l();
Andreas Hubercb67cd12011-08-26 16:02:19 -07001381 return;
1382 }
1383
1384 int64_t firstAudioTimeUs;
1385 int64_t firstVideoTimeUs;
1386 CHECK(firstAudioBuffer->meta()
1387 ->findInt64("timeUs", &firstAudioTimeUs));
1388 CHECK(firstVideoBuffer->meta()
1389 ->findInt64("timeUs", &firstVideoTimeUs));
1390
1391 int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1392
Steve Block3856b092011-10-20 11:56:00 +01001393 ALOGV("queueDiff = %.2f secs", diff / 1E6);
Andreas Hubercb67cd12011-08-26 16:02:19 -07001394
1395 if (diff > 100000ll) {
1396 // Audio data starts More than 0.1 secs before video.
1397 // Drop some audio.
1398
1399 (*mAudioQueue.begin()).mNotifyConsumed->post();
1400 mAudioQueue.erase(mAudioQueue.begin());
1401 return;
1402 }
1403
Wei Jiabc2fb722014-07-08 16:37:57 -07001404 syncQueuesDone_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001405}
1406
Wei Jiabc2fb722014-07-08 16:37:57 -07001407void NuPlayer::Renderer::syncQueuesDone_l() {
Andreas Huberf9334412010-12-15 15:17:42 -08001408 if (!mSyncQueues) {
1409 return;
1410 }
1411
1412 mSyncQueues = false;
1413
1414 if (!mAudioQueue.empty()) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001415 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001416 }
1417
1418 if (!mVideoQueue.empty()) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001419 mLock.unlock();
1420 postDrainVideoQueue();
1421 mLock.lock();
Andreas Huberf9334412010-12-15 15:17:42 -08001422 }
1423}
1424
1425void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1426 int32_t audio;
1427 CHECK(msg->findInt32("audio", &audio));
1428
Wei Jia7b15cb32015-02-03 17:46:06 -08001429 if (dropBufferIfStale(audio, msg)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001430 return;
1431 }
1432
1433 int32_t finalResult;
1434 CHECK(msg->findInt32("finalResult", &finalResult));
1435
1436 QueueEntry entry;
1437 entry.mOffset = 0;
1438 entry.mFinalResult = finalResult;
1439
1440 if (audio) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001441 Mutex::Autolock autoLock(mLock);
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001442 if (mAudioQueue.empty() && mSyncQueues) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001443 syncQueuesDone_l();
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001444 }
Andreas Huberf9334412010-12-15 15:17:42 -08001445 mAudioQueue.push_back(entry);
Wei Jiabc2fb722014-07-08 16:37:57 -07001446 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001447 } else {
Wei Jia7b15cb32015-02-03 17:46:06 -08001448 if (mVideoQueue.empty() && getSyncQueues()) {
1449 Mutex::Autolock autoLock(mLock);
Wei Jiabc2fb722014-07-08 16:37:57 -07001450 syncQueuesDone_l();
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001451 }
Andreas Huberf9334412010-12-15 15:17:42 -08001452 mVideoQueue.push_back(entry);
Wei Jia7b15cb32015-02-03 17:46:06 -08001453 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -08001454 }
1455}
1456
1457void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
Chong Zhang7137ec72014-11-12 16:41:05 -08001458 int32_t audio, notifyComplete;
Andreas Huberf9334412010-12-15 15:17:42 -08001459 CHECK(msg->findInt32("audio", &audio));
1460
Wei Jia28a8a9f2014-08-18 11:29:50 -07001461 {
Wei Jia7b15cb32015-02-03 17:46:06 -08001462 Mutex::Autolock autoLock(mLock);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001463 if (audio) {
Chong Zhang7137ec72014-11-12 16:41:05 -08001464 notifyComplete = mNotifyCompleteAudio;
1465 mNotifyCompleteAudio = false;
Wei Jia2995dc72015-07-24 16:00:33 -07001466 mLastAudioMediaTimeUs = -1;
Wei Jia28a8a9f2014-08-18 11:29:50 -07001467 } else {
Chong Zhang7137ec72014-11-12 16:41:05 -08001468 notifyComplete = mNotifyCompleteVideo;
1469 mNotifyCompleteVideo = false;
Wei Jia28a8a9f2014-08-18 11:29:50 -07001470 }
Wei Jia28a8a9f2014-08-18 11:29:50 -07001471
Wei Jia7b15cb32015-02-03 17:46:06 -08001472 // If we're currently syncing the queues, i.e. dropping audio while
1473 // aligning the first audio/video buffer times and only one of the
1474 // two queues has data, we may starve that queue by not requesting
1475 // more buffers from the decoder. If the other source then encounters
1476 // a discontinuity that leads to flushing, we'll never find the
1477 // corresponding discontinuity on the other queue.
1478 // Therefore we'll stop syncing the queues if at least one of them
1479 // is flushed.
1480 syncQueuesDone_l();
1481 clearAnchorTime_l();
Wei Jiabc2fb722014-07-08 16:37:57 -07001482 }
Andreas Huberf9334412010-12-15 15:17:42 -08001483
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001484 ALOGV("flushing %s", audio ? "audio" : "video");
Andreas Huberf9334412010-12-15 15:17:42 -08001485 if (audio) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001486 {
1487 Mutex::Autolock autoLock(mLock);
1488 flushQueue(&mAudioQueue);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001489
Wei Jia7b15cb32015-02-03 17:46:06 -08001490 ++mAudioDrainGeneration;
Wei Jia7c8d0e02015-08-27 17:40:21 -07001491 ++mAudioEOSGeneration;
Wei Jia7b15cb32015-02-03 17:46:06 -08001492 prepareForMediaRenderingStart_l();
Wei Jia28a8a9f2014-08-18 11:29:50 -07001493
Andy Hunga0b39712015-05-31 22:40:49 -07001494 // the frame count will be reset after flush.
1495 clearAudioFirstAnchorTime_l();
Wei Jiabc2fb722014-07-08 16:37:57 -07001496 }
Andreas Huberf9334412010-12-15 15:17:42 -08001497
Andreas Huberf9334412010-12-15 15:17:42 -08001498 mDrainAudioQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001499
Wei Jiabc2fb722014-07-08 16:37:57 -07001500 if (offloadingAudio()) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001501 mAudioSink->pause();
1502 mAudioSink->flush();
Andy Hung85e48142015-05-31 23:04:15 -07001503 if (!mPaused) {
1504 mAudioSink->start();
1505 }
Wei Jia9e7ed332015-05-01 16:35:43 -07001506 } else {
1507 mAudioSink->pause();
1508 mAudioSink->flush();
1509 // Call stop() to signal to the AudioSink to completely fill the
1510 // internal buffer before resuming playback.
Andy Hungb03dcb32015-08-27 16:18:59 -07001511 // FIXME: this is ignored after flush().
Wei Jia9e7ed332015-05-01 16:35:43 -07001512 mAudioSink->stop();
Andy Hungb03dcb32015-08-27 16:18:59 -07001513 if (mPaused) {
1514 // Race condition: if renderer is paused and audio sink is stopped,
1515 // we need to make sure that the audio track buffer fully drains
1516 // before delivering data.
1517 // FIXME: remove this if we can detect if stop() is complete.
1518 const int delayUs = 2 * 50 * 1000; // (2 full mixer thread cycles at 50ms)
1519 mPauseDrainAudioAllowedUs = ALooper::GetNowUs() + delayUs;
1520 } else {
Wei Jia9e7ed332015-05-01 16:35:43 -07001521 mAudioSink->start();
1522 }
1523 mNumFramesWritten = 0;
Wei Jiabc2fb722014-07-08 16:37:57 -07001524 }
Andy Hung528c8402016-01-12 12:39:34 -08001525 mNextAudioClockUpdateTimeUs = -1;
Andreas Huberf9334412010-12-15 15:17:42 -08001526 } else {
1527 flushQueue(&mVideoQueue);
1528
Andreas Huberf9334412010-12-15 15:17:42 -08001529 mDrainVideoQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001530
Lajos Molnarc851b5d2014-09-18 14:14:29 -07001531 if (mVideoScheduler != NULL) {
1532 mVideoScheduler->restart();
1533 }
1534
Wei Jia7b15cb32015-02-03 17:46:06 -08001535 Mutex::Autolock autoLock(mLock);
1536 ++mVideoDrainGeneration;
1537 prepareForMediaRenderingStart_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001538 }
1539
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001540 mVideoSampleReceived = false;
Chong Zhang7137ec72014-11-12 16:41:05 -08001541
1542 if (notifyComplete) {
1543 notifyFlushComplete(audio);
1544 }
Andreas Huberf9334412010-12-15 15:17:42 -08001545}
1546
1547void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1548 while (!queue->empty()) {
1549 QueueEntry *entry = &*queue->begin();
1550
1551 if (entry->mBuffer != NULL) {
1552 entry->mNotifyConsumed->post();
1553 }
1554
1555 queue->erase(queue->begin());
1556 entry = NULL;
1557 }
1558}
1559
1560void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1561 sp<AMessage> notify = mNotify->dup();
1562 notify->setInt32("what", kWhatFlushComplete);
1563 notify->setInt32("audio", static_cast<int32_t>(audio));
1564 notify->post();
1565}
1566
Wei Jia7b15cb32015-02-03 17:46:06 -08001567bool NuPlayer::Renderer::dropBufferIfStale(
Andreas Huberf9334412010-12-15 15:17:42 -08001568 bool audio, const sp<AMessage> &msg) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001569 int32_t queueGeneration;
1570 CHECK(msg->findInt32("queueGeneration", &queueGeneration));
Andreas Huberf9334412010-12-15 15:17:42 -08001571
Wei Jia7b15cb32015-02-03 17:46:06 -08001572 if (queueGeneration == getQueueGeneration(audio)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001573 return false;
1574 }
1575
1576 sp<AMessage> notifyConsumed;
1577 if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1578 notifyConsumed->post();
1579 }
1580
1581 return true;
1582}
1583
Andreas Huber3831a062010-12-21 10:22:33 -08001584void NuPlayer::Renderer::onAudioSinkChanged() {
Wei Jiabc2fb722014-07-08 16:37:57 -07001585 if (offloadingAudio()) {
1586 return;
1587 }
Andreas Huber3831a062010-12-21 10:22:33 -08001588 CHECK(!mDrainAudioQueuePending);
1589 mNumFramesWritten = 0;
Wei Jia7b15cb32015-02-03 17:46:06 -08001590 {
1591 Mutex::Autolock autoLock(mLock);
1592 mAnchorNumFramesWritten = -1;
1593 }
Marco Nelissen4110c102012-03-29 09:31:28 -07001594 uint32_t written;
1595 if (mAudioSink->getFramesWritten(&written) == OK) {
1596 mNumFramesWritten = written;
1597 }
Andreas Huber3831a062010-12-21 10:22:33 -08001598}
1599
Wei Jiabc2fb722014-07-08 16:37:57 -07001600void NuPlayer::Renderer::onDisableOffloadAudio() {
1601 Mutex::Autolock autoLock(mLock);
1602 mFlags &= ~FLAG_OFFLOAD_AUDIO;
Wei Jia7b15cb32015-02-03 17:46:06 -08001603 ++mAudioDrainGeneration;
Robert Shihe1d70192015-07-23 17:54:13 -07001604 if (mAudioRenderingStartGeneration != -1) {
1605 prepareForMediaRenderingStart_l();
1606 }
Wei Jiabc2fb722014-07-08 16:37:57 -07001607}
1608
Ronghua Wua10fd232014-11-06 16:15:20 -08001609void NuPlayer::Renderer::onEnableOffloadAudio() {
1610 Mutex::Autolock autoLock(mLock);
1611 mFlags |= FLAG_OFFLOAD_AUDIO;
Wei Jia7b15cb32015-02-03 17:46:06 -08001612 ++mAudioDrainGeneration;
Robert Shihe1d70192015-07-23 17:54:13 -07001613 if (mAudioRenderingStartGeneration != -1) {
1614 prepareForMediaRenderingStart_l();
1615 }
Ronghua Wua10fd232014-11-06 16:15:20 -08001616}
1617
Andreas Huberb4082222011-01-20 15:23:04 -08001618void NuPlayer::Renderer::onPause() {
Rachad8592dbb2014-09-09 13:10:28 -07001619 if (mPaused) {
Rachad8592dbb2014-09-09 13:10:28 -07001620 return;
1621 }
Lajos Molnar6d339f12015-04-17 16:15:53 -07001622
Wei Jia28a8a9f2014-08-18 11:29:50 -07001623 {
1624 Mutex::Autolock autoLock(mLock);
Andy Hung005e9d02015-05-31 23:16:07 -07001625 // we do not increment audio drain generation so that we fill audio buffer during pause.
Wei Jia7b15cb32015-02-03 17:46:06 -08001626 ++mVideoDrainGeneration;
1627 prepareForMediaRenderingStart_l();
Wei Jia73ddd212014-08-29 16:33:49 -07001628 mPaused = true;
Wei Jia98160162015-02-04 17:01:11 -08001629 mMediaClock->setPlaybackRate(0.0);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001630 }
1631
Andreas Huberb4082222011-01-20 15:23:04 -08001632 mDrainAudioQueuePending = false;
Andreas Huberb4082222011-01-20 15:23:04 -08001633 mDrainVideoQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001634
Andy Hung84200732015-12-03 14:41:06 -08001635 // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1636 mAudioSink->pause();
1637 startAudioOffloadPauseTimeout();
Andreas Huberb4082222011-01-20 15:23:04 -08001638
Lajos Molnar6d339f12015-04-17 16:15:53 -07001639 ALOGV("now paused audio queue has %zu entries, video has %zu entries",
Andreas Huberea9d51b2011-11-30 09:53:40 -08001640 mAudioQueue.size(), mVideoQueue.size());
Andreas Huberb4082222011-01-20 15:23:04 -08001641}
1642
1643void NuPlayer::Renderer::onResume() {
Andreas Huberb58ce9f2011-11-28 16:27:35 -08001644 if (!mPaused) {
1645 return;
1646 }
Andreas Huberb4082222011-01-20 15:23:04 -08001647
Andy Hung84200732015-12-03 14:41:06 -08001648 // Note: audio data may not have been decoded, and the AudioSink may not be opened.
1649 cancelAudioOffloadPauseTimeout();
1650 if (mAudioSink->ready()) {
Eric Laurent97c9f4f2015-08-11 18:04:14 -07001651 status_t err = mAudioSink->start();
1652 if (err != OK) {
Andy Hung58d315c2015-09-11 18:44:56 -07001653 ALOGE("cannot start AudioSink err %d", err);
Wei Jiaa05f1e32016-03-25 16:31:22 -07001654 notifyAudioTearDown(kDueToError);
Eric Laurent97c9f4f2015-08-11 18:04:14 -07001655 }
Andreas Huberb4082222011-01-20 15:23:04 -08001656 }
1657
Wei Jia7b15cb32015-02-03 17:46:06 -08001658 {
1659 Mutex::Autolock autoLock(mLock);
1660 mPaused = false;
Andy Hungb12ea0b2015-09-03 15:13:01 -07001661 // rendering started message may have been delayed if we were paused.
1662 if (mRenderingDataDelivered) {
1663 notifyIfMediaRenderingStarted_l();
1664 }
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001665 // configure audiosink as we did not do it when pausing
Wei Jia27ea08e2015-05-12 14:50:35 -07001666 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001667 mAudioSink->setPlaybackRate(mPlaybackSettings);
1668 }
1669
Wei Jia98160162015-02-04 17:01:11 -08001670 mMediaClock->setPlaybackRate(mPlaybackRate);
Andreas Huberb4082222011-01-20 15:23:04 -08001671
Wei Jia7b15cb32015-02-03 17:46:06 -08001672 if (!mAudioQueue.empty()) {
1673 postDrainAudioQueue_l();
1674 }
Andreas Huberb4082222011-01-20 15:23:04 -08001675 }
1676
1677 if (!mVideoQueue.empty()) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001678 postDrainVideoQueue();
Andreas Huberb4082222011-01-20 15:23:04 -08001679 }
1680}
1681
Lajos Molnarc851b5d2014-09-18 14:14:29 -07001682void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1683 if (mVideoScheduler == NULL) {
1684 mVideoScheduler = new VideoFrameScheduler();
1685 }
1686 mVideoScheduler->init(fps);
1687}
1688
Wei Jia7b15cb32015-02-03 17:46:06 -08001689int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1690 Mutex::Autolock autoLock(mLock);
1691 return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1692}
1693
1694int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1695 Mutex::Autolock autoLock(mLock);
1696 return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1697}
1698
1699bool NuPlayer::Renderer::getSyncQueues() {
1700 Mutex::Autolock autoLock(mLock);
1701 return mSyncQueues;
1702}
1703
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001704void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1705 if (mAudioTornDown) {
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001706 return;
1707 }
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001708 mAudioTornDown = true;
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001709
Ronghua Wua73d9e02014-10-08 15:13:29 -07001710 int64_t currentPositionUs;
Robert Shih1a5c8592015-08-04 18:07:44 -07001711 sp<AMessage> notify = mNotify->dup();
1712 if (getCurrentPosition(&currentPositionUs) == OK) {
1713 notify->setInt64("positionUs", currentPositionUs);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001714 }
Lajos Molnar06ad1522014-08-28 07:27:44 -07001715
Wei Jia3a2956d2014-07-22 16:01:33 -07001716 mAudioSink->stop();
1717 mAudioSink->flush();
1718
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001719 notify->setInt32("what", kWhatAudioTearDown);
Ronghua Wu08529172014-10-02 16:55:52 -07001720 notify->setInt32("reason", reason);
Wei Jia3a2956d2014-07-22 16:01:33 -07001721 notify->post();
1722}
1723
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001724void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1725 if (offloadingAudio()) {
Weiyin Jiang35d5af12015-01-28 16:14:02 +08001726 mWakeLock->acquire();
Lajos Molnar1d15ab52015-03-04 16:46:34 -08001727 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
Wei Jia7b15cb32015-02-03 17:46:06 -08001728 msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001729 msg->post(kOffloadPauseMaxUs);
1730 }
1731}
1732
1733void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
Andy Hung21afe802016-04-21 14:52:05 -07001734 // We may have called startAudioOffloadPauseTimeout() without
1735 // the AudioSink open and with offloadingAudio enabled.
1736 //
1737 // When we cancel, it may be that offloadingAudio is subsequently disabled, so regardless
1738 // we always release the wakelock and increment the pause timeout generation.
1739 //
1740 // Note: The acquired wakelock prevents the device from suspending
1741 // immediately after offload pause (in case a resume happens shortly thereafter).
1742 mWakeLock->release(true);
1743 ++mAudioOffloadPauseTimeoutGeneration;
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001744}
1745
Andy Hung202bce12014-12-03 11:47:36 -08001746status_t NuPlayer::Renderer::onOpenAudioSink(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001747 const sp<AMessage> &format,
1748 bool offloadOnly,
1749 bool hasVideo,
1750 uint32_t flags) {
1751 ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1752 offloadOnly, offloadingAudio());
1753 bool audioSinkChanged = false;
1754
1755 int32_t numChannels;
1756 CHECK(format->findInt32("channel-count", &numChannels));
1757
1758 int32_t channelMask;
1759 if (!format->findInt32("channel-mask", &channelMask)) {
1760 // signal to the AudioSink to derive the mask from count.
1761 channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1762 }
1763
1764 int32_t sampleRate;
1765 CHECK(format->findInt32("sample-rate", &sampleRate));
1766
1767 if (offloadingAudio()) {
1768 audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1769 AString mime;
1770 CHECK(format->findString("mime", &mime));
1771 status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1772
1773 if (err != OK) {
1774 ALOGE("Couldn't map mime \"%s\" to a valid "
1775 "audio_format", mime.c_str());
1776 onDisableOffloadAudio();
1777 } else {
1778 ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1779 mime.c_str(), audioFormat);
1780
1781 int avgBitRate = -1;
Lajos Molnar46f80162016-05-19 15:35:47 -07001782 format->findInt32("bitrate", &avgBitRate);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001783
1784 int32_t aacProfile = -1;
1785 if (audioFormat == AUDIO_FORMAT_AAC
1786 && format->findInt32("aac-profile", &aacProfile)) {
1787 // Redefine AAC format as per aac profile
1788 mapAACProfileToAudioFormat(
1789 audioFormat,
1790 aacProfile);
1791 }
1792
1793 audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1794 offloadInfo.duration_us = -1;
1795 format->findInt64(
1796 "durationUs", &offloadInfo.duration_us);
1797 offloadInfo.sample_rate = sampleRate;
1798 offloadInfo.channel_mask = channelMask;
1799 offloadInfo.format = audioFormat;
1800 offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1801 offloadInfo.bit_rate = avgBitRate;
1802 offloadInfo.has_video = hasVideo;
1803 offloadInfo.is_streaming = true;
1804
1805 if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1806 ALOGV("openAudioSink: no change in offload mode");
1807 // no change from previous configuration, everything ok.
Andy Hung202bce12014-12-03 11:47:36 -08001808 return OK;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001809 }
Andy Hungf0e83642014-12-19 17:55:56 -08001810 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1811
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001812 ALOGV("openAudioSink: try to open AudioSink in offload mode");
Eric Laurentd88c3ca2014-10-28 10:52:11 -07001813 uint32_t offloadFlags = flags;
1814 offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1815 offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001816 audioSinkChanged = true;
1817 mAudioSink->close();
Andy Hunga0b39712015-05-31 22:40:49 -07001818
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001819 err = mAudioSink->open(
1820 sampleRate,
1821 numChannels,
1822 (audio_channel_mask_t)channelMask,
1823 audioFormat,
Andy Hung179652e2015-05-31 22:49:46 -07001824 0 /* bufferCount - unused */,
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001825 &NuPlayer::Renderer::AudioSinkCallback,
1826 this,
Eric Laurentd88c3ca2014-10-28 10:52:11 -07001827 (audio_output_flags_t)offloadFlags,
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001828 &offloadInfo);
1829
1830 if (err == OK) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001831 err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1832 }
1833
1834 if (err == OK) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001835 // If the playback is offloaded to h/w, we pass
1836 // the HAL some metadata information.
1837 // We don't want to do this for PCM because it
1838 // will be going through the AudioFlinger mixer
1839 // before reaching the hardware.
1840 // TODO
1841 mCurrentOffloadInfo = offloadInfo;
Andy Hung85e48142015-05-31 23:04:15 -07001842 if (!mPaused) { // for preview mode, don't start if paused
1843 err = mAudioSink->start();
1844 }
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001845 ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1846 }
1847 if (err != OK) {
1848 // Clean up, fall back to non offload mode.
1849 mAudioSink->close();
1850 onDisableOffloadAudio();
1851 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1852 ALOGV("openAudioSink: offload failed");
Wei Jiaa05f1e32016-03-25 16:31:22 -07001853 if (offloadOnly) {
1854 notifyAudioTearDown(kForceNonOffload);
1855 }
Wei Jia3ab25452015-06-10 09:37:47 -07001856 } else {
1857 mUseAudioCallback = true; // offload mode transfers data through callback
1858 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message.
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001859 }
1860 }
1861 }
1862 if (!offloadOnly && !offloadingAudio()) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001863 ALOGV("openAudioSink: open AudioSink in NON-offload mode");
Eric Laurentd88c3ca2014-10-28 10:52:11 -07001864 uint32_t pcmFlags = flags;
1865 pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
Andy Hungf0e83642014-12-19 17:55:56 -08001866
1867 const PcmInfo info = {
1868 (audio_channel_mask_t)channelMask,
1869 (audio_output_flags_t)pcmFlags,
1870 AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1871 numChannels,
1872 sampleRate
1873 };
1874 if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1875 ALOGV("openAudioSink: no change in pcm mode");
1876 // no change from previous configuration, everything ok.
1877 return OK;
1878 }
1879
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001880 audioSinkChanged = true;
1881 mAudioSink->close();
1882 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
Andy Hunga0b39712015-05-31 22:40:49 -07001883 // Note: It is possible to set up the callback, but not use it to send audio data.
1884 // This requires a fix in AudioSink to explicitly specify the transfer mode.
1885 mUseAudioCallback = getUseAudioCallbackSetting();
Wei Jia3ab25452015-06-10 09:37:47 -07001886 if (mUseAudioCallback) {
1887 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message.
1888 }
Andy Hung179652e2015-05-31 22:49:46 -07001889
1890 // Compute the desired buffer size.
1891 // For callback mode, the amount of time before wakeup is about half the buffer size.
1892 const uint32_t frameCount =
1893 (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
1894
Ronghua Wu7665f582015-09-02 10:15:30 -07001895 // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
1896 // AudioSink. We don't want this when there's video because it will cause a video seek to
1897 // the previous I frame. But we do want this when there's only audio because it will give
1898 // NuPlayer a chance to switch from non-offload mode to offload mode.
1899 // So we only set doNotReconnect when there's no video.
1900 const bool doNotReconnect = !hasVideo;
Andy Hungff874dc2016-04-11 16:49:09 -07001901
1902 // We should always be able to set our playback settings if the sink is closed.
1903 LOG_ALWAYS_FATAL_IF(mAudioSink->setPlaybackRate(mPlaybackSettings) != OK,
1904 "onOpenAudioSink: can't set playback rate on closed sink");
Andy Hung202bce12014-12-03 11:47:36 -08001905 status_t err = mAudioSink->open(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001906 sampleRate,
1907 numChannels,
1908 (audio_channel_mask_t)channelMask,
1909 AUDIO_FORMAT_PCM_16_BIT,
Andy Hung179652e2015-05-31 22:49:46 -07001910 0 /* bufferCount - unused */,
Andy Hunga0b39712015-05-31 22:40:49 -07001911 mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
1912 mUseAudioCallback ? this : NULL,
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001913 (audio_output_flags_t)pcmFlags,
1914 NULL,
Ronghua Wu7665f582015-09-02 10:15:30 -07001915 doNotReconnect,
Andy Hung179652e2015-05-31 22:49:46 -07001916 frameCount);
Andy Hung202bce12014-12-03 11:47:36 -08001917 if (err != OK) {
1918 ALOGW("openAudioSink: non offloaded open failed status: %d", err);
Wei Jia4d7ac852015-08-31 18:33:14 -07001919 mAudioSink->close();
Andy Hungf0e83642014-12-19 17:55:56 -08001920 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
Andy Hung202bce12014-12-03 11:47:36 -08001921 return err;
1922 }
Andy Hungf0e83642014-12-19 17:55:56 -08001923 mCurrentPcmInfo = info;
Andy Hung005e9d02015-05-31 23:16:07 -07001924 if (!mPaused) { // for preview mode, don't start if paused
1925 mAudioSink->start();
1926 }
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001927 }
1928 if (audioSinkChanged) {
1929 onAudioSinkChanged();
1930 }
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001931 mAudioTornDown = false;
Andy Hung202bce12014-12-03 11:47:36 -08001932 return OK;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001933}
1934
1935void NuPlayer::Renderer::onCloseAudioSink() {
1936 mAudioSink->close();
1937 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
Andy Hungf0e83642014-12-19 17:55:56 -08001938 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001939}
1940
Andreas Huberf9334412010-12-15 15:17:42 -08001941} // namespace android
1942