blob: 481726c049a2d8d0539e07bc1475d4aa805e4468 [file] [log] [blame]
Andreas Huberf9334412010-12-15 15:17:42 -08001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
Andy Hunga0b39712015-05-31 22:40:49 -070022#include <cutils/properties.h>
Andreas Huberf9334412010-12-15 15:17:42 -080023#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
Andreas Huber5bc087c2010-12-23 10:27:40 -080025#include <media/stagefright/foundation/AMessage.h>
Ronghua Wua73d9e02014-10-08 15:13:29 -070026#include <media/stagefright/foundation/AUtils.h>
Weiyin Jiang35d5af12015-01-28 16:14:02 +080027#include <media/stagefright/foundation/AWakeLock.h>
Wei Jia5833b6a2015-03-09 17:01:47 -070028#include <media/stagefright/MediaClock.h>
Wei Jiabc2fb722014-07-08 16:37:57 -070029#include <media/stagefright/MediaErrors.h>
30#include <media/stagefright/MetaData.h>
Chong Zhang3b9eb1f2014-10-15 17:05:08 -070031#include <media/stagefright/Utils.h>
Lajos Molnara3725d72015-07-01 16:47:22 -070032#include <media/stagefright/VideoFrameScheduler.h>
Lajos Molnardc43dfa2014-05-07 15:33:04 -070033
Lajos Molnar09524832014-07-17 14:29:51 -070034#include <inttypes.h>
35
Andreas Huberf9334412010-12-15 15:17:42 -080036namespace android {
37
Andy Hunga0b39712015-05-31 22:40:49 -070038/*
39 * Example of common configuration settings in shell script form
40
41 #Turn offload audio off (use PCM for Play Music) -- AudioPolicyManager
42 adb shell setprop audio.offload.disable 1
43
44 #Allow offload audio with video (requires offloading to be enabled) -- AudioPolicyManager
45 adb shell setprop audio.offload.video 1
46
47 #Use audio callbacks for PCM data
48 adb shell setprop media.stagefright.audio.cbk 1
49
Andy Hung288da022015-05-31 22:55:59 -070050 #Use deep buffer for PCM data with video (it is generally enabled for audio-only)
51 adb shell setprop media.stagefright.audio.deep 1
52
Andy Hung179652e2015-05-31 22:49:46 -070053 #Set size of buffers for pcm audio sink in msec (example: 1000 msec)
54 adb shell setprop media.stagefright.audio.sink 1000
55
Andy Hunga0b39712015-05-31 22:40:49 -070056 * These configurations take effect for the next track played (not the current track).
57 */
58
59static inline bool getUseAudioCallbackSetting() {
60 return property_get_bool("media.stagefright.audio.cbk", false /* default_value */);
61}
62
Andy Hung179652e2015-05-31 22:49:46 -070063static inline int32_t getAudioSinkPcmMsSetting() {
64 return property_get_int32(
65 "media.stagefright.audio.sink", 500 /* default_value */);
66}
67
Ronghua Wuf5b1db12014-09-09 10:11:08 -070068// Maximum time in paused state when offloading audio decompression. When elapsed, the AudioSink
69// is closed to allow the audio DSP to power down.
Eric Laurenta5d316f2014-10-29 15:32:17 -070070static const int64_t kOffloadPauseMaxUs = 10000000ll;
Ronghua Wuf5b1db12014-09-09 10:11:08 -070071
Andreas Huber714aa7b2011-09-13 08:28:38 -070072// static
Andy Hungf0e83642014-12-19 17:55:56 -080073const NuPlayer::Renderer::PcmInfo NuPlayer::Renderer::AUDIO_PCMINFO_INITIALIZER = {
74 AUDIO_CHANNEL_NONE,
75 AUDIO_OUTPUT_FLAG_NONE,
76 AUDIO_FORMAT_INVALID,
77 0, // mNumChannels
78 0 // mSampleRate
79};
80
81// static
Andreas Huber714aa7b2011-09-13 08:28:38 -070082const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
83
Andreas Huberf9334412010-12-15 15:17:42 -080084NuPlayer::Renderer::Renderer(
85 const sp<MediaPlayerBase::AudioSink> &sink,
Andreas Huberd5e56232013-03-12 11:01:43 -070086 const sp<AMessage> &notify,
87 uint32_t flags)
Andreas Huberf9334412010-12-15 15:17:42 -080088 : mAudioSink(sink),
89 mNotify(notify),
Andreas Huberd5e56232013-03-12 11:01:43 -070090 mFlags(flags),
Andreas Huberf9334412010-12-15 15:17:42 -080091 mNumFramesWritten(0),
92 mDrainAudioQueuePending(false),
93 mDrainVideoQueuePending(false),
94 mAudioQueueGeneration(0),
95 mVideoQueueGeneration(0),
Wei Jia7b15cb32015-02-03 17:46:06 -080096 mAudioDrainGeneration(0),
97 mVideoDrainGeneration(0),
Wei Jia7c8d0e02015-08-27 17:40:21 -070098 mAudioEOSGeneration(0),
Lajos Molnar3a474aa2015-04-24 17:10:07 -070099 mPlaybackSettings(AUDIO_PLAYBACK_RATE_DEFAULT),
Ronghua Wua73d9e02014-10-08 15:13:29 -0700100 mAudioFirstAnchorTimeMediaUs(-1),
Ronghua Wueecb7802014-10-19 23:12:50 -0700101 mAnchorTimeMediaUs(-1),
Lajos Molnarf5926712014-10-21 09:36:43 -0700102 mAnchorNumFramesWritten(-1),
Ronghua Wua73d9e02014-10-08 15:13:29 -0700103 mVideoLateByUs(0ll),
Andreas Huberbc7f5b22011-01-21 10:15:23 -0800104 mHasAudio(false),
105 mHasVideo(false),
Chong Zhang7137ec72014-11-12 16:41:05 -0800106 mNotifyCompleteAudio(false),
107 mNotifyCompleteVideo(false),
Andreas Huberbc7f5b22011-01-21 10:15:23 -0800108 mSyncQueues(false),
Andreas Huber714aa7b2011-09-13 08:28:38 -0700109 mPaused(false),
Andy Hungb03dcb32015-08-27 16:18:59 -0700110 mPauseDrainAudioAllowedUs(0),
Andy Hung09e0c362014-09-12 15:12:16 -0700111 mVideoSampleReceived(false),
James Dongf57b4ea2012-07-20 13:38:36 -0700112 mVideoRenderingStarted(false),
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700113 mVideoRenderingStartGeneration(0),
114 mAudioRenderingStartGeneration(0),
Andy Hungb12ea0b2015-09-03 15:13:01 -0700115 mRenderingDataDelivered(false),
Wei Jia2995dc72015-07-24 16:00:33 -0700116 mLastAudioMediaTimeUs(-1),
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700117 mAudioOffloadPauseTimeoutGeneration(0),
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700118 mAudioTornDown(false),
Lajos Molnard5923402014-10-20 17:47:54 -0700119 mCurrentOffloadInfo(AUDIO_INFO_INITIALIZER),
Andy Hungf0e83642014-12-19 17:55:56 -0800120 mCurrentPcmInfo(AUDIO_PCMINFO_INITIALIZER),
Lajos Molnard5923402014-10-20 17:47:54 -0700121 mTotalBuffersQueued(0),
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800122 mLastAudioBufferDrained(0),
Andy Hunga0b39712015-05-31 22:40:49 -0700123 mUseAudioCallback(false),
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800124 mWakeLock(new AWakeLock()) {
Wei Jia7b15cb32015-02-03 17:46:06 -0800125 mMediaClock = new MediaClock;
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700126 mPlaybackRate = mPlaybackSettings.mSpeed;
127 mMediaClock->setPlaybackRate(mPlaybackRate);
Andreas Huberf9334412010-12-15 15:17:42 -0800128}
129
130NuPlayer::Renderer::~Renderer() {
Wei Jiabc2fb722014-07-08 16:37:57 -0700131 if (offloadingAudio()) {
132 mAudioSink->stop();
133 mAudioSink->flush();
134 mAudioSink->close();
135 }
Andreas Huberf9334412010-12-15 15:17:42 -0800136}
137
138void NuPlayer::Renderer::queueBuffer(
139 bool audio,
140 const sp<ABuffer> &buffer,
141 const sp<AMessage> &notifyConsumed) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800142 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800143 msg->setInt32("queueGeneration", getQueueGeneration(audio));
Andreas Huberf9334412010-12-15 15:17:42 -0800144 msg->setInt32("audio", static_cast<int32_t>(audio));
Andreas Huber2d8bedd2012-02-21 14:38:23 -0800145 msg->setBuffer("buffer", buffer);
Andreas Huberf9334412010-12-15 15:17:42 -0800146 msg->setMessage("notifyConsumed", notifyConsumed);
147 msg->post();
148}
149
150void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
151 CHECK_NE(finalResult, (status_t)OK);
152
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800153 sp<AMessage> msg = new AMessage(kWhatQueueEOS, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800154 msg->setInt32("queueGeneration", getQueueGeneration(audio));
Andreas Huberf9334412010-12-15 15:17:42 -0800155 msg->setInt32("audio", static_cast<int32_t>(audio));
156 msg->setInt32("finalResult", finalResult);
157 msg->post();
158}
159
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700160status_t NuPlayer::Renderer::setPlaybackSettings(const AudioPlaybackRate &rate) {
161 sp<AMessage> msg = new AMessage(kWhatConfigPlayback, this);
162 writeToAMessage(msg, rate);
163 sp<AMessage> response;
164 status_t err = msg->postAndAwaitResponse(&response);
165 if (err == OK && response != NULL) {
166 CHECK(response->findInt32("err", &err));
167 }
168 return err;
169}
170
171status_t NuPlayer::Renderer::onConfigPlayback(const AudioPlaybackRate &rate /* sanitized */) {
172 if (rate.mSpeed == 0.f) {
173 onPause();
174 // don't call audiosink's setPlaybackRate if pausing, as pitch does not
175 // have to correspond to the any non-0 speed (e.g old speed). Keep
176 // settings nonetheless, using the old speed, in case audiosink changes.
177 AudioPlaybackRate newRate = rate;
178 newRate.mSpeed = mPlaybackSettings.mSpeed;
179 mPlaybackSettings = newRate;
180 return OK;
181 }
182
Wei Jia27ea08e2015-05-12 14:50:35 -0700183 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700184 status_t err = mAudioSink->setPlaybackRate(rate);
185 if (err != OK) {
186 return err;
187 }
188 }
189 mPlaybackSettings = rate;
190 mPlaybackRate = rate.mSpeed;
191 mMediaClock->setPlaybackRate(mPlaybackRate);
192 return OK;
193}
194
195status_t NuPlayer::Renderer::getPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
196 sp<AMessage> msg = new AMessage(kWhatGetPlaybackSettings, this);
197 sp<AMessage> response;
198 status_t err = msg->postAndAwaitResponse(&response);
199 if (err == OK && response != NULL) {
200 CHECK(response->findInt32("err", &err));
201 if (err == OK) {
202 readFromAMessage(response, rate);
203 }
204 }
205 return err;
206}
207
208status_t NuPlayer::Renderer::onGetPlaybackSettings(AudioPlaybackRate *rate /* nonnull */) {
Wei Jia27ea08e2015-05-12 14:50:35 -0700209 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700210 status_t err = mAudioSink->getPlaybackRate(rate);
211 if (err == OK) {
212 if (!isAudioPlaybackRateEqual(*rate, mPlaybackSettings)) {
213 ALOGW("correcting mismatch in internal/external playback rate");
214 }
215 // get playback settings used by audiosink, as it may be
216 // slightly off due to audiosink not taking small changes.
217 mPlaybackSettings = *rate;
218 if (mPaused) {
219 rate->mSpeed = 0.f;
220 }
221 }
222 return err;
223 }
224 *rate = mPlaybackSettings;
225 return OK;
226}
227
228status_t NuPlayer::Renderer::setSyncSettings(const AVSyncSettings &sync, float videoFpsHint) {
229 sp<AMessage> msg = new AMessage(kWhatConfigSync, this);
230 writeToAMessage(msg, sync, videoFpsHint);
231 sp<AMessage> response;
232 status_t err = msg->postAndAwaitResponse(&response);
233 if (err == OK && response != NULL) {
234 CHECK(response->findInt32("err", &err));
235 }
236 return err;
237}
238
239status_t NuPlayer::Renderer::onConfigSync(const AVSyncSettings &sync, float videoFpsHint __unused) {
240 if (sync.mSource != AVSYNC_SOURCE_DEFAULT) {
241 return BAD_VALUE;
242 }
243 // TODO: support sync sources
244 return INVALID_OPERATION;
245}
246
247status_t NuPlayer::Renderer::getSyncSettings(AVSyncSettings *sync, float *videoFps) {
248 sp<AMessage> msg = new AMessage(kWhatGetSyncSettings, this);
249 sp<AMessage> response;
250 status_t err = msg->postAndAwaitResponse(&response);
251 if (err == OK && response != NULL) {
252 CHECK(response->findInt32("err", &err));
253 if (err == OK) {
254 readFromAMessage(response, sync, videoFps);
255 }
256 }
257 return err;
258}
259
260status_t NuPlayer::Renderer::onGetSyncSettings(
261 AVSyncSettings *sync /* nonnull */, float *videoFps /* nonnull */) {
262 *sync = mSyncSettings;
263 *videoFps = -1.f;
264 return OK;
Wei Jia98160162015-02-04 17:01:11 -0800265}
266
Chong Zhang7137ec72014-11-12 16:41:05 -0800267void NuPlayer::Renderer::flush(bool audio, bool notifyComplete) {
Andreas Huberf9334412010-12-15 15:17:42 -0800268 {
Wei Jia7b15cb32015-02-03 17:46:06 -0800269 Mutex::Autolock autoLock(mLock);
Andreas Huberf9334412010-12-15 15:17:42 -0800270 if (audio) {
Chong Zhang7137ec72014-11-12 16:41:05 -0800271 mNotifyCompleteAudio |= notifyComplete;
Wei Jia3b0cd262015-07-13 11:28:10 -0700272 clearAudioFirstAnchorTime_l();
Wei Jia7b15cb32015-02-03 17:46:06 -0800273 ++mAudioQueueGeneration;
274 ++mAudioDrainGeneration;
Andreas Huberf9334412010-12-15 15:17:42 -0800275 } else {
Chong Zhang7137ec72014-11-12 16:41:05 -0800276 mNotifyCompleteVideo |= notifyComplete;
Wei Jia7b15cb32015-02-03 17:46:06 -0800277 ++mVideoQueueGeneration;
278 ++mVideoDrainGeneration;
Andreas Huberf9334412010-12-15 15:17:42 -0800279 }
Wei Jia7b15cb32015-02-03 17:46:06 -0800280
281 clearAnchorTime_l();
Wei Jia7b15cb32015-02-03 17:46:06 -0800282 mVideoLateByUs = 0;
283 mSyncQueues = false;
Andreas Huberf9334412010-12-15 15:17:42 -0800284 }
285
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800286 sp<AMessage> msg = new AMessage(kWhatFlush, this);
Andreas Huberf9334412010-12-15 15:17:42 -0800287 msg->setInt32("audio", static_cast<int32_t>(audio));
288 msg->post();
289}
290
291void NuPlayer::Renderer::signalTimeDiscontinuity() {
Wei Jia28a8a9f2014-08-18 11:29:50 -0700292}
293
294void NuPlayer::Renderer::signalDisableOffloadAudio() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800295 (new AMessage(kWhatDisableOffloadAudio, this))->post();
Wei Jia28a8a9f2014-08-18 11:29:50 -0700296}
297
Ronghua Wua10fd232014-11-06 16:15:20 -0800298void NuPlayer::Renderer::signalEnableOffloadAudio() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800299 (new AMessage(kWhatEnableOffloadAudio, this))->post();
Ronghua Wua10fd232014-11-06 16:15:20 -0800300}
301
Andreas Huberb4082222011-01-20 15:23:04 -0800302void NuPlayer::Renderer::pause() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800303 (new AMessage(kWhatPause, this))->post();
Andreas Huberb4082222011-01-20 15:23:04 -0800304}
305
306void NuPlayer::Renderer::resume() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800307 (new AMessage(kWhatResume, this))->post();
Andreas Huberb4082222011-01-20 15:23:04 -0800308}
309
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700310void NuPlayer::Renderer::setVideoFrameRate(float fps) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800311 sp<AMessage> msg = new AMessage(kWhatSetVideoFrameRate, this);
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700312 msg->setFloat("frame-rate", fps);
313 msg->post();
314}
315
Wei Jiad4cdba12014-12-18 10:44:17 -0800316// Called on any threads.
Wei Jia7b15cb32015-02-03 17:46:06 -0800317status_t NuPlayer::Renderer::getCurrentPosition(int64_t *mediaUs) {
Wei Jia98160162015-02-04 17:01:11 -0800318 return mMediaClock->getMediaTime(ALooper::GetNowUs(), mediaUs);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700319}
320
Wei Jia7b15cb32015-02-03 17:46:06 -0800321void NuPlayer::Renderer::clearAudioFirstAnchorTime_l() {
322 mAudioFirstAnchorTimeMediaUs = -1;
323 mMediaClock->setStartingTimeMedia(-1);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700324}
325
Wei Jia7b15cb32015-02-03 17:46:06 -0800326void NuPlayer::Renderer::setAudioFirstAnchorTimeIfNeeded_l(int64_t mediaUs) {
Ronghua Wua73d9e02014-10-08 15:13:29 -0700327 if (mAudioFirstAnchorTimeMediaUs == -1) {
328 mAudioFirstAnchorTimeMediaUs = mediaUs;
Wei Jia7b15cb32015-02-03 17:46:06 -0800329 mMediaClock->setStartingTimeMedia(mediaUs);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700330 }
331}
332
Wei Jia7b15cb32015-02-03 17:46:06 -0800333void NuPlayer::Renderer::clearAnchorTime_l() {
334 mMediaClock->clearAnchor();
335 mAnchorTimeMediaUs = -1;
336 mAnchorNumFramesWritten = -1;
Ronghua Wua73d9e02014-10-08 15:13:29 -0700337}
338
339void NuPlayer::Renderer::setVideoLateByUs(int64_t lateUs) {
Wei Jia7b15cb32015-02-03 17:46:06 -0800340 Mutex::Autolock autoLock(mLock);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700341 mVideoLateByUs = lateUs;
342}
343
344int64_t NuPlayer::Renderer::getVideoLateByUs() {
Wei Jia7b15cb32015-02-03 17:46:06 -0800345 Mutex::Autolock autoLock(mLock);
Ronghua Wua73d9e02014-10-08 15:13:29 -0700346 return mVideoLateByUs;
347}
348
Andy Hung202bce12014-12-03 11:47:36 -0800349status_t NuPlayer::Renderer::openAudioSink(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700350 const sp<AMessage> &format,
351 bool offloadOnly,
352 bool hasVideo,
Andy Hung202bce12014-12-03 11:47:36 -0800353 uint32_t flags,
354 bool *isOffloaded) {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800355 sp<AMessage> msg = new AMessage(kWhatOpenAudioSink, this);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700356 msg->setMessage("format", format);
357 msg->setInt32("offload-only", offloadOnly);
358 msg->setInt32("has-video", hasVideo);
359 msg->setInt32("flags", flags);
360
361 sp<AMessage> response;
362 msg->postAndAwaitResponse(&response);
363
Andy Hung202bce12014-12-03 11:47:36 -0800364 int32_t err;
365 if (!response->findInt32("err", &err)) {
366 err = INVALID_OPERATION;
367 } else if (err == OK && isOffloaded != NULL) {
368 int32_t offload;
369 CHECK(response->findInt32("offload", &offload));
370 *isOffloaded = (offload != 0);
371 }
372 return err;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700373}
374
375void NuPlayer::Renderer::closeAudioSink() {
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800376 sp<AMessage> msg = new AMessage(kWhatCloseAudioSink, this);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700377
378 sp<AMessage> response;
379 msg->postAndAwaitResponse(&response);
380}
381
Andreas Huberf9334412010-12-15 15:17:42 -0800382void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
383 switch (msg->what()) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700384 case kWhatOpenAudioSink:
385 {
386 sp<AMessage> format;
387 CHECK(msg->findMessage("format", &format));
388
389 int32_t offloadOnly;
390 CHECK(msg->findInt32("offload-only", &offloadOnly));
391
392 int32_t hasVideo;
393 CHECK(msg->findInt32("has-video", &hasVideo));
394
395 uint32_t flags;
396 CHECK(msg->findInt32("flags", (int32_t *)&flags));
397
Andy Hung202bce12014-12-03 11:47:36 -0800398 status_t err = onOpenAudioSink(format, offloadOnly, hasVideo, flags);
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700399
400 sp<AMessage> response = new AMessage;
Andy Hung202bce12014-12-03 11:47:36 -0800401 response->setInt32("err", err);
402 response->setInt32("offload", offloadingAudio());
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700403
Lajos Molnar3f274362015-03-05 14:35:41 -0800404 sp<AReplyToken> replyID;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700405 CHECK(msg->senderAwaitsResponse(&replyID));
406 response->postReply(replyID);
407
408 break;
409 }
410
411 case kWhatCloseAudioSink:
412 {
Lajos Molnar3f274362015-03-05 14:35:41 -0800413 sp<AReplyToken> replyID;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -0700414 CHECK(msg->senderAwaitsResponse(&replyID));
415
416 onCloseAudioSink();
417
418 sp<AMessage> response = new AMessage;
419 response->postReply(replyID);
420 break;
421 }
422
Wei Jiabc2fb722014-07-08 16:37:57 -0700423 case kWhatStopAudioSink:
424 {
425 mAudioSink->stop();
426 break;
427 }
428
Andreas Huberf9334412010-12-15 15:17:42 -0800429 case kWhatDrainAudioQueue:
430 {
Wei Jia3ab25452015-06-10 09:37:47 -0700431 mDrainAudioQueuePending = false;
432
Andreas Huberf9334412010-12-15 15:17:42 -0800433 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800434 CHECK(msg->findInt32("drainGeneration", &generation));
435 if (generation != getDrainGeneration(true /* audio */)) {
Andreas Huberf9334412010-12-15 15:17:42 -0800436 break;
437 }
438
Andreas Huber078cfcf2011-09-15 12:25:04 -0700439 if (onDrainAudioQueue()) {
440 uint32_t numFramesPlayed;
441 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
442 (status_t)OK);
Andreas Huberf9334412010-12-15 15:17:42 -0800443
Andreas Huber078cfcf2011-09-15 12:25:04 -0700444 uint32_t numFramesPendingPlayout =
445 mNumFramesWritten - numFramesPlayed;
446
447 // This is how long the audio sink will have data to
448 // play back.
449 int64_t delayUs =
450 mAudioSink->msecsPerFrame()
451 * numFramesPendingPlayout * 1000ll;
Wei Jiad9c2e9c2015-04-10 19:04:09 -0700452 if (mPlaybackRate > 1.0f) {
453 delayUs /= mPlaybackRate;
454 }
Andreas Huber078cfcf2011-09-15 12:25:04 -0700455
456 // Let's give it more data after about half that time
457 // has elapsed.
Wei Jia7b15cb32015-02-03 17:46:06 -0800458 Mutex::Autolock autoLock(mLock);
Wei Jiabc2fb722014-07-08 16:37:57 -0700459 postDrainAudioQueue_l(delayUs / 2);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700460 }
Andreas Huberf9334412010-12-15 15:17:42 -0800461 break;
462 }
463
464 case kWhatDrainVideoQueue:
465 {
466 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800467 CHECK(msg->findInt32("drainGeneration", &generation));
468 if (generation != getDrainGeneration(false /* audio */)) {
Andreas Huberf9334412010-12-15 15:17:42 -0800469 break;
470 }
471
472 mDrainVideoQueuePending = false;
473
474 onDrainVideoQueue();
475
Wei Jia7b15cb32015-02-03 17:46:06 -0800476 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -0800477 break;
478 }
479
Lajos Molnard5923402014-10-20 17:47:54 -0700480 case kWhatPostDrainVideoQueue:
481 {
482 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800483 CHECK(msg->findInt32("drainGeneration", &generation));
484 if (generation != getDrainGeneration(false /* audio */)) {
Lajos Molnard5923402014-10-20 17:47:54 -0700485 break;
486 }
487
488 mDrainVideoQueuePending = false;
Wei Jia7b15cb32015-02-03 17:46:06 -0800489 postDrainVideoQueue();
Lajos Molnard5923402014-10-20 17:47:54 -0700490 break;
491 }
492
Andreas Huberf9334412010-12-15 15:17:42 -0800493 case kWhatQueueBuffer:
494 {
495 onQueueBuffer(msg);
496 break;
497 }
498
499 case kWhatQueueEOS:
500 {
501 onQueueEOS(msg);
502 break;
503 }
504
Wei Jia7c8d0e02015-08-27 17:40:21 -0700505 case kWhatEOS:
506 {
507 int32_t generation;
508 CHECK(msg->findInt32("audioEOSGeneration", &generation));
509 if (generation != mAudioEOSGeneration) {
510 break;
511 }
512 status_t finalResult;
513 CHECK(msg->findInt32("finalResult", &finalResult));
514 notifyEOS(true /* audio */, finalResult);
515 break;
516 }
517
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700518 case kWhatConfigPlayback:
Wei Jia98160162015-02-04 17:01:11 -0800519 {
Lajos Molnar3a474aa2015-04-24 17:10:07 -0700520 sp<AReplyToken> replyID;
521 CHECK(msg->senderAwaitsResponse(&replyID));
522 AudioPlaybackRate rate;
523 readFromAMessage(msg, &rate);
524 status_t err = onConfigPlayback(rate);
525 sp<AMessage> response = new AMessage;
526 response->setInt32("err", err);
527 response->postReply(replyID);
528 break;
529 }
530
531 case kWhatGetPlaybackSettings:
532 {
533 sp<AReplyToken> replyID;
534 CHECK(msg->senderAwaitsResponse(&replyID));
535 AudioPlaybackRate rate = AUDIO_PLAYBACK_RATE_DEFAULT;
536 status_t err = onGetPlaybackSettings(&rate);
537 sp<AMessage> response = new AMessage;
538 if (err == OK) {
539 writeToAMessage(response, rate);
540 }
541 response->setInt32("err", err);
542 response->postReply(replyID);
543 break;
544 }
545
546 case kWhatConfigSync:
547 {
548 sp<AReplyToken> replyID;
549 CHECK(msg->senderAwaitsResponse(&replyID));
550 AVSyncSettings sync;
551 float videoFpsHint;
552 readFromAMessage(msg, &sync, &videoFpsHint);
553 status_t err = onConfigSync(sync, videoFpsHint);
554 sp<AMessage> response = new AMessage;
555 response->setInt32("err", err);
556 response->postReply(replyID);
557 break;
558 }
559
560 case kWhatGetSyncSettings:
561 {
562 sp<AReplyToken> replyID;
563 CHECK(msg->senderAwaitsResponse(&replyID));
564
565 ALOGV("kWhatGetSyncSettings");
566 AVSyncSettings sync;
567 float videoFps = -1.f;
568 status_t err = onGetSyncSettings(&sync, &videoFps);
569 sp<AMessage> response = new AMessage;
570 if (err == OK) {
571 writeToAMessage(response, sync, videoFps);
572 }
573 response->setInt32("err", err);
574 response->postReply(replyID);
Wei Jia98160162015-02-04 17:01:11 -0800575 break;
576 }
577
Andreas Huberf9334412010-12-15 15:17:42 -0800578 case kWhatFlush:
579 {
580 onFlush(msg);
581 break;
582 }
583
Wei Jiabc2fb722014-07-08 16:37:57 -0700584 case kWhatDisableOffloadAudio:
585 {
586 onDisableOffloadAudio();
587 break;
588 }
589
Ronghua Wua10fd232014-11-06 16:15:20 -0800590 case kWhatEnableOffloadAudio:
591 {
592 onEnableOffloadAudio();
593 break;
594 }
595
Andreas Huberb4082222011-01-20 15:23:04 -0800596 case kWhatPause:
597 {
598 onPause();
599 break;
600 }
601
602 case kWhatResume:
603 {
604 onResume();
605 break;
606 }
607
Lajos Molnarc851b5d2014-09-18 14:14:29 -0700608 case kWhatSetVideoFrameRate:
609 {
610 float fps;
611 CHECK(msg->findFloat("frame-rate", &fps));
612 onSetVideoFrameRate(fps);
613 break;
614 }
615
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700616 case kWhatAudioTearDown:
Wei Jia3a2956d2014-07-22 16:01:33 -0700617 {
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700618 onAudioTearDown(kDueToError);
Wei Jia3a2956d2014-07-22 16:01:33 -0700619 break;
620 }
621
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700622 case kWhatAudioOffloadPauseTimeout:
623 {
624 int32_t generation;
Wei Jia7b15cb32015-02-03 17:46:06 -0800625 CHECK(msg->findInt32("drainGeneration", &generation));
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700626 if (generation != mAudioOffloadPauseTimeoutGeneration) {
627 break;
628 }
Ronghua Wu08529172014-10-02 16:55:52 -0700629 ALOGV("Audio Offload tear down due to pause timeout.");
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700630 onAudioTearDown(kDueToTimeout);
Weiyin Jiang35d5af12015-01-28 16:14:02 +0800631 mWakeLock->release();
Ronghua Wuf5b1db12014-09-09 10:11:08 -0700632 break;
633 }
634
Andreas Huberf9334412010-12-15 15:17:42 -0800635 default:
636 TRESPASS();
637 break;
638 }
639}
640
Wei Jiabc2fb722014-07-08 16:37:57 -0700641void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
Andy Hung005e9d02015-05-31 23:16:07 -0700642 if (mDrainAudioQueuePending || mSyncQueues || mUseAudioCallback) {
Andreas Huberf9334412010-12-15 15:17:42 -0800643 return;
644 }
645
646 if (mAudioQueue.empty()) {
647 return;
648 }
649
Andy Hungb03dcb32015-08-27 16:18:59 -0700650 // FIXME: if paused, wait until AudioTrack stop() is complete before delivering data.
651 if (mPaused) {
652 const int64_t diffUs = mPauseDrainAudioAllowedUs - ALooper::GetNowUs();
653 if (diffUs > delayUs) {
654 delayUs = diffUs;
655 }
656 }
657
Andreas Huberf9334412010-12-15 15:17:42 -0800658 mDrainAudioQueuePending = true;
Lajos Molnar1d15ab52015-03-04 16:46:34 -0800659 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, this);
Wei Jia7b15cb32015-02-03 17:46:06 -0800660 msg->setInt32("drainGeneration", mAudioDrainGeneration);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700661 msg->post(delayUs);
Andreas Huberf9334412010-12-15 15:17:42 -0800662}
663
Wei Jia7b15cb32015-02-03 17:46:06 -0800664void NuPlayer::Renderer::prepareForMediaRenderingStart_l() {
665 mAudioRenderingStartGeneration = mAudioDrainGeneration;
666 mVideoRenderingStartGeneration = mVideoDrainGeneration;
Andy Hungb12ea0b2015-09-03 15:13:01 -0700667 mRenderingDataDelivered = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700668}
669
Wei Jia7b15cb32015-02-03 17:46:06 -0800670void NuPlayer::Renderer::notifyIfMediaRenderingStarted_l() {
671 if (mVideoRenderingStartGeneration == mVideoDrainGeneration &&
672 mAudioRenderingStartGeneration == mAudioDrainGeneration) {
Andy Hungb12ea0b2015-09-03 15:13:01 -0700673 mRenderingDataDelivered = true;
674 if (mPaused) {
675 return;
676 }
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700677 mVideoRenderingStartGeneration = -1;
678 mAudioRenderingStartGeneration = -1;
679
680 sp<AMessage> notify = mNotify->dup();
681 notify->setInt32("what", kWhatMediaRenderingStart);
682 notify->post();
683 }
684}
685
Wei Jiabc2fb722014-07-08 16:37:57 -0700686// static
687size_t NuPlayer::Renderer::AudioSinkCallback(
688 MediaPlayerBase::AudioSink * /* audioSink */,
689 void *buffer,
690 size_t size,
691 void *cookie,
692 MediaPlayerBase::AudioSink::cb_event_t event) {
693 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
694
695 switch (event) {
696 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
697 {
698 return me->fillAudioBuffer(buffer, size);
699 break;
700 }
701
702 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
703 {
Andy Hunga0b39712015-05-31 22:40:49 -0700704 ALOGV("AudioSink::CB_EVENT_STREAM_END");
Wei Jiabc2fb722014-07-08 16:37:57 -0700705 me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
706 break;
707 }
708
709 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
710 {
Andy Hunga0b39712015-05-31 22:40:49 -0700711 ALOGV("AudioSink::CB_EVENT_TEAR_DOWN");
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700712 me->notifyAudioTearDown();
Wei Jiabc2fb722014-07-08 16:37:57 -0700713 break;
714 }
715 }
716
717 return 0;
718}
719
720size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
721 Mutex::Autolock autoLock(mLock);
722
Andy Hung85e48142015-05-31 23:04:15 -0700723 if (!mUseAudioCallback) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700724 return 0;
725 }
726
727 bool hasEOS = false;
728
729 size_t sizeCopied = 0;
Ronghua Wu3e5efb32014-08-18 16:27:08 -0700730 bool firstEntry = true;
Andy Hunga0b39712015-05-31 22:40:49 -0700731 QueueEntry *entry; // will be valid after while loop if hasEOS is set.
Wei Jiabc2fb722014-07-08 16:37:57 -0700732 while (sizeCopied < size && !mAudioQueue.empty()) {
Andy Hunga0b39712015-05-31 22:40:49 -0700733 entry = &*mAudioQueue.begin();
Wei Jiabc2fb722014-07-08 16:37:57 -0700734
735 if (entry->mBuffer == NULL) { // EOS
736 hasEOS = true;
737 mAudioQueue.erase(mAudioQueue.begin());
Wei Jiabc2fb722014-07-08 16:37:57 -0700738 break;
739 }
740
Ronghua Wu3e5efb32014-08-18 16:27:08 -0700741 if (firstEntry && entry->mOffset == 0) {
742 firstEntry = false;
Wei Jiabc2fb722014-07-08 16:37:57 -0700743 int64_t mediaTimeUs;
744 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andy Hunga0b39712015-05-31 22:40:49 -0700745 ALOGV("fillAudioBuffer: rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
Wei Jia7b15cb32015-02-03 17:46:06 -0800746 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
Wei Jiabc2fb722014-07-08 16:37:57 -0700747 }
748
749 size_t copy = entry->mBuffer->size() - entry->mOffset;
750 size_t sizeRemaining = size - sizeCopied;
751 if (copy > sizeRemaining) {
752 copy = sizeRemaining;
753 }
754
755 memcpy((char *)buffer + sizeCopied,
756 entry->mBuffer->data() + entry->mOffset,
757 copy);
758
759 entry->mOffset += copy;
760 if (entry->mOffset == entry->mBuffer->size()) {
761 entry->mNotifyConsumed->post();
762 mAudioQueue.erase(mAudioQueue.begin());
763 entry = NULL;
764 }
765 sizeCopied += copy;
Wei Jia7b15cb32015-02-03 17:46:06 -0800766
767 notifyIfMediaRenderingStarted_l();
Wei Jiabc2fb722014-07-08 16:37:57 -0700768 }
769
Lajos Molnarf5926712014-10-21 09:36:43 -0700770 if (mAudioFirstAnchorTimeMediaUs >= 0) {
771 int64_t nowUs = ALooper::GetNowUs();
Wei Jia98160162015-02-04 17:01:11 -0800772 int64_t nowMediaUs =
773 mAudioFirstAnchorTimeMediaUs + getPlayedOutAudioDurationUs(nowUs);
Wei Jia7b15cb32015-02-03 17:46:06 -0800774 // we don't know how much data we are queueing for offloaded tracks.
Wei Jia98160162015-02-04 17:01:11 -0800775 mMediaClock->updateAnchor(nowMediaUs, nowUs, INT64_MAX);
Lajos Molnarf5926712014-10-21 09:36:43 -0700776 }
777
Andy Hunga0b39712015-05-31 22:40:49 -0700778 // for non-offloaded audio, we need to compute the frames written because
779 // there is no EVENT_STREAM_END notification. The frames written gives
780 // an estimate on the pending played out duration.
781 if (!offloadingAudio()) {
782 mNumFramesWritten += sizeCopied / mAudioSink->frameSize();
Wei Jiabc2fb722014-07-08 16:37:57 -0700783 }
784
Andy Hunga0b39712015-05-31 22:40:49 -0700785 if (hasEOS) {
786 (new AMessage(kWhatStopAudioSink, this))->post();
787 // As there is currently no EVENT_STREAM_END callback notification for
788 // non-offloaded audio tracks, we need to post the EOS ourselves.
789 if (!offloadingAudio()) {
790 int64_t postEOSDelayUs = 0;
791 if (mAudioSink->needsTrailingPadding()) {
792 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
793 }
794 ALOGV("fillAudioBuffer: notifyEOS "
795 "mNumFramesWritten:%u finalResult:%d postEOSDelay:%lld",
796 mNumFramesWritten, entry->mFinalResult, (long long)postEOSDelayUs);
797 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
798 }
799 }
Wei Jiabc2fb722014-07-08 16:37:57 -0700800 return sizeCopied;
801}
802
Chong Zhang9da0ce42015-06-11 17:18:58 -0700803void NuPlayer::Renderer::drainAudioQueueUntilLastEOS() {
804 List<QueueEntry>::iterator it = mAudioQueue.begin(), itEOS = it;
805 bool foundEOS = false;
806 while (it != mAudioQueue.end()) {
807 int32_t eos;
808 QueueEntry *entry = &*it++;
809 if (entry->mBuffer == NULL
810 || (entry->mNotifyConsumed->findInt32("eos", &eos) && eos != 0)) {
811 itEOS = it;
812 foundEOS = true;
813 }
814 }
815
816 if (foundEOS) {
817 // post all replies before EOS and drop the samples
818 for (it = mAudioQueue.begin(); it != itEOS; it++) {
819 if (it->mBuffer == NULL) {
820 // delay doesn't matter as we don't even have an AudioTrack
821 notifyEOS(true /* audio */, it->mFinalResult);
822 } else {
823 it->mNotifyConsumed->post();
824 }
825 }
826 mAudioQueue.erase(mAudioQueue.begin(), itEOS);
827 }
828}
829
Andreas Huber078cfcf2011-09-15 12:25:04 -0700830bool NuPlayer::Renderer::onDrainAudioQueue() {
Wei Jia230b1882015-03-19 15:13:51 -0700831 // TODO: This call to getPosition checks if AudioTrack has been created
832 // in AudioSink before draining audio. If AudioTrack doesn't exist, then
833 // CHECKs on getPosition will fail.
834 // We still need to figure out why AudioTrack is not created when
835 // this function is called. One possible reason could be leftover
836 // audio. Another possible place is to check whether decoder
837 // has received INFO_FORMAT_CHANGED as the first buffer since
838 // AudioSink is opened there, and possible interactions with flush
839 // immediately after start. Investigate error message
840 // "vorbis_dsp_synthesis returned -135", along with RTSP.
Andreas Huber078cfcf2011-09-15 12:25:04 -0700841 uint32_t numFramesPlayed;
Andreas Huber2bfdd422011-10-11 15:24:07 -0700842 if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
Chong Zhang9da0ce42015-06-11 17:18:58 -0700843 // When getPosition fails, renderer will not reschedule the draining
844 // unless new samples are queued.
845 // If we have pending EOS (or "eos" marker for discontinuities), we need
846 // to post these now as NuPlayerDecoder might be waiting for it.
847 drainAudioQueueUntilLastEOS();
848
849 ALOGW("onDrainAudioQueue(): audio sink is not ready");
Andreas Huber2bfdd422011-10-11 15:24:07 -0700850 return false;
851 }
Andreas Huberc92fd242011-08-16 13:48:44 -0700852
Wei Jia230b1882015-03-19 15:13:51 -0700853#if 0
Andreas Huber078cfcf2011-09-15 12:25:04 -0700854 ssize_t numFramesAvailableToWrite =
855 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
856
Andreas Huber078cfcf2011-09-15 12:25:04 -0700857 if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
Steve Blockdf64d152012-01-04 20:05:49 +0000858 ALOGI("audio sink underrun");
Andreas Huber078cfcf2011-09-15 12:25:04 -0700859 } else {
Steve Block3856b092011-10-20 11:56:00 +0100860 ALOGV("audio queue has %d frames left to play",
Andreas Huber078cfcf2011-09-15 12:25:04 -0700861 mAudioSink->frameCount() - numFramesAvailableToWrite);
862 }
863#endif
864
Andy Hung005e9d02015-05-31 23:16:07 -0700865 uint32_t prevFramesWritten = mNumFramesWritten;
Wei Jia7d3f4df2015-03-03 15:28:00 -0800866 while (!mAudioQueue.empty()) {
Andreas Huberc92fd242011-08-16 13:48:44 -0700867 QueueEntry *entry = &*mAudioQueue.begin();
868
Lajos Molnard5923402014-10-20 17:47:54 -0700869 mLastAudioBufferDrained = entry->mBufferOrdinal;
870
Andreas Huberc92fd242011-08-16 13:48:44 -0700871 if (entry->mBuffer == NULL) {
872 // EOS
Ronghua Wu5095d702014-08-27 12:05:48 -0700873 int64_t postEOSDelayUs = 0;
874 if (mAudioSink->needsTrailingPadding()) {
Lajos Molnar06ad1522014-08-28 07:27:44 -0700875 postEOSDelayUs = getPendingAudioPlayoutDurationUs(ALooper::GetNowUs());
Ronghua Wu5095d702014-08-27 12:05:48 -0700876 }
877 notifyEOS(true /* audio */, entry->mFinalResult, postEOSDelayUs);
Wei Jia2995dc72015-07-24 16:00:33 -0700878 mLastAudioMediaTimeUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
Andreas Huberc92fd242011-08-16 13:48:44 -0700879
880 mAudioQueue.erase(mAudioQueue.begin());
881 entry = NULL;
Andy Hung34912322014-12-18 18:48:39 -0800882 if (mAudioSink->needsTrailingPadding()) {
883 // If we're not in gapless playback (i.e. through setNextPlayer), we
884 // need to stop the track here, because that will play out the last
885 // little bit at the end of the file. Otherwise short files won't play.
886 mAudioSink->stop();
887 mNumFramesWritten = 0;
888 }
Andreas Huber078cfcf2011-09-15 12:25:04 -0700889 return false;
Eric Laurent9b7d9502011-03-21 11:49:00 -0700890 }
891
Chong Zhang81636762015-04-21 09:33:47 -0700892 // ignore 0-sized buffer which could be EOS marker with no data
893 if (entry->mOffset == 0 && entry->mBuffer->size() > 0) {
Andreas Huberf9334412010-12-15 15:17:42 -0800894 int64_t mediaTimeUs;
895 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andy Hunga0b39712015-05-31 22:40:49 -0700896 ALOGV("onDrainAudioQueue: rendering audio at media time %.2f secs",
897 mediaTimeUs / 1E6);
Ronghua Wueecb7802014-10-19 23:12:50 -0700898 onNewAudioMediaTime(mediaTimeUs);
Andreas Huberf9334412010-12-15 15:17:42 -0800899 }
900
901 size_t copy = entry->mBuffer->size() - entry->mOffset;
Andreas Huberf9334412010-12-15 15:17:42 -0800902
Wei Jia7d3f4df2015-03-03 15:28:00 -0800903 ssize_t written = mAudioSink->write(entry->mBuffer->data() + entry->mOffset,
904 copy, false /* blocking */);
Andy Hunga31335a2014-08-20 17:37:59 -0700905 if (written < 0) {
Andy Hung202bce12014-12-03 11:47:36 -0800906 // An error in AudioSink write. Perhaps the AudioSink was not properly opened.
Wei Jia7d3f4df2015-03-03 15:28:00 -0800907 if (written == WOULD_BLOCK) {
Wei Jiadf809472015-06-09 11:21:13 -0700908 ALOGV("AudioSink write would block when writing %zu bytes", copy);
Wei Jia7d3f4df2015-03-03 15:28:00 -0800909 } else {
910 ALOGE("AudioSink write error(%zd) when writing %zu bytes", written, copy);
Ronghua Wu7665f582015-09-02 10:15:30 -0700911 // This can only happen when AudioSink was opened with doNotReconnect flag set to
912 // true, in which case the NuPlayer will handle the reconnect.
Ronghua Wufaeb0f22015-05-21 12:20:21 -0700913 notifyAudioTearDown();
Wei Jia7d3f4df2015-03-03 15:28:00 -0800914 }
Andy Hung202bce12014-12-03 11:47:36 -0800915 break;
Andy Hunga31335a2014-08-20 17:37:59 -0700916 }
Andreas Huberf9334412010-12-15 15:17:42 -0800917
Andy Hunga31335a2014-08-20 17:37:59 -0700918 entry->mOffset += written;
Andreas Huberf9334412010-12-15 15:17:42 -0800919 if (entry->mOffset == entry->mBuffer->size()) {
920 entry->mNotifyConsumed->post();
921 mAudioQueue.erase(mAudioQueue.begin());
Eric Laurent9b7d9502011-03-21 11:49:00 -0700922
Andreas Huberf9334412010-12-15 15:17:42 -0800923 entry = NULL;
924 }
925
Andy Hunga31335a2014-08-20 17:37:59 -0700926 size_t copiedFrames = written / mAudioSink->frameSize();
Andreas Huber078cfcf2011-09-15 12:25:04 -0700927 mNumFramesWritten += copiedFrames;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700928
Wei Jia7b15cb32015-02-03 17:46:06 -0800929 {
930 Mutex::Autolock autoLock(mLock);
931 notifyIfMediaRenderingStarted_l();
932 }
Andreas Huber43c3e6c2011-01-05 12:17:08 -0800933
Andy Hunga31335a2014-08-20 17:37:59 -0700934 if (written != (ssize_t)copy) {
935 // A short count was received from AudioSink::write()
936 //
Wei Jia7d3f4df2015-03-03 15:28:00 -0800937 // AudioSink write is called in non-blocking mode.
938 // It may return with a short count when:
Andy Hunga31335a2014-08-20 17:37:59 -0700939 //
940 // 1) Size to be copied is not a multiple of the frame size. We consider this fatal.
Wei Jia7d3f4df2015-03-03 15:28:00 -0800941 // 2) The data to be copied exceeds the available buffer in AudioSink.
942 // 3) An error occurs and data has been partially copied to the buffer in AudioSink.
943 // 4) AudioSink is an AudioCache for data retrieval, and the AudioCache is exceeded.
Andy Hunga31335a2014-08-20 17:37:59 -0700944
945 // (Case 1)
946 // Must be a multiple of the frame size. If it is not a multiple of a frame size, it
947 // needs to fail, as we should not carry over fractional frames between calls.
948 CHECK_EQ(copy % mAudioSink->frameSize(), 0);
949
Wei Jia7d3f4df2015-03-03 15:28:00 -0800950 // (Case 2, 3, 4)
Andy Hunga31335a2014-08-20 17:37:59 -0700951 // Return early to the caller.
952 // Beware of calling immediately again as this may busy-loop if you are not careful.
Wei Jia7d3f4df2015-03-03 15:28:00 -0800953 ALOGV("AudioSink write short frame count %zd < %zu", written, copy);
Andy Hunga31335a2014-08-20 17:37:59 -0700954 break;
955 }
956 }
Wei Jia7b15cb32015-02-03 17:46:06 -0800957 int64_t maxTimeMedia;
958 {
959 Mutex::Autolock autoLock(mLock);
960 maxTimeMedia =
961 mAnchorTimeMediaUs +
962 (int64_t)(max((long long)mNumFramesWritten - mAnchorNumFramesWritten, 0LL)
963 * 1000LL * mAudioSink->msecsPerFrame());
964 }
965 mMediaClock->updateMaxTimeMedia(maxTimeMedia);
Lajos Molnarf5926712014-10-21 09:36:43 -0700966
Andy Hung005e9d02015-05-31 23:16:07 -0700967 // calculate whether we need to reschedule another write.
968 bool reschedule = !mAudioQueue.empty()
969 && (!mPaused
970 || prevFramesWritten != mNumFramesWritten); // permit pause to fill buffers
971 //ALOGD("reschedule:%d empty:%d mPaused:%d prevFramesWritten:%u mNumFramesWritten:%u",
972 // reschedule, mAudioQueue.empty(), mPaused, prevFramesWritten, mNumFramesWritten);
973 return reschedule;
Andreas Huberf9334412010-12-15 15:17:42 -0800974}
975
Wei Jia98160162015-02-04 17:01:11 -0800976int64_t NuPlayer::Renderer::getDurationUsIfPlayedAtSampleRate(uint32_t numFrames) {
977 int32_t sampleRate = offloadingAudio() ?
978 mCurrentOffloadInfo.sample_rate : mCurrentPcmInfo.mSampleRate;
Wei Jia4d7ac852015-08-31 18:33:14 -0700979 if (sampleRate == 0) {
980 ALOGE("sampleRate is 0 in %s mode", offloadingAudio() ? "offload" : "non-offload");
981 return 0;
982 }
Wei Jia98160162015-02-04 17:01:11 -0800983 // TODO: remove the (int32_t) casting below as it may overflow at 12.4 hours.
984 return (int64_t)((int32_t)numFrames * 1000000LL / sampleRate);
985}
986
987// Calculate duration of pending samples if played at normal rate (i.e., 1.0).
Lajos Molnar06ad1522014-08-28 07:27:44 -0700988int64_t NuPlayer::Renderer::getPendingAudioPlayoutDurationUs(int64_t nowUs) {
Wei Jia98160162015-02-04 17:01:11 -0800989 int64_t writtenAudioDurationUs = getDurationUsIfPlayedAtSampleRate(mNumFramesWritten);
Lajos Molnar06ad1522014-08-28 07:27:44 -0700990 return writtenAudioDurationUs - getPlayedOutAudioDurationUs(nowUs);
Ronghua Wu5095d702014-08-27 12:05:48 -0700991}
992
Ronghua Wua73d9e02014-10-08 15:13:29 -0700993int64_t NuPlayer::Renderer::getRealTimeUs(int64_t mediaTimeUs, int64_t nowUs) {
Wei Jia98160162015-02-04 17:01:11 -0800994 int64_t realUs;
995 if (mMediaClock->getRealTimeFor(mediaTimeUs, &realUs) != OK) {
Wei Jia7b15cb32015-02-03 17:46:06 -0800996 // If failed to get current position, e.g. due to audio clock is
997 // not ready, then just play out video immediately without delay.
Ronghua Wueecb7802014-10-19 23:12:50 -0700998 return nowUs;
Ronghua Wua73d9e02014-10-08 15:13:29 -0700999 }
Wei Jia98160162015-02-04 17:01:11 -08001000 return realUs;
Ronghua Wua73d9e02014-10-08 15:13:29 -07001001}
1002
Ronghua Wueecb7802014-10-19 23:12:50 -07001003void NuPlayer::Renderer::onNewAudioMediaTime(int64_t mediaTimeUs) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001004 Mutex::Autolock autoLock(mLock);
Ronghua Wueecb7802014-10-19 23:12:50 -07001005 // TRICKY: vorbis decoder generates multiple frames with the same
1006 // timestamp, so only update on the first frame with a given timestamp
1007 if (mediaTimeUs == mAnchorTimeMediaUs) {
1008 return;
1009 }
Wei Jia7b15cb32015-02-03 17:46:06 -08001010 setAudioFirstAnchorTimeIfNeeded_l(mediaTimeUs);
Ronghua Wueecb7802014-10-19 23:12:50 -07001011 int64_t nowUs = ALooper::GetNowUs();
Wei Jia98160162015-02-04 17:01:11 -08001012 int64_t nowMediaUs = mediaTimeUs - getPendingAudioPlayoutDurationUs(nowUs);
1013 mMediaClock->updateAnchor(nowMediaUs, nowUs, mediaTimeUs);
Wei Jia8edb3f82015-03-11 11:07:45 -07001014 mAnchorNumFramesWritten = mNumFramesWritten;
Wei Jia7b15cb32015-02-03 17:46:06 -08001015 mAnchorTimeMediaUs = mediaTimeUs;
Ronghua Wueecb7802014-10-19 23:12:50 -07001016}
1017
Wei Jia7b15cb32015-02-03 17:46:06 -08001018// Called without mLock acquired.
1019void NuPlayer::Renderer::postDrainVideoQueue() {
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001020 if (mDrainVideoQueuePending
Wei Jia7b15cb32015-02-03 17:46:06 -08001021 || getSyncQueues()
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001022 || (mPaused && mVideoSampleReceived)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001023 return;
1024 }
1025
1026 if (mVideoQueue.empty()) {
1027 return;
1028 }
1029
1030 QueueEntry &entry = *mVideoQueue.begin();
1031
Lajos Molnar1d15ab52015-03-04 16:46:34 -08001032 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, this);
Wei Jia7b15cb32015-02-03 17:46:06 -08001033 msg->setInt32("drainGeneration", getDrainGeneration(false /* audio */));
Andreas Huberf9334412010-12-15 15:17:42 -08001034
Andreas Huberf9334412010-12-15 15:17:42 -08001035 if (entry.mBuffer == NULL) {
1036 // EOS doesn't carry a timestamp.
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001037 msg->post();
1038 mDrainVideoQueuePending = true;
1039 return;
1040 }
1041
1042 int64_t delayUs;
1043 int64_t nowUs = ALooper::GetNowUs();
1044 int64_t realTimeUs;
1045 if (mFlags & FLAG_REAL_TIME) {
Andreas Huberd5e56232013-03-12 11:01:43 -07001046 int64_t mediaTimeUs;
1047 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001048 realTimeUs = mediaTimeUs;
Andreas Huberf9334412010-12-15 15:17:42 -08001049 } else {
1050 int64_t mediaTimeUs;
1051 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
1052
Wei Jia7b15cb32015-02-03 17:46:06 -08001053 {
1054 Mutex::Autolock autoLock(mLock);
1055 if (mAnchorTimeMediaUs < 0) {
1056 mMediaClock->updateAnchor(mediaTimeUs, nowUs, mediaTimeUs);
1057 mAnchorTimeMediaUs = mediaTimeUs;
1058 realTimeUs = nowUs;
1059 } else {
1060 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
1061 }
Andreas Huberf9334412010-12-15 15:17:42 -08001062 }
Lajos Molnarf5926712014-10-21 09:36:43 -07001063 if (!mHasAudio) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001064 // smooth out videos >= 10fps
1065 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
Lajos Molnarf5926712014-10-21 09:36:43 -07001066 }
Lajos Molnard5923402014-10-20 17:47:54 -07001067
1068 // Heuristics to handle situation when media time changed without a
1069 // discontinuity. If we have not drained an audio buffer that was
1070 // received after this buffer, repost in 10 msec. Otherwise repost
1071 // in 500 msec.
1072 delayUs = realTimeUs - nowUs;
1073 if (delayUs > 500000) {
1074 int64_t postDelayUs = 500000;
1075 if (mHasAudio && (mLastAudioBufferDrained - entry.mBufferOrdinal) <= 0) {
1076 postDelayUs = 10000;
1077 }
1078 msg->setWhat(kWhatPostDrainVideoQueue);
1079 msg->post(postDelayUs);
1080 mVideoScheduler->restart();
1081 ALOGI("possible video time jump of %dms, retrying in %dms",
1082 (int)(delayUs / 1000), (int)(postDelayUs / 1000));
1083 mDrainVideoQueuePending = true;
1084 return;
1085 }
Andreas Huberf9334412010-12-15 15:17:42 -08001086 }
1087
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001088 realTimeUs = mVideoScheduler->schedule(realTimeUs * 1000) / 1000;
1089 int64_t twoVsyncsUs = 2 * (mVideoScheduler->getVsyncPeriod() / 1000);
1090
1091 delayUs = realTimeUs - nowUs;
1092
Lajos Molnar09524832014-07-17 14:29:51 -07001093 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001094 // post 2 display refreshes before rendering is due
1095 msg->post(delayUs > twoVsyncsUs ? delayUs - twoVsyncsUs : 0);
Andreas Huberf9334412010-12-15 15:17:42 -08001096
1097 mDrainVideoQueuePending = true;
1098}
1099
1100void NuPlayer::Renderer::onDrainVideoQueue() {
1101 if (mVideoQueue.empty()) {
1102 return;
1103 }
1104
1105 QueueEntry *entry = &*mVideoQueue.begin();
1106
1107 if (entry->mBuffer == NULL) {
1108 // EOS
1109
Andreas Huberc92fd242011-08-16 13:48:44 -07001110 notifyEOS(false /* audio */, entry->mFinalResult);
Andreas Huberf9334412010-12-15 15:17:42 -08001111
1112 mVideoQueue.erase(mVideoQueue.begin());
1113 entry = NULL;
Andreas Huber3fe62152011-09-16 15:09:22 -07001114
Ronghua Wua73d9e02014-10-08 15:13:29 -07001115 setVideoLateByUs(0);
Andreas Huberf9334412010-12-15 15:17:42 -08001116 return;
1117 }
1118
Ronghua Wua73d9e02014-10-08 15:13:29 -07001119 int64_t nowUs = -1;
Andreas Huberd5e56232013-03-12 11:01:43 -07001120 int64_t realTimeUs;
Wei Jia2995dc72015-07-24 16:00:33 -07001121 int64_t mediaTimeUs = -1;
Andreas Huberd5e56232013-03-12 11:01:43 -07001122 if (mFlags & FLAG_REAL_TIME) {
1123 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
1124 } else {
Andreas Huberd5e56232013-03-12 11:01:43 -07001125 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andreas Huberf9334412010-12-15 15:17:42 -08001126
Ronghua Wua73d9e02014-10-08 15:13:29 -07001127 nowUs = ALooper::GetNowUs();
1128 realTimeUs = getRealTimeUs(mediaTimeUs, nowUs);
Andreas Huberd5e56232013-03-12 11:01:43 -07001129 }
1130
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001131 bool tooLate = false;
Andreas Huber3fe62152011-09-16 15:09:22 -07001132
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001133 if (!mPaused) {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001134 if (nowUs == -1) {
1135 nowUs = ALooper::GetNowUs();
1136 }
1137 setVideoLateByUs(nowUs - realTimeUs);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001138 tooLate = (mVideoLateByUs > 40000);
1139
1140 if (tooLate) {
1141 ALOGV("video late by %lld us (%.2f secs)",
Lajos Molnar6d339f12015-04-17 16:15:53 -07001142 (long long)mVideoLateByUs, mVideoLateByUs / 1E6);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001143 } else {
Wei Jia98160162015-02-04 17:01:11 -08001144 int64_t mediaUs = 0;
1145 mMediaClock->getMediaTime(realTimeUs, &mediaUs);
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001146 ALOGV("rendering video at media time %.2f secs",
1147 (mFlags & FLAG_REAL_TIME ? realTimeUs :
Wei Jia98160162015-02-04 17:01:11 -08001148 mediaUs) / 1E6);
Wei Jia2995dc72015-07-24 16:00:33 -07001149
1150 if (!(mFlags & FLAG_REAL_TIME)
1151 && mLastAudioMediaTimeUs != -1
1152 && mediaTimeUs > mLastAudioMediaTimeUs) {
1153 // If audio ends before video, video continues to drive media clock.
1154 // Also smooth out videos >= 10fps.
1155 mMediaClock->updateMaxTimeMedia(mediaTimeUs + 100000);
1156 }
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001157 }
Andreas Huber078cfcf2011-09-15 12:25:04 -07001158 } else {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001159 setVideoLateByUs(0);
Ronghua Wueecb7802014-10-19 23:12:50 -07001160 if (!mVideoSampleReceived && !mHasAudio) {
Ronghua Wua73d9e02014-10-08 15:13:29 -07001161 // This will ensure that the first frame after a flush won't be used as anchor
1162 // when renderer is in paused state, because resume can happen any time after seek.
Wei Jia7b15cb32015-02-03 17:46:06 -08001163 Mutex::Autolock autoLock(mLock);
1164 clearAnchorTime_l();
Wei Jia49966ff2014-10-08 18:44:45 -07001165 }
Andreas Huber078cfcf2011-09-15 12:25:04 -07001166 }
Andreas Huberf9334412010-12-15 15:17:42 -08001167
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001168 entry->mNotifyConsumed->setInt64("timestampNs", realTimeUs * 1000ll);
Glenn Kasten683525b2011-11-04 18:05:35 -07001169 entry->mNotifyConsumed->setInt32("render", !tooLate);
Andreas Huberf9334412010-12-15 15:17:42 -08001170 entry->mNotifyConsumed->post();
1171 mVideoQueue.erase(mVideoQueue.begin());
1172 entry = NULL;
Andreas Huber43c3e6c2011-01-05 12:17:08 -08001173
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001174 mVideoSampleReceived = true;
James Dongf57b4ea2012-07-20 13:38:36 -07001175
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001176 if (!mPaused) {
1177 if (!mVideoRenderingStarted) {
1178 mVideoRenderingStarted = true;
1179 notifyVideoRenderingStart();
1180 }
Wei Jia7b15cb32015-02-03 17:46:06 -08001181 Mutex::Autolock autoLock(mLock);
1182 notifyIfMediaRenderingStarted_l();
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001183 }
Andreas Huberf9334412010-12-15 15:17:42 -08001184}
1185
James Dongf57b4ea2012-07-20 13:38:36 -07001186void NuPlayer::Renderer::notifyVideoRenderingStart() {
1187 sp<AMessage> notify = mNotify->dup();
1188 notify->setInt32("what", kWhatVideoRenderingStart);
1189 notify->post();
1190}
1191
Ronghua Wu5095d702014-08-27 12:05:48 -07001192void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult, int64_t delayUs) {
Wei Jia7c8d0e02015-08-27 17:40:21 -07001193 if (audio && delayUs > 0) {
1194 sp<AMessage> msg = new AMessage(kWhatEOS, this);
1195 msg->setInt32("audioEOSGeneration", mAudioEOSGeneration);
1196 msg->setInt32("finalResult", finalResult);
1197 msg->post(delayUs);
1198 return;
1199 }
Andreas Huberf9334412010-12-15 15:17:42 -08001200 sp<AMessage> notify = mNotify->dup();
1201 notify->setInt32("what", kWhatEOS);
1202 notify->setInt32("audio", static_cast<int32_t>(audio));
Andreas Huberc92fd242011-08-16 13:48:44 -07001203 notify->setInt32("finalResult", finalResult);
Ronghua Wu5095d702014-08-27 12:05:48 -07001204 notify->post(delayUs);
Andreas Huberf9334412010-12-15 15:17:42 -08001205}
1206
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001207void NuPlayer::Renderer::notifyAudioTearDown() {
1208 (new AMessage(kWhatAudioTearDown, this))->post();
Wei Jia3a2956d2014-07-22 16:01:33 -07001209}
1210
Andreas Huberf9334412010-12-15 15:17:42 -08001211void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
1212 int32_t audio;
1213 CHECK(msg->findInt32("audio", &audio));
1214
Wei Jia7b15cb32015-02-03 17:46:06 -08001215 if (dropBufferIfStale(audio, msg)) {
1216 return;
1217 }
1218
1219 if (audio) {
1220 mHasAudio = true;
1221 } else {
1222 mHasVideo = true;
1223 }
Ronghua Wua73d9e02014-10-08 15:13:29 -07001224
1225 if (mHasVideo) {
Lajos Molnardc43dfa2014-05-07 15:33:04 -07001226 if (mVideoScheduler == NULL) {
1227 mVideoScheduler = new VideoFrameScheduler();
1228 mVideoScheduler->init();
1229 }
Andreas Huberbc7f5b22011-01-21 10:15:23 -08001230 }
1231
Andreas Huber2d8bedd2012-02-21 14:38:23 -08001232 sp<ABuffer> buffer;
1233 CHECK(msg->findBuffer("buffer", &buffer));
Andreas Huberf9334412010-12-15 15:17:42 -08001234
1235 sp<AMessage> notifyConsumed;
1236 CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
1237
1238 QueueEntry entry;
1239 entry.mBuffer = buffer;
1240 entry.mNotifyConsumed = notifyConsumed;
1241 entry.mOffset = 0;
1242 entry.mFinalResult = OK;
Lajos Molnard5923402014-10-20 17:47:54 -07001243 entry.mBufferOrdinal = ++mTotalBuffersQueued;
Andreas Huberf9334412010-12-15 15:17:42 -08001244
1245 if (audio) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001246 Mutex::Autolock autoLock(mLock);
Andreas Huberf9334412010-12-15 15:17:42 -08001247 mAudioQueue.push_back(entry);
Wei Jiabc2fb722014-07-08 16:37:57 -07001248 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001249 } else {
1250 mVideoQueue.push_back(entry);
Wei Jia7b15cb32015-02-03 17:46:06 -08001251 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -08001252 }
1253
Wei Jia7b15cb32015-02-03 17:46:06 -08001254 Mutex::Autolock autoLock(mLock);
Andreas Hubercb67cd12011-08-26 16:02:19 -07001255 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
1256 return;
Andreas Huberf9334412010-12-15 15:17:42 -08001257 }
Andreas Hubercb67cd12011-08-26 16:02:19 -07001258
1259 sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
1260 sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
1261
1262 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
1263 // EOS signalled on either queue.
Wei Jiabc2fb722014-07-08 16:37:57 -07001264 syncQueuesDone_l();
Andreas Hubercb67cd12011-08-26 16:02:19 -07001265 return;
1266 }
1267
1268 int64_t firstAudioTimeUs;
1269 int64_t firstVideoTimeUs;
1270 CHECK(firstAudioBuffer->meta()
1271 ->findInt64("timeUs", &firstAudioTimeUs));
1272 CHECK(firstVideoBuffer->meta()
1273 ->findInt64("timeUs", &firstVideoTimeUs));
1274
1275 int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
1276
Steve Block3856b092011-10-20 11:56:00 +01001277 ALOGV("queueDiff = %.2f secs", diff / 1E6);
Andreas Hubercb67cd12011-08-26 16:02:19 -07001278
1279 if (diff > 100000ll) {
1280 // Audio data starts More than 0.1 secs before video.
1281 // Drop some audio.
1282
1283 (*mAudioQueue.begin()).mNotifyConsumed->post();
1284 mAudioQueue.erase(mAudioQueue.begin());
1285 return;
1286 }
1287
Wei Jiabc2fb722014-07-08 16:37:57 -07001288 syncQueuesDone_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001289}
1290
Wei Jiabc2fb722014-07-08 16:37:57 -07001291void NuPlayer::Renderer::syncQueuesDone_l() {
Andreas Huberf9334412010-12-15 15:17:42 -08001292 if (!mSyncQueues) {
1293 return;
1294 }
1295
1296 mSyncQueues = false;
1297
1298 if (!mAudioQueue.empty()) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001299 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001300 }
1301
1302 if (!mVideoQueue.empty()) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001303 mLock.unlock();
1304 postDrainVideoQueue();
1305 mLock.lock();
Andreas Huberf9334412010-12-15 15:17:42 -08001306 }
1307}
1308
1309void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
1310 int32_t audio;
1311 CHECK(msg->findInt32("audio", &audio));
1312
Wei Jia7b15cb32015-02-03 17:46:06 -08001313 if (dropBufferIfStale(audio, msg)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001314 return;
1315 }
1316
1317 int32_t finalResult;
1318 CHECK(msg->findInt32("finalResult", &finalResult));
1319
1320 QueueEntry entry;
1321 entry.mOffset = 0;
1322 entry.mFinalResult = finalResult;
1323
1324 if (audio) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001325 Mutex::Autolock autoLock(mLock);
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001326 if (mAudioQueue.empty() && mSyncQueues) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001327 syncQueuesDone_l();
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001328 }
Andreas Huberf9334412010-12-15 15:17:42 -08001329 mAudioQueue.push_back(entry);
Wei Jiabc2fb722014-07-08 16:37:57 -07001330 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001331 } else {
Wei Jia7b15cb32015-02-03 17:46:06 -08001332 if (mVideoQueue.empty() && getSyncQueues()) {
1333 Mutex::Autolock autoLock(mLock);
Wei Jiabc2fb722014-07-08 16:37:57 -07001334 syncQueuesDone_l();
Roger Jönssonb50e83e2013-01-21 16:26:41 +01001335 }
Andreas Huberf9334412010-12-15 15:17:42 -08001336 mVideoQueue.push_back(entry);
Wei Jia7b15cb32015-02-03 17:46:06 -08001337 postDrainVideoQueue();
Andreas Huberf9334412010-12-15 15:17:42 -08001338 }
1339}
1340
1341void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
Chong Zhang7137ec72014-11-12 16:41:05 -08001342 int32_t audio, notifyComplete;
Andreas Huberf9334412010-12-15 15:17:42 -08001343 CHECK(msg->findInt32("audio", &audio));
1344
Wei Jia28a8a9f2014-08-18 11:29:50 -07001345 {
Wei Jia7b15cb32015-02-03 17:46:06 -08001346 Mutex::Autolock autoLock(mLock);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001347 if (audio) {
Chong Zhang7137ec72014-11-12 16:41:05 -08001348 notifyComplete = mNotifyCompleteAudio;
1349 mNotifyCompleteAudio = false;
Wei Jia2995dc72015-07-24 16:00:33 -07001350 mLastAudioMediaTimeUs = -1;
Wei Jia28a8a9f2014-08-18 11:29:50 -07001351 } else {
Chong Zhang7137ec72014-11-12 16:41:05 -08001352 notifyComplete = mNotifyCompleteVideo;
1353 mNotifyCompleteVideo = false;
Wei Jia28a8a9f2014-08-18 11:29:50 -07001354 }
Wei Jia28a8a9f2014-08-18 11:29:50 -07001355
Wei Jia7b15cb32015-02-03 17:46:06 -08001356 // If we're currently syncing the queues, i.e. dropping audio while
1357 // aligning the first audio/video buffer times and only one of the
1358 // two queues has data, we may starve that queue by not requesting
1359 // more buffers from the decoder. If the other source then encounters
1360 // a discontinuity that leads to flushing, we'll never find the
1361 // corresponding discontinuity on the other queue.
1362 // Therefore we'll stop syncing the queues if at least one of them
1363 // is flushed.
1364 syncQueuesDone_l();
1365 clearAnchorTime_l();
Wei Jiabc2fb722014-07-08 16:37:57 -07001366 }
Andreas Huberf9334412010-12-15 15:17:42 -08001367
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001368 ALOGV("flushing %s", audio ? "audio" : "video");
Andreas Huberf9334412010-12-15 15:17:42 -08001369 if (audio) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001370 {
1371 Mutex::Autolock autoLock(mLock);
1372 flushQueue(&mAudioQueue);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001373
Wei Jia7b15cb32015-02-03 17:46:06 -08001374 ++mAudioDrainGeneration;
Wei Jia7c8d0e02015-08-27 17:40:21 -07001375 ++mAudioEOSGeneration;
Wei Jia7b15cb32015-02-03 17:46:06 -08001376 prepareForMediaRenderingStart_l();
Wei Jia28a8a9f2014-08-18 11:29:50 -07001377
Andy Hunga0b39712015-05-31 22:40:49 -07001378 // the frame count will be reset after flush.
1379 clearAudioFirstAnchorTime_l();
Wei Jiabc2fb722014-07-08 16:37:57 -07001380 }
Andreas Huberf9334412010-12-15 15:17:42 -08001381
Andreas Huberf9334412010-12-15 15:17:42 -08001382 mDrainAudioQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001383
Wei Jiabc2fb722014-07-08 16:37:57 -07001384 if (offloadingAudio()) {
Wei Jiabc2fb722014-07-08 16:37:57 -07001385 mAudioSink->pause();
1386 mAudioSink->flush();
Andy Hung85e48142015-05-31 23:04:15 -07001387 if (!mPaused) {
1388 mAudioSink->start();
1389 }
Wei Jia9e7ed332015-05-01 16:35:43 -07001390 } else {
1391 mAudioSink->pause();
1392 mAudioSink->flush();
1393 // Call stop() to signal to the AudioSink to completely fill the
1394 // internal buffer before resuming playback.
Andy Hungb03dcb32015-08-27 16:18:59 -07001395 // FIXME: this is ignored after flush().
Wei Jia9e7ed332015-05-01 16:35:43 -07001396 mAudioSink->stop();
Andy Hungb03dcb32015-08-27 16:18:59 -07001397 if (mPaused) {
1398 // Race condition: if renderer is paused and audio sink is stopped,
1399 // we need to make sure that the audio track buffer fully drains
1400 // before delivering data.
1401 // FIXME: remove this if we can detect if stop() is complete.
1402 const int delayUs = 2 * 50 * 1000; // (2 full mixer thread cycles at 50ms)
1403 mPauseDrainAudioAllowedUs = ALooper::GetNowUs() + delayUs;
1404 } else {
Wei Jia9e7ed332015-05-01 16:35:43 -07001405 mAudioSink->start();
1406 }
1407 mNumFramesWritten = 0;
Wei Jiabc2fb722014-07-08 16:37:57 -07001408 }
Andreas Huberf9334412010-12-15 15:17:42 -08001409 } else {
1410 flushQueue(&mVideoQueue);
1411
Andreas Huberf9334412010-12-15 15:17:42 -08001412 mDrainVideoQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001413
Lajos Molnarc851b5d2014-09-18 14:14:29 -07001414 if (mVideoScheduler != NULL) {
1415 mVideoScheduler->restart();
1416 }
1417
Wei Jia7b15cb32015-02-03 17:46:06 -08001418 Mutex::Autolock autoLock(mLock);
1419 ++mVideoDrainGeneration;
1420 prepareForMediaRenderingStart_l();
Andreas Huberf9334412010-12-15 15:17:42 -08001421 }
1422
Chong Zhangfbe8bef2014-08-29 18:34:17 -07001423 mVideoSampleReceived = false;
Chong Zhang7137ec72014-11-12 16:41:05 -08001424
1425 if (notifyComplete) {
1426 notifyFlushComplete(audio);
1427 }
Andreas Huberf9334412010-12-15 15:17:42 -08001428}
1429
1430void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
1431 while (!queue->empty()) {
1432 QueueEntry *entry = &*queue->begin();
1433
1434 if (entry->mBuffer != NULL) {
1435 entry->mNotifyConsumed->post();
1436 }
1437
1438 queue->erase(queue->begin());
1439 entry = NULL;
1440 }
1441}
1442
1443void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
1444 sp<AMessage> notify = mNotify->dup();
1445 notify->setInt32("what", kWhatFlushComplete);
1446 notify->setInt32("audio", static_cast<int32_t>(audio));
1447 notify->post();
1448}
1449
Wei Jia7b15cb32015-02-03 17:46:06 -08001450bool NuPlayer::Renderer::dropBufferIfStale(
Andreas Huberf9334412010-12-15 15:17:42 -08001451 bool audio, const sp<AMessage> &msg) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001452 int32_t queueGeneration;
1453 CHECK(msg->findInt32("queueGeneration", &queueGeneration));
Andreas Huberf9334412010-12-15 15:17:42 -08001454
Wei Jia7b15cb32015-02-03 17:46:06 -08001455 if (queueGeneration == getQueueGeneration(audio)) {
Andreas Huberf9334412010-12-15 15:17:42 -08001456 return false;
1457 }
1458
1459 sp<AMessage> notifyConsumed;
1460 if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
1461 notifyConsumed->post();
1462 }
1463
1464 return true;
1465}
1466
Andreas Huber3831a062010-12-21 10:22:33 -08001467void NuPlayer::Renderer::onAudioSinkChanged() {
Wei Jiabc2fb722014-07-08 16:37:57 -07001468 if (offloadingAudio()) {
1469 return;
1470 }
Andreas Huber3831a062010-12-21 10:22:33 -08001471 CHECK(!mDrainAudioQueuePending);
1472 mNumFramesWritten = 0;
Wei Jia7b15cb32015-02-03 17:46:06 -08001473 {
1474 Mutex::Autolock autoLock(mLock);
1475 mAnchorNumFramesWritten = -1;
1476 }
Marco Nelissen4110c102012-03-29 09:31:28 -07001477 uint32_t written;
1478 if (mAudioSink->getFramesWritten(&written) == OK) {
1479 mNumFramesWritten = written;
1480 }
Andreas Huber3831a062010-12-21 10:22:33 -08001481}
1482
Wei Jiabc2fb722014-07-08 16:37:57 -07001483void NuPlayer::Renderer::onDisableOffloadAudio() {
1484 Mutex::Autolock autoLock(mLock);
1485 mFlags &= ~FLAG_OFFLOAD_AUDIO;
Wei Jia7b15cb32015-02-03 17:46:06 -08001486 ++mAudioDrainGeneration;
Robert Shihe1d70192015-07-23 17:54:13 -07001487 if (mAudioRenderingStartGeneration != -1) {
1488 prepareForMediaRenderingStart_l();
1489 }
Wei Jiabc2fb722014-07-08 16:37:57 -07001490}
1491
Ronghua Wua10fd232014-11-06 16:15:20 -08001492void NuPlayer::Renderer::onEnableOffloadAudio() {
1493 Mutex::Autolock autoLock(mLock);
1494 mFlags |= FLAG_OFFLOAD_AUDIO;
Wei Jia7b15cb32015-02-03 17:46:06 -08001495 ++mAudioDrainGeneration;
Robert Shihe1d70192015-07-23 17:54:13 -07001496 if (mAudioRenderingStartGeneration != -1) {
1497 prepareForMediaRenderingStart_l();
1498 }
Ronghua Wua10fd232014-11-06 16:15:20 -08001499}
1500
Andreas Huberb4082222011-01-20 15:23:04 -08001501void NuPlayer::Renderer::onPause() {
Rachad8592dbb2014-09-09 13:10:28 -07001502 if (mPaused) {
Rachad8592dbb2014-09-09 13:10:28 -07001503 return;
1504 }
Lajos Molnar6d339f12015-04-17 16:15:53 -07001505
Wei Jia28a8a9f2014-08-18 11:29:50 -07001506 {
1507 Mutex::Autolock autoLock(mLock);
Andy Hung005e9d02015-05-31 23:16:07 -07001508 // we do not increment audio drain generation so that we fill audio buffer during pause.
Wei Jia7b15cb32015-02-03 17:46:06 -08001509 ++mVideoDrainGeneration;
1510 prepareForMediaRenderingStart_l();
Wei Jia73ddd212014-08-29 16:33:49 -07001511 mPaused = true;
Wei Jia98160162015-02-04 17:01:11 -08001512 mMediaClock->setPlaybackRate(0.0);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001513 }
1514
Andreas Huberb4082222011-01-20 15:23:04 -08001515 mDrainAudioQueuePending = false;
Andreas Huberb4082222011-01-20 15:23:04 -08001516 mDrainVideoQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -07001517
Andreas Huberb4082222011-01-20 15:23:04 -08001518 if (mHasAudio) {
1519 mAudioSink->pause();
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001520 startAudioOffloadPauseTimeout();
Andreas Huberb4082222011-01-20 15:23:04 -08001521 }
1522
Lajos Molnar6d339f12015-04-17 16:15:53 -07001523 ALOGV("now paused audio queue has %zu entries, video has %zu entries",
Andreas Huberea9d51b2011-11-30 09:53:40 -08001524 mAudioQueue.size(), mVideoQueue.size());
Andreas Huberb4082222011-01-20 15:23:04 -08001525}
1526
1527void NuPlayer::Renderer::onResume() {
Andreas Huberb58ce9f2011-11-28 16:27:35 -08001528 if (!mPaused) {
1529 return;
1530 }
Andreas Huberb4082222011-01-20 15:23:04 -08001531
1532 if (mHasAudio) {
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001533 cancelAudioOffloadPauseTimeout();
Eric Laurent97c9f4f2015-08-11 18:04:14 -07001534 status_t err = mAudioSink->start();
1535 if (err != OK) {
1536 notifyAudioTearDown();
1537 }
Andreas Huberb4082222011-01-20 15:23:04 -08001538 }
1539
Wei Jia7b15cb32015-02-03 17:46:06 -08001540 {
1541 Mutex::Autolock autoLock(mLock);
1542 mPaused = false;
Andy Hungb12ea0b2015-09-03 15:13:01 -07001543 // rendering started message may have been delayed if we were paused.
1544 if (mRenderingDataDelivered) {
1545 notifyIfMediaRenderingStarted_l();
1546 }
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001547 // configure audiosink as we did not do it when pausing
Wei Jia27ea08e2015-05-12 14:50:35 -07001548 if (mAudioSink != NULL && mAudioSink->ready()) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001549 mAudioSink->setPlaybackRate(mPlaybackSettings);
1550 }
1551
Wei Jia98160162015-02-04 17:01:11 -08001552 mMediaClock->setPlaybackRate(mPlaybackRate);
Andreas Huberb4082222011-01-20 15:23:04 -08001553
Wei Jia7b15cb32015-02-03 17:46:06 -08001554 if (!mAudioQueue.empty()) {
1555 postDrainAudioQueue_l();
1556 }
Andreas Huberb4082222011-01-20 15:23:04 -08001557 }
1558
1559 if (!mVideoQueue.empty()) {
Wei Jia7b15cb32015-02-03 17:46:06 -08001560 postDrainVideoQueue();
Andreas Huberb4082222011-01-20 15:23:04 -08001561 }
1562}
1563
Lajos Molnarc851b5d2014-09-18 14:14:29 -07001564void NuPlayer::Renderer::onSetVideoFrameRate(float fps) {
1565 if (mVideoScheduler == NULL) {
1566 mVideoScheduler = new VideoFrameScheduler();
1567 }
1568 mVideoScheduler->init(fps);
1569}
1570
Wei Jia7b15cb32015-02-03 17:46:06 -08001571int32_t NuPlayer::Renderer::getQueueGeneration(bool audio) {
1572 Mutex::Autolock autoLock(mLock);
1573 return (audio ? mAudioQueueGeneration : mVideoQueueGeneration);
1574}
1575
1576int32_t NuPlayer::Renderer::getDrainGeneration(bool audio) {
1577 Mutex::Autolock autoLock(mLock);
1578 return (audio ? mAudioDrainGeneration : mVideoDrainGeneration);
1579}
1580
1581bool NuPlayer::Renderer::getSyncQueues() {
1582 Mutex::Autolock autoLock(mLock);
1583 return mSyncQueues;
1584}
1585
Andy Hung09e0c362014-09-12 15:12:16 -07001586// TODO: Remove unnecessary calls to getPlayedOutAudioDurationUs()
1587// as it acquires locks and may query the audio driver.
1588//
Andy Hung09e0c362014-09-12 15:12:16 -07001589// Some calls could conceivably retrieve extrapolated data instead of
1590// accessing getTimestamp() or getPosition() every time a data buffer with
1591// a media time is received.
1592//
Wei Jia98160162015-02-04 17:01:11 -08001593// Calculate duration of played samples if played at normal rate (i.e., 1.0).
Lajos Molnar06ad1522014-08-28 07:27:44 -07001594int64_t NuPlayer::Renderer::getPlayedOutAudioDurationUs(int64_t nowUs) {
Wei Jia3a2956d2014-07-22 16:01:33 -07001595 uint32_t numFramesPlayed;
Lajos Molnar06ad1522014-08-28 07:27:44 -07001596 int64_t numFramesPlayedAt;
1597 AudioTimestamp ts;
Andy Hung09e0c362014-09-12 15:12:16 -07001598 static const int64_t kStaleTimestamp100ms = 100000;
1599
Lajos Molnar06ad1522014-08-28 07:27:44 -07001600 status_t res = mAudioSink->getTimestamp(ts);
Andy Hung09e0c362014-09-12 15:12:16 -07001601 if (res == OK) { // case 1: mixing audio tracks and offloaded tracks.
Lajos Molnar06ad1522014-08-28 07:27:44 -07001602 numFramesPlayed = ts.mPosition;
1603 numFramesPlayedAt =
1604 ts.mTime.tv_sec * 1000000LL + ts.mTime.tv_nsec / 1000;
Andy Hung09e0c362014-09-12 15:12:16 -07001605 const int64_t timestampAge = nowUs - numFramesPlayedAt;
1606 if (timestampAge > kStaleTimestamp100ms) {
1607 // This is an audio FIXME.
1608 // getTimestamp returns a timestamp which may come from audio mixing threads.
1609 // After pausing, the MixerThread may go idle, thus the mTime estimate may
1610 // become stale. Assuming that the MixerThread runs 20ms, with FastMixer at 5ms,
1611 // the max latency should be about 25ms with an average around 12ms (to be verified).
1612 // For safety we use 100ms.
Andy Hung2abde2c2014-09-30 14:40:32 -07001613 ALOGV("getTimestamp: returned stale timestamp nowUs(%lld) numFramesPlayedAt(%lld)",
Andy Hung09e0c362014-09-12 15:12:16 -07001614 (long long)nowUs, (long long)numFramesPlayedAt);
1615 numFramesPlayedAt = nowUs - kStaleTimestamp100ms;
1616 }
1617 //ALOGD("getTimestamp: OK %d %lld", numFramesPlayed, (long long)numFramesPlayedAt);
1618 } else if (res == WOULD_BLOCK) { // case 2: transitory state on start of a new track
1619 numFramesPlayed = 0;
1620 numFramesPlayedAt = nowUs;
1621 //ALOGD("getTimestamp: WOULD_BLOCK %d %lld",
1622 // numFramesPlayed, (long long)numFramesPlayedAt);
1623 } else { // case 3: transitory at new track or audio fast tracks.
Lajos Molnar06ad1522014-08-28 07:27:44 -07001624 res = mAudioSink->getPosition(&numFramesPlayed);
1625 CHECK_EQ(res, (status_t)OK);
1626 numFramesPlayedAt = nowUs;
1627 numFramesPlayedAt += 1000LL * mAudioSink->latency() / 2; /* XXX */
Lajos Molnar6d339f12015-04-17 16:15:53 -07001628 //ALOGD("getPosition: %u %lld", numFramesPlayed, (long long)numFramesPlayedAt);
Lajos Molnar06ad1522014-08-28 07:27:44 -07001629 }
Andy Hung09e0c362014-09-12 15:12:16 -07001630
Andy Hung09e0c362014-09-12 15:12:16 -07001631 //CHECK_EQ(numFramesPlayed & (1 << 31), 0); // can't be negative until 12.4 hrs, test
Wei Jia98160162015-02-04 17:01:11 -08001632 int64_t durationUs = getDurationUsIfPlayedAtSampleRate(numFramesPlayed)
Lajos Molnar06ad1522014-08-28 07:27:44 -07001633 + nowUs - numFramesPlayedAt;
Andy Hung09e0c362014-09-12 15:12:16 -07001634 if (durationUs < 0) {
1635 // Occurs when numFramesPlayed position is very small and the following:
1636 // (1) In case 1, the time nowUs is computed before getTimestamp() is called and
1637 // numFramesPlayedAt is greater than nowUs by time more than numFramesPlayed.
1638 // (2) In case 3, using getPosition and adding mAudioSink->latency() to
1639 // numFramesPlayedAt, by a time amount greater than numFramesPlayed.
1640 //
1641 // Both of these are transitory conditions.
Andy Hung2abde2c2014-09-30 14:40:32 -07001642 ALOGV("getPlayedOutAudioDurationUs: negative duration %lld set to zero", (long long)durationUs);
Andy Hung09e0c362014-09-12 15:12:16 -07001643 durationUs = 0;
1644 }
1645 ALOGV("getPlayedOutAudioDurationUs(%lld) nowUs(%lld) frames(%u) framesAt(%lld)",
1646 (long long)durationUs, (long long)nowUs, numFramesPlayed, (long long)numFramesPlayedAt);
1647 return durationUs;
Lajos Molnar06ad1522014-08-28 07:27:44 -07001648}
Wei Jia3a2956d2014-07-22 16:01:33 -07001649
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001650void NuPlayer::Renderer::onAudioTearDown(AudioTearDownReason reason) {
1651 if (mAudioTornDown) {
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001652 return;
1653 }
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001654 mAudioTornDown = true;
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001655
Ronghua Wua73d9e02014-10-08 15:13:29 -07001656 int64_t currentPositionUs;
Robert Shih1a5c8592015-08-04 18:07:44 -07001657 sp<AMessage> notify = mNotify->dup();
1658 if (getCurrentPosition(&currentPositionUs) == OK) {
1659 notify->setInt64("positionUs", currentPositionUs);
Wei Jia28a8a9f2014-08-18 11:29:50 -07001660 }
Lajos Molnar06ad1522014-08-28 07:27:44 -07001661
Wei Jia3a2956d2014-07-22 16:01:33 -07001662 mAudioSink->stop();
1663 mAudioSink->flush();
1664
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001665 notify->setInt32("what", kWhatAudioTearDown);
Ronghua Wu08529172014-10-02 16:55:52 -07001666 notify->setInt32("reason", reason);
Wei Jia3a2956d2014-07-22 16:01:33 -07001667 notify->post();
1668}
1669
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001670void NuPlayer::Renderer::startAudioOffloadPauseTimeout() {
1671 if (offloadingAudio()) {
Weiyin Jiang35d5af12015-01-28 16:14:02 +08001672 mWakeLock->acquire();
Lajos Molnar1d15ab52015-03-04 16:46:34 -08001673 sp<AMessage> msg = new AMessage(kWhatAudioOffloadPauseTimeout, this);
Wei Jia7b15cb32015-02-03 17:46:06 -08001674 msg->setInt32("drainGeneration", mAudioOffloadPauseTimeoutGeneration);
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001675 msg->post(kOffloadPauseMaxUs);
1676 }
1677}
1678
1679void NuPlayer::Renderer::cancelAudioOffloadPauseTimeout() {
1680 if (offloadingAudio()) {
Weiyin Jiang35d5af12015-01-28 16:14:02 +08001681 mWakeLock->release(true);
Ronghua Wuf5b1db12014-09-09 10:11:08 -07001682 ++mAudioOffloadPauseTimeoutGeneration;
1683 }
1684}
1685
Andy Hung202bce12014-12-03 11:47:36 -08001686status_t NuPlayer::Renderer::onOpenAudioSink(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001687 const sp<AMessage> &format,
1688 bool offloadOnly,
1689 bool hasVideo,
1690 uint32_t flags) {
1691 ALOGV("openAudioSink: offloadOnly(%d) offloadingAudio(%d)",
1692 offloadOnly, offloadingAudio());
1693 bool audioSinkChanged = false;
1694
1695 int32_t numChannels;
1696 CHECK(format->findInt32("channel-count", &numChannels));
1697
1698 int32_t channelMask;
1699 if (!format->findInt32("channel-mask", &channelMask)) {
1700 // signal to the AudioSink to derive the mask from count.
1701 channelMask = CHANNEL_MASK_USE_CHANNEL_ORDER;
1702 }
1703
1704 int32_t sampleRate;
1705 CHECK(format->findInt32("sample-rate", &sampleRate));
1706
1707 if (offloadingAudio()) {
1708 audio_format_t audioFormat = AUDIO_FORMAT_PCM_16_BIT;
1709 AString mime;
1710 CHECK(format->findString("mime", &mime));
1711 status_t err = mapMimeToAudioFormat(audioFormat, mime.c_str());
1712
1713 if (err != OK) {
1714 ALOGE("Couldn't map mime \"%s\" to a valid "
1715 "audio_format", mime.c_str());
1716 onDisableOffloadAudio();
1717 } else {
1718 ALOGV("Mime \"%s\" mapped to audio_format 0x%x",
1719 mime.c_str(), audioFormat);
1720
1721 int avgBitRate = -1;
1722 format->findInt32("bit-rate", &avgBitRate);
1723
1724 int32_t aacProfile = -1;
1725 if (audioFormat == AUDIO_FORMAT_AAC
1726 && format->findInt32("aac-profile", &aacProfile)) {
1727 // Redefine AAC format as per aac profile
1728 mapAACProfileToAudioFormat(
1729 audioFormat,
1730 aacProfile);
1731 }
1732
1733 audio_offload_info_t offloadInfo = AUDIO_INFO_INITIALIZER;
1734 offloadInfo.duration_us = -1;
1735 format->findInt64(
1736 "durationUs", &offloadInfo.duration_us);
1737 offloadInfo.sample_rate = sampleRate;
1738 offloadInfo.channel_mask = channelMask;
1739 offloadInfo.format = audioFormat;
1740 offloadInfo.stream_type = AUDIO_STREAM_MUSIC;
1741 offloadInfo.bit_rate = avgBitRate;
1742 offloadInfo.has_video = hasVideo;
1743 offloadInfo.is_streaming = true;
1744
1745 if (memcmp(&mCurrentOffloadInfo, &offloadInfo, sizeof(offloadInfo)) == 0) {
1746 ALOGV("openAudioSink: no change in offload mode");
1747 // no change from previous configuration, everything ok.
Andy Hung202bce12014-12-03 11:47:36 -08001748 return OK;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001749 }
Andy Hungf0e83642014-12-19 17:55:56 -08001750 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
1751
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001752 ALOGV("openAudioSink: try to open AudioSink in offload mode");
Eric Laurentd88c3ca2014-10-28 10:52:11 -07001753 uint32_t offloadFlags = flags;
1754 offloadFlags |= AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
1755 offloadFlags &= ~AUDIO_OUTPUT_FLAG_DEEP_BUFFER;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001756 audioSinkChanged = true;
1757 mAudioSink->close();
Andy Hunga0b39712015-05-31 22:40:49 -07001758
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001759 err = mAudioSink->open(
1760 sampleRate,
1761 numChannels,
1762 (audio_channel_mask_t)channelMask,
1763 audioFormat,
Andy Hung179652e2015-05-31 22:49:46 -07001764 0 /* bufferCount - unused */,
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001765 &NuPlayer::Renderer::AudioSinkCallback,
1766 this,
Eric Laurentd88c3ca2014-10-28 10:52:11 -07001767 (audio_output_flags_t)offloadFlags,
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001768 &offloadInfo);
1769
1770 if (err == OK) {
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001771 err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1772 }
1773
1774 if (err == OK) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001775 // If the playback is offloaded to h/w, we pass
1776 // the HAL some metadata information.
1777 // We don't want to do this for PCM because it
1778 // will be going through the AudioFlinger mixer
1779 // before reaching the hardware.
1780 // TODO
1781 mCurrentOffloadInfo = offloadInfo;
Andy Hung85e48142015-05-31 23:04:15 -07001782 if (!mPaused) { // for preview mode, don't start if paused
1783 err = mAudioSink->start();
1784 }
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001785 ALOGV_IF(err == OK, "openAudioSink: offload succeeded");
1786 }
1787 if (err != OK) {
1788 // Clean up, fall back to non offload mode.
1789 mAudioSink->close();
1790 onDisableOffloadAudio();
1791 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
1792 ALOGV("openAudioSink: offload failed");
Wei Jia3ab25452015-06-10 09:37:47 -07001793 } else {
1794 mUseAudioCallback = true; // offload mode transfers data through callback
1795 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message.
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001796 }
1797 }
1798 }
1799 if (!offloadOnly && !offloadingAudio()) {
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001800 ALOGV("openAudioSink: open AudioSink in NON-offload mode");
Eric Laurentd88c3ca2014-10-28 10:52:11 -07001801 uint32_t pcmFlags = flags;
1802 pcmFlags &= ~AUDIO_OUTPUT_FLAG_COMPRESS_OFFLOAD;
Andy Hungf0e83642014-12-19 17:55:56 -08001803
1804 const PcmInfo info = {
1805 (audio_channel_mask_t)channelMask,
1806 (audio_output_flags_t)pcmFlags,
1807 AUDIO_FORMAT_PCM_16_BIT, // TODO: change to audioFormat
1808 numChannels,
1809 sampleRate
1810 };
1811 if (memcmp(&mCurrentPcmInfo, &info, sizeof(info)) == 0) {
1812 ALOGV("openAudioSink: no change in pcm mode");
1813 // no change from previous configuration, everything ok.
1814 return OK;
1815 }
1816
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001817 audioSinkChanged = true;
1818 mAudioSink->close();
1819 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
Andy Hunga0b39712015-05-31 22:40:49 -07001820 // Note: It is possible to set up the callback, but not use it to send audio data.
1821 // This requires a fix in AudioSink to explicitly specify the transfer mode.
1822 mUseAudioCallback = getUseAudioCallbackSetting();
Wei Jia3ab25452015-06-10 09:37:47 -07001823 if (mUseAudioCallback) {
1824 ++mAudioDrainGeneration; // discard pending kWhatDrainAudioQueue message.
1825 }
Andy Hung179652e2015-05-31 22:49:46 -07001826
1827 // Compute the desired buffer size.
1828 // For callback mode, the amount of time before wakeup is about half the buffer size.
1829 const uint32_t frameCount =
1830 (unsigned long long)sampleRate * getAudioSinkPcmMsSetting() / 1000;
1831
Ronghua Wu7665f582015-09-02 10:15:30 -07001832 // The doNotReconnect means AudioSink will signal back and let NuPlayer to re-construct
1833 // AudioSink. We don't want this when there's video because it will cause a video seek to
1834 // the previous I frame. But we do want this when there's only audio because it will give
1835 // NuPlayer a chance to switch from non-offload mode to offload mode.
1836 // So we only set doNotReconnect when there's no video.
1837 const bool doNotReconnect = !hasVideo;
Andy Hung202bce12014-12-03 11:47:36 -08001838 status_t err = mAudioSink->open(
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001839 sampleRate,
1840 numChannels,
1841 (audio_channel_mask_t)channelMask,
1842 AUDIO_FORMAT_PCM_16_BIT,
Andy Hung179652e2015-05-31 22:49:46 -07001843 0 /* bufferCount - unused */,
Andy Hunga0b39712015-05-31 22:40:49 -07001844 mUseAudioCallback ? &NuPlayer::Renderer::AudioSinkCallback : NULL,
1845 mUseAudioCallback ? this : NULL,
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001846 (audio_output_flags_t)pcmFlags,
1847 NULL,
Ronghua Wu7665f582015-09-02 10:15:30 -07001848 doNotReconnect,
Andy Hung179652e2015-05-31 22:49:46 -07001849 frameCount);
Lajos Molnar3a474aa2015-04-24 17:10:07 -07001850 if (err == OK) {
1851 err = mAudioSink->setPlaybackRate(mPlaybackSettings);
1852 }
Andy Hung202bce12014-12-03 11:47:36 -08001853 if (err != OK) {
1854 ALOGW("openAudioSink: non offloaded open failed status: %d", err);
Wei Jia4d7ac852015-08-31 18:33:14 -07001855 mAudioSink->close();
Andy Hungf0e83642014-12-19 17:55:56 -08001856 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
Andy Hung202bce12014-12-03 11:47:36 -08001857 return err;
1858 }
Andy Hungf0e83642014-12-19 17:55:56 -08001859 mCurrentPcmInfo = info;
Andy Hung005e9d02015-05-31 23:16:07 -07001860 if (!mPaused) { // for preview mode, don't start if paused
1861 mAudioSink->start();
1862 }
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001863 }
1864 if (audioSinkChanged) {
1865 onAudioSinkChanged();
1866 }
Ronghua Wufaeb0f22015-05-21 12:20:21 -07001867 mAudioTornDown = false;
Andy Hung202bce12014-12-03 11:47:36 -08001868 return OK;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001869}
1870
1871void NuPlayer::Renderer::onCloseAudioSink() {
1872 mAudioSink->close();
1873 mCurrentOffloadInfo = AUDIO_INFO_INITIALIZER;
Andy Hungf0e83642014-12-19 17:55:56 -08001874 mCurrentPcmInfo = AUDIO_PCMINFO_INITIALIZER;
Chong Zhang3b9eb1f2014-10-15 17:05:08 -07001875}
1876
Andreas Huberf9334412010-12-15 15:17:42 -08001877} // namespace android
1878