blob: 8391b9e715a41142735f19a4f56195bacff66af7 [file] [log] [blame]
Andreas Huberf9334412010-12-15 15:17:42 -08001/*
2 * Copyright (C) 2010 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "NuPlayerRenderer"
19#include <utils/Log.h>
20
21#include "NuPlayerRenderer.h"
22
23#include <media/stagefright/foundation/ABuffer.h>
24#include <media/stagefright/foundation/ADebug.h>
Andreas Huber5bc087c2010-12-23 10:27:40 -080025#include <media/stagefright/foundation/AMessage.h>
Wei Jiabc2fb722014-07-08 16:37:57 -070026#include <media/stagefright/MediaErrors.h>
27#include <media/stagefright/MetaData.h>
Andreas Huberf9334412010-12-15 15:17:42 -080028
Lajos Molnar09524832014-07-17 14:29:51 -070029#include <inttypes.h>
30
Andreas Huberf9334412010-12-15 15:17:42 -080031namespace android {
32
Andreas Huber714aa7b2011-09-13 08:28:38 -070033// static
34const int64_t NuPlayer::Renderer::kMinPositionUpdateDelayUs = 100000ll;
35
Andreas Huberf9334412010-12-15 15:17:42 -080036NuPlayer::Renderer::Renderer(
37 const sp<MediaPlayerBase::AudioSink> &sink,
Andreas Huberd5e56232013-03-12 11:01:43 -070038 const sp<AMessage> &notify,
39 uint32_t flags)
Andreas Huberf9334412010-12-15 15:17:42 -080040 : mAudioSink(sink),
41 mNotify(notify),
Andreas Huberd5e56232013-03-12 11:01:43 -070042 mFlags(flags),
Andreas Huberf9334412010-12-15 15:17:42 -080043 mNumFramesWritten(0),
44 mDrainAudioQueuePending(false),
45 mDrainVideoQueuePending(false),
46 mAudioQueueGeneration(0),
47 mVideoQueueGeneration(0),
Wei Jiabc2fb722014-07-08 16:37:57 -070048 mFirstAudioTimeUs(-1),
Andreas Huberf9334412010-12-15 15:17:42 -080049 mAnchorTimeMediaUs(-1),
50 mAnchorTimeRealUs(-1),
51 mFlushingAudio(false),
52 mFlushingVideo(false),
Andreas Huberbc7f5b22011-01-21 10:15:23 -080053 mHasAudio(false),
54 mHasVideo(false),
55 mSyncQueues(false),
Andreas Huber714aa7b2011-09-13 08:28:38 -070056 mPaused(false),
James Dongf57b4ea2012-07-20 13:38:36 -070057 mVideoRenderingStarted(false),
Lajos Molnarcbaffcf2013-08-14 18:30:38 -070058 mVideoRenderingStartGeneration(0),
59 mAudioRenderingStartGeneration(0),
Andreas Huber3fe62152011-09-16 15:09:22 -070060 mLastPositionUpdateUs(-1ll),
61 mVideoLateByUs(0ll) {
Andreas Huberf9334412010-12-15 15:17:42 -080062}
63
64NuPlayer::Renderer::~Renderer() {
Wei Jiabc2fb722014-07-08 16:37:57 -070065 if (offloadingAudio()) {
66 mAudioSink->stop();
67 mAudioSink->flush();
68 mAudioSink->close();
69 }
Andreas Huberf9334412010-12-15 15:17:42 -080070}
71
72void NuPlayer::Renderer::queueBuffer(
73 bool audio,
74 const sp<ABuffer> &buffer,
75 const sp<AMessage> &notifyConsumed) {
76 sp<AMessage> msg = new AMessage(kWhatQueueBuffer, id());
77 msg->setInt32("audio", static_cast<int32_t>(audio));
Andreas Huber2d8bedd2012-02-21 14:38:23 -080078 msg->setBuffer("buffer", buffer);
Andreas Huberf9334412010-12-15 15:17:42 -080079 msg->setMessage("notifyConsumed", notifyConsumed);
80 msg->post();
81}
82
83void NuPlayer::Renderer::queueEOS(bool audio, status_t finalResult) {
84 CHECK_NE(finalResult, (status_t)OK);
85
86 sp<AMessage> msg = new AMessage(kWhatQueueEOS, id());
87 msg->setInt32("audio", static_cast<int32_t>(audio));
88 msg->setInt32("finalResult", finalResult);
89 msg->post();
90}
91
92void NuPlayer::Renderer::flush(bool audio) {
93 {
94 Mutex::Autolock autoLock(mFlushLock);
95 if (audio) {
Wei Jia28a8a9f2014-08-18 11:29:50 -070096 if (mFlushingAudio) {
97 return;
98 }
Andreas Huberf9334412010-12-15 15:17:42 -080099 mFlushingAudio = true;
100 } else {
Wei Jia28a8a9f2014-08-18 11:29:50 -0700101 if (mFlushingVideo) {
102 return;
103 }
Andreas Huberf9334412010-12-15 15:17:42 -0800104 mFlushingVideo = true;
105 }
106 }
107
108 sp<AMessage> msg = new AMessage(kWhatFlush, id());
109 msg->setInt32("audio", static_cast<int32_t>(audio));
110 msg->post();
111}
112
113void NuPlayer::Renderer::signalTimeDiscontinuity() {
Wei Jiabc2fb722014-07-08 16:37:57 -0700114 Mutex::Autolock autoLock(mLock);
Andreas Huber14f76722013-01-15 09:04:18 -0800115 // CHECK(mAudioQueue.empty());
116 // CHECK(mVideoQueue.empty());
Andreas Huberf9334412010-12-15 15:17:42 -0800117 mAnchorTimeMediaUs = -1;
118 mAnchorTimeRealUs = -1;
Andreas Huber14f76722013-01-15 09:04:18 -0800119 mSyncQueues = false;
Andreas Huberf9334412010-12-15 15:17:42 -0800120}
121
Wei Jia28a8a9f2014-08-18 11:29:50 -0700122void NuPlayer::Renderer::signalAudioSinkChanged() {
123 (new AMessage(kWhatAudioSinkChanged, id()))->post();
124}
125
126void NuPlayer::Renderer::signalDisableOffloadAudio() {
127 (new AMessage(kWhatDisableOffloadAudio, id()))->post();
128}
129
Andreas Huberb4082222011-01-20 15:23:04 -0800130void NuPlayer::Renderer::pause() {
131 (new AMessage(kWhatPause, id()))->post();
132}
133
134void NuPlayer::Renderer::resume() {
135 (new AMessage(kWhatResume, id()))->post();
136}
137
Andreas Huberf9334412010-12-15 15:17:42 -0800138void NuPlayer::Renderer::onMessageReceived(const sp<AMessage> &msg) {
139 switch (msg->what()) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700140 case kWhatStopAudioSink:
141 {
142 mAudioSink->stop();
143 break;
144 }
145
Andreas Huberf9334412010-12-15 15:17:42 -0800146 case kWhatDrainAudioQueue:
147 {
148 int32_t generation;
149 CHECK(msg->findInt32("generation", &generation));
150 if (generation != mAudioQueueGeneration) {
151 break;
152 }
153
154 mDrainAudioQueuePending = false;
155
Andreas Huber078cfcf2011-09-15 12:25:04 -0700156 if (onDrainAudioQueue()) {
157 uint32_t numFramesPlayed;
158 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed),
159 (status_t)OK);
Andreas Huberf9334412010-12-15 15:17:42 -0800160
Andreas Huber078cfcf2011-09-15 12:25:04 -0700161 uint32_t numFramesPendingPlayout =
162 mNumFramesWritten - numFramesPlayed;
163
164 // This is how long the audio sink will have data to
165 // play back.
166 int64_t delayUs =
167 mAudioSink->msecsPerFrame()
168 * numFramesPendingPlayout * 1000ll;
169
170 // Let's give it more data after about half that time
171 // has elapsed.
Wei Jiabc2fb722014-07-08 16:37:57 -0700172 // kWhatDrainAudioQueue is used for non-offloading mode,
173 // and mLock is used only for offloading mode. Therefore,
174 // no need to acquire mLock here.
175 postDrainAudioQueue_l(delayUs / 2);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700176 }
Andreas Huberf9334412010-12-15 15:17:42 -0800177 break;
178 }
179
180 case kWhatDrainVideoQueue:
181 {
182 int32_t generation;
183 CHECK(msg->findInt32("generation", &generation));
184 if (generation != mVideoQueueGeneration) {
185 break;
186 }
187
188 mDrainVideoQueuePending = false;
189
190 onDrainVideoQueue();
191
192 postDrainVideoQueue();
193 break;
194 }
195
196 case kWhatQueueBuffer:
197 {
198 onQueueBuffer(msg);
199 break;
200 }
201
202 case kWhatQueueEOS:
203 {
204 onQueueEOS(msg);
205 break;
206 }
207
208 case kWhatFlush:
209 {
210 onFlush(msg);
211 break;
212 }
213
Andreas Huber3831a062010-12-21 10:22:33 -0800214 case kWhatAudioSinkChanged:
215 {
216 onAudioSinkChanged();
217 break;
218 }
219
Wei Jiabc2fb722014-07-08 16:37:57 -0700220 case kWhatDisableOffloadAudio:
221 {
222 onDisableOffloadAudio();
223 break;
224 }
225
Andreas Huberb4082222011-01-20 15:23:04 -0800226 case kWhatPause:
227 {
228 onPause();
229 break;
230 }
231
232 case kWhatResume:
233 {
234 onResume();
235 break;
236 }
237
Wei Jia3a2956d2014-07-22 16:01:33 -0700238 case kWhatAudioOffloadTearDown:
239 {
240 onAudioOffloadTearDown();
241 break;
242 }
243
Andreas Huberf9334412010-12-15 15:17:42 -0800244 default:
245 TRESPASS();
246 break;
247 }
248}
249
Wei Jiabc2fb722014-07-08 16:37:57 -0700250void NuPlayer::Renderer::postDrainAudioQueue_l(int64_t delayUs) {
251 if (mDrainAudioQueuePending || mSyncQueues || mPaused
252 || offloadingAudio()) {
Andreas Huberf9334412010-12-15 15:17:42 -0800253 return;
254 }
255
256 if (mAudioQueue.empty()) {
257 return;
258 }
259
260 mDrainAudioQueuePending = true;
261 sp<AMessage> msg = new AMessage(kWhatDrainAudioQueue, id());
262 msg->setInt32("generation", mAudioQueueGeneration);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700263 msg->post(delayUs);
Andreas Huberf9334412010-12-15 15:17:42 -0800264}
265
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700266void NuPlayer::Renderer::prepareForMediaRenderingStart() {
267 mAudioRenderingStartGeneration = mAudioQueueGeneration;
268 mVideoRenderingStartGeneration = mVideoQueueGeneration;
269}
270
271void NuPlayer::Renderer::notifyIfMediaRenderingStarted() {
272 if (mVideoRenderingStartGeneration == mVideoQueueGeneration &&
273 mAudioRenderingStartGeneration == mAudioQueueGeneration) {
274 mVideoRenderingStartGeneration = -1;
275 mAudioRenderingStartGeneration = -1;
276
277 sp<AMessage> notify = mNotify->dup();
278 notify->setInt32("what", kWhatMediaRenderingStart);
279 notify->post();
280 }
281}
282
Wei Jiabc2fb722014-07-08 16:37:57 -0700283// static
284size_t NuPlayer::Renderer::AudioSinkCallback(
285 MediaPlayerBase::AudioSink * /* audioSink */,
286 void *buffer,
287 size_t size,
288 void *cookie,
289 MediaPlayerBase::AudioSink::cb_event_t event) {
290 NuPlayer::Renderer *me = (NuPlayer::Renderer *)cookie;
291
292 switch (event) {
293 case MediaPlayerBase::AudioSink::CB_EVENT_FILL_BUFFER:
294 {
295 return me->fillAudioBuffer(buffer, size);
296 break;
297 }
298
299 case MediaPlayerBase::AudioSink::CB_EVENT_STREAM_END:
300 {
301 me->notifyEOS(true /* audio */, ERROR_END_OF_STREAM);
302 break;
303 }
304
305 case MediaPlayerBase::AudioSink::CB_EVENT_TEAR_DOWN:
306 {
Wei Jia3a2956d2014-07-22 16:01:33 -0700307 me->notifyAudioOffloadTearDown();
Wei Jiabc2fb722014-07-08 16:37:57 -0700308 break;
309 }
310 }
311
312 return 0;
313}
314
315size_t NuPlayer::Renderer::fillAudioBuffer(void *buffer, size_t size) {
316 Mutex::Autolock autoLock(mLock);
317
318 if (!offloadingAudio()) {
319 return 0;
320 }
321
322 bool hasEOS = false;
323
324 size_t sizeCopied = 0;
325 while (sizeCopied < size && !mAudioQueue.empty()) {
326 QueueEntry *entry = &*mAudioQueue.begin();
327
328 if (entry->mBuffer == NULL) { // EOS
329 hasEOS = true;
330 mAudioQueue.erase(mAudioQueue.begin());
331 entry = NULL;
332 break;
333 }
334
335 if (entry->mOffset == 0) {
336 int64_t mediaTimeUs;
337 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
338 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
339 if (mFirstAudioTimeUs == -1) {
340 mFirstAudioTimeUs = mediaTimeUs;
341 }
342 mAnchorTimeMediaUs = mediaTimeUs;
343
344 uint32_t numFramesPlayed;
345 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
346
347 // TODO: figure out how to calculate initial latency.
348 // Otherwise, the initial time is not correct till the first sample
349 // is played.
350 mAnchorTimeMediaUs = mFirstAudioTimeUs
351 + (numFramesPlayed * mAudioSink->msecsPerFrame()) * 1000ll;
352 mAnchorTimeRealUs = ALooper::GetNowUs();
353 }
354
355 size_t copy = entry->mBuffer->size() - entry->mOffset;
356 size_t sizeRemaining = size - sizeCopied;
357 if (copy > sizeRemaining) {
358 copy = sizeRemaining;
359 }
360
361 memcpy((char *)buffer + sizeCopied,
362 entry->mBuffer->data() + entry->mOffset,
363 copy);
364
365 entry->mOffset += copy;
366 if (entry->mOffset == entry->mBuffer->size()) {
367 entry->mNotifyConsumed->post();
368 mAudioQueue.erase(mAudioQueue.begin());
369 entry = NULL;
370 }
371 sizeCopied += copy;
372 notifyIfMediaRenderingStarted();
373 }
374
375 if (sizeCopied != 0) {
376 notifyPosition();
377 }
378
379 if (hasEOS) {
380 (new AMessage(kWhatStopAudioSink, id()))->post();
381 }
382
383 return sizeCopied;
384}
385
Andreas Huber078cfcf2011-09-15 12:25:04 -0700386bool NuPlayer::Renderer::onDrainAudioQueue() {
387 uint32_t numFramesPlayed;
Andreas Huber2bfdd422011-10-11 15:24:07 -0700388 if (mAudioSink->getPosition(&numFramesPlayed) != OK) {
389 return false;
390 }
Andreas Huberc92fd242011-08-16 13:48:44 -0700391
Andreas Huber078cfcf2011-09-15 12:25:04 -0700392 ssize_t numFramesAvailableToWrite =
393 mAudioSink->frameCount() - (mNumFramesWritten - numFramesPlayed);
394
395#if 0
396 if (numFramesAvailableToWrite == mAudioSink->frameCount()) {
Steve Blockdf64d152012-01-04 20:05:49 +0000397 ALOGI("audio sink underrun");
Andreas Huber078cfcf2011-09-15 12:25:04 -0700398 } else {
Steve Block3856b092011-10-20 11:56:00 +0100399 ALOGV("audio queue has %d frames left to play",
Andreas Huber078cfcf2011-09-15 12:25:04 -0700400 mAudioSink->frameCount() - numFramesAvailableToWrite);
401 }
402#endif
403
404 size_t numBytesAvailableToWrite =
405 numFramesAvailableToWrite * mAudioSink->frameSize();
406
407 while (numBytesAvailableToWrite > 0 && !mAudioQueue.empty()) {
Andreas Huberc92fd242011-08-16 13:48:44 -0700408 QueueEntry *entry = &*mAudioQueue.begin();
409
410 if (entry->mBuffer == NULL) {
411 // EOS
412
413 notifyEOS(true /* audio */, entry->mFinalResult);
414
415 mAudioQueue.erase(mAudioQueue.begin());
416 entry = NULL;
Andreas Huber078cfcf2011-09-15 12:25:04 -0700417 return false;
Eric Laurent9b7d9502011-03-21 11:49:00 -0700418 }
419
Andreas Huberf9334412010-12-15 15:17:42 -0800420 if (entry->mOffset == 0) {
421 int64_t mediaTimeUs;
422 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
423
Steve Block3856b092011-10-20 11:56:00 +0100424 ALOGV("rendering audio at media time %.2f secs", mediaTimeUs / 1E6);
Andreas Huberf9334412010-12-15 15:17:42 -0800425
426 mAnchorTimeMediaUs = mediaTimeUs;
427
428 uint32_t numFramesPlayed;
429 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
430
431 uint32_t numFramesPendingPlayout =
432 mNumFramesWritten - numFramesPlayed;
433
434 int64_t realTimeOffsetUs =
435 (mAudioSink->latency() / 2 /* XXX */
436 + numFramesPendingPlayout
437 * mAudioSink->msecsPerFrame()) * 1000ll;
438
Steve Blockdf64d152012-01-04 20:05:49 +0000439 // ALOGI("realTimeOffsetUs = %lld us", realTimeOffsetUs);
Andreas Huberf9334412010-12-15 15:17:42 -0800440
441 mAnchorTimeRealUs =
442 ALooper::GetNowUs() + realTimeOffsetUs;
443 }
444
445 size_t copy = entry->mBuffer->size() - entry->mOffset;
446 if (copy > numBytesAvailableToWrite) {
447 copy = numBytesAvailableToWrite;
448 }
449
450 CHECK_EQ(mAudioSink->write(
451 entry->mBuffer->data() + entry->mOffset, copy),
452 (ssize_t)copy);
453
454 entry->mOffset += copy;
455 if (entry->mOffset == entry->mBuffer->size()) {
456 entry->mNotifyConsumed->post();
457 mAudioQueue.erase(mAudioQueue.begin());
Eric Laurent9b7d9502011-03-21 11:49:00 -0700458
Andreas Huberf9334412010-12-15 15:17:42 -0800459 entry = NULL;
460 }
461
Andreas Huber078cfcf2011-09-15 12:25:04 -0700462 numBytesAvailableToWrite -= copy;
463 size_t copiedFrames = copy / mAudioSink->frameSize();
464 mNumFramesWritten += copiedFrames;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700465
466 notifyIfMediaRenderingStarted();
Andreas Huberf9334412010-12-15 15:17:42 -0800467 }
Andreas Huber43c3e6c2011-01-05 12:17:08 -0800468
469 notifyPosition();
Andreas Huber078cfcf2011-09-15 12:25:04 -0700470
471 return !mAudioQueue.empty();
Andreas Huberf9334412010-12-15 15:17:42 -0800472}
473
474void NuPlayer::Renderer::postDrainVideoQueue() {
Andreas Huberb4082222011-01-20 15:23:04 -0800475 if (mDrainVideoQueuePending || mSyncQueues || mPaused) {
Andreas Huberf9334412010-12-15 15:17:42 -0800476 return;
477 }
478
479 if (mVideoQueue.empty()) {
480 return;
481 }
482
483 QueueEntry &entry = *mVideoQueue.begin();
484
485 sp<AMessage> msg = new AMessage(kWhatDrainVideoQueue, id());
486 msg->setInt32("generation", mVideoQueueGeneration);
487
488 int64_t delayUs;
489
490 if (entry.mBuffer == NULL) {
491 // EOS doesn't carry a timestamp.
492 delayUs = 0;
Andreas Huberd5e56232013-03-12 11:01:43 -0700493 } else if (mFlags & FLAG_REAL_TIME) {
494 int64_t mediaTimeUs;
495 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
496
497 delayUs = mediaTimeUs - ALooper::GetNowUs();
Andreas Huberf9334412010-12-15 15:17:42 -0800498 } else {
499 int64_t mediaTimeUs;
500 CHECK(entry.mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
501
502 if (mAnchorTimeMediaUs < 0) {
503 delayUs = 0;
504
Andreas Huber3831a062010-12-21 10:22:33 -0800505 if (!mHasAudio) {
Andreas Huberf9334412010-12-15 15:17:42 -0800506 mAnchorTimeMediaUs = mediaTimeUs;
507 mAnchorTimeRealUs = ALooper::GetNowUs();
508 }
509 } else {
510 int64_t realTimeUs =
511 (mediaTimeUs - mAnchorTimeMediaUs) + mAnchorTimeRealUs;
512
513 delayUs = realTimeUs - ALooper::GetNowUs();
514 }
515 }
516
Lajos Molnar09524832014-07-17 14:29:51 -0700517 ALOGW_IF(delayUs > 500000, "unusually high delayUs: %" PRId64, delayUs);
Andreas Huberf9334412010-12-15 15:17:42 -0800518 msg->post(delayUs);
519
520 mDrainVideoQueuePending = true;
521}
522
523void NuPlayer::Renderer::onDrainVideoQueue() {
524 if (mVideoQueue.empty()) {
525 return;
526 }
527
528 QueueEntry *entry = &*mVideoQueue.begin();
529
530 if (entry->mBuffer == NULL) {
531 // EOS
532
Andreas Huberc92fd242011-08-16 13:48:44 -0700533 notifyEOS(false /* audio */, entry->mFinalResult);
Andreas Huberf9334412010-12-15 15:17:42 -0800534
535 mVideoQueue.erase(mVideoQueue.begin());
536 entry = NULL;
Andreas Huber3fe62152011-09-16 15:09:22 -0700537
538 mVideoLateByUs = 0ll;
539
540 notifyPosition();
Andreas Huberf9334412010-12-15 15:17:42 -0800541 return;
542 }
543
Andreas Huberd5e56232013-03-12 11:01:43 -0700544 int64_t realTimeUs;
545 if (mFlags & FLAG_REAL_TIME) {
546 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &realTimeUs));
547 } else {
548 int64_t mediaTimeUs;
549 CHECK(entry->mBuffer->meta()->findInt64("timeUs", &mediaTimeUs));
Andreas Huberf9334412010-12-15 15:17:42 -0800550
Andreas Huberd5e56232013-03-12 11:01:43 -0700551 realTimeUs = mediaTimeUs - mAnchorTimeMediaUs + mAnchorTimeRealUs;
552 }
553
Andreas Huber3fe62152011-09-16 15:09:22 -0700554 mVideoLateByUs = ALooper::GetNowUs() - realTimeUs;
Andreas Huber3fe62152011-09-16 15:09:22 -0700555 bool tooLate = (mVideoLateByUs > 40000);
556
557 if (tooLate) {
Andreas Huberafed0e12011-09-20 15:39:58 -0700558 ALOGV("video late by %lld us (%.2f secs)",
559 mVideoLateByUs, mVideoLateByUs / 1E6);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700560 } else {
Oscar Rydhéd6074f02013-10-15 09:54:08 +0200561 ALOGV("rendering video at media time %.2f secs",
562 (mFlags & FLAG_REAL_TIME ? realTimeUs :
563 (realTimeUs + mAnchorTimeMediaUs - mAnchorTimeRealUs)) / 1E6);
Andreas Huber078cfcf2011-09-15 12:25:04 -0700564 }
Andreas Huberf9334412010-12-15 15:17:42 -0800565
Glenn Kasten683525b2011-11-04 18:05:35 -0700566 entry->mNotifyConsumed->setInt32("render", !tooLate);
Andreas Huberf9334412010-12-15 15:17:42 -0800567 entry->mNotifyConsumed->post();
568 mVideoQueue.erase(mVideoQueue.begin());
569 entry = NULL;
Andreas Huber43c3e6c2011-01-05 12:17:08 -0800570
James Dongf57b4ea2012-07-20 13:38:36 -0700571 if (!mVideoRenderingStarted) {
572 mVideoRenderingStarted = true;
573 notifyVideoRenderingStart();
574 }
575
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700576 notifyIfMediaRenderingStarted();
577
Andreas Huber43c3e6c2011-01-05 12:17:08 -0800578 notifyPosition();
Andreas Huberf9334412010-12-15 15:17:42 -0800579}
580
James Dongf57b4ea2012-07-20 13:38:36 -0700581void NuPlayer::Renderer::notifyVideoRenderingStart() {
582 sp<AMessage> notify = mNotify->dup();
583 notify->setInt32("what", kWhatVideoRenderingStart);
584 notify->post();
585}
586
Andreas Huberc92fd242011-08-16 13:48:44 -0700587void NuPlayer::Renderer::notifyEOS(bool audio, status_t finalResult) {
Andreas Huberf9334412010-12-15 15:17:42 -0800588 sp<AMessage> notify = mNotify->dup();
589 notify->setInt32("what", kWhatEOS);
590 notify->setInt32("audio", static_cast<int32_t>(audio));
Andreas Huberc92fd242011-08-16 13:48:44 -0700591 notify->setInt32("finalResult", finalResult);
Andreas Huberf9334412010-12-15 15:17:42 -0800592 notify->post();
593}
594
Wei Jia3a2956d2014-07-22 16:01:33 -0700595void NuPlayer::Renderer::notifyAudioOffloadTearDown() {
596 (new AMessage(kWhatAudioOffloadTearDown, id()))->post();
597}
598
Andreas Huberf9334412010-12-15 15:17:42 -0800599void NuPlayer::Renderer::onQueueBuffer(const sp<AMessage> &msg) {
600 int32_t audio;
601 CHECK(msg->findInt32("audio", &audio));
602
Andreas Huberbc7f5b22011-01-21 10:15:23 -0800603 if (audio) {
604 mHasAudio = true;
605 } else {
606 mHasVideo = true;
607 }
608
Andreas Huberf9334412010-12-15 15:17:42 -0800609 if (dropBufferWhileFlushing(audio, msg)) {
610 return;
611 }
612
Andreas Huber2d8bedd2012-02-21 14:38:23 -0800613 sp<ABuffer> buffer;
614 CHECK(msg->findBuffer("buffer", &buffer));
Andreas Huberf9334412010-12-15 15:17:42 -0800615
616 sp<AMessage> notifyConsumed;
617 CHECK(msg->findMessage("notifyConsumed", &notifyConsumed));
618
619 QueueEntry entry;
620 entry.mBuffer = buffer;
621 entry.mNotifyConsumed = notifyConsumed;
622 entry.mOffset = 0;
623 entry.mFinalResult = OK;
624
625 if (audio) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700626 Mutex::Autolock autoLock(mLock);
Andreas Huberf9334412010-12-15 15:17:42 -0800627 mAudioQueue.push_back(entry);
Wei Jiabc2fb722014-07-08 16:37:57 -0700628 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -0800629 } else {
630 mVideoQueue.push_back(entry);
631 postDrainVideoQueue();
632 }
633
Wei Jiabc2fb722014-07-08 16:37:57 -0700634 Mutex::Autolock autoLock(mLock);
Andreas Hubercb67cd12011-08-26 16:02:19 -0700635 if (!mSyncQueues || mAudioQueue.empty() || mVideoQueue.empty()) {
636 return;
Andreas Huberf9334412010-12-15 15:17:42 -0800637 }
Andreas Hubercb67cd12011-08-26 16:02:19 -0700638
639 sp<ABuffer> firstAudioBuffer = (*mAudioQueue.begin()).mBuffer;
640 sp<ABuffer> firstVideoBuffer = (*mVideoQueue.begin()).mBuffer;
641
642 if (firstAudioBuffer == NULL || firstVideoBuffer == NULL) {
643 // EOS signalled on either queue.
Wei Jiabc2fb722014-07-08 16:37:57 -0700644 syncQueuesDone_l();
Andreas Hubercb67cd12011-08-26 16:02:19 -0700645 return;
646 }
647
648 int64_t firstAudioTimeUs;
649 int64_t firstVideoTimeUs;
650 CHECK(firstAudioBuffer->meta()
651 ->findInt64("timeUs", &firstAudioTimeUs));
652 CHECK(firstVideoBuffer->meta()
653 ->findInt64("timeUs", &firstVideoTimeUs));
654
655 int64_t diff = firstVideoTimeUs - firstAudioTimeUs;
656
Steve Block3856b092011-10-20 11:56:00 +0100657 ALOGV("queueDiff = %.2f secs", diff / 1E6);
Andreas Hubercb67cd12011-08-26 16:02:19 -0700658
659 if (diff > 100000ll) {
660 // Audio data starts More than 0.1 secs before video.
661 // Drop some audio.
662
663 (*mAudioQueue.begin()).mNotifyConsumed->post();
664 mAudioQueue.erase(mAudioQueue.begin());
665 return;
666 }
667
Wei Jiabc2fb722014-07-08 16:37:57 -0700668 syncQueuesDone_l();
Andreas Huberf9334412010-12-15 15:17:42 -0800669}
670
Wei Jiabc2fb722014-07-08 16:37:57 -0700671void NuPlayer::Renderer::syncQueuesDone_l() {
Andreas Huberf9334412010-12-15 15:17:42 -0800672 if (!mSyncQueues) {
673 return;
674 }
675
676 mSyncQueues = false;
677
678 if (!mAudioQueue.empty()) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700679 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -0800680 }
681
682 if (!mVideoQueue.empty()) {
683 postDrainVideoQueue();
684 }
685}
686
687void NuPlayer::Renderer::onQueueEOS(const sp<AMessage> &msg) {
688 int32_t audio;
689 CHECK(msg->findInt32("audio", &audio));
690
691 if (dropBufferWhileFlushing(audio, msg)) {
692 return;
693 }
694
695 int32_t finalResult;
696 CHECK(msg->findInt32("finalResult", &finalResult));
697
698 QueueEntry entry;
699 entry.mOffset = 0;
700 entry.mFinalResult = finalResult;
701
702 if (audio) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700703 Mutex::Autolock autoLock(mLock);
Roger Jönssonb50e83e2013-01-21 16:26:41 +0100704 if (mAudioQueue.empty() && mSyncQueues) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700705 syncQueuesDone_l();
Roger Jönssonb50e83e2013-01-21 16:26:41 +0100706 }
Andreas Huberf9334412010-12-15 15:17:42 -0800707 mAudioQueue.push_back(entry);
Wei Jiabc2fb722014-07-08 16:37:57 -0700708 postDrainAudioQueue_l();
Andreas Huberf9334412010-12-15 15:17:42 -0800709 } else {
Roger Jönssonb50e83e2013-01-21 16:26:41 +0100710 if (mVideoQueue.empty() && mSyncQueues) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700711 Mutex::Autolock autoLock(mLock);
712 syncQueuesDone_l();
Roger Jönssonb50e83e2013-01-21 16:26:41 +0100713 }
Andreas Huberf9334412010-12-15 15:17:42 -0800714 mVideoQueue.push_back(entry);
715 postDrainVideoQueue();
716 }
717}
718
719void NuPlayer::Renderer::onFlush(const sp<AMessage> &msg) {
720 int32_t audio;
721 CHECK(msg->findInt32("audio", &audio));
722
Wei Jia28a8a9f2014-08-18 11:29:50 -0700723 {
724 Mutex::Autolock autoLock(mFlushLock);
725 if (audio) {
726 mFlushingAudio = false;
727 } else {
728 mFlushingVideo = false;
729 }
730 }
731
Andreas Huberf9334412010-12-15 15:17:42 -0800732 // If we're currently syncing the queues, i.e. dropping audio while
733 // aligning the first audio/video buffer times and only one of the
734 // two queues has data, we may starve that queue by not requesting
735 // more buffers from the decoder. If the other source then encounters
736 // a discontinuity that leads to flushing, we'll never find the
737 // corresponding discontinuity on the other queue.
738 // Therefore we'll stop syncing the queues if at least one of them
739 // is flushed.
Wei Jiabc2fb722014-07-08 16:37:57 -0700740 {
741 Mutex::Autolock autoLock(mLock);
742 syncQueuesDone_l();
743 }
Andreas Huberf9334412010-12-15 15:17:42 -0800744
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700745 ALOGV("flushing %s", audio ? "audio" : "video");
Andreas Huberf9334412010-12-15 15:17:42 -0800746 if (audio) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700747 {
748 Mutex::Autolock autoLock(mLock);
749 flushQueue(&mAudioQueue);
Wei Jia28a8a9f2014-08-18 11:29:50 -0700750
751 ++mAudioQueueGeneration;
752 prepareForMediaRenderingStart();
753
754 if (offloadingAudio()) {
755 mFirstAudioTimeUs = -1;
756 }
Wei Jiabc2fb722014-07-08 16:37:57 -0700757 }
Andreas Huberf9334412010-12-15 15:17:42 -0800758
Andreas Huberf9334412010-12-15 15:17:42 -0800759 mDrainAudioQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700760
Wei Jiabc2fb722014-07-08 16:37:57 -0700761 if (offloadingAudio()) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700762 mAudioSink->pause();
763 mAudioSink->flush();
764 mAudioSink->start();
765 }
Andreas Huberf9334412010-12-15 15:17:42 -0800766 } else {
767 flushQueue(&mVideoQueue);
768
Andreas Huberf9334412010-12-15 15:17:42 -0800769 mDrainVideoQueuePending = false;
770 ++mVideoQueueGeneration;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700771
772 prepareForMediaRenderingStart();
Andreas Huberf9334412010-12-15 15:17:42 -0800773 }
774
775 notifyFlushComplete(audio);
776}
777
778void NuPlayer::Renderer::flushQueue(List<QueueEntry> *queue) {
779 while (!queue->empty()) {
780 QueueEntry *entry = &*queue->begin();
781
782 if (entry->mBuffer != NULL) {
783 entry->mNotifyConsumed->post();
784 }
785
786 queue->erase(queue->begin());
787 entry = NULL;
788 }
789}
790
791void NuPlayer::Renderer::notifyFlushComplete(bool audio) {
792 sp<AMessage> notify = mNotify->dup();
793 notify->setInt32("what", kWhatFlushComplete);
794 notify->setInt32("audio", static_cast<int32_t>(audio));
795 notify->post();
796}
797
798bool NuPlayer::Renderer::dropBufferWhileFlushing(
799 bool audio, const sp<AMessage> &msg) {
800 bool flushing = false;
801
802 {
803 Mutex::Autolock autoLock(mFlushLock);
804 if (audio) {
805 flushing = mFlushingAudio;
806 } else {
807 flushing = mFlushingVideo;
808 }
809 }
810
811 if (!flushing) {
812 return false;
813 }
814
815 sp<AMessage> notifyConsumed;
816 if (msg->findMessage("notifyConsumed", &notifyConsumed)) {
817 notifyConsumed->post();
818 }
819
820 return true;
821}
822
Andreas Huber3831a062010-12-21 10:22:33 -0800823void NuPlayer::Renderer::onAudioSinkChanged() {
Wei Jiabc2fb722014-07-08 16:37:57 -0700824 if (offloadingAudio()) {
825 return;
826 }
Andreas Huber3831a062010-12-21 10:22:33 -0800827 CHECK(!mDrainAudioQueuePending);
828 mNumFramesWritten = 0;
Marco Nelissen4110c102012-03-29 09:31:28 -0700829 uint32_t written;
830 if (mAudioSink->getFramesWritten(&written) == OK) {
831 mNumFramesWritten = written;
832 }
Andreas Huber3831a062010-12-21 10:22:33 -0800833}
834
Wei Jiabc2fb722014-07-08 16:37:57 -0700835void NuPlayer::Renderer::onDisableOffloadAudio() {
836 Mutex::Autolock autoLock(mLock);
837 mFlags &= ~FLAG_OFFLOAD_AUDIO;
Wei Jia3a2956d2014-07-22 16:01:33 -0700838 ++mAudioQueueGeneration;
Wei Jiabc2fb722014-07-08 16:37:57 -0700839}
840
Andreas Huber43c3e6c2011-01-05 12:17:08 -0800841void NuPlayer::Renderer::notifyPosition() {
842 if (mAnchorTimeRealUs < 0 || mAnchorTimeMediaUs < 0) {
843 return;
844 }
845
846 int64_t nowUs = ALooper::GetNowUs();
Andreas Huber714aa7b2011-09-13 08:28:38 -0700847
848 if (mLastPositionUpdateUs >= 0
849 && nowUs < mLastPositionUpdateUs + kMinPositionUpdateDelayUs) {
850 return;
851 }
852 mLastPositionUpdateUs = nowUs;
853
Andreas Huber43c3e6c2011-01-05 12:17:08 -0800854 int64_t positionUs = (nowUs - mAnchorTimeRealUs) + mAnchorTimeMediaUs;
855
856 sp<AMessage> notify = mNotify->dup();
857 notify->setInt32("what", kWhatPosition);
858 notify->setInt64("positionUs", positionUs);
Andreas Huber3fe62152011-09-16 15:09:22 -0700859 notify->setInt64("videoLateByUs", mVideoLateByUs);
Andreas Huber43c3e6c2011-01-05 12:17:08 -0800860 notify->post();
861}
862
Andreas Huberb4082222011-01-20 15:23:04 -0800863void NuPlayer::Renderer::onPause() {
864 CHECK(!mPaused);
865
Wei Jia28a8a9f2014-08-18 11:29:50 -0700866 {
867 Mutex::Autolock autoLock(mLock);
868 ++mAudioQueueGeneration;
869 ++mVideoQueueGeneration;
870 prepareForMediaRenderingStart();
871 }
872
Andreas Huberb4082222011-01-20 15:23:04 -0800873 mDrainAudioQueuePending = false;
Andreas Huberb4082222011-01-20 15:23:04 -0800874 mDrainVideoQueuePending = false;
Lajos Molnarcbaffcf2013-08-14 18:30:38 -0700875
Andreas Huberb4082222011-01-20 15:23:04 -0800876 if (mHasAudio) {
877 mAudioSink->pause();
878 }
879
Andreas Huberea9d51b2011-11-30 09:53:40 -0800880 ALOGV("now paused audio queue has %d entries, video has %d entries",
881 mAudioQueue.size(), mVideoQueue.size());
Andreas Huberb58ce9f2011-11-28 16:27:35 -0800882
Andreas Huberb4082222011-01-20 15:23:04 -0800883 mPaused = true;
884}
885
886void NuPlayer::Renderer::onResume() {
Andreas Huberb58ce9f2011-11-28 16:27:35 -0800887 if (!mPaused) {
888 return;
889 }
Andreas Huberb4082222011-01-20 15:23:04 -0800890
891 if (mHasAudio) {
892 mAudioSink->start();
893 }
894
895 mPaused = false;
896
Wei Jiabc2fb722014-07-08 16:37:57 -0700897 Mutex::Autolock autoLock(mLock);
Andreas Huberb4082222011-01-20 15:23:04 -0800898 if (!mAudioQueue.empty()) {
Wei Jiabc2fb722014-07-08 16:37:57 -0700899 postDrainAudioQueue_l();
Andreas Huberb4082222011-01-20 15:23:04 -0800900 }
901
902 if (!mVideoQueue.empty()) {
903 postDrainVideoQueue();
904 }
905}
906
Wei Jia3a2956d2014-07-22 16:01:33 -0700907void NuPlayer::Renderer::onAudioOffloadTearDown() {
908 uint32_t numFramesPlayed;
909 CHECK_EQ(mAudioSink->getPosition(&numFramesPlayed), (status_t)OK);
910
Wei Jia28a8a9f2014-08-18 11:29:50 -0700911 int64_t firstAudioTimeUs;
912 {
913 Mutex::Autolock autoLock(mLock);
914 firstAudioTimeUs = mFirstAudioTimeUs;
915 }
916 int64_t currentPositionUs = firstAudioTimeUs
Wei Jia3a2956d2014-07-22 16:01:33 -0700917 + (numFramesPlayed * mAudioSink->msecsPerFrame()) * 1000ll;
918
919 mAudioSink->stop();
920 mAudioSink->flush();
921
922 sp<AMessage> notify = mNotify->dup();
923 notify->setInt32("what", kWhatAudioOffloadTearDown);
924 notify->setInt64("positionUs", currentPositionUs);
925 notify->post();
926}
927
Andreas Huberf9334412010-12-15 15:17:42 -0800928} // namespace android
929