blob: 3c3b41d31e6d3a8dad5d364c3a5957f02bb0e1c6 [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
Pawin Vongmasae7bb8612020-06-04 06:15:22 -070021#include <algorithm>
22#include <list>
Pawin Vongmasa36653902018-11-15 00:10:25 -080023#include <numeric>
24
25#include <C2AllocatorGralloc.h>
26#include <C2PlatformSupport.h>
27#include <C2BlockInternal.h>
28#include <C2Config.h>
29#include <C2Debug.h>
30
31#include <android/hardware/cas/native/1.0/IDescrambler.h>
Robert Shih895fba92019-07-16 16:29:44 -070032#include <android/hardware/drm/1.0/types.h>
Josh Houcd2f7582021-02-02 16:26:53 +080033#include <android-base/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080034#include <android-base/stringprintf.h>
Wonsik Kimfb7a7672019-12-27 17:13:33 -080035#include <binder/MemoryBase.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080036#include <binder/MemoryDealer.h>
Ray Essick18ea0452019-08-27 16:07:27 -070037#include <cutils/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080038#include <gui/Surface.h>
Robert Shih895fba92019-07-16 16:29:44 -070039#include <hidlmemory/FrameworkUtils.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080040#include <media/openmax/OMX_Core.h>
41#include <media/stagefright/foundation/ABuffer.h>
42#include <media/stagefright/foundation/ALookup.h>
43#include <media/stagefright/foundation/AMessage.h>
44#include <media/stagefright/foundation/AUtils.h>
45#include <media/stagefright/foundation/hexdump.h>
46#include <media/stagefright/MediaCodec.h>
47#include <media/stagefright/MediaCodecConstants.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070048#include <media/stagefright/SkipCutBuffer.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080049#include <media/MediaCodecBuffer.h>
Wonsik Kim41d83432020-04-27 16:40:49 -070050#include <mediadrm/ICrypto.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080051#include <system/window.h>
52
53#include "CCodecBufferChannel.h"
54#include "Codec2Buffer.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080055
56namespace android {
57
58using android::base::StringPrintf;
59using hardware::hidl_handle;
60using hardware::hidl_string;
61using hardware::hidl_vec;
Robert Shih895fba92019-07-16 16:29:44 -070062using hardware::fromHeap;
63using hardware::HidlMemory;
64
Pawin Vongmasa36653902018-11-15 00:10:25 -080065using namespace hardware::cas::V1_0;
66using namespace hardware::cas::native::V1_0;
67
68using CasStatus = hardware::cas::V1_0::Status;
Robert Shih895fba92019-07-16 16:29:44 -070069using DrmBufferType = hardware::drm::V1_0::BufferType;
Pawin Vongmasa36653902018-11-15 00:10:25 -080070
Pawin Vongmasa36653902018-11-15 00:10:25 -080071namespace {
72
Wonsik Kim469c8342019-04-11 16:46:09 -070073constexpr size_t kSmoothnessFactor = 4;
74constexpr size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -080075
Sungtak Leeab6f2f32019-02-15 14:43:51 -080076// This is for keeping IGBP's buffer dropping logic in legacy mode other
77// than making it non-blocking. Do not change this value.
78const static size_t kDequeueTimeoutNs = 0;
79
Pawin Vongmasa36653902018-11-15 00:10:25 -080080} // namespace
81
82CCodecBufferChannel::QueueGuard::QueueGuard(
83 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
84 Mutex::Autolock l(mSync.mGuardLock);
85 // At this point it's guaranteed that mSync is not under state transition,
86 // as we are holding its mutex.
87
88 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
89 if (count->value == -1) {
90 mRunning = false;
91 } else {
92 ++count->value;
93 mRunning = true;
94 }
95}
96
97CCodecBufferChannel::QueueGuard::~QueueGuard() {
98 if (mRunning) {
99 // We are not holding mGuardLock at this point so that QueueSync::stop() can
100 // keep holding the lock until mCount reaches zero.
101 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
102 --count->value;
103 count->cond.broadcast();
104 }
105}
106
107void CCodecBufferChannel::QueueSync::start() {
108 Mutex::Autolock l(mGuardLock);
109 // If stopped, it goes to running state; otherwise no-op.
110 Mutexed<Counter>::Locked count(mCount);
111 if (count->value == -1) {
112 count->value = 0;
113 }
114}
115
116void CCodecBufferChannel::QueueSync::stop() {
117 Mutex::Autolock l(mGuardLock);
118 Mutexed<Counter>::Locked count(mCount);
119 if (count->value == -1) {
120 // no-op
121 return;
122 }
123 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
124 // mCount can only decrement. In other words, threads that acquired the lock
125 // are allowed to finish execution but additional threads trying to acquire
126 // the lock at this point will block, and then get QueueGuard at STOPPED
127 // state.
128 while (count->value != 0) {
129 count.waitForCondition(count->cond);
130 }
131 count->value = -1;
132}
133
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700134// Input
135
136CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
137
Pawin Vongmasa36653902018-11-15 00:10:25 -0800138// CCodecBufferChannel
139
140CCodecBufferChannel::CCodecBufferChannel(
141 const std::shared_ptr<CCodecCallback> &callback)
142 : mHeapSeqNum(-1),
143 mCCodecCallback(callback),
144 mFrameIndex(0u),
145 mFirstValidFrameIndex(0u),
146 mMetaMode(MODE_NONE),
Sungtak Lee04b30352020-07-27 13:57:25 -0700147 mInputMetEos(false),
148 mSendEncryptedInfoBuffer(false) {
Sungtak Leed7463d12019-09-04 16:01:00 -0700149 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700150 {
151 Mutexed<Input>::Locked input(mInput);
152 input->buffers.reset(new DummyInputBuffers(""));
153 input->extraBuffers.flush();
154 input->inputDelay = 0u;
155 input->pipelineDelay = 0u;
156 input->numSlots = kSmoothnessFactor;
157 input->numExtraSlots = 0u;
158 }
159 {
160 Mutexed<Output>::Locked output(mOutput);
161 output->outputDelay = 0u;
162 output->numSlots = kSmoothnessFactor;
163 }
David Stevensc3fbb282021-01-18 18:11:20 +0900164 {
165 Mutexed<BlockPools>::Locked pools(mBlockPools);
166 pools->outputPoolId = C2BlockPool::BASIC_LINEAR;
167 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800168}
169
170CCodecBufferChannel::~CCodecBufferChannel() {
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800171 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800172 mCrypto->unsetHeap(mHeapSeqNum);
173 }
174}
175
176void CCodecBufferChannel::setComponent(
177 const std::shared_ptr<Codec2Client::Component> &component) {
178 mComponent = component;
179 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
180 mName = mComponentName.c_str();
181}
182
183status_t CCodecBufferChannel::setInputSurface(
184 const std::shared_ptr<InputSurfaceWrapper> &surface) {
185 ALOGV("[%s] setInputSurface", mName);
186 mInputSurface = surface;
187 return mInputSurface->connect(mComponent);
188}
189
190status_t CCodecBufferChannel::signalEndOfInputStream() {
191 if (mInputSurface == nullptr) {
192 return INVALID_OPERATION;
193 }
194 return mInputSurface->signalEndOfInputStream();
195}
196
Sungtak Lee04b30352020-07-27 13:57:25 -0700197status_t CCodecBufferChannel::queueInputBufferInternal(
198 sp<MediaCodecBuffer> buffer,
199 std::shared_ptr<C2LinearBlock> encryptedBlock,
200 size_t blockSize) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800201 int64_t timeUs;
202 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
203
204 if (mInputMetEos) {
205 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
206 return OK;
207 }
208
209 int32_t flags = 0;
210 int32_t tmp = 0;
211 bool eos = false;
212 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
213 eos = true;
214 mInputMetEos = true;
215 ALOGV("[%s] input EOS", mName);
216 }
217 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
218 flags |= C2FrameData::FLAG_CODEC_CONFIG;
219 }
220 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
Wonsik Kim0379ae82020-11-24 15:01:33 -0800221 std::list<std::unique_ptr<C2Work>> items;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800222 std::unique_ptr<C2Work> work(new C2Work);
223 work->input.ordinal.timestamp = timeUs;
224 work->input.ordinal.frameIndex = mFrameIndex++;
225 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
226 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
227 // Keep client timestamp in customOrdinal
228 work->input.ordinal.customOrdinal = timeUs;
229 work->input.buffers.clear();
230
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700231 sp<Codec2Buffer> copy;
Wonsik Kim0379ae82020-11-24 15:01:33 -0800232 bool usesFrameReassembler = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800233
Pawin Vongmasa36653902018-11-15 00:10:25 -0800234 if (buffer->size() > 0u) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700235 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800236 std::shared_ptr<C2Buffer> c2buffer;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700237 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800238 return -ENOENT;
239 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700240 // TODO: we want to delay copying buffers.
241 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
242 copy = input->buffers->cloneAndReleaseBuffer(buffer);
243 if (copy != nullptr) {
244 (void)input->extraBuffers.assignSlot(copy);
245 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
246 return UNKNOWN_ERROR;
247 }
248 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
249 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
250 mName, released ? "" : "not ");
251 buffer.clear();
252 } else {
253 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
254 "buffer starvation on component.", mName);
255 }
256 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800257 if (input->frameReassembler) {
258 usesFrameReassembler = true;
259 input->frameReassembler.process(buffer, &items);
260 } else {
261 int32_t cvo = 0;
262 if (buffer->meta()->findInt32("cvo", &cvo)) {
263 int32_t rotation = cvo % 360;
264 // change rotation to counter-clock wise.
265 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
266
267 Mutexed<OutputSurface>::Locked output(mOutputSurface);
268 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
269 output->rotation[frameIndex] = rotation;
270 }
271 work->input.buffers.push_back(c2buffer);
272 if (encryptedBlock) {
273 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
274 kParamIndexEncryptedBuffer,
275 encryptedBlock->share(0, blockSize, C2Fence())));
276 }
Byeongjo Park2eef13e2020-06-12 17:24:21 +0900277 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800278 } else if (eos) {
279 flags |= C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800280 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800281 if (usesFrameReassembler) {
282 if (!items.empty()) {
283 items.front()->input.configUpdate = std::move(mParamsToBeSet);
284 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
285 }
286 } else {
287 work->input.flags = (C2FrameData::flags_t)flags;
288 // TODO: fill info's
Pawin Vongmasa36653902018-11-15 00:10:25 -0800289
Wonsik Kim0379ae82020-11-24 15:01:33 -0800290 work->input.configUpdate = std::move(mParamsToBeSet);
291 work->worklets.clear();
292 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800293
Wonsik Kim0379ae82020-11-24 15:01:33 -0800294 items.push_back(std::move(work));
295
296 eos = eos && buffer->size() > 0u;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800297 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800298 if (eos) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800299 work.reset(new C2Work);
300 work->input.ordinal.timestamp = timeUs;
301 work->input.ordinal.frameIndex = mFrameIndex++;
302 // WORKAROUND: keep client timestamp in customOrdinal
303 work->input.ordinal.customOrdinal = timeUs;
304 work->input.buffers.clear();
305 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800306 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800307 items.push_back(std::move(work));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800308 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800309 c2_status_t err = C2_OK;
310 if (!items.empty()) {
311 {
312 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
313 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
314 for (const std::unique_ptr<C2Work> &work : items) {
315 watcher->onWorkQueued(
316 work->input.ordinal.frameIndex.peeku(),
317 std::vector(work->input.buffers),
318 now);
319 }
320 }
321 err = mComponent->queue(&items);
322 }
323 if (err != C2_OK) {
324 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
325 for (const std::unique_ptr<C2Work> &work : items) {
326 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
327 }
328 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700329 Mutexed<Input>::Locked input(mInput);
330 bool released = false;
331 if (buffer) {
332 released = input->buffers->releaseBuffer(buffer, nullptr, true);
333 } else if (copy) {
334 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
335 }
336 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
337 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800338 }
339
340 feedInputBufferIfAvailableInternal();
341 return err;
342}
343
344status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
345 QueueGuard guard(mSync);
346 if (!guard.isRunning()) {
347 ALOGD("[%s] setParameters is only supported in the running state.", mName);
348 return -ENOSYS;
349 }
350 mParamsToBeSet.insert(mParamsToBeSet.end(),
351 std::make_move_iterator(params.begin()),
352 std::make_move_iterator(params.end()));
353 params.clear();
354 return OK;
355}
356
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800357status_t CCodecBufferChannel::attachBuffer(
358 const std::shared_ptr<C2Buffer> &c2Buffer,
359 const sp<MediaCodecBuffer> &buffer) {
360 if (!buffer->copy(c2Buffer)) {
361 return -ENOSYS;
362 }
363 return OK;
364}
365
366void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
367 if (!mDecryptDestination || mDecryptDestination->size() < size) {
368 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
369 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
370 mCrypto->unsetHeap(mHeapSeqNum);
371 }
372 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
373 if (mCrypto) {
374 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
375 }
376 }
377}
378
379int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
380 CHECK(mCrypto);
381 auto it = mHeapSeqNumMap.find(memory);
382 int32_t heapSeqNum = -1;
383 if (it == mHeapSeqNumMap.end()) {
384 heapSeqNum = mCrypto->setHeap(memory);
385 mHeapSeqNumMap.emplace(memory, heapSeqNum);
386 } else {
387 heapSeqNum = it->second;
388 }
389 return heapSeqNum;
390}
391
392status_t CCodecBufferChannel::attachEncryptedBuffer(
393 const sp<hardware::HidlMemory> &memory,
394 bool secure,
395 const uint8_t *key,
396 const uint8_t *iv,
397 CryptoPlugin::Mode mode,
398 CryptoPlugin::Pattern pattern,
399 size_t offset,
400 const CryptoPlugin::SubSample *subSamples,
401 size_t numSubSamples,
402 const sp<MediaCodecBuffer> &buffer) {
403 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
404 static const C2MemoryUsage kDefaultReadWriteUsage{
405 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
406
407 size_t size = 0;
408 for (size_t i = 0; i < numSubSamples; ++i) {
409 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
410 }
411 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
412 std::shared_ptr<C2LinearBlock> block;
413 c2_status_t err = pool->fetchLinearBlock(
414 size,
415 secure ? kSecureUsage : kDefaultReadWriteUsage,
416 &block);
417 if (err != C2_OK) {
418 return NO_MEMORY;
419 }
420 if (!secure) {
421 ensureDecryptDestination(size);
422 }
423 ssize_t result = -1;
424 ssize_t codecDataOffset = 0;
425 if (mCrypto) {
426 AString errorDetailMsg;
427 int32_t heapSeqNum = getHeapSeqNum(memory);
428 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
429 hardware::drm::V1_0::DestinationBuffer dst;
430 if (secure) {
431 dst.type = DrmBufferType::NATIVE_HANDLE;
432 dst.secureMemory = hardware::hidl_handle(block->handle());
433 } else {
434 dst.type = DrmBufferType::SHARED_MEMORY;
435 IMemoryToSharedBuffer(
436 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
437 }
438 result = mCrypto->decrypt(
439 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
440 dst, &errorDetailMsg);
441 if (result < 0) {
442 return result;
443 }
444 if (dst.type == DrmBufferType::SHARED_MEMORY) {
445 C2WriteView view = block->map().get();
446 if (view.error() != C2_OK) {
447 return false;
448 }
449 if (view.size() < result) {
450 return false;
451 }
452 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
453 }
454 } else {
455 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
456 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
457 hidl_vec<SubSample> hidlSubSamples;
458 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
459
460 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
461 hardware::cas::native::V1_0::DestinationBuffer dst;
462 if (secure) {
463 dst.type = BufferType::NATIVE_HANDLE;
464 dst.secureMemory = hardware::hidl_handle(block->handle());
465 } else {
466 dst.type = BufferType::SHARED_MEMORY;
467 dst.nonsecureMemory = src;
468 }
469
470 CasStatus status = CasStatus::OK;
471 hidl_string detailedError;
472 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
473
474 if (key != nullptr) {
475 sctrl = (ScramblingControl)key[0];
476 // Adjust for the PES offset
477 codecDataOffset = key[2] | (key[3] << 8);
478 }
479
480 auto returnVoid = mDescrambler->descramble(
481 sctrl,
482 hidlSubSamples,
483 src,
484 0,
485 dst,
486 0,
487 [&status, &result, &detailedError] (
488 CasStatus _status, uint32_t _bytesWritten,
489 const hidl_string& _detailedError) {
490 status = _status;
491 result = (ssize_t)_bytesWritten;
492 detailedError = _detailedError;
493 });
494
495 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
496 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
497 mName, returnVoid.description().c_str(), status, result);
498 return UNKNOWN_ERROR;
499 }
500
501 if (result < codecDataOffset) {
502 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
503 return BAD_VALUE;
504 }
505 }
506 if (!secure) {
507 C2WriteView view = block->map().get();
508 if (view.error() != C2_OK) {
509 return UNKNOWN_ERROR;
510 }
511 if (view.size() < result) {
512 return UNKNOWN_ERROR;
513 }
514 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
515 }
516 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
517 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
518 if (!buffer->copy(c2Buffer)) {
519 return -ENOSYS;
520 }
521 return OK;
522}
523
Pawin Vongmasa36653902018-11-15 00:10:25 -0800524status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
525 QueueGuard guard(mSync);
526 if (!guard.isRunning()) {
527 ALOGD("[%s] No more buffers should be queued at current state.", mName);
528 return -ENOSYS;
529 }
530 return queueInputBufferInternal(buffer);
531}
532
533status_t CCodecBufferChannel::queueSecureInputBuffer(
534 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
535 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
536 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
537 AString *errorDetailMsg) {
538 QueueGuard guard(mSync);
539 if (!guard.isRunning()) {
540 ALOGD("[%s] No more buffers should be queued at current state.", mName);
541 return -ENOSYS;
542 }
543
544 if (!hasCryptoOrDescrambler()) {
545 return -ENOSYS;
546 }
547 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
548
Sungtak Lee04b30352020-07-27 13:57:25 -0700549 std::shared_ptr<C2LinearBlock> block;
550 size_t allocSize = buffer->size();
551 size_t bufferSize = 0;
552 c2_status_t blockRes = C2_OK;
553 bool copied = false;
554 if (mSendEncryptedInfoBuffer) {
555 static const C2MemoryUsage kDefaultReadWriteUsage{
556 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
557 constexpr int kAllocGranule0 = 1024 * 64;
558 constexpr int kAllocGranule1 = 1024 * 1024;
559 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
560 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
561 if (allocSize <= kAllocGranule1) {
562 bufferSize = align(allocSize, kAllocGranule0);
563 } else {
564 bufferSize = align(allocSize, kAllocGranule1);
565 }
566 blockRes = pool->fetchLinearBlock(
567 bufferSize, kDefaultReadWriteUsage, &block);
568
569 if (blockRes == C2_OK) {
570 C2WriteView view = block->map().get();
571 if (view.error() == C2_OK && view.size() == bufferSize) {
572 copied = true;
573 // TODO: only copy clear sections
574 memcpy(view.data(), buffer->data(), allocSize);
575 }
576 }
577 }
578
579 if (!copied) {
580 block.reset();
581 }
582
Pawin Vongmasa36653902018-11-15 00:10:25 -0800583 ssize_t result = -1;
584 ssize_t codecDataOffset = 0;
Wonsik Kim557c88c2020-03-13 11:03:52 -0700585 if (numSubSamples == 1
586 && subSamples[0].mNumBytesOfClearData == 0
587 && subSamples[0].mNumBytesOfEncryptedData == 0) {
588 // We don't need to go through crypto or descrambler if the input is empty.
589 result = 0;
590 } else if (mCrypto != nullptr) {
Robert Shih895fba92019-07-16 16:29:44 -0700591 hardware::drm::V1_0::DestinationBuffer destination;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800592 if (secure) {
Robert Shih895fba92019-07-16 16:29:44 -0700593 destination.type = DrmBufferType::NATIVE_HANDLE;
594 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800595 } else {
Robert Shih895fba92019-07-16 16:29:44 -0700596 destination.type = DrmBufferType::SHARED_MEMORY;
597 IMemoryToSharedBuffer(
598 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800599 }
Robert Shih895fba92019-07-16 16:29:44 -0700600 hardware::drm::V1_0::SharedBuffer source;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800601 encryptedBuffer->fillSourceBuffer(&source);
602 result = mCrypto->decrypt(
603 key, iv, mode, pattern, source, buffer->offset(),
604 subSamples, numSubSamples, destination, errorDetailMsg);
605 if (result < 0) {
Wonsik Kim557c88c2020-03-13 11:03:52 -0700606 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800607 return result;
608 }
Robert Shih895fba92019-07-16 16:29:44 -0700609 if (destination.type == DrmBufferType::SHARED_MEMORY) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800610 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
611 }
612 } else {
613 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
614 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
615 hidl_vec<SubSample> hidlSubSamples;
616 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
617
618 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
619 encryptedBuffer->fillSourceBuffer(&srcBuffer);
620
621 DestinationBuffer dstBuffer;
622 if (secure) {
623 dstBuffer.type = BufferType::NATIVE_HANDLE;
624 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
625 } else {
626 dstBuffer.type = BufferType::SHARED_MEMORY;
627 dstBuffer.nonsecureMemory = srcBuffer;
628 }
629
630 CasStatus status = CasStatus::OK;
631 hidl_string detailedError;
632 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
633
634 if (key != nullptr) {
635 sctrl = (ScramblingControl)key[0];
636 // Adjust for the PES offset
637 codecDataOffset = key[2] | (key[3] << 8);
638 }
639
640 auto returnVoid = mDescrambler->descramble(
641 sctrl,
642 hidlSubSamples,
643 srcBuffer,
644 0,
645 dstBuffer,
646 0,
647 [&status, &result, &detailedError] (
648 CasStatus _status, uint32_t _bytesWritten,
649 const hidl_string& _detailedError) {
650 status = _status;
651 result = (ssize_t)_bytesWritten;
652 detailedError = _detailedError;
653 });
654
655 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
656 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
657 mName, returnVoid.description().c_str(), status, result);
658 return UNKNOWN_ERROR;
659 }
660
661 if (result < codecDataOffset) {
662 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
663 return BAD_VALUE;
664 }
665
666 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
667
668 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
669 encryptedBuffer->copyDecryptedContentFromMemory(result);
670 }
671 }
672
673 buffer->setRange(codecDataOffset, result - codecDataOffset);
Sungtak Lee04b30352020-07-27 13:57:25 -0700674
675 return queueInputBufferInternal(buffer, block, bufferSize);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800676}
677
678void CCodecBufferChannel::feedInputBufferIfAvailable() {
679 QueueGuard guard(mSync);
680 if (!guard.isRunning()) {
681 ALOGV("[%s] We're not running --- no input buffer reported", mName);
682 return;
683 }
684 feedInputBufferIfAvailableInternal();
685}
686
687void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
Taehwan Kimda0517d2020-09-16 17:29:37 +0900688 if (mInputMetEos) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800689 return;
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700690 }
691 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700692 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700693 if (!output->buffers ||
694 output->buffers->hasPending() ||
Wonsik Kim0487b782020-10-28 11:45:50 -0700695 output->buffers->numActiveSlots() >= output->numSlots) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800696 return;
697 }
698 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700699 size_t numActiveSlots = 0;
700 while (!mPipelineWatcher.lock()->pipelineFull()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800701 sp<MediaCodecBuffer> inBuffer;
702 size_t index;
703 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700704 Mutexed<Input>::Locked input(mInput);
Wonsik Kim0487b782020-10-28 11:45:50 -0700705 numActiveSlots = input->buffers->numActiveSlots();
706 if (numActiveSlots >= input->numSlots) {
707 break;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800708 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700709 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800710 ALOGV("[%s] no new buffer available", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800711 break;
712 }
713 }
714 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
715 mCallback->onInputBufferAvailable(index, inBuffer);
716 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700717 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800718}
719
720status_t CCodecBufferChannel::renderOutputBuffer(
721 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800722 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800723 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800724 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800725 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700726 Mutexed<Output>::Locked output(mOutput);
727 if (output->buffers) {
728 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800729 }
730 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800731 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
732 // set to true.
733 sendOutputBuffers();
734 // input buffer feeding may have been gated by pending output buffers
735 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800736 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800737 if (released) {
Wonsik Kimf7529dd2019-04-18 17:35:53 -0700738 std::call_once(mRenderWarningFlag, [this] {
739 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
740 "timestamp or render=true with non-video buffers. Apps should "
741 "call releaseOutputBuffer() with render=false for those.",
742 mName);
743 });
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800744 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800745 return INVALID_OPERATION;
746 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800747
748#if 0
749 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
750 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
751 for (const std::shared_ptr<const C2Info> &info : infoParams) {
752 AString res;
753 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
754 if (ix) res.append(", ");
755 res.append(*((int32_t*)info.get() + (ix / 4)));
756 }
757 ALOGV(" [%s]", res.c_str());
758 }
759#endif
760 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
761 std::static_pointer_cast<const C2StreamRotationInfo::output>(
762 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
763 bool flip = rotation && (rotation->flip & 1);
764 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
Byeongjo Park2eef13e2020-06-12 17:24:21 +0900765
766 {
767 Mutexed<OutputSurface>::Locked output(mOutputSurface);
768 if (output->surface == nullptr) {
769 ALOGI("[%s] cannot render buffer without surface", mName);
770 return OK;
771 }
772 int64_t frameIndex;
773 buffer->meta()->findInt64("frameIndex", &frameIndex);
774 if (output->rotation.count(frameIndex) != 0) {
775 auto it = output->rotation.find(frameIndex);
776 quarters = (it->second / 90) & 3;
777 output->rotation.erase(it);
778 }
779 }
780
Pawin Vongmasa36653902018-11-15 00:10:25 -0800781 uint32_t transform = 0;
782 switch (quarters) {
783 case 0: // no rotation
784 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
785 break;
786 case 1: // 90 degrees counter-clockwise
787 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
788 : HAL_TRANSFORM_ROT_270;
789 break;
790 case 2: // 180 degrees
791 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
792 break;
793 case 3: // 90 degrees clockwise
794 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
795 : HAL_TRANSFORM_ROT_90;
796 break;
797 }
798
799 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
800 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
801 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
802 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
803 if (surfaceScaling) {
804 videoScalingMode = surfaceScaling->value;
805 }
806
807 // Use dataspace from format as it has the default aspects already applied
808 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
809 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
810
811 // HDR static info
812 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
813 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
814 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
815
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800816 // HDR10 plus info
817 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
818 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
819 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
Yichi Chen54be23c2020-06-15 14:30:53 +0800820 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
821 hdr10PlusInfo.reset();
822 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800823
Pawin Vongmasa36653902018-11-15 00:10:25 -0800824 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
825 if (blocks.size() != 1u) {
826 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
827 return UNKNOWN_ERROR;
828 }
829 const C2ConstGraphicBlock &block = blocks.front();
830
831 // TODO: revisit this after C2Fence implementation.
832 android::IGraphicBufferProducer::QueueBufferInput qbi(
833 timestampNs,
834 false, // droppable
835 dataSpace,
836 Rect(blocks.front().crop().left,
837 blocks.front().crop().top,
838 blocks.front().crop().right(),
839 blocks.front().crop().bottom()),
840 videoScalingMode,
841 transform,
842 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800843 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800844 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800845 if (hdrStaticInfo) {
wenchangliuf3f92882020-05-14 00:02:01 +0800846 // If mastering max and min luminance fields are 0, do not use them.
847 // It indicates the value may not be present in the stream.
848 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
849 hdrStaticInfo->mastering.minLuminance > 0.0f) {
850 struct android_smpte2086_metadata smpte2086_meta = {
851 .displayPrimaryRed = {
852 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
853 },
854 .displayPrimaryGreen = {
855 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
856 },
857 .displayPrimaryBlue = {
858 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
859 },
860 .whitePoint = {
861 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
862 },
863 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
864 .minLuminance = hdrStaticInfo->mastering.minLuminance,
865 };
Yichi Chen54be23c2020-06-15 14:30:53 +0800866 hdr.validTypes |= HdrMetadata::SMPTE2086;
wenchangliuf3f92882020-05-14 00:02:01 +0800867 hdr.smpte2086 = smpte2086_meta;
868 }
Chong Zhang3bb2a7f2020-04-21 10:35:12 -0700869 // If the content light level fields are 0, do not use them, it
870 // indicates the value may not be present in the stream.
871 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
872 struct android_cta861_3_metadata cta861_meta = {
873 .maxContentLightLevel = hdrStaticInfo->maxCll,
874 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
875 };
876 hdr.validTypes |= HdrMetadata::CTA861_3;
877 hdr.cta8613 = cta861_meta;
878 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800879 }
880 if (hdr10PlusInfo) {
881 hdr.validTypes |= HdrMetadata::HDR10PLUS;
882 hdr.hdr10plus.assign(
883 hdr10PlusInfo->m.value,
884 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
885 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800886 qbi.setHdrMetadata(hdr);
887 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800888 // we don't have dirty regions
889 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800890 android::IGraphicBufferProducer::QueueBufferOutput qbo;
891 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
892 if (result != OK) {
893 ALOGI("[%s] queueBuffer failed: %d", mName, result);
Sungtak Lee47c018a2020-11-07 01:02:49 -0800894 if (result == NO_INIT) {
895 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
896 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800897 return result;
898 }
Josh Houcd2f7582021-02-02 16:26:53 +0800899
900 if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
901 ALOGD("[%s] queue buffer successful", mName);
902 } else {
903 ALOGV("[%s] queue buffer successful", mName);
904 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800905
906 int64_t mediaTimeUs = 0;
907 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
908 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
909
910 return OK;
911}
912
913status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
914 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
915 bool released = false;
916 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700917 Mutexed<Input>::Locked input(mInput);
918 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800919 released = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800920 }
921 }
922 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700923 Mutexed<Output>::Locked output(mOutput);
924 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800925 released = true;
926 }
927 }
928 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800929 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800930 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800931 } else {
932 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
933 }
934 return OK;
935}
936
937void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
938 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700939 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800940
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700941 if (!input->buffers->isArrayMode()) {
942 input->buffers = input->buffers->toArrayMode(input->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800943 }
944
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700945 input->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800946}
947
948void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
949 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700950 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800951
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700952 if (!output->buffers->isArrayMode()) {
953 output->buffers = output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800954 }
955
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700956 output->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800957}
958
959status_t CCodecBufferChannel::start(
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800960 const sp<AMessage> &inputFormat,
961 const sp<AMessage> &outputFormat,
962 bool buffersBoundToCodec) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800963 C2StreamBufferTypeSetting::input iStreamFormat(0u);
964 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim0379ae82020-11-24 15:01:33 -0800965 C2ComponentKindSetting kind;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800966 C2PortReorderBufferDepthTuning::output reorderDepth;
967 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800968 C2PortActualDelayTuning::input inputDelay(0);
969 C2PortActualDelayTuning::output outputDelay(0);
970 C2ActualPipelineDelayTuning pipelineDelay(0);
Sungtak Lee04b30352020-07-27 13:57:25 -0700971 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
Wonsik Kim078b58e2019-01-09 15:08:06 -0800972
Pawin Vongmasa36653902018-11-15 00:10:25 -0800973 c2_status_t err = mComponent->query(
974 {
975 &iStreamFormat,
976 &oStreamFormat,
Wonsik Kim0379ae82020-11-24 15:01:33 -0800977 &kind,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800978 &reorderDepth,
979 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800980 &inputDelay,
981 &pipelineDelay,
982 &outputDelay,
Sungtak Lee04b30352020-07-27 13:57:25 -0700983 &secureMode,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800984 },
985 {},
986 C2_DONT_BLOCK,
987 nullptr);
988 if (err == C2_BAD_INDEX) {
Wonsik Kim0379ae82020-11-24 15:01:33 -0800989 if (!iStreamFormat || !oStreamFormat || !kind) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800990 return UNKNOWN_ERROR;
991 }
992 } else if (err != C2_OK) {
993 return UNKNOWN_ERROR;
994 }
995
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -0800996 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
997 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
998 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
999
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001000 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
1001 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
Wonsik Kim078b58e2019-01-09 15:08:06 -08001002
Pawin Vongmasa36653902018-11-15 00:10:25 -08001003 // TODO: get this from input format
1004 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1005
Sungtak Lee04b30352020-07-27 13:57:25 -07001006 // secure mode is a static parameter (shall not change in the executing state)
1007 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1008
Pawin Vongmasa36653902018-11-15 00:10:25 -08001009 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001010 int poolMask = GetCodec2PoolMask();
1011 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001012
1013 if (inputFormat != nullptr) {
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001014 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
Wonsik Kim0379ae82020-11-24 15:01:33 -08001015 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001016 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1017 API_REFLECTION |
1018 API_VALUES |
1019 API_CURRENT_VALUES |
1020 API_DEPENDENCY |
1021 API_SAME_INPUT_BUFFER);
Wonsik Kim0379ae82020-11-24 15:01:33 -08001022 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1023 C2StreamSampleRateInfo::input sampleRate(0u);
1024 C2StreamChannelCountInfo::input channelCount(0u);
1025 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001026 std::shared_ptr<C2BlockPool> pool;
1027 {
1028 Mutexed<BlockPools>::Locked pools(mBlockPools);
1029
1030 // set default allocator ID.
1031 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001032 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001033
1034 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1035 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1036 std::vector<std::unique_ptr<C2Param>> params;
Wonsik Kimffb889a2020-05-28 11:32:25 -07001037 C2ApiFeaturesSetting featuresSetting{apiFeatures};
Wonsik Kim0379ae82020-11-24 15:01:33 -08001038 std::vector<C2Param *> stackParams({&featuresSetting});
1039 if (audioEncoder) {
1040 stackParams.push_back(&encoderFrameSize);
1041 stackParams.push_back(&sampleRate);
1042 stackParams.push_back(&channelCount);
1043 stackParams.push_back(&pcmEncoding);
1044 } else {
1045 encoderFrameSize.invalidate();
1046 sampleRate.invalidate();
1047 channelCount.invalidate();
1048 pcmEncoding.invalidate();
1049 }
1050 err = mComponent->query(stackParams,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001051 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1052 C2_DONT_BLOCK,
1053 &params);
1054 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1055 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1056 mName, params.size(), asString(err), err);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001057 } else if (params.size() == 1) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001058 C2PortAllocatorsTuning::input *inputAllocators =
1059 C2PortAllocatorsTuning::input::From(params[0].get());
1060 if (inputAllocators && inputAllocators->flexCount() > 0) {
1061 std::shared_ptr<C2Allocator> allocator;
1062 // verify allocator IDs and resolve default allocator
1063 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1064 if (allocator) {
1065 pools->inputAllocatorId = allocator->getId();
1066 } else {
1067 ALOGD("[%s] component requested invalid input allocator ID %u",
1068 mName, inputAllocators->m.values[0]);
1069 }
1070 }
1071 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001072 if (featuresSetting) {
1073 apiFeatures = featuresSetting.value;
1074 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001075
1076 // TODO: use C2Component wrapper to associate this pool with ourselves
1077 if ((poolMask >> pools->inputAllocatorId) & 1) {
1078 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1079 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1080 mName, pools->inputAllocatorId,
1081 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1082 asString(err), err);
1083 } else {
1084 err = C2_NOT_FOUND;
1085 }
1086 if (err != C2_OK) {
1087 C2BlockPool::local_id_t inputPoolId =
1088 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1089 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1090 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1091 mName, (unsigned long long)inputPoolId,
1092 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1093 asString(err), err);
1094 if (err != C2_OK) {
1095 return NO_MEMORY;
1096 }
1097 }
1098 pools->inputPool = pool;
1099 }
1100
Wonsik Kim51051262018-11-28 13:59:05 -08001101 bool forceArrayMode = false;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001102 Mutexed<Input>::Locked input(mInput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001103 input->inputDelay = inputDelayValue;
1104 input->pipelineDelay = pipelineDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001105 input->numSlots = numInputSlots;
1106 input->extraBuffers.flush();
1107 input->numExtraSlots = 0u;
Wonsik Kim0379ae82020-11-24 15:01:33 -08001108 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1109 input->frameReassembler.init(
1110 pool,
1111 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1112 encoderFrameSize.value,
1113 sampleRate.value,
1114 channelCount.value,
1115 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1116 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001117 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1118 // For encrypted content, framework decrypts source buffer (ashmem) into
1119 // C2Buffers. Thus non-conforming codecs can process these.
Wonsik Kim0379ae82020-11-24 15:01:33 -08001120 if (!buffersBoundToCodec
1121 && !input->frameReassembler
1122 && (hasCryptoOrDescrambler() || conforming)) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001123 input->buffers.reset(new SlotInputBuffers(mName));
1124 } else if (graphic) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001125 if (mInputSurface) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001126 input->buffers.reset(new DummyInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001127 } else if (mMetaMode == MODE_ANW) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001128 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
Wonsik Kim1221fd12019-07-12 12:52:05 -07001129 // This is to ensure buffers do not get released prematurely.
1130 // TODO: handle this without going into array mode
1131 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001132 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001133 input->buffers.reset(new GraphicInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001134 }
1135 } else {
1136 if (hasCryptoOrDescrambler()) {
1137 int32_t capacity = kLinearBufferSize;
1138 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1139 if ((size_t)capacity > kMaxLinearBufferSize) {
1140 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1141 capacity = kMaxLinearBufferSize;
1142 }
1143 if (mDealer == nullptr) {
1144 mDealer = new MemoryDealer(
1145 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001146 * (numInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001147 "EncryptedLinearInputBuffers");
1148 mDecryptDestination = mDealer->allocate((size_t)capacity);
1149 }
1150 if (mCrypto != nullptr && mHeapSeqNum < 0) {
Robert Shih895fba92019-07-16 16:29:44 -07001151 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1152 mHeapSeqNum = mCrypto->setHeap(heap);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001153 } else {
1154 mHeapSeqNum = -1;
1155 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001156 input->buffers.reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001157 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001158 numInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08001159 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001160 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001161 input->buffers.reset(new LinearInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001162 }
1163 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001164 input->buffers->setFormat(inputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001165
1166 if (err == C2_OK) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001167 input->buffers->setPool(pool);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001168 } else {
1169 // TODO: error
1170 }
Wonsik Kim51051262018-11-28 13:59:05 -08001171
1172 if (forceArrayMode) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001173 input->buffers = input->buffers->toArrayMode(numInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08001174 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001175 }
1176
1177 if (outputFormat != nullptr) {
1178 sp<IGraphicBufferProducer> outputSurface;
1179 uint32_t outputGeneration;
Sungtak Leea714f112021-03-16 05:40:03 -07001180 int maxDequeueCount = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001181 {
1182 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Sungtak Leea714f112021-03-16 05:40:03 -07001183 maxDequeueCount = output->maxDequeueBuffers = numOutputSlots +
Sungtak Lee7a7b7422019-07-16 17:40:40 -07001184 reorderDepth.value + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001185 outputSurface = output->surface ?
1186 output->surface->getIGraphicBufferProducer() : nullptr;
Wonsik Kimf5e5c832019-02-21 11:36:05 -08001187 if (outputSurface) {
1188 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1189 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001190 outputGeneration = output->generation;
1191 }
1192
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001193 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001194 C2BlockPool::local_id_t outputPoolId_;
David Stevensc3fbb282021-01-18 18:11:20 +09001195 C2BlockPool::local_id_t prevOutputPoolId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001196
1197 {
1198 Mutexed<BlockPools>::Locked pools(mBlockPools);
1199
David Stevensc3fbb282021-01-18 18:11:20 +09001200 prevOutputPoolId = pools->outputPoolId;
1201
Pawin Vongmasa36653902018-11-15 00:10:25 -08001202 // set default allocator ID.
1203 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001204 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001205
1206 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1207 // unsuccessful.
1208 std::vector<std::unique_ptr<C2Param>> params;
1209 err = mComponent->query({ },
1210 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1211 C2_DONT_BLOCK,
1212 &params);
1213 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1214 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1215 mName, params.size(), asString(err), err);
1216 } else if (err == C2_OK && params.size() == 1) {
1217 C2PortAllocatorsTuning::output *outputAllocators =
1218 C2PortAllocatorsTuning::output::From(params[0].get());
1219 if (outputAllocators && outputAllocators->flexCount() > 0) {
1220 std::shared_ptr<C2Allocator> allocator;
1221 // verify allocator IDs and resolve default allocator
1222 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1223 if (allocator) {
1224 pools->outputAllocatorId = allocator->getId();
1225 } else {
1226 ALOGD("[%s] component requested invalid output allocator ID %u",
1227 mName, outputAllocators->m.values[0]);
1228 }
1229 }
1230 }
1231
1232 // use bufferqueue if outputting to a surface.
1233 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1234 // if unsuccessful.
1235 if (outputSurface) {
1236 params.clear();
1237 err = mComponent->query({ },
1238 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1239 C2_DONT_BLOCK,
1240 &params);
1241 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1242 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1243 mName, params.size(), asString(err), err);
1244 } else if (err == C2_OK && params.size() == 1) {
1245 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1246 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1247 if (surfaceAllocator) {
1248 std::shared_ptr<C2Allocator> allocator;
1249 // verify allocator IDs and resolve default allocator
1250 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1251 if (allocator) {
1252 pools->outputAllocatorId = allocator->getId();
1253 } else {
1254 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1255 mName, surfaceAllocator->value);
1256 err = C2_BAD_VALUE;
1257 }
1258 }
1259 }
1260 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1261 && err != C2_OK
1262 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1263 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1264 }
1265 }
1266
1267 if ((poolMask >> pools->outputAllocatorId) & 1) {
1268 err = mComponent->createBlockPool(
1269 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1270 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1271 mName, pools->outputAllocatorId,
1272 (unsigned long long)pools->outputPoolId,
1273 asString(err));
1274 } else {
1275 err = C2_NOT_FOUND;
1276 }
1277 if (err != C2_OK) {
1278 // use basic pool instead
1279 pools->outputPoolId =
1280 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1281 }
1282
1283 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1284 // component.
1285 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1286 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1287
1288 std::vector<std::unique_ptr<C2SettingResult>> failures;
1289 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1290 ALOGD("[%s] Configured output block pool ids %llu => %s",
1291 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1292 outputPoolId_ = pools->outputPoolId;
1293 }
1294
David Stevensc3fbb282021-01-18 18:11:20 +09001295 if (prevOutputPoolId != C2BlockPool::BASIC_LINEAR
1296 && prevOutputPoolId != C2BlockPool::BASIC_GRAPHIC) {
1297 c2_status_t err = mComponent->destroyBlockPool(prevOutputPoolId);
1298 if (err != C2_OK) {
1299 ALOGW("Failed to clean up previous block pool %llu - %s (%d)\n",
1300 (unsigned long long) prevOutputPoolId, asString(err), err);
1301 }
1302 }
1303
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001304 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001305 output->outputDelay = outputDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001306 output->numSlots = numOutputSlots;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001307 if (graphic) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001308 if (outputSurface || !buffersBoundToCodec) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001309 output->buffers.reset(new GraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001310 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001311 output->buffers.reset(new RawGraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001312 }
1313 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001314 output->buffers.reset(new LinearOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001315 }
Wonsik Kime4716c02020-02-28 10:42:21 -08001316 output->buffers->setFormat(outputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001317
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001318 output->buffers->clearStash();
1319 if (reorderDepth) {
1320 output->buffers->setReorderDepth(reorderDepth.value);
1321 }
1322 if (reorderKey) {
1323 output->buffers->setReorderKey(reorderKey.value);
1324 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001325
1326 // Try to set output surface to created block pool if given.
1327 if (outputSurface) {
1328 mComponent->setOutputSurface(
1329 outputPoolId_,
1330 outputSurface,
Sungtak Leedb14cba2021-04-10 00:50:23 -07001331 outputGeneration,
1332 maxDequeueCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001333 }
1334
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001335 if (oStreamFormat.value == C2BufferData::LINEAR) {
Wonsik Kim58713302020-01-29 22:25:23 -08001336 if (buffersBoundToCodec) {
1337 // WORKAROUND: if we're using early CSD workaround we convert to
1338 // array mode, to appease apps assuming the output
1339 // buffers to be of the same size.
1340 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1341 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001342
1343 int32_t channelCount;
1344 int32_t sampleRate;
1345 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1346 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1347 int32_t delay = 0;
1348 int32_t padding = 0;;
1349 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1350 delay = 0;
1351 }
1352 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1353 padding = 0;
1354 }
1355 if (delay || padding) {
1356 // We need write access to the buffers, and we're already in
1357 // array mode.
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001358 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001359 }
1360 }
1361 }
1362 }
1363
1364 // Set up pipeline control. This has to be done after mInputBuffers and
1365 // mOutputBuffers are initialized to make sure that lingering callbacks
1366 // about buffers from the previous generation do not interfere with the
1367 // newly initialized pipeline capacity.
1368
Wonsik Kim62545252021-01-20 11:25:41 -08001369 if (inputFormat || outputFormat) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08001370 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001371 watcher->inputDelay(inputDelayValue)
1372 .pipelineDelay(pipelineDelayValue)
1373 .outputDelay(outputDelayValue)
Wonsik Kimab34ed62019-01-31 15:28:46 -08001374 .smoothnessFactor(kSmoothnessFactor);
1375 watcher->flush();
1376 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001377
1378 mInputMetEos = false;
1379 mSync.start();
1380 return OK;
1381}
1382
1383status_t CCodecBufferChannel::requestInitialInputBuffers() {
1384 if (mInputSurface) {
1385 return OK;
1386 }
1387
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001388 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001389 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1390 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1391 if (err != C2_OK && err != C2_BAD_INDEX) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001392 return UNKNOWN_ERROR;
1393 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001394 size_t numInputSlots = mInput.lock()->numSlots;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001395
1396 struct ClientInputBuffer {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001397 size_t index;
1398 sp<MediaCodecBuffer> buffer;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001399 size_t capacity;
1400 };
1401 std::list<ClientInputBuffer> clientInputBuffers;
1402
1403 {
1404 Mutexed<Input>::Locked input(mInput);
1405 while (clientInputBuffers.size() < numInputSlots) {
1406 ClientInputBuffer clientInputBuffer;
1407 if (!input->buffers->requestNewBuffer(&clientInputBuffer.index,
1408 &clientInputBuffer.buffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001409 break;
1410 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001411 clientInputBuffer.capacity = clientInputBuffer.buffer->capacity();
1412 clientInputBuffers.emplace_back(std::move(clientInputBuffer));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001413 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001414 }
1415 if (clientInputBuffers.empty()) {
1416 ALOGW("[%s] start: cannot allocate memory at all", mName);
1417 return NO_MEMORY;
1418 } else if (clientInputBuffers.size() < numInputSlots) {
1419 ALOGD("[%s] start: cannot allocate memory for all slots, "
1420 "only %zu buffers allocated",
1421 mName, clientInputBuffers.size());
1422 } else {
1423 ALOGV("[%s] %zu initial input buffers available",
1424 mName, clientInputBuffers.size());
1425 }
1426 // Sort input buffers by their capacities in increasing order.
1427 clientInputBuffers.sort(
1428 [](const ClientInputBuffer& a, const ClientInputBuffer& b) {
1429 return a.capacity < b.capacity;
1430 });
1431
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001432 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1433 mFlushedConfigs.lock()->swap(flushedConfigs);
1434 if (!flushedConfigs.empty()) {
1435 err = mComponent->queue(&flushedConfigs);
1436 if (err != C2_OK) {
1437 ALOGW("[%s] Error while queueing a flushed config", mName);
1438 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001439 }
1440 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001441 if (oStreamFormat.value == C2BufferData::LINEAR &&
1442 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE)) {
1443 sp<MediaCodecBuffer> buffer = clientInputBuffers.front().buffer;
1444 // WORKAROUND: Some apps expect CSD available without queueing
1445 // any input. Queue an empty buffer to get the CSD.
1446 buffer->setRange(0, 0);
1447 buffer->meta()->clear();
1448 buffer->meta()->setInt64("timeUs", 0);
1449 if (queueInputBufferInternal(buffer) != OK) {
1450 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1451 mName);
1452 return UNKNOWN_ERROR;
1453 }
1454 clientInputBuffers.pop_front();
1455 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001456
1457 for (const ClientInputBuffer& clientInputBuffer: clientInputBuffers) {
1458 mCallback->onInputBufferAvailable(
1459 clientInputBuffer.index,
1460 clientInputBuffer.buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001461 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001462
Pawin Vongmasa36653902018-11-15 00:10:25 -08001463 return OK;
1464}
1465
1466void CCodecBufferChannel::stop() {
1467 mSync.stop();
1468 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001469}
1470
Wonsik Kim936a89c2020-05-08 16:07:50 -07001471void CCodecBufferChannel::reset() {
1472 stop();
Wonsik Kim62545252021-01-20 11:25:41 -08001473 if (mInputSurface != nullptr) {
1474 mInputSurface.reset();
1475 }
1476 mPipelineWatcher.lock()->flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001477 {
1478 Mutexed<Input>::Locked input(mInput);
1479 input->buffers.reset(new DummyInputBuffers(""));
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001480 input->extraBuffers.flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001481 }
1482 {
1483 Mutexed<Output>::Locked output(mOutput);
1484 output->buffers.reset();
1485 }
1486}
1487
1488void CCodecBufferChannel::release() {
1489 mComponent.reset();
1490 mInputAllocator.reset();
1491 mOutputSurface.lock()->surface.clear();
1492 {
1493 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1494 blockPools->inputPool.reset();
1495 blockPools->outputPoolIntf.reset();
1496 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001497 setCrypto(nullptr);
1498 setDescrambler(nullptr);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001499}
1500
1501
Pawin Vongmasa36653902018-11-15 00:10:25 -08001502void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1503 ALOGV("[%s] flush", mName);
Wonsik Kim62545252021-01-20 11:25:41 -08001504 std::vector<uint64_t> indices;
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001505 std::list<std::unique_ptr<C2Work>> configs;
1506 for (const std::unique_ptr<C2Work> &work : flushedWork) {
Wonsik Kim62545252021-01-20 11:25:41 -08001507 indices.push_back(work->input.ordinal.frameIndex.peeku());
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001508 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1509 continue;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001510 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001511 if (work->input.buffers.empty()
1512 || work->input.buffers.front() == nullptr
1513 || work->input.buffers.front()->data().linearBlocks().empty()) {
1514 ALOGD("[%s] no linear codec config data found", mName);
1515 continue;
1516 }
1517 std::unique_ptr<C2Work> copy(new C2Work);
1518 copy->input.flags = C2FrameData::flags_t(work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1519 copy->input.ordinal = work->input.ordinal;
Wonsik Kim62545252021-01-20 11:25:41 -08001520 copy->input.ordinal.frameIndex = mFrameIndex++;
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001521 copy->input.buffers.insert(
1522 copy->input.buffers.begin(),
1523 work->input.buffers.begin(),
1524 work->input.buffers.end());
1525 for (const std::unique_ptr<C2Param> &param : work->input.configUpdate) {
1526 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1527 }
1528 copy->input.infoBuffers.insert(
1529 copy->input.infoBuffers.begin(),
1530 work->input.infoBuffers.begin(),
1531 work->input.infoBuffers.end());
1532 copy->worklets.emplace_back(new C2Worklet);
1533 configs.push_back(std::move(copy));
1534 ALOGV("[%s] stashed flushed codec config data", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001535 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001536 mFlushedConfigs.lock()->swap(configs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001537 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001538 Mutexed<Input>::Locked input(mInput);
1539 input->buffers->flush();
1540 input->extraBuffers.flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001541 }
1542 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001543 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001544 if (output->buffers) {
1545 output->buffers->flush(flushedWork);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001546 output->buffers->flushStash();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001547 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001548 }
Wonsik Kim62545252021-01-20 11:25:41 -08001549 {
1550 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1551 for (uint64_t index : indices) {
1552 watcher->onWorkDone(index);
1553 }
1554 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001555}
1556
1557void CCodecBufferChannel::onWorkDone(
1558 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
Wonsik Kimab34ed62019-01-31 15:28:46 -08001559 const C2StreamInitDataInfo::output *initData) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001560 if (handleWork(std::move(work), outputFormat, initData)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001561 feedInputBufferIfAvailable();
1562 }
1563}
1564
1565void CCodecBufferChannel::onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -08001566 uint64_t frameIndex, size_t arrayIndex) {
Pawin Vongmasa8e2cfb52019-05-15 05:20:52 -07001567 if (mInputSurface) {
1568 return;
1569 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001570 std::shared_ptr<C2Buffer> buffer =
1571 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001572 bool newInputSlotAvailable;
1573 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001574 Mutexed<Input>::Locked input(mInput);
1575 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1576 if (!newInputSlotAvailable) {
1577 (void)input->extraBuffers.expireComponentBuffer(buffer);
1578 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001579 }
1580 if (newInputSlotAvailable) {
1581 feedInputBufferIfAvailable();
1582 }
1583}
1584
1585bool CCodecBufferChannel::handleWork(
1586 std::unique_ptr<C2Work> work,
1587 const sp<AMessage> &outputFormat,
1588 const C2StreamInitDataInfo::output *initData) {
Wonsik Kim936a89c2020-05-08 16:07:50 -07001589 {
Wonsik Kima4e049d2020-04-28 19:42:23 +00001590 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001591 if (!output->buffers) {
1592 return false;
1593 }
Wonsik Kime75a5da2020-02-14 17:29:03 -08001594 }
1595
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001596 // Whether the output buffer should be reported to the client or not.
1597 bool notifyClient = false;
1598
1599 if (work->result == C2_OK){
1600 notifyClient = true;
1601 } else if (work->result == C2_NOT_FOUND) {
1602 ALOGD("[%s] flushed work; ignored.", mName);
1603 } else {
1604 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1605 // the config update.
1606 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1607 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1608 return false;
1609 }
1610
1611 if ((work->input.ordinal.frameIndex -
1612 mFirstValidFrameIndex.load()).peek() < 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001613 // Discard frames from previous generation.
1614 ALOGD("[%s] Discard frames from previous generation.", mName);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001615 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001616 }
1617
Wonsik Kim524b0582019-03-12 11:28:57 -07001618 if (mInputSurface == nullptr && (work->worklets.size() != 1u
Pawin Vongmasa36653902018-11-15 00:10:25 -08001619 || !work->worklets.front()
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001620 || !(work->worklets.front()->output.flags &
1621 C2FrameData::FLAG_INCOMPLETE))) {
1622 mPipelineWatcher.lock()->onWorkDone(
1623 work->input.ordinal.frameIndex.peeku());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001624 }
1625
1626 // NOTE: MediaCodec usage supposedly have only one worklet
1627 if (work->worklets.size() != 1u) {
1628 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1629 mName, work->worklets.size());
1630 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1631 return false;
1632 }
1633
1634 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1635
1636 std::shared_ptr<C2Buffer> buffer;
1637 // NOTE: MediaCodec usage supposedly have only one output stream.
1638 if (worklet->output.buffers.size() > 1u) {
1639 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1640 mName, worklet->output.buffers.size());
1641 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1642 return false;
1643 } else if (worklet->output.buffers.size() == 1u) {
1644 buffer = worklet->output.buffers[0];
1645 if (!buffer) {
1646 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1647 }
1648 }
1649
Wonsik Kim84f439f2021-05-03 10:57:09 -07001650 std::optional<uint32_t> newInputDelay, newPipelineDelay, newOutputDelay, newReorderDepth;
1651 std::optional<C2Config::ordinal_key_t> newReorderKey;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001652 bool needMaxDequeueBufferCountUpdate = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001653 while (!worklet->output.configUpdate.empty()) {
1654 std::unique_ptr<C2Param> param;
1655 worklet->output.configUpdate.back().swap(param);
1656 worklet->output.configUpdate.pop_back();
1657 switch (param->coreIndex().coreIndex()) {
1658 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1659 C2PortReorderBufferDepthTuning::output reorderDepth;
1660 if (reorderDepth.updateFrom(*param)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001661 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1662 mName, reorderDepth.value);
Wonsik Kim84f439f2021-05-03 10:57:09 -07001663 newReorderDepth = reorderDepth.value;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001664 needMaxDequeueBufferCountUpdate = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001665 } else {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001666 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1667 mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001668 }
1669 break;
1670 }
1671 case C2PortReorderKeySetting::CORE_INDEX: {
1672 C2PortReorderKeySetting::output reorderKey;
1673 if (reorderKey.updateFrom(*param)) {
Wonsik Kim84f439f2021-05-03 10:57:09 -07001674 newReorderKey = reorderKey.value;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001675 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1676 mName, reorderKey.value);
1677 } else {
1678 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1679 }
1680 break;
1681 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001682 case C2PortActualDelayTuning::CORE_INDEX: {
1683 if (param->isGlobal()) {
1684 C2ActualPipelineDelayTuning pipelineDelay;
1685 if (pipelineDelay.updateFrom(*param)) {
1686 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1687 mName, pipelineDelay.value);
1688 newPipelineDelay = pipelineDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001689 (void)mPipelineWatcher.lock()->pipelineDelay(
1690 pipelineDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001691 }
1692 }
1693 if (param->forInput()) {
1694 C2PortActualDelayTuning::input inputDelay;
1695 if (inputDelay.updateFrom(*param)) {
1696 ALOGV("[%s] onWorkDone: updating input delay %u",
1697 mName, inputDelay.value);
1698 newInputDelay = inputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001699 (void)mPipelineWatcher.lock()->inputDelay(
1700 inputDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001701 }
1702 }
1703 if (param->forOutput()) {
1704 C2PortActualDelayTuning::output outputDelay;
1705 if (outputDelay.updateFrom(*param)) {
1706 ALOGV("[%s] onWorkDone: updating output delay %u",
1707 mName, outputDelay.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001708 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
Wonsik Kim84f439f2021-05-03 10:57:09 -07001709 newOutputDelay = outputDelay.value;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001710 needMaxDequeueBufferCountUpdate = true;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001711
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001712 }
1713 }
1714 break;
1715 }
ted.sunb1fbfdb2020-06-23 14:03:41 +08001716 case C2PortTunnelSystemTime::CORE_INDEX: {
1717 C2PortTunnelSystemTime::output frameRenderTime;
1718 if (frameRenderTime.updateFrom(*param)) {
1719 ALOGV("[%s] onWorkDone: frame rendered (sys:%lld ns, media:%lld us)",
1720 mName, (long long)frameRenderTime.value,
1721 (long long)worklet->output.ordinal.timestamp.peekll());
1722 mCCodecCallback->onOutputFramesRendered(
1723 worklet->output.ordinal.timestamp.peek(), frameRenderTime.value);
1724 }
1725 break;
1726 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001727 default:
1728 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1729 mName, param->index());
1730 break;
1731 }
1732 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001733 if (newInputDelay || newPipelineDelay) {
1734 Mutexed<Input>::Locked input(mInput);
1735 size_t newNumSlots =
1736 newInputDelay.value_or(input->inputDelay) +
1737 newPipelineDelay.value_or(input->pipelineDelay) +
1738 kSmoothnessFactor;
1739 if (input->buffers->isArrayMode()) {
1740 if (input->numSlots >= newNumSlots) {
1741 input->numExtraSlots = 0;
1742 } else {
1743 input->numExtraSlots = newNumSlots - input->numSlots;
1744 }
1745 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1746 mName, input->numExtraSlots);
1747 } else {
1748 input->numSlots = newNumSlots;
1749 }
1750 }
Wonsik Kim84f439f2021-05-03 10:57:09 -07001751 size_t numOutputSlots = 0;
1752 uint32_t reorderDepth = 0;
1753 bool outputBuffersChanged = false;
1754 if (newReorderKey || newReorderDepth || needMaxDequeueBufferCountUpdate) {
1755 Mutexed<Output>::Locked output(mOutput);
1756 if (!output->buffers) {
1757 return false;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001758 }
Wonsik Kim84f439f2021-05-03 10:57:09 -07001759 numOutputSlots = output->numSlots;
1760 if (newReorderKey) {
1761 output->buffers->setReorderKey(newReorderKey.value());
1762 }
1763 if (newReorderDepth) {
1764 output->buffers->setReorderDepth(newReorderDepth.value());
1765 }
1766 reorderDepth = output->buffers->getReorderDepth();
1767 if (newOutputDelay) {
1768 output->outputDelay = newOutputDelay.value();
1769 numOutputSlots = newOutputDelay.value() + kSmoothnessFactor;
1770 if (output->numSlots < numOutputSlots) {
1771 output->numSlots = numOutputSlots;
1772 if (output->buffers->isArrayMode()) {
1773 OutputBuffersArray *array =
1774 (OutputBuffersArray *)output->buffers.get();
1775 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1776 mName, numOutputSlots);
1777 array->grow(numOutputSlots);
1778 outputBuffersChanged = true;
1779 }
1780 }
1781 }
1782 numOutputSlots = output->numSlots;
1783 }
1784 if (outputBuffersChanged) {
1785 mCCodecCallback->onOutputBuffersChanged();
1786 }
1787 if (needMaxDequeueBufferCountUpdate) {
1788 int maxDequeueCount = 0;
Sungtak Leea714f112021-03-16 05:40:03 -07001789 {
1790 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1791 maxDequeueCount = output->maxDequeueBuffers =
1792 numOutputSlots + reorderDepth + kRenderingDepth;
1793 if (output->surface) {
1794 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1795 }
1796 }
1797 if (maxDequeueCount > 0) {
1798 mComponent->setOutputSurfaceMaxDequeueCount(maxDequeueCount);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001799 }
1800 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001801
Pawin Vongmasa36653902018-11-15 00:10:25 -08001802 int32_t flags = 0;
1803 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1804 flags |= MediaCodec::BUFFER_FLAG_EOS;
1805 ALOGV("[%s] onWorkDone: output EOS", mName);
1806 }
1807
Pawin Vongmasa36653902018-11-15 00:10:25 -08001808 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1809 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1810 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1811 // shall correspond to the client input timesamp (in customOrdinal). By using the
1812 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1813 // produces multiple output.
1814 c2_cntr64_t timestamp =
1815 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1816 - work->input.ordinal.timestamp;
Wonsik Kim95ba0162019-03-19 15:51:54 -07001817 if (mInputSurface != nullptr) {
1818 // When using input surface we need to restore the original input timestamp.
1819 timestamp = work->input.ordinal.customOrdinal;
1820 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001821 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1822 mName,
1823 work->input.ordinal.customOrdinal.peekll(),
1824 work->input.ordinal.timestamp.peekll(),
1825 worklet->output.ordinal.timestamp.peekll(),
1826 timestamp.peekll());
1827
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001828 // csd cannot be re-ordered and will always arrive first.
Pawin Vongmasa36653902018-11-15 00:10:25 -08001829 if (initData != nullptr) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001830 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001831 if (output->buffers && outputFormat) {
1832 output->buffers->updateSkipCutBuffer(outputFormat);
1833 output->buffers->setFormat(outputFormat);
1834 }
1835 if (!notifyClient) {
1836 return false;
1837 }
1838 size_t index;
1839 sp<MediaCodecBuffer> outBuffer;
Wonsik Kim936a89c2020-05-08 16:07:50 -07001840 if (output->buffers && output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001841 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1842 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1843 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1844
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001845 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001846 mCallback->onOutputBufferAvailable(index, outBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001847 } else {
1848 ALOGD("[%s] onWorkDone: unable to register csd", mName);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001849 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001850 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001851 return false;
1852 }
1853 }
1854
ted.sunb8fe01e2020-06-23 14:03:41 +08001855 bool drop = false;
1856 if (worklet->output.flags & C2FrameData::FLAG_DROP_FRAME) {
1857 ALOGV("[%s] onWorkDone: drop buffer but keep metadata", mName);
1858 drop = true;
1859 }
1860
ted.sun04698a32020-06-23 14:03:41 +08001861 if (notifyClient && !buffer && !flags && !(drop && outputFormat)) {
Wonsik Kim35bf5732020-05-14 17:40:29 +00001862 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
Pawin Vongmasa36653902018-11-15 00:10:25 -08001863 mName, work->input.ordinal.frameIndex.peekull());
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001864 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001865 }
1866
1867 if (buffer) {
1868 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1869 // TODO: properly translate these to metadata
1870 switch (info->coreIndex().coreIndex()) {
1871 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001872 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001873 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1874 }
1875 break;
1876 default:
1877 break;
1878 }
1879 }
1880 }
1881
1882 {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001883 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimc23cc402020-05-28 14:53:40 -07001884 if (!output->buffers) {
1885 return false;
1886 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001887 output->buffers->pushToStash(
ted.sun04698a32020-06-23 14:03:41 +08001888 drop ? nullptr : buffer,
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001889 notifyClient,
1890 timestamp.peek(),
1891 flags,
1892 outputFormat,
1893 worklet->output.ordinal);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001894 }
1895 sendOutputBuffers();
1896 return true;
1897}
1898
1899void CCodecBufferChannel::sendOutputBuffers() {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001900 OutputBuffers::BufferAction action;
Wonsik Kima4e049d2020-04-28 19:42:23 +00001901 size_t index;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001902 sp<MediaCodecBuffer> outBuffer;
1903 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001904
1905 while (true) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001906 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001907 if (!output->buffers) {
1908 return;
1909 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001910 action = output->buffers->popFromStashAndRegister(
1911 &c2Buffer, &index, &outBuffer);
1912 switch (action) {
1913 case OutputBuffers::SKIP:
1914 return;
1915 case OutputBuffers::DISCARD:
1916 break;
1917 case OutputBuffers::NOTIFY_CLIENT:
Wonsik Kima4e049d2020-04-28 19:42:23 +00001918 output.unlock();
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001919 mCallback->onOutputBufferAvailable(index, outBuffer);
1920 break;
1921 case OutputBuffers::REALLOCATE:
1922 if (!output->buffers->isArrayMode()) {
1923 output->buffers =
1924 output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001925 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001926 static_cast<OutputBuffersArray*>(output->buffers.get())->
1927 realloc(c2Buffer);
1928 output.unlock();
1929 mCCodecCallback->onOutputBuffersChanged();
Wonsik Kim4ada73d2020-05-26 14:58:07 -07001930 break;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001931 case OutputBuffers::RETRY:
1932 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
1933 mName);
1934 return;
1935 default:
1936 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
1937 "corrupted BufferAction value (%d) "
1938 "returned from popFromStashAndRegister.",
1939 mName, int(action));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001940 return;
1941 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001942 }
1943}
1944
1945status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1946 static std::atomic_uint32_t surfaceGeneration{0};
1947 uint32_t generation = (getpid() << 10) |
1948 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1949 & ((1 << 10) - 1));
1950
1951 sp<IGraphicBufferProducer> producer;
Sungtak Leedb14cba2021-04-10 00:50:23 -07001952 int maxDequeueCount = mOutputSurface.lock()->maxDequeueBuffers;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001953 if (newSurface) {
1954 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Sungtak Leeab6f2f32019-02-15 14:43:51 -08001955 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
Sungtak Leedb14cba2021-04-10 00:50:23 -07001956 newSurface->setMaxDequeuedBufferCount(maxDequeueCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001957 producer = newSurface->getIGraphicBufferProducer();
1958 producer->setGenerationNumber(generation);
1959 } else {
1960 ALOGE("[%s] setting output surface to null", mName);
1961 return INVALID_OPERATION;
1962 }
1963
1964 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
1965 C2BlockPool::local_id_t outputPoolId;
1966 {
1967 Mutexed<BlockPools>::Locked pools(mBlockPools);
1968 outputPoolId = pools->outputPoolId;
1969 outputPoolIntf = pools->outputPoolIntf;
1970 }
1971
1972 if (outputPoolIntf) {
1973 if (mComponent->setOutputSurface(
1974 outputPoolId,
1975 producer,
Sungtak Leedb14cba2021-04-10 00:50:23 -07001976 generation,
1977 maxDequeueCount) != C2_OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001978 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
1979 return INVALID_OPERATION;
1980 }
1981 }
1982
1983 {
1984 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1985 output->surface = newSurface;
1986 output->generation = generation;
1987 }
1988
1989 return OK;
1990}
1991
Wonsik Kimab34ed62019-01-31 15:28:46 -08001992PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001993 // When client pushed EOS, we want all the work to be done quickly.
1994 // Otherwise, component may have stalled work due to input starvation up to
1995 // the sum of the delay in the pipeline.
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001996 size_t n = 0;
1997 if (!mInputMetEos) {
1998 size_t outputDelay = mOutput.lock()->outputDelay;
1999 Mutexed<Input>::Locked input(mInput);
2000 n = input->inputDelay + input->pipelineDelay + outputDelay;
2001 }
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08002002 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
Wonsik Kimab34ed62019-01-31 15:28:46 -08002003}
2004
Pawin Vongmasa36653902018-11-15 00:10:25 -08002005void CCodecBufferChannel::setMetaMode(MetaMode mode) {
2006 mMetaMode = mode;
2007}
2008
Wonsik Kim596187e2019-10-25 12:44:10 -07002009void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002010 if (mCrypto != nullptr) {
2011 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
2012 mCrypto->unsetHeap(entry.second);
2013 }
2014 mHeapSeqNumMap.clear();
2015 if (mHeapSeqNum >= 0) {
2016 mCrypto->unsetHeap(mHeapSeqNum);
2017 mHeapSeqNum = -1;
2018 }
2019 }
Wonsik Kim596187e2019-10-25 12:44:10 -07002020 mCrypto = crypto;
2021}
2022
2023void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
2024 mDescrambler = descrambler;
2025}
2026
Pawin Vongmasa36653902018-11-15 00:10:25 -08002027status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2028 // C2_OK is always translated to OK.
2029 if (c2s == C2_OK) {
2030 return OK;
2031 }
2032
2033 // Operation-dependent translation
2034 // TODO: Add as necessary
2035 switch (c2op) {
2036 case C2_OPERATION_Component_start:
2037 switch (c2s) {
2038 case C2_NO_MEMORY:
2039 return NO_MEMORY;
2040 default:
2041 return UNKNOWN_ERROR;
2042 }
2043 default:
2044 break;
2045 }
2046
2047 // Backup operation-agnostic translation
2048 switch (c2s) {
2049 case C2_BAD_INDEX:
2050 return BAD_INDEX;
2051 case C2_BAD_VALUE:
2052 return BAD_VALUE;
2053 case C2_BLOCKING:
2054 return WOULD_BLOCK;
2055 case C2_DUPLICATE:
2056 return ALREADY_EXISTS;
2057 case C2_NO_INIT:
2058 return NO_INIT;
2059 case C2_NO_MEMORY:
2060 return NO_MEMORY;
2061 case C2_NOT_FOUND:
2062 return NAME_NOT_FOUND;
2063 case C2_TIMED_OUT:
2064 return TIMED_OUT;
2065 case C2_BAD_STATE:
2066 case C2_CANCELED:
2067 case C2_CANNOT_DO:
2068 case C2_CORRUPTED:
2069 case C2_OMITTED:
2070 case C2_REFUSED:
2071 return UNKNOWN_ERROR;
2072 default:
2073 return -static_cast<status_t>(c2s);
2074 }
2075}
2076
2077} // namespace android