blob: f88408e64cec16d66ce9b4827c97d310dd2fc692 [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
Pawin Vongmasae7bb8612020-06-04 06:15:22 -070021#include <algorithm>
22#include <list>
Pawin Vongmasa36653902018-11-15 00:10:25 -080023#include <numeric>
24
25#include <C2AllocatorGralloc.h>
26#include <C2PlatformSupport.h>
27#include <C2BlockInternal.h>
28#include <C2Config.h>
29#include <C2Debug.h>
30
31#include <android/hardware/cas/native/1.0/IDescrambler.h>
Robert Shih895fba92019-07-16 16:29:44 -070032#include <android/hardware/drm/1.0/types.h>
Josh Houcd2f7582021-02-02 16:26:53 +080033#include <android-base/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080034#include <android-base/stringprintf.h>
Wonsik Kimfb7a7672019-12-27 17:13:33 -080035#include <binder/MemoryBase.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080036#include <binder/MemoryDealer.h>
Ray Essick18ea0452019-08-27 16:07:27 -070037#include <cutils/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080038#include <gui/Surface.h>
Robert Shih895fba92019-07-16 16:29:44 -070039#include <hidlmemory/FrameworkUtils.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080040#include <media/openmax/OMX_Core.h>
41#include <media/stagefright/foundation/ABuffer.h>
42#include <media/stagefright/foundation/ALookup.h>
43#include <media/stagefright/foundation/AMessage.h>
44#include <media/stagefright/foundation/AUtils.h>
45#include <media/stagefright/foundation/hexdump.h>
46#include <media/stagefright/MediaCodec.h>
47#include <media/stagefright/MediaCodecConstants.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070048#include <media/stagefright/SkipCutBuffer.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080049#include <media/MediaCodecBuffer.h>
Wonsik Kim41d83432020-04-27 16:40:49 -070050#include <mediadrm/ICrypto.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080051#include <system/window.h>
52
53#include "CCodecBufferChannel.h"
54#include "Codec2Buffer.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080055
56namespace android {
57
58using android::base::StringPrintf;
59using hardware::hidl_handle;
60using hardware::hidl_string;
61using hardware::hidl_vec;
Robert Shih895fba92019-07-16 16:29:44 -070062using hardware::fromHeap;
63using hardware::HidlMemory;
64
Pawin Vongmasa36653902018-11-15 00:10:25 -080065using namespace hardware::cas::V1_0;
66using namespace hardware::cas::native::V1_0;
67
68using CasStatus = hardware::cas::V1_0::Status;
Robert Shih895fba92019-07-16 16:29:44 -070069using DrmBufferType = hardware::drm::V1_0::BufferType;
Pawin Vongmasa36653902018-11-15 00:10:25 -080070
Pawin Vongmasa36653902018-11-15 00:10:25 -080071namespace {
72
Wonsik Kim469c8342019-04-11 16:46:09 -070073constexpr size_t kSmoothnessFactor = 4;
74constexpr size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -080075
Sungtak Leeab6f2f32019-02-15 14:43:51 -080076// This is for keeping IGBP's buffer dropping logic in legacy mode other
77// than making it non-blocking. Do not change this value.
78const static size_t kDequeueTimeoutNs = 0;
79
Pawin Vongmasa36653902018-11-15 00:10:25 -080080} // namespace
81
82CCodecBufferChannel::QueueGuard::QueueGuard(
83 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
84 Mutex::Autolock l(mSync.mGuardLock);
85 // At this point it's guaranteed that mSync is not under state transition,
86 // as we are holding its mutex.
87
88 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
89 if (count->value == -1) {
90 mRunning = false;
91 } else {
92 ++count->value;
93 mRunning = true;
94 }
95}
96
97CCodecBufferChannel::QueueGuard::~QueueGuard() {
98 if (mRunning) {
99 // We are not holding mGuardLock at this point so that QueueSync::stop() can
100 // keep holding the lock until mCount reaches zero.
101 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
102 --count->value;
103 count->cond.broadcast();
104 }
105}
106
107void CCodecBufferChannel::QueueSync::start() {
108 Mutex::Autolock l(mGuardLock);
109 // If stopped, it goes to running state; otherwise no-op.
110 Mutexed<Counter>::Locked count(mCount);
111 if (count->value == -1) {
112 count->value = 0;
113 }
114}
115
116void CCodecBufferChannel::QueueSync::stop() {
117 Mutex::Autolock l(mGuardLock);
118 Mutexed<Counter>::Locked count(mCount);
119 if (count->value == -1) {
120 // no-op
121 return;
122 }
123 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
124 // mCount can only decrement. In other words, threads that acquired the lock
125 // are allowed to finish execution but additional threads trying to acquire
126 // the lock at this point will block, and then get QueueGuard at STOPPED
127 // state.
128 while (count->value != 0) {
129 count.waitForCondition(count->cond);
130 }
131 count->value = -1;
132}
133
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700134// Input
135
136CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
137
Pawin Vongmasa36653902018-11-15 00:10:25 -0800138// CCodecBufferChannel
139
140CCodecBufferChannel::CCodecBufferChannel(
141 const std::shared_ptr<CCodecCallback> &callback)
142 : mHeapSeqNum(-1),
143 mCCodecCallback(callback),
144 mFrameIndex(0u),
145 mFirstValidFrameIndex(0u),
146 mMetaMode(MODE_NONE),
Sungtak Lee04b30352020-07-27 13:57:25 -0700147 mInputMetEos(false),
148 mSendEncryptedInfoBuffer(false) {
Sungtak Leed7463d12019-09-04 16:01:00 -0700149 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700150 {
151 Mutexed<Input>::Locked input(mInput);
152 input->buffers.reset(new DummyInputBuffers(""));
153 input->extraBuffers.flush();
154 input->inputDelay = 0u;
155 input->pipelineDelay = 0u;
156 input->numSlots = kSmoothnessFactor;
157 input->numExtraSlots = 0u;
158 }
159 {
160 Mutexed<Output>::Locked output(mOutput);
161 output->outputDelay = 0u;
162 output->numSlots = kSmoothnessFactor;
163 }
David Stevensc3fbb282021-01-18 18:11:20 +0900164 {
165 Mutexed<BlockPools>::Locked pools(mBlockPools);
166 pools->outputPoolId = C2BlockPool::BASIC_LINEAR;
167 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800168}
169
170CCodecBufferChannel::~CCodecBufferChannel() {
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800171 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800172 mCrypto->unsetHeap(mHeapSeqNum);
173 }
174}
175
176void CCodecBufferChannel::setComponent(
177 const std::shared_ptr<Codec2Client::Component> &component) {
178 mComponent = component;
179 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
180 mName = mComponentName.c_str();
181}
182
183status_t CCodecBufferChannel::setInputSurface(
184 const std::shared_ptr<InputSurfaceWrapper> &surface) {
185 ALOGV("[%s] setInputSurface", mName);
186 mInputSurface = surface;
187 return mInputSurface->connect(mComponent);
188}
189
190status_t CCodecBufferChannel::signalEndOfInputStream() {
191 if (mInputSurface == nullptr) {
192 return INVALID_OPERATION;
193 }
194 return mInputSurface->signalEndOfInputStream();
195}
196
Sungtak Lee04b30352020-07-27 13:57:25 -0700197status_t CCodecBufferChannel::queueInputBufferInternal(
198 sp<MediaCodecBuffer> buffer,
199 std::shared_ptr<C2LinearBlock> encryptedBlock,
200 size_t blockSize) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800201 int64_t timeUs;
202 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
203
204 if (mInputMetEos) {
205 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
206 return OK;
207 }
208
209 int32_t flags = 0;
210 int32_t tmp = 0;
211 bool eos = false;
Guillaume Chelfi5ffbcb32021-04-12 14:23:43 +0200212 bool tunnelFirstFrame = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800213 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
214 eos = true;
215 mInputMetEos = true;
216 ALOGV("[%s] input EOS", mName);
217 }
218 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
219 flags |= C2FrameData::FLAG_CODEC_CONFIG;
220 }
Guillaume Chelfi5ffbcb32021-04-12 14:23:43 +0200221 if (buffer->meta()->findInt32("tunnel-first-frame", &tmp) && tmp) {
222 tunnelFirstFrame = true;
223 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800224 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
Wonsik Kim0379ae82020-11-24 15:01:33 -0800225 std::list<std::unique_ptr<C2Work>> items;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800226 std::unique_ptr<C2Work> work(new C2Work);
227 work->input.ordinal.timestamp = timeUs;
228 work->input.ordinal.frameIndex = mFrameIndex++;
229 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
230 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
231 // Keep client timestamp in customOrdinal
232 work->input.ordinal.customOrdinal = timeUs;
233 work->input.buffers.clear();
234
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700235 sp<Codec2Buffer> copy;
Wonsik Kim0379ae82020-11-24 15:01:33 -0800236 bool usesFrameReassembler = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800237
Pawin Vongmasa36653902018-11-15 00:10:25 -0800238 if (buffer->size() > 0u) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700239 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800240 std::shared_ptr<C2Buffer> c2buffer;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700241 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800242 return -ENOENT;
243 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700244 // TODO: we want to delay copying buffers.
245 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
246 copy = input->buffers->cloneAndReleaseBuffer(buffer);
247 if (copy != nullptr) {
248 (void)input->extraBuffers.assignSlot(copy);
249 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
250 return UNKNOWN_ERROR;
251 }
252 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
253 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
254 mName, released ? "" : "not ");
255 buffer.clear();
256 } else {
257 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
258 "buffer starvation on component.", mName);
259 }
260 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800261 if (input->frameReassembler) {
262 usesFrameReassembler = true;
263 input->frameReassembler.process(buffer, &items);
264 } else {
265 int32_t cvo = 0;
266 if (buffer->meta()->findInt32("cvo", &cvo)) {
267 int32_t rotation = cvo % 360;
268 // change rotation to counter-clock wise.
269 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
270
271 Mutexed<OutputSurface>::Locked output(mOutputSurface);
272 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
273 output->rotation[frameIndex] = rotation;
274 }
275 work->input.buffers.push_back(c2buffer);
276 if (encryptedBlock) {
277 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
278 kParamIndexEncryptedBuffer,
279 encryptedBlock->share(0, blockSize, C2Fence())));
280 }
Byeongjo Park2eef13e2020-06-12 17:24:21 +0900281 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800282 } else if (eos) {
283 flags |= C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800284 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800285 if (usesFrameReassembler) {
286 if (!items.empty()) {
287 items.front()->input.configUpdate = std::move(mParamsToBeSet);
288 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
289 }
290 } else {
291 work->input.flags = (C2FrameData::flags_t)flags;
292 // TODO: fill info's
Pawin Vongmasa36653902018-11-15 00:10:25 -0800293
Wonsik Kim0379ae82020-11-24 15:01:33 -0800294 work->input.configUpdate = std::move(mParamsToBeSet);
Guillaume Chelfi5ffbcb32021-04-12 14:23:43 +0200295 if (tunnelFirstFrame) {
296 C2StreamTunnelHoldRender::input tunnelHoldRender{
297 0u /* stream */,
298 C2_TRUE /* value */
299 };
300 work->input.configUpdate.push_back(C2Param::Copy(tunnelHoldRender));
301 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800302 work->worklets.clear();
303 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800304
Wonsik Kim0379ae82020-11-24 15:01:33 -0800305 items.push_back(std::move(work));
306
307 eos = eos && buffer->size() > 0u;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800308 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800309 if (eos) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800310 work.reset(new C2Work);
311 work->input.ordinal.timestamp = timeUs;
312 work->input.ordinal.frameIndex = mFrameIndex++;
313 // WORKAROUND: keep client timestamp in customOrdinal
314 work->input.ordinal.customOrdinal = timeUs;
315 work->input.buffers.clear();
316 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800317 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800318 items.push_back(std::move(work));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800319 }
Wonsik Kim0379ae82020-11-24 15:01:33 -0800320 c2_status_t err = C2_OK;
321 if (!items.empty()) {
322 {
323 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
324 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
325 for (const std::unique_ptr<C2Work> &work : items) {
326 watcher->onWorkQueued(
327 work->input.ordinal.frameIndex.peeku(),
328 std::vector(work->input.buffers),
329 now);
330 }
331 }
332 err = mComponent->queue(&items);
333 }
334 if (err != C2_OK) {
335 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
336 for (const std::unique_ptr<C2Work> &work : items) {
337 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
338 }
339 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700340 Mutexed<Input>::Locked input(mInput);
341 bool released = false;
342 if (buffer) {
343 released = input->buffers->releaseBuffer(buffer, nullptr, true);
344 } else if (copy) {
345 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
346 }
347 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
348 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800349 }
350
351 feedInputBufferIfAvailableInternal();
352 return err;
353}
354
355status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
356 QueueGuard guard(mSync);
357 if (!guard.isRunning()) {
358 ALOGD("[%s] setParameters is only supported in the running state.", mName);
359 return -ENOSYS;
360 }
361 mParamsToBeSet.insert(mParamsToBeSet.end(),
362 std::make_move_iterator(params.begin()),
363 std::make_move_iterator(params.end()));
364 params.clear();
365 return OK;
366}
367
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800368status_t CCodecBufferChannel::attachBuffer(
369 const std::shared_ptr<C2Buffer> &c2Buffer,
370 const sp<MediaCodecBuffer> &buffer) {
371 if (!buffer->copy(c2Buffer)) {
372 return -ENOSYS;
373 }
374 return OK;
375}
376
377void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
378 if (!mDecryptDestination || mDecryptDestination->size() < size) {
379 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
380 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
381 mCrypto->unsetHeap(mHeapSeqNum);
382 }
383 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
384 if (mCrypto) {
385 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
386 }
387 }
388}
389
390int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
391 CHECK(mCrypto);
392 auto it = mHeapSeqNumMap.find(memory);
393 int32_t heapSeqNum = -1;
394 if (it == mHeapSeqNumMap.end()) {
395 heapSeqNum = mCrypto->setHeap(memory);
396 mHeapSeqNumMap.emplace(memory, heapSeqNum);
397 } else {
398 heapSeqNum = it->second;
399 }
400 return heapSeqNum;
401}
402
403status_t CCodecBufferChannel::attachEncryptedBuffer(
404 const sp<hardware::HidlMemory> &memory,
405 bool secure,
406 const uint8_t *key,
407 const uint8_t *iv,
408 CryptoPlugin::Mode mode,
409 CryptoPlugin::Pattern pattern,
410 size_t offset,
411 const CryptoPlugin::SubSample *subSamples,
412 size_t numSubSamples,
413 const sp<MediaCodecBuffer> &buffer) {
414 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
415 static const C2MemoryUsage kDefaultReadWriteUsage{
416 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
417
418 size_t size = 0;
419 for (size_t i = 0; i < numSubSamples; ++i) {
420 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
421 }
422 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
423 std::shared_ptr<C2LinearBlock> block;
424 c2_status_t err = pool->fetchLinearBlock(
425 size,
426 secure ? kSecureUsage : kDefaultReadWriteUsage,
427 &block);
428 if (err != C2_OK) {
429 return NO_MEMORY;
430 }
431 if (!secure) {
432 ensureDecryptDestination(size);
433 }
434 ssize_t result = -1;
435 ssize_t codecDataOffset = 0;
436 if (mCrypto) {
437 AString errorDetailMsg;
438 int32_t heapSeqNum = getHeapSeqNum(memory);
439 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
440 hardware::drm::V1_0::DestinationBuffer dst;
441 if (secure) {
442 dst.type = DrmBufferType::NATIVE_HANDLE;
443 dst.secureMemory = hardware::hidl_handle(block->handle());
444 } else {
445 dst.type = DrmBufferType::SHARED_MEMORY;
446 IMemoryToSharedBuffer(
447 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
448 }
449 result = mCrypto->decrypt(
450 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
451 dst, &errorDetailMsg);
452 if (result < 0) {
453 return result;
454 }
455 if (dst.type == DrmBufferType::SHARED_MEMORY) {
456 C2WriteView view = block->map().get();
457 if (view.error() != C2_OK) {
458 return false;
459 }
460 if (view.size() < result) {
461 return false;
462 }
463 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
464 }
465 } else {
466 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
467 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
468 hidl_vec<SubSample> hidlSubSamples;
469 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
470
471 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
472 hardware::cas::native::V1_0::DestinationBuffer dst;
473 if (secure) {
474 dst.type = BufferType::NATIVE_HANDLE;
475 dst.secureMemory = hardware::hidl_handle(block->handle());
476 } else {
477 dst.type = BufferType::SHARED_MEMORY;
478 dst.nonsecureMemory = src;
479 }
480
481 CasStatus status = CasStatus::OK;
482 hidl_string detailedError;
483 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
484
485 if (key != nullptr) {
486 sctrl = (ScramblingControl)key[0];
487 // Adjust for the PES offset
488 codecDataOffset = key[2] | (key[3] << 8);
489 }
490
491 auto returnVoid = mDescrambler->descramble(
492 sctrl,
493 hidlSubSamples,
494 src,
495 0,
496 dst,
497 0,
498 [&status, &result, &detailedError] (
499 CasStatus _status, uint32_t _bytesWritten,
500 const hidl_string& _detailedError) {
501 status = _status;
502 result = (ssize_t)_bytesWritten;
503 detailedError = _detailedError;
504 });
505
506 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
507 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
508 mName, returnVoid.description().c_str(), status, result);
509 return UNKNOWN_ERROR;
510 }
511
512 if (result < codecDataOffset) {
513 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
514 return BAD_VALUE;
515 }
516 }
517 if (!secure) {
518 C2WriteView view = block->map().get();
519 if (view.error() != C2_OK) {
520 return UNKNOWN_ERROR;
521 }
522 if (view.size() < result) {
523 return UNKNOWN_ERROR;
524 }
525 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
526 }
527 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
528 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
529 if (!buffer->copy(c2Buffer)) {
530 return -ENOSYS;
531 }
532 return OK;
533}
534
Pawin Vongmasa36653902018-11-15 00:10:25 -0800535status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
536 QueueGuard guard(mSync);
537 if (!guard.isRunning()) {
538 ALOGD("[%s] No more buffers should be queued at current state.", mName);
539 return -ENOSYS;
540 }
541 return queueInputBufferInternal(buffer);
542}
543
544status_t CCodecBufferChannel::queueSecureInputBuffer(
545 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
546 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
547 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
548 AString *errorDetailMsg) {
549 QueueGuard guard(mSync);
550 if (!guard.isRunning()) {
551 ALOGD("[%s] No more buffers should be queued at current state.", mName);
552 return -ENOSYS;
553 }
554
555 if (!hasCryptoOrDescrambler()) {
556 return -ENOSYS;
557 }
558 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
559
Sungtak Lee04b30352020-07-27 13:57:25 -0700560 std::shared_ptr<C2LinearBlock> block;
561 size_t allocSize = buffer->size();
562 size_t bufferSize = 0;
563 c2_status_t blockRes = C2_OK;
564 bool copied = false;
565 if (mSendEncryptedInfoBuffer) {
566 static const C2MemoryUsage kDefaultReadWriteUsage{
567 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
568 constexpr int kAllocGranule0 = 1024 * 64;
569 constexpr int kAllocGranule1 = 1024 * 1024;
570 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
571 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
572 if (allocSize <= kAllocGranule1) {
573 bufferSize = align(allocSize, kAllocGranule0);
574 } else {
575 bufferSize = align(allocSize, kAllocGranule1);
576 }
577 blockRes = pool->fetchLinearBlock(
578 bufferSize, kDefaultReadWriteUsage, &block);
579
580 if (blockRes == C2_OK) {
581 C2WriteView view = block->map().get();
582 if (view.error() == C2_OK && view.size() == bufferSize) {
583 copied = true;
584 // TODO: only copy clear sections
585 memcpy(view.data(), buffer->data(), allocSize);
586 }
587 }
588 }
589
590 if (!copied) {
591 block.reset();
592 }
593
Pawin Vongmasa36653902018-11-15 00:10:25 -0800594 ssize_t result = -1;
595 ssize_t codecDataOffset = 0;
Wonsik Kim557c88c2020-03-13 11:03:52 -0700596 if (numSubSamples == 1
597 && subSamples[0].mNumBytesOfClearData == 0
598 && subSamples[0].mNumBytesOfEncryptedData == 0) {
599 // We don't need to go through crypto or descrambler if the input is empty.
600 result = 0;
601 } else if (mCrypto != nullptr) {
Robert Shih895fba92019-07-16 16:29:44 -0700602 hardware::drm::V1_0::DestinationBuffer destination;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800603 if (secure) {
Robert Shih895fba92019-07-16 16:29:44 -0700604 destination.type = DrmBufferType::NATIVE_HANDLE;
605 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800606 } else {
Robert Shih895fba92019-07-16 16:29:44 -0700607 destination.type = DrmBufferType::SHARED_MEMORY;
608 IMemoryToSharedBuffer(
609 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800610 }
Robert Shih895fba92019-07-16 16:29:44 -0700611 hardware::drm::V1_0::SharedBuffer source;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800612 encryptedBuffer->fillSourceBuffer(&source);
613 result = mCrypto->decrypt(
614 key, iv, mode, pattern, source, buffer->offset(),
615 subSamples, numSubSamples, destination, errorDetailMsg);
616 if (result < 0) {
Wonsik Kim557c88c2020-03-13 11:03:52 -0700617 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800618 return result;
619 }
Robert Shih895fba92019-07-16 16:29:44 -0700620 if (destination.type == DrmBufferType::SHARED_MEMORY) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800621 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
622 }
623 } else {
624 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
625 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
626 hidl_vec<SubSample> hidlSubSamples;
627 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
628
629 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
630 encryptedBuffer->fillSourceBuffer(&srcBuffer);
631
632 DestinationBuffer dstBuffer;
633 if (secure) {
634 dstBuffer.type = BufferType::NATIVE_HANDLE;
635 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
636 } else {
637 dstBuffer.type = BufferType::SHARED_MEMORY;
638 dstBuffer.nonsecureMemory = srcBuffer;
639 }
640
641 CasStatus status = CasStatus::OK;
642 hidl_string detailedError;
643 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
644
645 if (key != nullptr) {
646 sctrl = (ScramblingControl)key[0];
647 // Adjust for the PES offset
648 codecDataOffset = key[2] | (key[3] << 8);
649 }
650
651 auto returnVoid = mDescrambler->descramble(
652 sctrl,
653 hidlSubSamples,
654 srcBuffer,
655 0,
656 dstBuffer,
657 0,
658 [&status, &result, &detailedError] (
659 CasStatus _status, uint32_t _bytesWritten,
660 const hidl_string& _detailedError) {
661 status = _status;
662 result = (ssize_t)_bytesWritten;
663 detailedError = _detailedError;
664 });
665
666 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
667 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
668 mName, returnVoid.description().c_str(), status, result);
669 return UNKNOWN_ERROR;
670 }
671
672 if (result < codecDataOffset) {
673 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
674 return BAD_VALUE;
675 }
676
677 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
678
679 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
680 encryptedBuffer->copyDecryptedContentFromMemory(result);
681 }
682 }
683
684 buffer->setRange(codecDataOffset, result - codecDataOffset);
Sungtak Lee04b30352020-07-27 13:57:25 -0700685
686 return queueInputBufferInternal(buffer, block, bufferSize);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800687}
688
689void CCodecBufferChannel::feedInputBufferIfAvailable() {
690 QueueGuard guard(mSync);
691 if (!guard.isRunning()) {
692 ALOGV("[%s] We're not running --- no input buffer reported", mName);
693 return;
694 }
695 feedInputBufferIfAvailableInternal();
696}
697
698void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
Taehwan Kimda0517d2020-09-16 17:29:37 +0900699 if (mInputMetEos) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800700 return;
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700701 }
702 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700703 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700704 if (!output->buffers ||
705 output->buffers->hasPending() ||
Wonsik Kim0487b782020-10-28 11:45:50 -0700706 output->buffers->numActiveSlots() >= output->numSlots) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800707 return;
708 }
709 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700710 size_t numActiveSlots = 0;
711 while (!mPipelineWatcher.lock()->pipelineFull()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800712 sp<MediaCodecBuffer> inBuffer;
713 size_t index;
714 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700715 Mutexed<Input>::Locked input(mInput);
Wonsik Kim0487b782020-10-28 11:45:50 -0700716 numActiveSlots = input->buffers->numActiveSlots();
717 if (numActiveSlots >= input->numSlots) {
718 break;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800719 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700720 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800721 ALOGV("[%s] no new buffer available", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800722 break;
723 }
724 }
725 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
726 mCallback->onInputBufferAvailable(index, inBuffer);
727 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700728 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800729}
730
731status_t CCodecBufferChannel::renderOutputBuffer(
732 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800733 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800734 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800735 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800736 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700737 Mutexed<Output>::Locked output(mOutput);
738 if (output->buffers) {
739 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800740 }
741 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800742 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
743 // set to true.
744 sendOutputBuffers();
745 // input buffer feeding may have been gated by pending output buffers
746 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800747 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800748 if (released) {
Wonsik Kimf7529dd2019-04-18 17:35:53 -0700749 std::call_once(mRenderWarningFlag, [this] {
750 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
751 "timestamp or render=true with non-video buffers. Apps should "
752 "call releaseOutputBuffer() with render=false for those.",
753 mName);
754 });
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800755 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800756 return INVALID_OPERATION;
757 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800758
759#if 0
760 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
761 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
762 for (const std::shared_ptr<const C2Info> &info : infoParams) {
763 AString res;
764 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
765 if (ix) res.append(", ");
766 res.append(*((int32_t*)info.get() + (ix / 4)));
767 }
768 ALOGV(" [%s]", res.c_str());
769 }
770#endif
771 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
772 std::static_pointer_cast<const C2StreamRotationInfo::output>(
773 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
774 bool flip = rotation && (rotation->flip & 1);
775 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
Byeongjo Park2eef13e2020-06-12 17:24:21 +0900776
777 {
778 Mutexed<OutputSurface>::Locked output(mOutputSurface);
779 if (output->surface == nullptr) {
780 ALOGI("[%s] cannot render buffer without surface", mName);
781 return OK;
782 }
783 int64_t frameIndex;
784 buffer->meta()->findInt64("frameIndex", &frameIndex);
785 if (output->rotation.count(frameIndex) != 0) {
786 auto it = output->rotation.find(frameIndex);
787 quarters = (it->second / 90) & 3;
788 output->rotation.erase(it);
789 }
790 }
791
Pawin Vongmasa36653902018-11-15 00:10:25 -0800792 uint32_t transform = 0;
793 switch (quarters) {
794 case 0: // no rotation
795 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
796 break;
797 case 1: // 90 degrees counter-clockwise
798 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
799 : HAL_TRANSFORM_ROT_270;
800 break;
801 case 2: // 180 degrees
802 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
803 break;
804 case 3: // 90 degrees clockwise
805 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
806 : HAL_TRANSFORM_ROT_90;
807 break;
808 }
809
810 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
811 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
812 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
813 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
814 if (surfaceScaling) {
815 videoScalingMode = surfaceScaling->value;
816 }
817
818 // Use dataspace from format as it has the default aspects already applied
819 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
820 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
821
822 // HDR static info
823 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
824 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
825 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
826
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800827 // HDR10 plus info
828 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
829 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
830 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
Yichi Chen54be23c2020-06-15 14:30:53 +0800831 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
832 hdr10PlusInfo.reset();
833 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800834
Pawin Vongmasa36653902018-11-15 00:10:25 -0800835 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
836 if (blocks.size() != 1u) {
837 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
838 return UNKNOWN_ERROR;
839 }
840 const C2ConstGraphicBlock &block = blocks.front();
841
842 // TODO: revisit this after C2Fence implementation.
843 android::IGraphicBufferProducer::QueueBufferInput qbi(
844 timestampNs,
845 false, // droppable
846 dataSpace,
847 Rect(blocks.front().crop().left,
848 blocks.front().crop().top,
849 blocks.front().crop().right(),
850 blocks.front().crop().bottom()),
851 videoScalingMode,
852 transform,
853 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800854 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800855 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800856 if (hdrStaticInfo) {
wenchangliuf3f92882020-05-14 00:02:01 +0800857 // If mastering max and min luminance fields are 0, do not use them.
858 // It indicates the value may not be present in the stream.
859 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
860 hdrStaticInfo->mastering.minLuminance > 0.0f) {
861 struct android_smpte2086_metadata smpte2086_meta = {
862 .displayPrimaryRed = {
863 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
864 },
865 .displayPrimaryGreen = {
866 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
867 },
868 .displayPrimaryBlue = {
869 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
870 },
871 .whitePoint = {
872 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
873 },
874 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
875 .minLuminance = hdrStaticInfo->mastering.minLuminance,
876 };
Yichi Chen54be23c2020-06-15 14:30:53 +0800877 hdr.validTypes |= HdrMetadata::SMPTE2086;
wenchangliuf3f92882020-05-14 00:02:01 +0800878 hdr.smpte2086 = smpte2086_meta;
879 }
Chong Zhang3bb2a7f2020-04-21 10:35:12 -0700880 // If the content light level fields are 0, do not use them, it
881 // indicates the value may not be present in the stream.
882 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
883 struct android_cta861_3_metadata cta861_meta = {
884 .maxContentLightLevel = hdrStaticInfo->maxCll,
885 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
886 };
887 hdr.validTypes |= HdrMetadata::CTA861_3;
888 hdr.cta8613 = cta861_meta;
889 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800890 }
891 if (hdr10PlusInfo) {
892 hdr.validTypes |= HdrMetadata::HDR10PLUS;
893 hdr.hdr10plus.assign(
894 hdr10PlusInfo->m.value,
895 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
896 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800897 qbi.setHdrMetadata(hdr);
898 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800899 // we don't have dirty regions
900 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800901 android::IGraphicBufferProducer::QueueBufferOutput qbo;
902 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
903 if (result != OK) {
904 ALOGI("[%s] queueBuffer failed: %d", mName, result);
Sungtak Lee47c018a2020-11-07 01:02:49 -0800905 if (result == NO_INIT) {
906 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
907 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800908 return result;
909 }
Josh Houcd2f7582021-02-02 16:26:53 +0800910
911 if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
912 ALOGD("[%s] queue buffer successful", mName);
913 } else {
914 ALOGV("[%s] queue buffer successful", mName);
915 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800916
917 int64_t mediaTimeUs = 0;
918 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
919 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
920
921 return OK;
922}
923
924status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
925 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
926 bool released = false;
927 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700928 Mutexed<Input>::Locked input(mInput);
929 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800930 released = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800931 }
932 }
933 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700934 Mutexed<Output>::Locked output(mOutput);
935 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800936 released = true;
937 }
938 }
939 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800940 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800941 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800942 } else {
943 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
944 }
945 return OK;
946}
947
948void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
949 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700950 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800951
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700952 if (!input->buffers->isArrayMode()) {
953 input->buffers = input->buffers->toArrayMode(input->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800954 }
955
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700956 input->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800957}
958
959void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
960 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700961 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800962
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700963 if (!output->buffers->isArrayMode()) {
964 output->buffers = output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800965 }
966
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700967 output->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800968}
969
970status_t CCodecBufferChannel::start(
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800971 const sp<AMessage> &inputFormat,
972 const sp<AMessage> &outputFormat,
973 bool buffersBoundToCodec) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800974 C2StreamBufferTypeSetting::input iStreamFormat(0u);
975 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim0379ae82020-11-24 15:01:33 -0800976 C2ComponentKindSetting kind;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800977 C2PortReorderBufferDepthTuning::output reorderDepth;
978 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800979 C2PortActualDelayTuning::input inputDelay(0);
980 C2PortActualDelayTuning::output outputDelay(0);
981 C2ActualPipelineDelayTuning pipelineDelay(0);
Sungtak Lee04b30352020-07-27 13:57:25 -0700982 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
Wonsik Kim078b58e2019-01-09 15:08:06 -0800983
Pawin Vongmasa36653902018-11-15 00:10:25 -0800984 c2_status_t err = mComponent->query(
985 {
986 &iStreamFormat,
987 &oStreamFormat,
Wonsik Kim0379ae82020-11-24 15:01:33 -0800988 &kind,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800989 &reorderDepth,
990 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800991 &inputDelay,
992 &pipelineDelay,
993 &outputDelay,
Sungtak Lee04b30352020-07-27 13:57:25 -0700994 &secureMode,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800995 },
996 {},
997 C2_DONT_BLOCK,
998 nullptr);
999 if (err == C2_BAD_INDEX) {
Wonsik Kim0379ae82020-11-24 15:01:33 -08001000 if (!iStreamFormat || !oStreamFormat || !kind) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001001 return UNKNOWN_ERROR;
1002 }
1003 } else if (err != C2_OK) {
1004 return UNKNOWN_ERROR;
1005 }
1006
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001007 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
1008 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
1009 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
1010
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001011 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
1012 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
Wonsik Kim078b58e2019-01-09 15:08:06 -08001013
Pawin Vongmasa36653902018-11-15 00:10:25 -08001014 // TODO: get this from input format
1015 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1016
Sungtak Lee04b30352020-07-27 13:57:25 -07001017 // secure mode is a static parameter (shall not change in the executing state)
1018 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1019
Pawin Vongmasa36653902018-11-15 00:10:25 -08001020 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001021 int poolMask = GetCodec2PoolMask();
1022 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001023
1024 if (inputFormat != nullptr) {
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001025 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
Wonsik Kim0379ae82020-11-24 15:01:33 -08001026 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001027 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1028 API_REFLECTION |
1029 API_VALUES |
1030 API_CURRENT_VALUES |
1031 API_DEPENDENCY |
1032 API_SAME_INPUT_BUFFER);
Wonsik Kim0379ae82020-11-24 15:01:33 -08001033 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1034 C2StreamSampleRateInfo::input sampleRate(0u);
1035 C2StreamChannelCountInfo::input channelCount(0u);
1036 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001037 std::shared_ptr<C2BlockPool> pool;
1038 {
1039 Mutexed<BlockPools>::Locked pools(mBlockPools);
1040
1041 // set default allocator ID.
1042 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001043 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001044
1045 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1046 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1047 std::vector<std::unique_ptr<C2Param>> params;
Wonsik Kimffb889a2020-05-28 11:32:25 -07001048 C2ApiFeaturesSetting featuresSetting{apiFeatures};
Wonsik Kim0379ae82020-11-24 15:01:33 -08001049 std::vector<C2Param *> stackParams({&featuresSetting});
1050 if (audioEncoder) {
1051 stackParams.push_back(&encoderFrameSize);
1052 stackParams.push_back(&sampleRate);
1053 stackParams.push_back(&channelCount);
1054 stackParams.push_back(&pcmEncoding);
1055 } else {
1056 encoderFrameSize.invalidate();
1057 sampleRate.invalidate();
1058 channelCount.invalidate();
1059 pcmEncoding.invalidate();
1060 }
1061 err = mComponent->query(stackParams,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001062 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1063 C2_DONT_BLOCK,
1064 &params);
1065 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1066 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1067 mName, params.size(), asString(err), err);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001068 } else if (params.size() == 1) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001069 C2PortAllocatorsTuning::input *inputAllocators =
1070 C2PortAllocatorsTuning::input::From(params[0].get());
1071 if (inputAllocators && inputAllocators->flexCount() > 0) {
1072 std::shared_ptr<C2Allocator> allocator;
1073 // verify allocator IDs and resolve default allocator
1074 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1075 if (allocator) {
1076 pools->inputAllocatorId = allocator->getId();
1077 } else {
1078 ALOGD("[%s] component requested invalid input allocator ID %u",
1079 mName, inputAllocators->m.values[0]);
1080 }
1081 }
1082 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001083 if (featuresSetting) {
1084 apiFeatures = featuresSetting.value;
1085 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001086
1087 // TODO: use C2Component wrapper to associate this pool with ourselves
1088 if ((poolMask >> pools->inputAllocatorId) & 1) {
1089 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1090 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1091 mName, pools->inputAllocatorId,
1092 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1093 asString(err), err);
1094 } else {
1095 err = C2_NOT_FOUND;
1096 }
1097 if (err != C2_OK) {
1098 C2BlockPool::local_id_t inputPoolId =
1099 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1100 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1101 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1102 mName, (unsigned long long)inputPoolId,
1103 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1104 asString(err), err);
1105 if (err != C2_OK) {
1106 return NO_MEMORY;
1107 }
1108 }
1109 pools->inputPool = pool;
1110 }
1111
Wonsik Kim51051262018-11-28 13:59:05 -08001112 bool forceArrayMode = false;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001113 Mutexed<Input>::Locked input(mInput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001114 input->inputDelay = inputDelayValue;
1115 input->pipelineDelay = pipelineDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001116 input->numSlots = numInputSlots;
1117 input->extraBuffers.flush();
1118 input->numExtraSlots = 0u;
Wonsik Kim0379ae82020-11-24 15:01:33 -08001119 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1120 input->frameReassembler.init(
1121 pool,
1122 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1123 encoderFrameSize.value,
1124 sampleRate.value,
1125 channelCount.value,
1126 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1127 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001128 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1129 // For encrypted content, framework decrypts source buffer (ashmem) into
1130 // C2Buffers. Thus non-conforming codecs can process these.
Wonsik Kim0379ae82020-11-24 15:01:33 -08001131 if (!buffersBoundToCodec
1132 && !input->frameReassembler
1133 && (hasCryptoOrDescrambler() || conforming)) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001134 input->buffers.reset(new SlotInputBuffers(mName));
1135 } else if (graphic) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001136 if (mInputSurface) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001137 input->buffers.reset(new DummyInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001138 } else if (mMetaMode == MODE_ANW) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001139 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
Wonsik Kim1221fd12019-07-12 12:52:05 -07001140 // This is to ensure buffers do not get released prematurely.
1141 // TODO: handle this without going into array mode
1142 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001143 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001144 input->buffers.reset(new GraphicInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001145 }
1146 } else {
1147 if (hasCryptoOrDescrambler()) {
1148 int32_t capacity = kLinearBufferSize;
1149 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1150 if ((size_t)capacity > kMaxLinearBufferSize) {
1151 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1152 capacity = kMaxLinearBufferSize;
1153 }
1154 if (mDealer == nullptr) {
1155 mDealer = new MemoryDealer(
1156 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001157 * (numInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001158 "EncryptedLinearInputBuffers");
1159 mDecryptDestination = mDealer->allocate((size_t)capacity);
1160 }
1161 if (mCrypto != nullptr && mHeapSeqNum < 0) {
Robert Shih895fba92019-07-16 16:29:44 -07001162 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1163 mHeapSeqNum = mCrypto->setHeap(heap);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001164 } else {
1165 mHeapSeqNum = -1;
1166 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001167 input->buffers.reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001168 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001169 numInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08001170 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001171 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001172 input->buffers.reset(new LinearInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001173 }
1174 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001175 input->buffers->setFormat(inputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001176
1177 if (err == C2_OK) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001178 input->buffers->setPool(pool);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001179 } else {
1180 // TODO: error
1181 }
Wonsik Kim51051262018-11-28 13:59:05 -08001182
1183 if (forceArrayMode) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001184 input->buffers = input->buffers->toArrayMode(numInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08001185 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001186 }
1187
1188 if (outputFormat != nullptr) {
1189 sp<IGraphicBufferProducer> outputSurface;
1190 uint32_t outputGeneration;
Sungtak Leea714f112021-03-16 05:40:03 -07001191 int maxDequeueCount = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001192 {
1193 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Sungtak Leea714f112021-03-16 05:40:03 -07001194 maxDequeueCount = output->maxDequeueBuffers = numOutputSlots +
Sungtak Lee7a7b7422019-07-16 17:40:40 -07001195 reorderDepth.value + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001196 outputSurface = output->surface ?
1197 output->surface->getIGraphicBufferProducer() : nullptr;
Wonsik Kimf5e5c832019-02-21 11:36:05 -08001198 if (outputSurface) {
1199 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1200 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001201 outputGeneration = output->generation;
1202 }
1203
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001204 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001205 C2BlockPool::local_id_t outputPoolId_;
David Stevensc3fbb282021-01-18 18:11:20 +09001206 C2BlockPool::local_id_t prevOutputPoolId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001207
1208 {
1209 Mutexed<BlockPools>::Locked pools(mBlockPools);
1210
David Stevensc3fbb282021-01-18 18:11:20 +09001211 prevOutputPoolId = pools->outputPoolId;
1212
Pawin Vongmasa36653902018-11-15 00:10:25 -08001213 // set default allocator ID.
1214 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001215 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001216
1217 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1218 // unsuccessful.
1219 std::vector<std::unique_ptr<C2Param>> params;
1220 err = mComponent->query({ },
1221 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1222 C2_DONT_BLOCK,
1223 &params);
1224 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1225 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1226 mName, params.size(), asString(err), err);
1227 } else if (err == C2_OK && params.size() == 1) {
1228 C2PortAllocatorsTuning::output *outputAllocators =
1229 C2PortAllocatorsTuning::output::From(params[0].get());
1230 if (outputAllocators && outputAllocators->flexCount() > 0) {
1231 std::shared_ptr<C2Allocator> allocator;
1232 // verify allocator IDs and resolve default allocator
1233 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1234 if (allocator) {
1235 pools->outputAllocatorId = allocator->getId();
1236 } else {
1237 ALOGD("[%s] component requested invalid output allocator ID %u",
1238 mName, outputAllocators->m.values[0]);
1239 }
1240 }
1241 }
1242
1243 // use bufferqueue if outputting to a surface.
1244 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1245 // if unsuccessful.
1246 if (outputSurface) {
1247 params.clear();
1248 err = mComponent->query({ },
1249 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1250 C2_DONT_BLOCK,
1251 &params);
1252 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1253 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1254 mName, params.size(), asString(err), err);
1255 } else if (err == C2_OK && params.size() == 1) {
1256 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1257 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1258 if (surfaceAllocator) {
1259 std::shared_ptr<C2Allocator> allocator;
1260 // verify allocator IDs and resolve default allocator
1261 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1262 if (allocator) {
1263 pools->outputAllocatorId = allocator->getId();
1264 } else {
1265 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1266 mName, surfaceAllocator->value);
1267 err = C2_BAD_VALUE;
1268 }
1269 }
1270 }
1271 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1272 && err != C2_OK
1273 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1274 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1275 }
1276 }
1277
1278 if ((poolMask >> pools->outputAllocatorId) & 1) {
1279 err = mComponent->createBlockPool(
1280 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1281 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1282 mName, pools->outputAllocatorId,
1283 (unsigned long long)pools->outputPoolId,
1284 asString(err));
1285 } else {
1286 err = C2_NOT_FOUND;
1287 }
1288 if (err != C2_OK) {
1289 // use basic pool instead
1290 pools->outputPoolId =
1291 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1292 }
1293
1294 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1295 // component.
1296 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1297 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1298
1299 std::vector<std::unique_ptr<C2SettingResult>> failures;
1300 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1301 ALOGD("[%s] Configured output block pool ids %llu => %s",
1302 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1303 outputPoolId_ = pools->outputPoolId;
1304 }
1305
David Stevensc3fbb282021-01-18 18:11:20 +09001306 if (prevOutputPoolId != C2BlockPool::BASIC_LINEAR
1307 && prevOutputPoolId != C2BlockPool::BASIC_GRAPHIC) {
1308 c2_status_t err = mComponent->destroyBlockPool(prevOutputPoolId);
1309 if (err != C2_OK) {
1310 ALOGW("Failed to clean up previous block pool %llu - %s (%d)\n",
1311 (unsigned long long) prevOutputPoolId, asString(err), err);
1312 }
1313 }
1314
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001315 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001316 output->outputDelay = outputDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001317 output->numSlots = numOutputSlots;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001318 if (graphic) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001319 if (outputSurface || !buffersBoundToCodec) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001320 output->buffers.reset(new GraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001321 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001322 output->buffers.reset(new RawGraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001323 }
1324 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001325 output->buffers.reset(new LinearOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001326 }
Wonsik Kime4716c02020-02-28 10:42:21 -08001327 output->buffers->setFormat(outputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001328
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001329 output->buffers->clearStash();
1330 if (reorderDepth) {
1331 output->buffers->setReorderDepth(reorderDepth.value);
1332 }
1333 if (reorderKey) {
1334 output->buffers->setReorderKey(reorderKey.value);
1335 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001336
1337 // Try to set output surface to created block pool if given.
1338 if (outputSurface) {
1339 mComponent->setOutputSurface(
1340 outputPoolId_,
1341 outputSurface,
Sungtak Leedb14cba2021-04-10 00:50:23 -07001342 outputGeneration,
1343 maxDequeueCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001344 }
1345
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001346 if (oStreamFormat.value == C2BufferData::LINEAR) {
Wonsik Kim58713302020-01-29 22:25:23 -08001347 if (buffersBoundToCodec) {
1348 // WORKAROUND: if we're using early CSD workaround we convert to
1349 // array mode, to appease apps assuming the output
1350 // buffers to be of the same size.
1351 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1352 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001353
1354 int32_t channelCount;
1355 int32_t sampleRate;
1356 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1357 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1358 int32_t delay = 0;
1359 int32_t padding = 0;;
1360 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1361 delay = 0;
1362 }
1363 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1364 padding = 0;
1365 }
1366 if (delay || padding) {
1367 // We need write access to the buffers, and we're already in
1368 // array mode.
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001369 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001370 }
1371 }
1372 }
1373 }
1374
1375 // Set up pipeline control. This has to be done after mInputBuffers and
1376 // mOutputBuffers are initialized to make sure that lingering callbacks
1377 // about buffers from the previous generation do not interfere with the
1378 // newly initialized pipeline capacity.
1379
Wonsik Kim62545252021-01-20 11:25:41 -08001380 if (inputFormat || outputFormat) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08001381 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001382 watcher->inputDelay(inputDelayValue)
1383 .pipelineDelay(pipelineDelayValue)
1384 .outputDelay(outputDelayValue)
Wonsik Kimab34ed62019-01-31 15:28:46 -08001385 .smoothnessFactor(kSmoothnessFactor);
1386 watcher->flush();
1387 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001388
1389 mInputMetEos = false;
1390 mSync.start();
1391 return OK;
1392}
1393
1394status_t CCodecBufferChannel::requestInitialInputBuffers() {
1395 if (mInputSurface) {
1396 return OK;
1397 }
1398
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001399 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001400 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1401 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1402 if (err != C2_OK && err != C2_BAD_INDEX) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001403 return UNKNOWN_ERROR;
1404 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001405 size_t numInputSlots = mInput.lock()->numSlots;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001406
1407 struct ClientInputBuffer {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001408 size_t index;
1409 sp<MediaCodecBuffer> buffer;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001410 size_t capacity;
1411 };
1412 std::list<ClientInputBuffer> clientInputBuffers;
1413
1414 {
1415 Mutexed<Input>::Locked input(mInput);
1416 while (clientInputBuffers.size() < numInputSlots) {
1417 ClientInputBuffer clientInputBuffer;
1418 if (!input->buffers->requestNewBuffer(&clientInputBuffer.index,
1419 &clientInputBuffer.buffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001420 break;
1421 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001422 clientInputBuffer.capacity = clientInputBuffer.buffer->capacity();
1423 clientInputBuffers.emplace_back(std::move(clientInputBuffer));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001424 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001425 }
1426 if (clientInputBuffers.empty()) {
1427 ALOGW("[%s] start: cannot allocate memory at all", mName);
1428 return NO_MEMORY;
1429 } else if (clientInputBuffers.size() < numInputSlots) {
1430 ALOGD("[%s] start: cannot allocate memory for all slots, "
1431 "only %zu buffers allocated",
1432 mName, clientInputBuffers.size());
1433 } else {
1434 ALOGV("[%s] %zu initial input buffers available",
1435 mName, clientInputBuffers.size());
1436 }
1437 // Sort input buffers by their capacities in increasing order.
1438 clientInputBuffers.sort(
1439 [](const ClientInputBuffer& a, const ClientInputBuffer& b) {
1440 return a.capacity < b.capacity;
1441 });
1442
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001443 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1444 mFlushedConfigs.lock()->swap(flushedConfigs);
1445 if (!flushedConfigs.empty()) {
1446 err = mComponent->queue(&flushedConfigs);
1447 if (err != C2_OK) {
1448 ALOGW("[%s] Error while queueing a flushed config", mName);
1449 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001450 }
1451 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001452 if (oStreamFormat.value == C2BufferData::LINEAR &&
1453 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE)) {
1454 sp<MediaCodecBuffer> buffer = clientInputBuffers.front().buffer;
1455 // WORKAROUND: Some apps expect CSD available without queueing
1456 // any input. Queue an empty buffer to get the CSD.
1457 buffer->setRange(0, 0);
1458 buffer->meta()->clear();
1459 buffer->meta()->setInt64("timeUs", 0);
1460 if (queueInputBufferInternal(buffer) != OK) {
1461 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1462 mName);
1463 return UNKNOWN_ERROR;
1464 }
1465 clientInputBuffers.pop_front();
1466 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001467
1468 for (const ClientInputBuffer& clientInputBuffer: clientInputBuffers) {
1469 mCallback->onInputBufferAvailable(
1470 clientInputBuffer.index,
1471 clientInputBuffer.buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001472 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001473
Pawin Vongmasa36653902018-11-15 00:10:25 -08001474 return OK;
1475}
1476
1477void CCodecBufferChannel::stop() {
1478 mSync.stop();
1479 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001480}
1481
Wonsik Kim936a89c2020-05-08 16:07:50 -07001482void CCodecBufferChannel::reset() {
1483 stop();
Wonsik Kim62545252021-01-20 11:25:41 -08001484 if (mInputSurface != nullptr) {
1485 mInputSurface.reset();
1486 }
1487 mPipelineWatcher.lock()->flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001488 {
1489 Mutexed<Input>::Locked input(mInput);
1490 input->buffers.reset(new DummyInputBuffers(""));
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001491 input->extraBuffers.flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001492 }
1493 {
1494 Mutexed<Output>::Locked output(mOutput);
1495 output->buffers.reset();
1496 }
1497}
1498
1499void CCodecBufferChannel::release() {
1500 mComponent.reset();
1501 mInputAllocator.reset();
1502 mOutputSurface.lock()->surface.clear();
1503 {
1504 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1505 blockPools->inputPool.reset();
1506 blockPools->outputPoolIntf.reset();
1507 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001508 setCrypto(nullptr);
1509 setDescrambler(nullptr);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001510}
1511
1512
Pawin Vongmasa36653902018-11-15 00:10:25 -08001513void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1514 ALOGV("[%s] flush", mName);
Wonsik Kim62545252021-01-20 11:25:41 -08001515 std::vector<uint64_t> indices;
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001516 std::list<std::unique_ptr<C2Work>> configs;
1517 for (const std::unique_ptr<C2Work> &work : flushedWork) {
Wonsik Kim62545252021-01-20 11:25:41 -08001518 indices.push_back(work->input.ordinal.frameIndex.peeku());
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001519 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1520 continue;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001521 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001522 if (work->input.buffers.empty()
1523 || work->input.buffers.front() == nullptr
1524 || work->input.buffers.front()->data().linearBlocks().empty()) {
1525 ALOGD("[%s] no linear codec config data found", mName);
1526 continue;
1527 }
1528 std::unique_ptr<C2Work> copy(new C2Work);
1529 copy->input.flags = C2FrameData::flags_t(work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1530 copy->input.ordinal = work->input.ordinal;
Wonsik Kim62545252021-01-20 11:25:41 -08001531 copy->input.ordinal.frameIndex = mFrameIndex++;
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001532 copy->input.buffers.insert(
1533 copy->input.buffers.begin(),
1534 work->input.buffers.begin(),
1535 work->input.buffers.end());
1536 for (const std::unique_ptr<C2Param> &param : work->input.configUpdate) {
1537 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1538 }
1539 copy->input.infoBuffers.insert(
1540 copy->input.infoBuffers.begin(),
1541 work->input.infoBuffers.begin(),
1542 work->input.infoBuffers.end());
1543 copy->worklets.emplace_back(new C2Worklet);
1544 configs.push_back(std::move(copy));
1545 ALOGV("[%s] stashed flushed codec config data", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001546 }
Wonsik Kimf34e4e02021-01-05 18:58:15 -08001547 mFlushedConfigs.lock()->swap(configs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001548 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001549 Mutexed<Input>::Locked input(mInput);
1550 input->buffers->flush();
1551 input->extraBuffers.flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001552 }
1553 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001554 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001555 if (output->buffers) {
1556 output->buffers->flush(flushedWork);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001557 output->buffers->flushStash();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001558 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001559 }
Wonsik Kim62545252021-01-20 11:25:41 -08001560 {
1561 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1562 for (uint64_t index : indices) {
1563 watcher->onWorkDone(index);
1564 }
1565 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001566}
1567
1568void CCodecBufferChannel::onWorkDone(
1569 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
Wonsik Kimab34ed62019-01-31 15:28:46 -08001570 const C2StreamInitDataInfo::output *initData) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001571 if (handleWork(std::move(work), outputFormat, initData)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001572 feedInputBufferIfAvailable();
1573 }
1574}
1575
1576void CCodecBufferChannel::onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -08001577 uint64_t frameIndex, size_t arrayIndex) {
Pawin Vongmasa8e2cfb52019-05-15 05:20:52 -07001578 if (mInputSurface) {
1579 return;
1580 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001581 std::shared_ptr<C2Buffer> buffer =
1582 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001583 bool newInputSlotAvailable;
1584 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001585 Mutexed<Input>::Locked input(mInput);
1586 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1587 if (!newInputSlotAvailable) {
1588 (void)input->extraBuffers.expireComponentBuffer(buffer);
1589 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001590 }
1591 if (newInputSlotAvailable) {
1592 feedInputBufferIfAvailable();
1593 }
1594}
1595
1596bool CCodecBufferChannel::handleWork(
1597 std::unique_ptr<C2Work> work,
1598 const sp<AMessage> &outputFormat,
1599 const C2StreamInitDataInfo::output *initData) {
Wonsik Kim936a89c2020-05-08 16:07:50 -07001600 {
Wonsik Kima4e049d2020-04-28 19:42:23 +00001601 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001602 if (!output->buffers) {
1603 return false;
1604 }
Wonsik Kime75a5da2020-02-14 17:29:03 -08001605 }
1606
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001607 // Whether the output buffer should be reported to the client or not.
1608 bool notifyClient = false;
1609
1610 if (work->result == C2_OK){
1611 notifyClient = true;
1612 } else if (work->result == C2_NOT_FOUND) {
1613 ALOGD("[%s] flushed work; ignored.", mName);
1614 } else {
1615 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1616 // the config update.
1617 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1618 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1619 return false;
1620 }
1621
1622 if ((work->input.ordinal.frameIndex -
1623 mFirstValidFrameIndex.load()).peek() < 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001624 // Discard frames from previous generation.
1625 ALOGD("[%s] Discard frames from previous generation.", mName);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001626 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001627 }
1628
Wonsik Kim524b0582019-03-12 11:28:57 -07001629 if (mInputSurface == nullptr && (work->worklets.size() != 1u
Pawin Vongmasa36653902018-11-15 00:10:25 -08001630 || !work->worklets.front()
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001631 || !(work->worklets.front()->output.flags &
1632 C2FrameData::FLAG_INCOMPLETE))) {
1633 mPipelineWatcher.lock()->onWorkDone(
1634 work->input.ordinal.frameIndex.peeku());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001635 }
1636
1637 // NOTE: MediaCodec usage supposedly have only one worklet
1638 if (work->worklets.size() != 1u) {
1639 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1640 mName, work->worklets.size());
1641 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1642 return false;
1643 }
1644
1645 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1646
1647 std::shared_ptr<C2Buffer> buffer;
1648 // NOTE: MediaCodec usage supposedly have only one output stream.
1649 if (worklet->output.buffers.size() > 1u) {
1650 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1651 mName, worklet->output.buffers.size());
1652 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1653 return false;
1654 } else if (worklet->output.buffers.size() == 1u) {
1655 buffer = worklet->output.buffers[0];
1656 if (!buffer) {
1657 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1658 }
1659 }
1660
Wonsik Kim84f439f2021-05-03 10:57:09 -07001661 std::optional<uint32_t> newInputDelay, newPipelineDelay, newOutputDelay, newReorderDepth;
1662 std::optional<C2Config::ordinal_key_t> newReorderKey;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001663 bool needMaxDequeueBufferCountUpdate = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001664 while (!worklet->output.configUpdate.empty()) {
1665 std::unique_ptr<C2Param> param;
1666 worklet->output.configUpdate.back().swap(param);
1667 worklet->output.configUpdate.pop_back();
1668 switch (param->coreIndex().coreIndex()) {
1669 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1670 C2PortReorderBufferDepthTuning::output reorderDepth;
1671 if (reorderDepth.updateFrom(*param)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001672 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1673 mName, reorderDepth.value);
Wonsik Kim84f439f2021-05-03 10:57:09 -07001674 newReorderDepth = reorderDepth.value;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001675 needMaxDequeueBufferCountUpdate = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001676 } else {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001677 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1678 mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001679 }
1680 break;
1681 }
1682 case C2PortReorderKeySetting::CORE_INDEX: {
1683 C2PortReorderKeySetting::output reorderKey;
1684 if (reorderKey.updateFrom(*param)) {
Wonsik Kim84f439f2021-05-03 10:57:09 -07001685 newReorderKey = reorderKey.value;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001686 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1687 mName, reorderKey.value);
1688 } else {
1689 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1690 }
1691 break;
1692 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001693 case C2PortActualDelayTuning::CORE_INDEX: {
1694 if (param->isGlobal()) {
1695 C2ActualPipelineDelayTuning pipelineDelay;
1696 if (pipelineDelay.updateFrom(*param)) {
1697 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1698 mName, pipelineDelay.value);
1699 newPipelineDelay = pipelineDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001700 (void)mPipelineWatcher.lock()->pipelineDelay(
1701 pipelineDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001702 }
1703 }
1704 if (param->forInput()) {
1705 C2PortActualDelayTuning::input inputDelay;
1706 if (inputDelay.updateFrom(*param)) {
1707 ALOGV("[%s] onWorkDone: updating input delay %u",
1708 mName, inputDelay.value);
1709 newInputDelay = inputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001710 (void)mPipelineWatcher.lock()->inputDelay(
1711 inputDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001712 }
1713 }
1714 if (param->forOutput()) {
1715 C2PortActualDelayTuning::output outputDelay;
1716 if (outputDelay.updateFrom(*param)) {
1717 ALOGV("[%s] onWorkDone: updating output delay %u",
1718 mName, outputDelay.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001719 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
Wonsik Kim84f439f2021-05-03 10:57:09 -07001720 newOutputDelay = outputDelay.value;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001721 needMaxDequeueBufferCountUpdate = true;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001722
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001723 }
1724 }
1725 break;
1726 }
ted.sunb1fbfdb2020-06-23 14:03:41 +08001727 case C2PortTunnelSystemTime::CORE_INDEX: {
1728 C2PortTunnelSystemTime::output frameRenderTime;
1729 if (frameRenderTime.updateFrom(*param)) {
1730 ALOGV("[%s] onWorkDone: frame rendered (sys:%lld ns, media:%lld us)",
1731 mName, (long long)frameRenderTime.value,
1732 (long long)worklet->output.ordinal.timestamp.peekll());
1733 mCCodecCallback->onOutputFramesRendered(
1734 worklet->output.ordinal.timestamp.peek(), frameRenderTime.value);
1735 }
1736 break;
1737 }
Guillaume Chelfi5ffbcb32021-04-12 14:23:43 +02001738 case C2StreamTunnelHoldRender::CORE_INDEX: {
1739 C2StreamTunnelHoldRender::output firstTunnelFrameHoldRender;
1740 if (!(worklet->output.flags & C2FrameData::FLAG_INCOMPLETE)) break;
1741 if (!firstTunnelFrameHoldRender.updateFrom(*param)) break;
1742 if (firstTunnelFrameHoldRender.value != C2_TRUE) break;
1743 ALOGV("[%s] onWorkDone: first tunnel frame ready", mName);
1744 mCCodecCallback->onFirstTunnelFrameReady();
1745 break;
1746 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001747 default:
1748 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1749 mName, param->index());
1750 break;
1751 }
1752 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001753 if (newInputDelay || newPipelineDelay) {
1754 Mutexed<Input>::Locked input(mInput);
1755 size_t newNumSlots =
1756 newInputDelay.value_or(input->inputDelay) +
1757 newPipelineDelay.value_or(input->pipelineDelay) +
1758 kSmoothnessFactor;
1759 if (input->buffers->isArrayMode()) {
1760 if (input->numSlots >= newNumSlots) {
1761 input->numExtraSlots = 0;
1762 } else {
1763 input->numExtraSlots = newNumSlots - input->numSlots;
1764 }
1765 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1766 mName, input->numExtraSlots);
1767 } else {
1768 input->numSlots = newNumSlots;
1769 }
1770 }
Wonsik Kim84f439f2021-05-03 10:57:09 -07001771 size_t numOutputSlots = 0;
1772 uint32_t reorderDepth = 0;
1773 bool outputBuffersChanged = false;
1774 if (newReorderKey || newReorderDepth || needMaxDequeueBufferCountUpdate) {
1775 Mutexed<Output>::Locked output(mOutput);
1776 if (!output->buffers) {
1777 return false;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001778 }
Wonsik Kim84f439f2021-05-03 10:57:09 -07001779 numOutputSlots = output->numSlots;
1780 if (newReorderKey) {
1781 output->buffers->setReorderKey(newReorderKey.value());
1782 }
1783 if (newReorderDepth) {
1784 output->buffers->setReorderDepth(newReorderDepth.value());
1785 }
1786 reorderDepth = output->buffers->getReorderDepth();
1787 if (newOutputDelay) {
1788 output->outputDelay = newOutputDelay.value();
1789 numOutputSlots = newOutputDelay.value() + kSmoothnessFactor;
1790 if (output->numSlots < numOutputSlots) {
1791 output->numSlots = numOutputSlots;
1792 if (output->buffers->isArrayMode()) {
1793 OutputBuffersArray *array =
1794 (OutputBuffersArray *)output->buffers.get();
1795 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1796 mName, numOutputSlots);
1797 array->grow(numOutputSlots);
1798 outputBuffersChanged = true;
1799 }
1800 }
1801 }
1802 numOutputSlots = output->numSlots;
1803 }
1804 if (outputBuffersChanged) {
1805 mCCodecCallback->onOutputBuffersChanged();
1806 }
1807 if (needMaxDequeueBufferCountUpdate) {
1808 int maxDequeueCount = 0;
Sungtak Leea714f112021-03-16 05:40:03 -07001809 {
1810 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1811 maxDequeueCount = output->maxDequeueBuffers =
1812 numOutputSlots + reorderDepth + kRenderingDepth;
1813 if (output->surface) {
1814 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1815 }
1816 }
1817 if (maxDequeueCount > 0) {
1818 mComponent->setOutputSurfaceMaxDequeueCount(maxDequeueCount);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001819 }
1820 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001821
Pawin Vongmasa36653902018-11-15 00:10:25 -08001822 int32_t flags = 0;
1823 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1824 flags |= MediaCodec::BUFFER_FLAG_EOS;
1825 ALOGV("[%s] onWorkDone: output EOS", mName);
1826 }
1827
Pawin Vongmasa36653902018-11-15 00:10:25 -08001828 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1829 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1830 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1831 // shall correspond to the client input timesamp (in customOrdinal). By using the
1832 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1833 // produces multiple output.
1834 c2_cntr64_t timestamp =
1835 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1836 - work->input.ordinal.timestamp;
Wonsik Kim95ba0162019-03-19 15:51:54 -07001837 if (mInputSurface != nullptr) {
1838 // When using input surface we need to restore the original input timestamp.
1839 timestamp = work->input.ordinal.customOrdinal;
1840 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001841 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1842 mName,
1843 work->input.ordinal.customOrdinal.peekll(),
1844 work->input.ordinal.timestamp.peekll(),
1845 worklet->output.ordinal.timestamp.peekll(),
1846 timestamp.peekll());
1847
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001848 // csd cannot be re-ordered and will always arrive first.
Pawin Vongmasa36653902018-11-15 00:10:25 -08001849 if (initData != nullptr) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001850 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001851 if (output->buffers && outputFormat) {
1852 output->buffers->updateSkipCutBuffer(outputFormat);
1853 output->buffers->setFormat(outputFormat);
1854 }
1855 if (!notifyClient) {
1856 return false;
1857 }
1858 size_t index;
1859 sp<MediaCodecBuffer> outBuffer;
Wonsik Kim936a89c2020-05-08 16:07:50 -07001860 if (output->buffers && output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001861 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1862 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1863 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1864
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001865 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001866 mCallback->onOutputBufferAvailable(index, outBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001867 } else {
1868 ALOGD("[%s] onWorkDone: unable to register csd", mName);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001869 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001870 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001871 return false;
1872 }
1873 }
1874
ted.sunb8fe01e2020-06-23 14:03:41 +08001875 bool drop = false;
1876 if (worklet->output.flags & C2FrameData::FLAG_DROP_FRAME) {
1877 ALOGV("[%s] onWorkDone: drop buffer but keep metadata", mName);
1878 drop = true;
1879 }
1880
ted.sun04698a32020-06-23 14:03:41 +08001881 if (notifyClient && !buffer && !flags && !(drop && outputFormat)) {
Wonsik Kim35bf5732020-05-14 17:40:29 +00001882 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
Pawin Vongmasa36653902018-11-15 00:10:25 -08001883 mName, work->input.ordinal.frameIndex.peekull());
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001884 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001885 }
1886
1887 if (buffer) {
1888 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1889 // TODO: properly translate these to metadata
1890 switch (info->coreIndex().coreIndex()) {
1891 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001892 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001893 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1894 }
1895 break;
1896 default:
1897 break;
1898 }
1899 }
1900 }
1901
1902 {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001903 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimc23cc402020-05-28 14:53:40 -07001904 if (!output->buffers) {
1905 return false;
1906 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001907 output->buffers->pushToStash(
ted.sun04698a32020-06-23 14:03:41 +08001908 drop ? nullptr : buffer,
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001909 notifyClient,
1910 timestamp.peek(),
1911 flags,
1912 outputFormat,
1913 worklet->output.ordinal);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001914 }
1915 sendOutputBuffers();
1916 return true;
1917}
1918
1919void CCodecBufferChannel::sendOutputBuffers() {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001920 OutputBuffers::BufferAction action;
Wonsik Kima4e049d2020-04-28 19:42:23 +00001921 size_t index;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001922 sp<MediaCodecBuffer> outBuffer;
1923 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001924
1925 while (true) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001926 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001927 if (!output->buffers) {
1928 return;
1929 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001930 action = output->buffers->popFromStashAndRegister(
1931 &c2Buffer, &index, &outBuffer);
1932 switch (action) {
1933 case OutputBuffers::SKIP:
1934 return;
1935 case OutputBuffers::DISCARD:
1936 break;
1937 case OutputBuffers::NOTIFY_CLIENT:
Wonsik Kima4e049d2020-04-28 19:42:23 +00001938 output.unlock();
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001939 mCallback->onOutputBufferAvailable(index, outBuffer);
1940 break;
1941 case OutputBuffers::REALLOCATE:
1942 if (!output->buffers->isArrayMode()) {
1943 output->buffers =
1944 output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001945 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001946 static_cast<OutputBuffersArray*>(output->buffers.get())->
1947 realloc(c2Buffer);
1948 output.unlock();
1949 mCCodecCallback->onOutputBuffersChanged();
Wonsik Kim4ada73d2020-05-26 14:58:07 -07001950 break;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001951 case OutputBuffers::RETRY:
1952 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
1953 mName);
1954 return;
1955 default:
1956 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
1957 "corrupted BufferAction value (%d) "
1958 "returned from popFromStashAndRegister.",
1959 mName, int(action));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001960 return;
1961 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001962 }
1963}
1964
1965status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1966 static std::atomic_uint32_t surfaceGeneration{0};
1967 uint32_t generation = (getpid() << 10) |
1968 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1969 & ((1 << 10) - 1));
1970
1971 sp<IGraphicBufferProducer> producer;
Sungtak Leedb14cba2021-04-10 00:50:23 -07001972 int maxDequeueCount = mOutputSurface.lock()->maxDequeueBuffers;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001973 if (newSurface) {
1974 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Sungtak Leeab6f2f32019-02-15 14:43:51 -08001975 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
Sungtak Leedb14cba2021-04-10 00:50:23 -07001976 newSurface->setMaxDequeuedBufferCount(maxDequeueCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001977 producer = newSurface->getIGraphicBufferProducer();
1978 producer->setGenerationNumber(generation);
1979 } else {
1980 ALOGE("[%s] setting output surface to null", mName);
1981 return INVALID_OPERATION;
1982 }
1983
1984 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
1985 C2BlockPool::local_id_t outputPoolId;
1986 {
1987 Mutexed<BlockPools>::Locked pools(mBlockPools);
1988 outputPoolId = pools->outputPoolId;
1989 outputPoolIntf = pools->outputPoolIntf;
1990 }
1991
1992 if (outputPoolIntf) {
1993 if (mComponent->setOutputSurface(
1994 outputPoolId,
1995 producer,
Sungtak Leedb14cba2021-04-10 00:50:23 -07001996 generation,
1997 maxDequeueCount) != C2_OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001998 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
1999 return INVALID_OPERATION;
2000 }
2001 }
2002
2003 {
2004 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2005 output->surface = newSurface;
2006 output->generation = generation;
2007 }
2008
2009 return OK;
2010}
2011
Wonsik Kimab34ed62019-01-31 15:28:46 -08002012PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08002013 // When client pushed EOS, we want all the work to be done quickly.
2014 // Otherwise, component may have stalled work due to input starvation up to
2015 // the sum of the delay in the pipeline.
Wonsik Kimf0e7d222019-06-28 12:33:16 -07002016 size_t n = 0;
2017 if (!mInputMetEos) {
2018 size_t outputDelay = mOutput.lock()->outputDelay;
2019 Mutexed<Input>::Locked input(mInput);
2020 n = input->inputDelay + input->pipelineDelay + outputDelay;
2021 }
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08002022 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
Wonsik Kimab34ed62019-01-31 15:28:46 -08002023}
2024
Pawin Vongmasa36653902018-11-15 00:10:25 -08002025void CCodecBufferChannel::setMetaMode(MetaMode mode) {
2026 mMetaMode = mode;
2027}
2028
Wonsik Kim596187e2019-10-25 12:44:10 -07002029void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002030 if (mCrypto != nullptr) {
2031 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
2032 mCrypto->unsetHeap(entry.second);
2033 }
2034 mHeapSeqNumMap.clear();
2035 if (mHeapSeqNum >= 0) {
2036 mCrypto->unsetHeap(mHeapSeqNum);
2037 mHeapSeqNum = -1;
2038 }
2039 }
Wonsik Kim596187e2019-10-25 12:44:10 -07002040 mCrypto = crypto;
2041}
2042
2043void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
2044 mDescrambler = descrambler;
2045}
2046
Pawin Vongmasa36653902018-11-15 00:10:25 -08002047status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2048 // C2_OK is always translated to OK.
2049 if (c2s == C2_OK) {
2050 return OK;
2051 }
2052
2053 // Operation-dependent translation
2054 // TODO: Add as necessary
2055 switch (c2op) {
2056 case C2_OPERATION_Component_start:
2057 switch (c2s) {
2058 case C2_NO_MEMORY:
2059 return NO_MEMORY;
2060 default:
2061 return UNKNOWN_ERROR;
2062 }
2063 default:
2064 break;
2065 }
2066
2067 // Backup operation-agnostic translation
2068 switch (c2s) {
2069 case C2_BAD_INDEX:
2070 return BAD_INDEX;
2071 case C2_BAD_VALUE:
2072 return BAD_VALUE;
2073 case C2_BLOCKING:
2074 return WOULD_BLOCK;
2075 case C2_DUPLICATE:
2076 return ALREADY_EXISTS;
2077 case C2_NO_INIT:
2078 return NO_INIT;
2079 case C2_NO_MEMORY:
2080 return NO_MEMORY;
2081 case C2_NOT_FOUND:
2082 return NAME_NOT_FOUND;
2083 case C2_TIMED_OUT:
2084 return TIMED_OUT;
2085 case C2_BAD_STATE:
2086 case C2_CANCELED:
2087 case C2_CANNOT_DO:
2088 case C2_CORRUPTED:
2089 case C2_OMITTED:
2090 case C2_REFUSED:
2091 return UNKNOWN_ERROR;
2092 default:
2093 return -static_cast<status_t>(c2s);
2094 }
2095}
2096
2097} // namespace android