blob: e29ec1123caf7c83e030661c338b55a0c0b6a8f9 [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
Pawin Vongmasae7bb8612020-06-04 06:15:22 -070021#include <algorithm>
22#include <list>
Pawin Vongmasa36653902018-11-15 00:10:25 -080023#include <numeric>
24
25#include <C2AllocatorGralloc.h>
26#include <C2PlatformSupport.h>
27#include <C2BlockInternal.h>
28#include <C2Config.h>
29#include <C2Debug.h>
30
31#include <android/hardware/cas/native/1.0/IDescrambler.h>
Robert Shih895fba92019-07-16 16:29:44 -070032#include <android/hardware/drm/1.0/types.h>
Josh Hou8eddf4b2021-02-02 16:26:53 +080033#include <android-base/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080034#include <android-base/stringprintf.h>
Wonsik Kimfb7a7672019-12-27 17:13:33 -080035#include <binder/MemoryBase.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080036#include <binder/MemoryDealer.h>
Ray Essick18ea0452019-08-27 16:07:27 -070037#include <cutils/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080038#include <gui/Surface.h>
Robert Shih895fba92019-07-16 16:29:44 -070039#include <hidlmemory/FrameworkUtils.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080040#include <media/openmax/OMX_Core.h>
41#include <media/stagefright/foundation/ABuffer.h>
42#include <media/stagefright/foundation/ALookup.h>
43#include <media/stagefright/foundation/AMessage.h>
44#include <media/stagefright/foundation/AUtils.h>
45#include <media/stagefright/foundation/hexdump.h>
46#include <media/stagefright/MediaCodec.h>
47#include <media/stagefright/MediaCodecConstants.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070048#include <media/stagefright/SkipCutBuffer.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080049#include <media/MediaCodecBuffer.h>
Wonsik Kim41d83432020-04-27 16:40:49 -070050#include <mediadrm/ICrypto.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080051#include <system/window.h>
52
53#include "CCodecBufferChannel.h"
54#include "Codec2Buffer.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080055
56namespace android {
57
58using android::base::StringPrintf;
59using hardware::hidl_handle;
60using hardware::hidl_string;
61using hardware::hidl_vec;
Robert Shih895fba92019-07-16 16:29:44 -070062using hardware::fromHeap;
63using hardware::HidlMemory;
64
Pawin Vongmasa36653902018-11-15 00:10:25 -080065using namespace hardware::cas::V1_0;
66using namespace hardware::cas::native::V1_0;
67
68using CasStatus = hardware::cas::V1_0::Status;
Robert Shih895fba92019-07-16 16:29:44 -070069using DrmBufferType = hardware::drm::V1_0::BufferType;
Pawin Vongmasa36653902018-11-15 00:10:25 -080070
Pawin Vongmasa36653902018-11-15 00:10:25 -080071namespace {
72
Wonsik Kim469c8342019-04-11 16:46:09 -070073constexpr size_t kSmoothnessFactor = 4;
74constexpr size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -080075
Sungtak Leeab6f2f32019-02-15 14:43:51 -080076// This is for keeping IGBP's buffer dropping logic in legacy mode other
77// than making it non-blocking. Do not change this value.
78const static size_t kDequeueTimeoutNs = 0;
79
Pawin Vongmasa36653902018-11-15 00:10:25 -080080} // namespace
81
82CCodecBufferChannel::QueueGuard::QueueGuard(
83 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
84 Mutex::Autolock l(mSync.mGuardLock);
85 // At this point it's guaranteed that mSync is not under state transition,
86 // as we are holding its mutex.
87
88 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
89 if (count->value == -1) {
90 mRunning = false;
91 } else {
92 ++count->value;
93 mRunning = true;
94 }
95}
96
97CCodecBufferChannel::QueueGuard::~QueueGuard() {
98 if (mRunning) {
99 // We are not holding mGuardLock at this point so that QueueSync::stop() can
100 // keep holding the lock until mCount reaches zero.
101 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
102 --count->value;
103 count->cond.broadcast();
104 }
105}
106
107void CCodecBufferChannel::QueueSync::start() {
108 Mutex::Autolock l(mGuardLock);
109 // If stopped, it goes to running state; otherwise no-op.
110 Mutexed<Counter>::Locked count(mCount);
111 if (count->value == -1) {
112 count->value = 0;
113 }
114}
115
116void CCodecBufferChannel::QueueSync::stop() {
117 Mutex::Autolock l(mGuardLock);
118 Mutexed<Counter>::Locked count(mCount);
119 if (count->value == -1) {
120 // no-op
121 return;
122 }
123 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
124 // mCount can only decrement. In other words, threads that acquired the lock
125 // are allowed to finish execution but additional threads trying to acquire
126 // the lock at this point will block, and then get QueueGuard at STOPPED
127 // state.
128 while (count->value != 0) {
129 count.waitForCondition(count->cond);
130 }
131 count->value = -1;
132}
133
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700134// Input
135
136CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
137
Pawin Vongmasa36653902018-11-15 00:10:25 -0800138// CCodecBufferChannel
139
140CCodecBufferChannel::CCodecBufferChannel(
141 const std::shared_ptr<CCodecCallback> &callback)
142 : mHeapSeqNum(-1),
143 mCCodecCallback(callback),
144 mFrameIndex(0u),
145 mFirstValidFrameIndex(0u),
146 mMetaMode(MODE_NONE),
Sungtak Lee04b30352020-07-27 13:57:25 -0700147 mInputMetEos(false),
148 mSendEncryptedInfoBuffer(false) {
Sungtak Leed7463d12019-09-04 16:01:00 -0700149 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700150 {
151 Mutexed<Input>::Locked input(mInput);
152 input->buffers.reset(new DummyInputBuffers(""));
153 input->extraBuffers.flush();
154 input->inputDelay = 0u;
155 input->pipelineDelay = 0u;
156 input->numSlots = kSmoothnessFactor;
157 input->numExtraSlots = 0u;
158 }
159 {
160 Mutexed<Output>::Locked output(mOutput);
161 output->outputDelay = 0u;
162 output->numSlots = kSmoothnessFactor;
163 }
David Stevensc3fbb282021-01-18 18:11:20 +0900164 {
165 Mutexed<BlockPools>::Locked pools(mBlockPools);
166 pools->outputPoolId = C2BlockPool::BASIC_LINEAR;
167 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800168}
169
170CCodecBufferChannel::~CCodecBufferChannel() {
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800171 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800172 mCrypto->unsetHeap(mHeapSeqNum);
173 }
174}
175
176void CCodecBufferChannel::setComponent(
177 const std::shared_ptr<Codec2Client::Component> &component) {
178 mComponent = component;
179 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
180 mName = mComponentName.c_str();
181}
182
183status_t CCodecBufferChannel::setInputSurface(
184 const std::shared_ptr<InputSurfaceWrapper> &surface) {
185 ALOGV("[%s] setInputSurface", mName);
186 mInputSurface = surface;
187 return mInputSurface->connect(mComponent);
188}
189
190status_t CCodecBufferChannel::signalEndOfInputStream() {
191 if (mInputSurface == nullptr) {
192 return INVALID_OPERATION;
193 }
194 return mInputSurface->signalEndOfInputStream();
195}
196
Sungtak Lee04b30352020-07-27 13:57:25 -0700197status_t CCodecBufferChannel::queueInputBufferInternal(
198 sp<MediaCodecBuffer> buffer,
199 std::shared_ptr<C2LinearBlock> encryptedBlock,
200 size_t blockSize) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800201 int64_t timeUs;
202 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
203
204 if (mInputMetEos) {
205 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
206 return OK;
207 }
208
209 int32_t flags = 0;
210 int32_t tmp = 0;
211 bool eos = false;
Guillaume Chelfi867d4dd2021-07-01 18:38:45 +0200212 bool tunnelFirstFrame = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800213 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
214 eos = true;
215 mInputMetEos = true;
216 ALOGV("[%s] input EOS", mName);
217 }
218 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
219 flags |= C2FrameData::FLAG_CODEC_CONFIG;
220 }
Guillaume Chelfi867d4dd2021-07-01 18:38:45 +0200221 if (buffer->meta()->findInt32("tunnel-first-frame", &tmp) && tmp) {
222 tunnelFirstFrame = true;
223 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800224 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
Wonsik Kime1104ca2020-11-24 15:01:33 -0800225 std::list<std::unique_ptr<C2Work>> items;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800226 std::unique_ptr<C2Work> work(new C2Work);
227 work->input.ordinal.timestamp = timeUs;
228 work->input.ordinal.frameIndex = mFrameIndex++;
229 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
230 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
231 // Keep client timestamp in customOrdinal
232 work->input.ordinal.customOrdinal = timeUs;
233 work->input.buffers.clear();
234
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700235 sp<Codec2Buffer> copy;
Wonsik Kime1104ca2020-11-24 15:01:33 -0800236 bool usesFrameReassembler = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800237
Pawin Vongmasa36653902018-11-15 00:10:25 -0800238 if (buffer->size() > 0u) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700239 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800240 std::shared_ptr<C2Buffer> c2buffer;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700241 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800242 return -ENOENT;
243 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700244 // TODO: we want to delay copying buffers.
245 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
246 copy = input->buffers->cloneAndReleaseBuffer(buffer);
247 if (copy != nullptr) {
248 (void)input->extraBuffers.assignSlot(copy);
249 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
250 return UNKNOWN_ERROR;
251 }
252 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
253 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
254 mName, released ? "" : "not ");
Wonsik Kimfb5ca492021-08-11 14:18:19 -0700255 buffer = copy;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700256 } else {
257 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
258 "buffer starvation on component.", mName);
259 }
260 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800261 if (input->frameReassembler) {
262 usesFrameReassembler = true;
263 input->frameReassembler.process(buffer, &items);
264 } else {
Byeongjo Park25c3a3d2020-06-12 17:24:21 +0900265 int32_t cvo = 0;
266 if (buffer->meta()->findInt32("cvo", &cvo)) {
267 int32_t rotation = cvo % 360;
268 // change rotation to counter-clock wise.
269 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
270
271 Mutexed<OutputSurface>::Locked output(mOutputSurface);
272 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
273 output->rotation[frameIndex] = rotation;
274 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800275 work->input.buffers.push_back(c2buffer);
276 if (encryptedBlock) {
277 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
278 kParamIndexEncryptedBuffer,
279 encryptedBlock->share(0, blockSize, C2Fence())));
280 }
Sungtak Lee04b30352020-07-27 13:57:25 -0700281 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800282 } else if (eos) {
Wonsik Kimcc59ad82021-08-11 18:15:19 -0700283 Mutexed<Input>::Locked input(mInput);
284 if (input->frameReassembler) {
285 usesFrameReassembler = true;
286 // drain any pending items with eos
287 input->frameReassembler.process(buffer, &items);
288 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800289 flags |= C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800290 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800291 if (usesFrameReassembler) {
292 if (!items.empty()) {
293 items.front()->input.configUpdate = std::move(mParamsToBeSet);
294 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
295 }
296 } else {
297 work->input.flags = (C2FrameData::flags_t)flags;
298 // TODO: fill info's
Pawin Vongmasa36653902018-11-15 00:10:25 -0800299
Wonsik Kime1104ca2020-11-24 15:01:33 -0800300 work->input.configUpdate = std::move(mParamsToBeSet);
Guillaume Chelfi867d4dd2021-07-01 18:38:45 +0200301 if (tunnelFirstFrame) {
302 C2StreamTunnelHoldRender::input tunnelHoldRender{
303 0u /* stream */,
304 C2_TRUE /* value */
305 };
306 work->input.configUpdate.push_back(C2Param::Copy(tunnelHoldRender));
307 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800308 work->worklets.clear();
309 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800310
Wonsik Kime1104ca2020-11-24 15:01:33 -0800311 items.push_back(std::move(work));
312
313 eos = eos && buffer->size() > 0u;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800314 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800315 if (eos) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800316 work.reset(new C2Work);
317 work->input.ordinal.timestamp = timeUs;
318 work->input.ordinal.frameIndex = mFrameIndex++;
319 // WORKAROUND: keep client timestamp in customOrdinal
320 work->input.ordinal.customOrdinal = timeUs;
321 work->input.buffers.clear();
322 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800323 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800324 items.push_back(std::move(work));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800325 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800326 c2_status_t err = C2_OK;
327 if (!items.empty()) {
328 {
329 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
330 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
331 for (const std::unique_ptr<C2Work> &work : items) {
332 watcher->onWorkQueued(
333 work->input.ordinal.frameIndex.peeku(),
334 std::vector(work->input.buffers),
335 now);
336 }
337 }
338 err = mComponent->queue(&items);
339 }
340 if (err != C2_OK) {
341 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
342 for (const std::unique_ptr<C2Work> &work : items) {
343 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
344 }
345 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700346 Mutexed<Input>::Locked input(mInput);
347 bool released = false;
Wonsik Kimfb5ca492021-08-11 14:18:19 -0700348 if (copy) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700349 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
Wonsik Kimfb5ca492021-08-11 14:18:19 -0700350 } else if (buffer) {
351 released = input->buffers->releaseBuffer(buffer, nullptr, true);
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700352 }
353 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
354 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800355 }
356
357 feedInputBufferIfAvailableInternal();
358 return err;
359}
360
361status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
362 QueueGuard guard(mSync);
363 if (!guard.isRunning()) {
364 ALOGD("[%s] setParameters is only supported in the running state.", mName);
365 return -ENOSYS;
366 }
367 mParamsToBeSet.insert(mParamsToBeSet.end(),
368 std::make_move_iterator(params.begin()),
369 std::make_move_iterator(params.end()));
370 params.clear();
371 return OK;
372}
373
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800374status_t CCodecBufferChannel::attachBuffer(
375 const std::shared_ptr<C2Buffer> &c2Buffer,
376 const sp<MediaCodecBuffer> &buffer) {
377 if (!buffer->copy(c2Buffer)) {
378 return -ENOSYS;
379 }
380 return OK;
381}
382
383void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
384 if (!mDecryptDestination || mDecryptDestination->size() < size) {
385 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
386 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
387 mCrypto->unsetHeap(mHeapSeqNum);
388 }
389 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
390 if (mCrypto) {
391 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
392 }
393 }
394}
395
396int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
397 CHECK(mCrypto);
398 auto it = mHeapSeqNumMap.find(memory);
399 int32_t heapSeqNum = -1;
400 if (it == mHeapSeqNumMap.end()) {
401 heapSeqNum = mCrypto->setHeap(memory);
402 mHeapSeqNumMap.emplace(memory, heapSeqNum);
403 } else {
404 heapSeqNum = it->second;
405 }
406 return heapSeqNum;
407}
408
409status_t CCodecBufferChannel::attachEncryptedBuffer(
410 const sp<hardware::HidlMemory> &memory,
411 bool secure,
412 const uint8_t *key,
413 const uint8_t *iv,
414 CryptoPlugin::Mode mode,
415 CryptoPlugin::Pattern pattern,
416 size_t offset,
417 const CryptoPlugin::SubSample *subSamples,
418 size_t numSubSamples,
419 const sp<MediaCodecBuffer> &buffer) {
420 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
421 static const C2MemoryUsage kDefaultReadWriteUsage{
422 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
423
424 size_t size = 0;
425 for (size_t i = 0; i < numSubSamples; ++i) {
426 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
427 }
428 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
429 std::shared_ptr<C2LinearBlock> block;
430 c2_status_t err = pool->fetchLinearBlock(
431 size,
432 secure ? kSecureUsage : kDefaultReadWriteUsage,
433 &block);
434 if (err != C2_OK) {
435 return NO_MEMORY;
436 }
437 if (!secure) {
438 ensureDecryptDestination(size);
439 }
440 ssize_t result = -1;
441 ssize_t codecDataOffset = 0;
442 if (mCrypto) {
443 AString errorDetailMsg;
444 int32_t heapSeqNum = getHeapSeqNum(memory);
445 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
446 hardware::drm::V1_0::DestinationBuffer dst;
447 if (secure) {
448 dst.type = DrmBufferType::NATIVE_HANDLE;
449 dst.secureMemory = hardware::hidl_handle(block->handle());
450 } else {
451 dst.type = DrmBufferType::SHARED_MEMORY;
452 IMemoryToSharedBuffer(
453 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
454 }
455 result = mCrypto->decrypt(
456 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
457 dst, &errorDetailMsg);
458 if (result < 0) {
459 return result;
460 }
461 if (dst.type == DrmBufferType::SHARED_MEMORY) {
462 C2WriteView view = block->map().get();
463 if (view.error() != C2_OK) {
464 return false;
465 }
466 if (view.size() < result) {
467 return false;
468 }
469 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
470 }
471 } else {
472 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
473 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
474 hidl_vec<SubSample> hidlSubSamples;
475 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
476
477 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
478 hardware::cas::native::V1_0::DestinationBuffer dst;
479 if (secure) {
480 dst.type = BufferType::NATIVE_HANDLE;
481 dst.secureMemory = hardware::hidl_handle(block->handle());
482 } else {
483 dst.type = BufferType::SHARED_MEMORY;
484 dst.nonsecureMemory = src;
485 }
486
487 CasStatus status = CasStatus::OK;
488 hidl_string detailedError;
489 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
490
491 if (key != nullptr) {
492 sctrl = (ScramblingControl)key[0];
493 // Adjust for the PES offset
494 codecDataOffset = key[2] | (key[3] << 8);
495 }
496
497 auto returnVoid = mDescrambler->descramble(
498 sctrl,
499 hidlSubSamples,
500 src,
501 0,
502 dst,
503 0,
504 [&status, &result, &detailedError] (
505 CasStatus _status, uint32_t _bytesWritten,
506 const hidl_string& _detailedError) {
507 status = _status;
508 result = (ssize_t)_bytesWritten;
509 detailedError = _detailedError;
510 });
511
512 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
513 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
514 mName, returnVoid.description().c_str(), status, result);
515 return UNKNOWN_ERROR;
516 }
517
518 if (result < codecDataOffset) {
519 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
520 return BAD_VALUE;
521 }
522 }
523 if (!secure) {
524 C2WriteView view = block->map().get();
525 if (view.error() != C2_OK) {
526 return UNKNOWN_ERROR;
527 }
528 if (view.size() < result) {
529 return UNKNOWN_ERROR;
530 }
531 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
532 }
533 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
534 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
535 if (!buffer->copy(c2Buffer)) {
536 return -ENOSYS;
537 }
538 return OK;
539}
540
Pawin Vongmasa36653902018-11-15 00:10:25 -0800541status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
542 QueueGuard guard(mSync);
543 if (!guard.isRunning()) {
544 ALOGD("[%s] No more buffers should be queued at current state.", mName);
545 return -ENOSYS;
546 }
547 return queueInputBufferInternal(buffer);
548}
549
550status_t CCodecBufferChannel::queueSecureInputBuffer(
551 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
552 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
553 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
554 AString *errorDetailMsg) {
555 QueueGuard guard(mSync);
556 if (!guard.isRunning()) {
557 ALOGD("[%s] No more buffers should be queued at current state.", mName);
558 return -ENOSYS;
559 }
560
561 if (!hasCryptoOrDescrambler()) {
562 return -ENOSYS;
563 }
564 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
565
Sungtak Lee04b30352020-07-27 13:57:25 -0700566 std::shared_ptr<C2LinearBlock> block;
567 size_t allocSize = buffer->size();
568 size_t bufferSize = 0;
569 c2_status_t blockRes = C2_OK;
570 bool copied = false;
571 if (mSendEncryptedInfoBuffer) {
572 static const C2MemoryUsage kDefaultReadWriteUsage{
573 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
574 constexpr int kAllocGranule0 = 1024 * 64;
575 constexpr int kAllocGranule1 = 1024 * 1024;
576 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
577 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
578 if (allocSize <= kAllocGranule1) {
579 bufferSize = align(allocSize, kAllocGranule0);
580 } else {
581 bufferSize = align(allocSize, kAllocGranule1);
582 }
583 blockRes = pool->fetchLinearBlock(
584 bufferSize, kDefaultReadWriteUsage, &block);
585
586 if (blockRes == C2_OK) {
587 C2WriteView view = block->map().get();
588 if (view.error() == C2_OK && view.size() == bufferSize) {
589 copied = true;
590 // TODO: only copy clear sections
591 memcpy(view.data(), buffer->data(), allocSize);
592 }
593 }
594 }
595
596 if (!copied) {
597 block.reset();
598 }
599
Pawin Vongmasa36653902018-11-15 00:10:25 -0800600 ssize_t result = -1;
601 ssize_t codecDataOffset = 0;
Wonsik Kim557c88c2020-03-13 11:03:52 -0700602 if (numSubSamples == 1
603 && subSamples[0].mNumBytesOfClearData == 0
604 && subSamples[0].mNumBytesOfEncryptedData == 0) {
605 // We don't need to go through crypto or descrambler if the input is empty.
606 result = 0;
607 } else if (mCrypto != nullptr) {
Robert Shih895fba92019-07-16 16:29:44 -0700608 hardware::drm::V1_0::DestinationBuffer destination;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800609 if (secure) {
Robert Shih895fba92019-07-16 16:29:44 -0700610 destination.type = DrmBufferType::NATIVE_HANDLE;
611 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800612 } else {
Robert Shih895fba92019-07-16 16:29:44 -0700613 destination.type = DrmBufferType::SHARED_MEMORY;
614 IMemoryToSharedBuffer(
615 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800616 }
Robert Shih895fba92019-07-16 16:29:44 -0700617 hardware::drm::V1_0::SharedBuffer source;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800618 encryptedBuffer->fillSourceBuffer(&source);
619 result = mCrypto->decrypt(
620 key, iv, mode, pattern, source, buffer->offset(),
621 subSamples, numSubSamples, destination, errorDetailMsg);
622 if (result < 0) {
Wonsik Kim557c88c2020-03-13 11:03:52 -0700623 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800624 return result;
625 }
Robert Shih895fba92019-07-16 16:29:44 -0700626 if (destination.type == DrmBufferType::SHARED_MEMORY) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800627 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
628 }
629 } else {
630 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
631 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
632 hidl_vec<SubSample> hidlSubSamples;
633 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
634
635 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
636 encryptedBuffer->fillSourceBuffer(&srcBuffer);
637
638 DestinationBuffer dstBuffer;
639 if (secure) {
640 dstBuffer.type = BufferType::NATIVE_HANDLE;
641 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
642 } else {
643 dstBuffer.type = BufferType::SHARED_MEMORY;
644 dstBuffer.nonsecureMemory = srcBuffer;
645 }
646
647 CasStatus status = CasStatus::OK;
648 hidl_string detailedError;
649 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
650
651 if (key != nullptr) {
652 sctrl = (ScramblingControl)key[0];
653 // Adjust for the PES offset
654 codecDataOffset = key[2] | (key[3] << 8);
655 }
656
657 auto returnVoid = mDescrambler->descramble(
658 sctrl,
659 hidlSubSamples,
660 srcBuffer,
661 0,
662 dstBuffer,
663 0,
664 [&status, &result, &detailedError] (
665 CasStatus _status, uint32_t _bytesWritten,
666 const hidl_string& _detailedError) {
667 status = _status;
668 result = (ssize_t)_bytesWritten;
669 detailedError = _detailedError;
670 });
671
672 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
673 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
674 mName, returnVoid.description().c_str(), status, result);
675 return UNKNOWN_ERROR;
676 }
677
678 if (result < codecDataOffset) {
679 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
680 return BAD_VALUE;
681 }
682
683 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
684
685 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
686 encryptedBuffer->copyDecryptedContentFromMemory(result);
687 }
688 }
689
690 buffer->setRange(codecDataOffset, result - codecDataOffset);
Sungtak Lee04b30352020-07-27 13:57:25 -0700691
692 return queueInputBufferInternal(buffer, block, bufferSize);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800693}
694
695void CCodecBufferChannel::feedInputBufferIfAvailable() {
696 QueueGuard guard(mSync);
697 if (!guard.isRunning()) {
698 ALOGV("[%s] We're not running --- no input buffer reported", mName);
699 return;
700 }
701 feedInputBufferIfAvailableInternal();
702}
703
704void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
Taehwan Kimda0517d2020-09-16 17:29:37 +0900705 if (mInputMetEos) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800706 return;
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700707 }
708 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700709 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700710 if (!output->buffers ||
711 output->buffers->hasPending() ||
Wonsik Kim0487b782020-10-28 11:45:50 -0700712 output->buffers->numActiveSlots() >= output->numSlots) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800713 return;
714 }
715 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700716 size_t numActiveSlots = 0;
717 while (!mPipelineWatcher.lock()->pipelineFull()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800718 sp<MediaCodecBuffer> inBuffer;
719 size_t index;
720 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700721 Mutexed<Input>::Locked input(mInput);
Wonsik Kim0487b782020-10-28 11:45:50 -0700722 numActiveSlots = input->buffers->numActiveSlots();
723 if (numActiveSlots >= input->numSlots) {
724 break;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800725 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700726 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800727 ALOGV("[%s] no new buffer available", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800728 break;
729 }
730 }
731 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
732 mCallback->onInputBufferAvailable(index, inBuffer);
733 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700734 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800735}
736
737status_t CCodecBufferChannel::renderOutputBuffer(
738 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800739 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800740 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800741 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800742 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700743 Mutexed<Output>::Locked output(mOutput);
744 if (output->buffers) {
745 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800746 }
747 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800748 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
749 // set to true.
750 sendOutputBuffers();
751 // input buffer feeding may have been gated by pending output buffers
752 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800753 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800754 if (released) {
Wonsik Kimf7529dd2019-04-18 17:35:53 -0700755 std::call_once(mRenderWarningFlag, [this] {
756 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
757 "timestamp or render=true with non-video buffers. Apps should "
758 "call releaseOutputBuffer() with render=false for those.",
759 mName);
760 });
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800761 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800762 return INVALID_OPERATION;
763 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800764
765#if 0
766 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
767 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
768 for (const std::shared_ptr<const C2Info> &info : infoParams) {
769 AString res;
770 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
771 if (ix) res.append(", ");
772 res.append(*((int32_t*)info.get() + (ix / 4)));
773 }
774 ALOGV(" [%s]", res.c_str());
775 }
776#endif
777 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
778 std::static_pointer_cast<const C2StreamRotationInfo::output>(
779 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
780 bool flip = rotation && (rotation->flip & 1);
781 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
Byeongjo Park25c3a3d2020-06-12 17:24:21 +0900782
783 {
784 Mutexed<OutputSurface>::Locked output(mOutputSurface);
785 if (output->surface == nullptr) {
786 ALOGI("[%s] cannot render buffer without surface", mName);
787 return OK;
788 }
789 int64_t frameIndex;
790 buffer->meta()->findInt64("frameIndex", &frameIndex);
791 if (output->rotation.count(frameIndex) != 0) {
792 auto it = output->rotation.find(frameIndex);
793 quarters = (it->second / 90) & 3;
794 output->rotation.erase(it);
795 }
796 }
797
Pawin Vongmasa36653902018-11-15 00:10:25 -0800798 uint32_t transform = 0;
799 switch (quarters) {
800 case 0: // no rotation
801 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
802 break;
803 case 1: // 90 degrees counter-clockwise
804 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
805 : HAL_TRANSFORM_ROT_270;
806 break;
807 case 2: // 180 degrees
808 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
809 break;
810 case 3: // 90 degrees clockwise
811 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
812 : HAL_TRANSFORM_ROT_90;
813 break;
814 }
815
816 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
817 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
818 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
819 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
820 if (surfaceScaling) {
821 videoScalingMode = surfaceScaling->value;
822 }
823
824 // Use dataspace from format as it has the default aspects already applied
825 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
826 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
827
828 // HDR static info
829 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
830 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
831 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
832
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800833 // HDR10 plus info
834 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
835 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
836 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
Yichi Chen54be23c2020-06-15 14:30:53 +0800837 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
838 hdr10PlusInfo.reset();
839 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800840
Pawin Vongmasa36653902018-11-15 00:10:25 -0800841 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
842 if (blocks.size() != 1u) {
843 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
844 return UNKNOWN_ERROR;
845 }
846 const C2ConstGraphicBlock &block = blocks.front();
847
848 // TODO: revisit this after C2Fence implementation.
849 android::IGraphicBufferProducer::QueueBufferInput qbi(
850 timestampNs,
851 false, // droppable
852 dataSpace,
853 Rect(blocks.front().crop().left,
854 blocks.front().crop().top,
855 blocks.front().crop().right(),
856 blocks.front().crop().bottom()),
857 videoScalingMode,
858 transform,
859 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800860 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800861 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800862 if (hdrStaticInfo) {
wenchangliuf3f92882020-05-14 00:02:01 +0800863 // If mastering max and min luminance fields are 0, do not use them.
864 // It indicates the value may not be present in the stream.
865 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
866 hdrStaticInfo->mastering.minLuminance > 0.0f) {
867 struct android_smpte2086_metadata smpte2086_meta = {
868 .displayPrimaryRed = {
869 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
870 },
871 .displayPrimaryGreen = {
872 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
873 },
874 .displayPrimaryBlue = {
875 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
876 },
877 .whitePoint = {
878 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
879 },
880 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
881 .minLuminance = hdrStaticInfo->mastering.minLuminance,
882 };
Yichi Chen54be23c2020-06-15 14:30:53 +0800883 hdr.validTypes |= HdrMetadata::SMPTE2086;
wenchangliuf3f92882020-05-14 00:02:01 +0800884 hdr.smpte2086 = smpte2086_meta;
885 }
Chong Zhang3bb2a7f2020-04-21 10:35:12 -0700886 // If the content light level fields are 0, do not use them, it
887 // indicates the value may not be present in the stream.
888 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
889 struct android_cta861_3_metadata cta861_meta = {
890 .maxContentLightLevel = hdrStaticInfo->maxCll,
891 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
892 };
893 hdr.validTypes |= HdrMetadata::CTA861_3;
894 hdr.cta8613 = cta861_meta;
895 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800896 }
897 if (hdr10PlusInfo) {
898 hdr.validTypes |= HdrMetadata::HDR10PLUS;
899 hdr.hdr10plus.assign(
900 hdr10PlusInfo->m.value,
901 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
902 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800903 qbi.setHdrMetadata(hdr);
904 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800905 // we don't have dirty regions
906 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800907 android::IGraphicBufferProducer::QueueBufferOutput qbo;
908 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
909 if (result != OK) {
910 ALOGI("[%s] queueBuffer failed: %d", mName, result);
Sungtak Lee47c018a2020-11-07 01:02:49 -0800911 if (result == NO_INIT) {
912 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
913 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800914 return result;
915 }
Josh Hou8eddf4b2021-02-02 16:26:53 +0800916
917 if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
918 ALOGD("[%s] queue buffer successful", mName);
919 } else {
920 ALOGV("[%s] queue buffer successful", mName);
921 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800922
923 int64_t mediaTimeUs = 0;
924 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
925 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
926
927 return OK;
928}
929
930status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
931 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
932 bool released = false;
933 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700934 Mutexed<Input>::Locked input(mInput);
935 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800936 released = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800937 }
938 }
939 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700940 Mutexed<Output>::Locked output(mOutput);
941 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800942 released = true;
943 }
944 }
945 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800946 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800947 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800948 } else {
949 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
950 }
951 return OK;
952}
953
954void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
955 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700956 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800957
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700958 if (!input->buffers->isArrayMode()) {
959 input->buffers = input->buffers->toArrayMode(input->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800960 }
961
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700962 input->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800963}
964
965void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
966 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700967 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800968
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700969 if (!output->buffers->isArrayMode()) {
970 output->buffers = output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800971 }
972
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700973 output->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800974}
975
976status_t CCodecBufferChannel::start(
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800977 const sp<AMessage> &inputFormat,
978 const sp<AMessage> &outputFormat,
979 bool buffersBoundToCodec) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800980 C2StreamBufferTypeSetting::input iStreamFormat(0u);
981 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kime1104ca2020-11-24 15:01:33 -0800982 C2ComponentKindSetting kind;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800983 C2PortReorderBufferDepthTuning::output reorderDepth;
984 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800985 C2PortActualDelayTuning::input inputDelay(0);
986 C2PortActualDelayTuning::output outputDelay(0);
987 C2ActualPipelineDelayTuning pipelineDelay(0);
Sungtak Lee04b30352020-07-27 13:57:25 -0700988 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
Wonsik Kim078b58e2019-01-09 15:08:06 -0800989
Pawin Vongmasa36653902018-11-15 00:10:25 -0800990 c2_status_t err = mComponent->query(
991 {
992 &iStreamFormat,
993 &oStreamFormat,
Wonsik Kime1104ca2020-11-24 15:01:33 -0800994 &kind,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800995 &reorderDepth,
996 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800997 &inputDelay,
998 &pipelineDelay,
999 &outputDelay,
Sungtak Lee04b30352020-07-27 13:57:25 -07001000 &secureMode,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001001 },
1002 {},
1003 C2_DONT_BLOCK,
1004 nullptr);
1005 if (err == C2_BAD_INDEX) {
Wonsik Kime1104ca2020-11-24 15:01:33 -08001006 if (!iStreamFormat || !oStreamFormat || !kind) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001007 return UNKNOWN_ERROR;
1008 }
1009 } else if (err != C2_OK) {
1010 return UNKNOWN_ERROR;
1011 }
1012
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001013 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
1014 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
1015 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
1016
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001017 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
1018 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
Wonsik Kim078b58e2019-01-09 15:08:06 -08001019
Pawin Vongmasa36653902018-11-15 00:10:25 -08001020 // TODO: get this from input format
1021 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1022
Sungtak Lee04b30352020-07-27 13:57:25 -07001023 // secure mode is a static parameter (shall not change in the executing state)
1024 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1025
Pawin Vongmasa36653902018-11-15 00:10:25 -08001026 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001027 int poolMask = GetCodec2PoolMask();
1028 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001029
1030 if (inputFormat != nullptr) {
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001031 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
Wonsik Kime1104ca2020-11-24 15:01:33 -08001032 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001033 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1034 API_REFLECTION |
1035 API_VALUES |
1036 API_CURRENT_VALUES |
1037 API_DEPENDENCY |
1038 API_SAME_INPUT_BUFFER);
Wonsik Kime1104ca2020-11-24 15:01:33 -08001039 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1040 C2StreamSampleRateInfo::input sampleRate(0u);
1041 C2StreamChannelCountInfo::input channelCount(0u);
1042 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001043 std::shared_ptr<C2BlockPool> pool;
1044 {
1045 Mutexed<BlockPools>::Locked pools(mBlockPools);
1046
1047 // set default allocator ID.
1048 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001049 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001050
1051 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1052 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1053 std::vector<std::unique_ptr<C2Param>> params;
Wonsik Kimffb889a2020-05-28 11:32:25 -07001054 C2ApiFeaturesSetting featuresSetting{apiFeatures};
Wonsik Kime1104ca2020-11-24 15:01:33 -08001055 std::vector<C2Param *> stackParams({&featuresSetting});
1056 if (audioEncoder) {
1057 stackParams.push_back(&encoderFrameSize);
1058 stackParams.push_back(&sampleRate);
1059 stackParams.push_back(&channelCount);
1060 stackParams.push_back(&pcmEncoding);
1061 } else {
1062 encoderFrameSize.invalidate();
1063 sampleRate.invalidate();
1064 channelCount.invalidate();
1065 pcmEncoding.invalidate();
1066 }
1067 err = mComponent->query(stackParams,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001068 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1069 C2_DONT_BLOCK,
1070 &params);
1071 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1072 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1073 mName, params.size(), asString(err), err);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001074 } else if (params.size() == 1) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001075 C2PortAllocatorsTuning::input *inputAllocators =
1076 C2PortAllocatorsTuning::input::From(params[0].get());
1077 if (inputAllocators && inputAllocators->flexCount() > 0) {
1078 std::shared_ptr<C2Allocator> allocator;
1079 // verify allocator IDs and resolve default allocator
1080 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1081 if (allocator) {
1082 pools->inputAllocatorId = allocator->getId();
1083 } else {
1084 ALOGD("[%s] component requested invalid input allocator ID %u",
1085 mName, inputAllocators->m.values[0]);
1086 }
1087 }
1088 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001089 if (featuresSetting) {
1090 apiFeatures = featuresSetting.value;
1091 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001092
1093 // TODO: use C2Component wrapper to associate this pool with ourselves
1094 if ((poolMask >> pools->inputAllocatorId) & 1) {
1095 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1096 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1097 mName, pools->inputAllocatorId,
1098 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1099 asString(err), err);
1100 } else {
1101 err = C2_NOT_FOUND;
1102 }
1103 if (err != C2_OK) {
1104 C2BlockPool::local_id_t inputPoolId =
1105 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1106 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1107 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1108 mName, (unsigned long long)inputPoolId,
1109 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1110 asString(err), err);
1111 if (err != C2_OK) {
1112 return NO_MEMORY;
1113 }
1114 }
1115 pools->inputPool = pool;
1116 }
1117
Wonsik Kim51051262018-11-28 13:59:05 -08001118 bool forceArrayMode = false;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001119 Mutexed<Input>::Locked input(mInput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001120 input->inputDelay = inputDelayValue;
1121 input->pipelineDelay = pipelineDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001122 input->numSlots = numInputSlots;
1123 input->extraBuffers.flush();
1124 input->numExtraSlots = 0u;
Wonsik Kime1104ca2020-11-24 15:01:33 -08001125 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1126 input->frameReassembler.init(
1127 pool,
1128 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1129 encoderFrameSize.value,
1130 sampleRate.value,
1131 channelCount.value,
1132 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1133 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001134 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1135 // For encrypted content, framework decrypts source buffer (ashmem) into
1136 // C2Buffers. Thus non-conforming codecs can process these.
Wonsik Kime1104ca2020-11-24 15:01:33 -08001137 if (!buffersBoundToCodec
1138 && !input->frameReassembler
1139 && (hasCryptoOrDescrambler() || conforming)) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001140 input->buffers.reset(new SlotInputBuffers(mName));
1141 } else if (graphic) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001142 if (mInputSurface) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001143 input->buffers.reset(new DummyInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001144 } else if (mMetaMode == MODE_ANW) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001145 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
Wonsik Kim1221fd12019-07-12 12:52:05 -07001146 // This is to ensure buffers do not get released prematurely.
1147 // TODO: handle this without going into array mode
1148 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001149 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001150 input->buffers.reset(new GraphicInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001151 }
1152 } else {
1153 if (hasCryptoOrDescrambler()) {
1154 int32_t capacity = kLinearBufferSize;
1155 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1156 if ((size_t)capacity > kMaxLinearBufferSize) {
1157 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1158 capacity = kMaxLinearBufferSize;
1159 }
1160 if (mDealer == nullptr) {
1161 mDealer = new MemoryDealer(
1162 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001163 * (numInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001164 "EncryptedLinearInputBuffers");
1165 mDecryptDestination = mDealer->allocate((size_t)capacity);
1166 }
1167 if (mCrypto != nullptr && mHeapSeqNum < 0) {
Robert Shih895fba92019-07-16 16:29:44 -07001168 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1169 mHeapSeqNum = mCrypto->setHeap(heap);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001170 } else {
1171 mHeapSeqNum = -1;
1172 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001173 input->buffers.reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001174 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001175 numInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08001176 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001177 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001178 input->buffers.reset(new LinearInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001179 }
1180 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001181 input->buffers->setFormat(inputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001182
1183 if (err == C2_OK) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001184 input->buffers->setPool(pool);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001185 } else {
1186 // TODO: error
1187 }
Wonsik Kim51051262018-11-28 13:59:05 -08001188
1189 if (forceArrayMode) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001190 input->buffers = input->buffers->toArrayMode(numInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08001191 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001192 }
1193
1194 if (outputFormat != nullptr) {
1195 sp<IGraphicBufferProducer> outputSurface;
1196 uint32_t outputGeneration;
Sungtak Leea714f112021-03-16 05:40:03 -07001197 int maxDequeueCount = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001198 {
1199 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Sungtak Leea714f112021-03-16 05:40:03 -07001200 maxDequeueCount = output->maxDequeueBuffers = numOutputSlots +
Sungtak Lee7a7b7422019-07-16 17:40:40 -07001201 reorderDepth.value + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001202 outputSurface = output->surface ?
1203 output->surface->getIGraphicBufferProducer() : nullptr;
Wonsik Kimf5e5c832019-02-21 11:36:05 -08001204 if (outputSurface) {
1205 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1206 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001207 outputGeneration = output->generation;
1208 }
1209
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001210 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001211 C2BlockPool::local_id_t outputPoolId_;
David Stevensc3fbb282021-01-18 18:11:20 +09001212 C2BlockPool::local_id_t prevOutputPoolId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001213
1214 {
1215 Mutexed<BlockPools>::Locked pools(mBlockPools);
1216
David Stevensc3fbb282021-01-18 18:11:20 +09001217 prevOutputPoolId = pools->outputPoolId;
1218
Pawin Vongmasa36653902018-11-15 00:10:25 -08001219 // set default allocator ID.
1220 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001221 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001222
1223 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1224 // unsuccessful.
1225 std::vector<std::unique_ptr<C2Param>> params;
1226 err = mComponent->query({ },
1227 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1228 C2_DONT_BLOCK,
1229 &params);
1230 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1231 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1232 mName, params.size(), asString(err), err);
1233 } else if (err == C2_OK && params.size() == 1) {
1234 C2PortAllocatorsTuning::output *outputAllocators =
1235 C2PortAllocatorsTuning::output::From(params[0].get());
1236 if (outputAllocators && outputAllocators->flexCount() > 0) {
1237 std::shared_ptr<C2Allocator> allocator;
1238 // verify allocator IDs and resolve default allocator
1239 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1240 if (allocator) {
1241 pools->outputAllocatorId = allocator->getId();
1242 } else {
1243 ALOGD("[%s] component requested invalid output allocator ID %u",
1244 mName, outputAllocators->m.values[0]);
1245 }
1246 }
1247 }
1248
1249 // use bufferqueue if outputting to a surface.
1250 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1251 // if unsuccessful.
1252 if (outputSurface) {
1253 params.clear();
1254 err = mComponent->query({ },
1255 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1256 C2_DONT_BLOCK,
1257 &params);
1258 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1259 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1260 mName, params.size(), asString(err), err);
1261 } else if (err == C2_OK && params.size() == 1) {
1262 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1263 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1264 if (surfaceAllocator) {
1265 std::shared_ptr<C2Allocator> allocator;
1266 // verify allocator IDs and resolve default allocator
1267 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1268 if (allocator) {
1269 pools->outputAllocatorId = allocator->getId();
1270 } else {
1271 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1272 mName, surfaceAllocator->value);
1273 err = C2_BAD_VALUE;
1274 }
1275 }
1276 }
1277 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1278 && err != C2_OK
1279 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1280 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1281 }
1282 }
1283
1284 if ((poolMask >> pools->outputAllocatorId) & 1) {
1285 err = mComponent->createBlockPool(
1286 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1287 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1288 mName, pools->outputAllocatorId,
1289 (unsigned long long)pools->outputPoolId,
1290 asString(err));
1291 } else {
1292 err = C2_NOT_FOUND;
1293 }
1294 if (err != C2_OK) {
1295 // use basic pool instead
1296 pools->outputPoolId =
1297 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1298 }
1299
1300 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1301 // component.
1302 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1303 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1304
1305 std::vector<std::unique_ptr<C2SettingResult>> failures;
1306 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1307 ALOGD("[%s] Configured output block pool ids %llu => %s",
1308 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1309 outputPoolId_ = pools->outputPoolId;
1310 }
1311
David Stevensc3fbb282021-01-18 18:11:20 +09001312 if (prevOutputPoolId != C2BlockPool::BASIC_LINEAR
1313 && prevOutputPoolId != C2BlockPool::BASIC_GRAPHIC) {
1314 c2_status_t err = mComponent->destroyBlockPool(prevOutputPoolId);
1315 if (err != C2_OK) {
1316 ALOGW("Failed to clean up previous block pool %llu - %s (%d)\n",
1317 (unsigned long long) prevOutputPoolId, asString(err), err);
1318 }
1319 }
1320
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001321 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001322 output->outputDelay = outputDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001323 output->numSlots = numOutputSlots;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001324 if (graphic) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001325 if (outputSurface || !buffersBoundToCodec) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001326 output->buffers.reset(new GraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001327 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001328 output->buffers.reset(new RawGraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001329 }
1330 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001331 output->buffers.reset(new LinearOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001332 }
Wonsik Kime4716c02020-02-28 10:42:21 -08001333 output->buffers->setFormat(outputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001334
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001335 output->buffers->clearStash();
1336 if (reorderDepth) {
1337 output->buffers->setReorderDepth(reorderDepth.value);
1338 }
1339 if (reorderKey) {
1340 output->buffers->setReorderKey(reorderKey.value);
1341 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001342
1343 // Try to set output surface to created block pool if given.
1344 if (outputSurface) {
1345 mComponent->setOutputSurface(
1346 outputPoolId_,
1347 outputSurface,
Sungtak Leedb14cba2021-04-10 00:50:23 -07001348 outputGeneration,
1349 maxDequeueCount);
Lajos Molnar78aa7c92021-02-18 21:39:01 -08001350 } else {
1351 // configure CPU read consumer usage
1352 C2StreamUsageTuning::output outputUsage{0u, C2MemoryUsage::CPU_READ};
1353 std::vector<std::unique_ptr<C2SettingResult>> failures;
1354 err = mComponent->config({ &outputUsage }, C2_MAY_BLOCK, &failures);
1355 // do not print error message for now as most components may not yet
1356 // support this setting
1357 ALOGD_IF(err != C2_BAD_INDEX, "[%s] Configured output usage [%#llx]",
1358 mName, (long long)outputUsage.value);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001359 }
1360
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001361 if (oStreamFormat.value == C2BufferData::LINEAR) {
Wonsik Kim58713302020-01-29 22:25:23 -08001362 if (buffersBoundToCodec) {
1363 // WORKAROUND: if we're using early CSD workaround we convert to
1364 // array mode, to appease apps assuming the output
1365 // buffers to be of the same size.
1366 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1367 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001368
1369 int32_t channelCount;
1370 int32_t sampleRate;
1371 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1372 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1373 int32_t delay = 0;
1374 int32_t padding = 0;;
1375 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1376 delay = 0;
1377 }
1378 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1379 padding = 0;
1380 }
1381 if (delay || padding) {
1382 // We need write access to the buffers, and we're already in
1383 // array mode.
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001384 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001385 }
1386 }
1387 }
Wonsik Kimec585c32021-10-01 01:11:00 -07001388
1389 int32_t tunneled = 0;
1390 if (!outputFormat->findInt32("android._tunneled", &tunneled)) {
1391 tunneled = 0;
1392 }
1393 mTunneled = (tunneled != 0);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001394 }
1395
1396 // Set up pipeline control. This has to be done after mInputBuffers and
1397 // mOutputBuffers are initialized to make sure that lingering callbacks
1398 // about buffers from the previous generation do not interfere with the
1399 // newly initialized pipeline capacity.
1400
Wonsik Kim62545252021-01-20 11:25:41 -08001401 if (inputFormat || outputFormat) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08001402 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001403 watcher->inputDelay(inputDelayValue)
1404 .pipelineDelay(pipelineDelayValue)
1405 .outputDelay(outputDelayValue)
Wonsik Kimab34ed62019-01-31 15:28:46 -08001406 .smoothnessFactor(kSmoothnessFactor);
1407 watcher->flush();
1408 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001409
1410 mInputMetEos = false;
1411 mSync.start();
1412 return OK;
1413}
1414
1415status_t CCodecBufferChannel::requestInitialInputBuffers() {
1416 if (mInputSurface) {
1417 return OK;
1418 }
1419
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001420 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001421 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1422 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1423 if (err != C2_OK && err != C2_BAD_INDEX) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001424 return UNKNOWN_ERROR;
1425 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001426 size_t numInputSlots = mInput.lock()->numSlots;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001427
1428 struct ClientInputBuffer {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001429 size_t index;
1430 sp<MediaCodecBuffer> buffer;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001431 size_t capacity;
1432 };
1433 std::list<ClientInputBuffer> clientInputBuffers;
1434
1435 {
1436 Mutexed<Input>::Locked input(mInput);
1437 while (clientInputBuffers.size() < numInputSlots) {
1438 ClientInputBuffer clientInputBuffer;
1439 if (!input->buffers->requestNewBuffer(&clientInputBuffer.index,
1440 &clientInputBuffer.buffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001441 break;
1442 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001443 clientInputBuffer.capacity = clientInputBuffer.buffer->capacity();
1444 clientInputBuffers.emplace_back(std::move(clientInputBuffer));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001445 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001446 }
1447 if (clientInputBuffers.empty()) {
1448 ALOGW("[%s] start: cannot allocate memory at all", mName);
1449 return NO_MEMORY;
1450 } else if (clientInputBuffers.size() < numInputSlots) {
1451 ALOGD("[%s] start: cannot allocate memory for all slots, "
1452 "only %zu buffers allocated",
1453 mName, clientInputBuffers.size());
1454 } else {
1455 ALOGV("[%s] %zu initial input buffers available",
1456 mName, clientInputBuffers.size());
1457 }
1458 // Sort input buffers by their capacities in increasing order.
1459 clientInputBuffers.sort(
1460 [](const ClientInputBuffer& a, const ClientInputBuffer& b) {
1461 return a.capacity < b.capacity;
1462 });
1463
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001464 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1465 mFlushedConfigs.lock()->swap(flushedConfigs);
1466 if (!flushedConfigs.empty()) {
1467 err = mComponent->queue(&flushedConfigs);
1468 if (err != C2_OK) {
1469 ALOGW("[%s] Error while queueing a flushed config", mName);
1470 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001471 }
1472 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001473 if (oStreamFormat.value == C2BufferData::LINEAR &&
1474 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE)) {
1475 sp<MediaCodecBuffer> buffer = clientInputBuffers.front().buffer;
1476 // WORKAROUND: Some apps expect CSD available without queueing
1477 // any input. Queue an empty buffer to get the CSD.
1478 buffer->setRange(0, 0);
1479 buffer->meta()->clear();
1480 buffer->meta()->setInt64("timeUs", 0);
1481 if (queueInputBufferInternal(buffer) != OK) {
1482 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1483 mName);
1484 return UNKNOWN_ERROR;
1485 }
1486 clientInputBuffers.pop_front();
1487 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001488
1489 for (const ClientInputBuffer& clientInputBuffer: clientInputBuffers) {
1490 mCallback->onInputBufferAvailable(
1491 clientInputBuffer.index,
1492 clientInputBuffer.buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001493 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001494
Pawin Vongmasa36653902018-11-15 00:10:25 -08001495 return OK;
1496}
1497
1498void CCodecBufferChannel::stop() {
1499 mSync.stop();
1500 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001501}
1502
Wonsik Kim936a89c2020-05-08 16:07:50 -07001503void CCodecBufferChannel::reset() {
1504 stop();
Wonsik Kim62545252021-01-20 11:25:41 -08001505 if (mInputSurface != nullptr) {
1506 mInputSurface.reset();
1507 }
1508 mPipelineWatcher.lock()->flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001509 {
1510 Mutexed<Input>::Locked input(mInput);
1511 input->buffers.reset(new DummyInputBuffers(""));
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001512 input->extraBuffers.flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001513 }
1514 {
1515 Mutexed<Output>::Locked output(mOutput);
1516 output->buffers.reset();
1517 }
1518}
1519
1520void CCodecBufferChannel::release() {
1521 mComponent.reset();
1522 mInputAllocator.reset();
1523 mOutputSurface.lock()->surface.clear();
1524 {
1525 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1526 blockPools->inputPool.reset();
1527 blockPools->outputPoolIntf.reset();
1528 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001529 setCrypto(nullptr);
1530 setDescrambler(nullptr);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001531}
1532
1533
Pawin Vongmasa36653902018-11-15 00:10:25 -08001534void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1535 ALOGV("[%s] flush", mName);
Wonsik Kim62545252021-01-20 11:25:41 -08001536 std::vector<uint64_t> indices;
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001537 std::list<std::unique_ptr<C2Work>> configs;
1538 for (const std::unique_ptr<C2Work> &work : flushedWork) {
Wonsik Kim62545252021-01-20 11:25:41 -08001539 indices.push_back(work->input.ordinal.frameIndex.peeku());
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001540 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1541 continue;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001542 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001543 if (work->input.buffers.empty()
1544 || work->input.buffers.front() == nullptr
1545 || work->input.buffers.front()->data().linearBlocks().empty()) {
1546 ALOGD("[%s] no linear codec config data found", mName);
1547 continue;
1548 }
1549 std::unique_ptr<C2Work> copy(new C2Work);
1550 copy->input.flags = C2FrameData::flags_t(work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1551 copy->input.ordinal = work->input.ordinal;
Wonsik Kim62545252021-01-20 11:25:41 -08001552 copy->input.ordinal.frameIndex = mFrameIndex++;
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001553 copy->input.buffers.insert(
1554 copy->input.buffers.begin(),
1555 work->input.buffers.begin(),
1556 work->input.buffers.end());
1557 for (const std::unique_ptr<C2Param> &param : work->input.configUpdate) {
1558 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1559 }
1560 copy->input.infoBuffers.insert(
1561 copy->input.infoBuffers.begin(),
1562 work->input.infoBuffers.begin(),
1563 work->input.infoBuffers.end());
1564 copy->worklets.emplace_back(new C2Worklet);
1565 configs.push_back(std::move(copy));
1566 ALOGV("[%s] stashed flushed codec config data", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001567 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001568 mFlushedConfigs.lock()->swap(configs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001569 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001570 Mutexed<Input>::Locked input(mInput);
1571 input->buffers->flush();
1572 input->extraBuffers.flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001573 }
1574 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001575 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001576 if (output->buffers) {
1577 output->buffers->flush(flushedWork);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001578 output->buffers->flushStash();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001579 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001580 }
Wonsik Kim62545252021-01-20 11:25:41 -08001581 {
1582 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
1583 for (uint64_t index : indices) {
1584 watcher->onWorkDone(index);
1585 }
1586 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001587}
1588
1589void CCodecBufferChannel::onWorkDone(
1590 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
Wonsik Kimab34ed62019-01-31 15:28:46 -08001591 const C2StreamInitDataInfo::output *initData) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001592 if (handleWork(std::move(work), outputFormat, initData)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001593 feedInputBufferIfAvailable();
1594 }
1595}
1596
1597void CCodecBufferChannel::onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -08001598 uint64_t frameIndex, size_t arrayIndex) {
Pawin Vongmasa8e2cfb52019-05-15 05:20:52 -07001599 if (mInputSurface) {
1600 return;
1601 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001602 std::shared_ptr<C2Buffer> buffer =
1603 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001604 bool newInputSlotAvailable;
1605 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001606 Mutexed<Input>::Locked input(mInput);
1607 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1608 if (!newInputSlotAvailable) {
1609 (void)input->extraBuffers.expireComponentBuffer(buffer);
1610 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001611 }
1612 if (newInputSlotAvailable) {
1613 feedInputBufferIfAvailable();
1614 }
1615}
1616
1617bool CCodecBufferChannel::handleWork(
1618 std::unique_ptr<C2Work> work,
1619 const sp<AMessage> &outputFormat,
1620 const C2StreamInitDataInfo::output *initData) {
Wonsik Kim936a89c2020-05-08 16:07:50 -07001621 {
Wonsik Kima4e049d2020-04-28 19:42:23 +00001622 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001623 if (!output->buffers) {
1624 return false;
1625 }
Wonsik Kime75a5da2020-02-14 17:29:03 -08001626 }
1627
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001628 // Whether the output buffer should be reported to the client or not.
1629 bool notifyClient = false;
1630
1631 if (work->result == C2_OK){
1632 notifyClient = true;
1633 } else if (work->result == C2_NOT_FOUND) {
1634 ALOGD("[%s] flushed work; ignored.", mName);
1635 } else {
1636 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1637 // the config update.
1638 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1639 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1640 return false;
1641 }
1642
1643 if ((work->input.ordinal.frameIndex -
1644 mFirstValidFrameIndex.load()).peek() < 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001645 // Discard frames from previous generation.
1646 ALOGD("[%s] Discard frames from previous generation.", mName);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001647 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001648 }
1649
Wonsik Kim524b0582019-03-12 11:28:57 -07001650 if (mInputSurface == nullptr && (work->worklets.size() != 1u
Pawin Vongmasa36653902018-11-15 00:10:25 -08001651 || !work->worklets.front()
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001652 || !(work->worklets.front()->output.flags &
1653 C2FrameData::FLAG_INCOMPLETE))) {
1654 mPipelineWatcher.lock()->onWorkDone(
1655 work->input.ordinal.frameIndex.peeku());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001656 }
1657
1658 // NOTE: MediaCodec usage supposedly have only one worklet
1659 if (work->worklets.size() != 1u) {
1660 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1661 mName, work->worklets.size());
1662 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1663 return false;
1664 }
1665
1666 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1667
1668 std::shared_ptr<C2Buffer> buffer;
1669 // NOTE: MediaCodec usage supposedly have only one output stream.
1670 if (worklet->output.buffers.size() > 1u) {
1671 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1672 mName, worklet->output.buffers.size());
1673 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1674 return false;
1675 } else if (worklet->output.buffers.size() == 1u) {
1676 buffer = worklet->output.buffers[0];
1677 if (!buffer) {
1678 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1679 }
1680 }
1681
Wonsik Kim3dedf682021-05-03 10:57:09 -07001682 std::optional<uint32_t> newInputDelay, newPipelineDelay, newOutputDelay, newReorderDepth;
1683 std::optional<C2Config::ordinal_key_t> newReorderKey;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001684 bool needMaxDequeueBufferCountUpdate = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001685 while (!worklet->output.configUpdate.empty()) {
1686 std::unique_ptr<C2Param> param;
1687 worklet->output.configUpdate.back().swap(param);
1688 worklet->output.configUpdate.pop_back();
1689 switch (param->coreIndex().coreIndex()) {
1690 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1691 C2PortReorderBufferDepthTuning::output reorderDepth;
1692 if (reorderDepth.updateFrom(*param)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001693 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1694 mName, reorderDepth.value);
Wonsik Kim3dedf682021-05-03 10:57:09 -07001695 newReorderDepth = reorderDepth.value;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001696 needMaxDequeueBufferCountUpdate = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001697 } else {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001698 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1699 mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001700 }
1701 break;
1702 }
1703 case C2PortReorderKeySetting::CORE_INDEX: {
1704 C2PortReorderKeySetting::output reorderKey;
1705 if (reorderKey.updateFrom(*param)) {
Wonsik Kim3dedf682021-05-03 10:57:09 -07001706 newReorderKey = reorderKey.value;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001707 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1708 mName, reorderKey.value);
1709 } else {
1710 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1711 }
1712 break;
1713 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001714 case C2PortActualDelayTuning::CORE_INDEX: {
1715 if (param->isGlobal()) {
1716 C2ActualPipelineDelayTuning pipelineDelay;
1717 if (pipelineDelay.updateFrom(*param)) {
1718 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1719 mName, pipelineDelay.value);
1720 newPipelineDelay = pipelineDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001721 (void)mPipelineWatcher.lock()->pipelineDelay(
1722 pipelineDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001723 }
1724 }
1725 if (param->forInput()) {
1726 C2PortActualDelayTuning::input inputDelay;
1727 if (inputDelay.updateFrom(*param)) {
1728 ALOGV("[%s] onWorkDone: updating input delay %u",
1729 mName, inputDelay.value);
1730 newInputDelay = inputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001731 (void)mPipelineWatcher.lock()->inputDelay(
1732 inputDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001733 }
1734 }
1735 if (param->forOutput()) {
1736 C2PortActualDelayTuning::output outputDelay;
1737 if (outputDelay.updateFrom(*param)) {
1738 ALOGV("[%s] onWorkDone: updating output delay %u",
1739 mName, outputDelay.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001740 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
Wonsik Kim3dedf682021-05-03 10:57:09 -07001741 newOutputDelay = outputDelay.value;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001742 needMaxDequeueBufferCountUpdate = true;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001743
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001744 }
1745 }
1746 break;
1747 }
ted.sunb1fbfdb2020-06-23 14:03:41 +08001748 case C2PortTunnelSystemTime::CORE_INDEX: {
1749 C2PortTunnelSystemTime::output frameRenderTime;
1750 if (frameRenderTime.updateFrom(*param)) {
1751 ALOGV("[%s] onWorkDone: frame rendered (sys:%lld ns, media:%lld us)",
1752 mName, (long long)frameRenderTime.value,
1753 (long long)worklet->output.ordinal.timestamp.peekll());
1754 mCCodecCallback->onOutputFramesRendered(
1755 worklet->output.ordinal.timestamp.peek(), frameRenderTime.value);
1756 }
1757 break;
1758 }
Guillaume Chelfi867d4dd2021-07-01 18:38:45 +02001759 case C2StreamTunnelHoldRender::CORE_INDEX: {
1760 C2StreamTunnelHoldRender::output firstTunnelFrameHoldRender;
1761 if (!(worklet->output.flags & C2FrameData::FLAG_INCOMPLETE)) break;
1762 if (!firstTunnelFrameHoldRender.updateFrom(*param)) break;
1763 if (firstTunnelFrameHoldRender.value != C2_TRUE) break;
1764 ALOGV("[%s] onWorkDone: first tunnel frame ready", mName);
1765 mCCodecCallback->onFirstTunnelFrameReady();
1766 break;
1767 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001768 default:
1769 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1770 mName, param->index());
1771 break;
1772 }
1773 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001774 if (newInputDelay || newPipelineDelay) {
1775 Mutexed<Input>::Locked input(mInput);
1776 size_t newNumSlots =
1777 newInputDelay.value_or(input->inputDelay) +
1778 newPipelineDelay.value_or(input->pipelineDelay) +
1779 kSmoothnessFactor;
1780 if (input->buffers->isArrayMode()) {
1781 if (input->numSlots >= newNumSlots) {
1782 input->numExtraSlots = 0;
1783 } else {
1784 input->numExtraSlots = newNumSlots - input->numSlots;
1785 }
1786 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1787 mName, input->numExtraSlots);
1788 } else {
1789 input->numSlots = newNumSlots;
1790 }
1791 }
Wonsik Kim3dedf682021-05-03 10:57:09 -07001792 size_t numOutputSlots = 0;
1793 uint32_t reorderDepth = 0;
1794 bool outputBuffersChanged = false;
1795 if (newReorderKey || newReorderDepth || needMaxDequeueBufferCountUpdate) {
1796 Mutexed<Output>::Locked output(mOutput);
1797 if (!output->buffers) {
1798 return false;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001799 }
Wonsik Kim3dedf682021-05-03 10:57:09 -07001800 numOutputSlots = output->numSlots;
1801 if (newReorderKey) {
1802 output->buffers->setReorderKey(newReorderKey.value());
1803 }
1804 if (newReorderDepth) {
1805 output->buffers->setReorderDepth(newReorderDepth.value());
1806 }
1807 reorderDepth = output->buffers->getReorderDepth();
1808 if (newOutputDelay) {
1809 output->outputDelay = newOutputDelay.value();
1810 numOutputSlots = newOutputDelay.value() + kSmoothnessFactor;
1811 if (output->numSlots < numOutputSlots) {
1812 output->numSlots = numOutputSlots;
1813 if (output->buffers->isArrayMode()) {
1814 OutputBuffersArray *array =
1815 (OutputBuffersArray *)output->buffers.get();
1816 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1817 mName, numOutputSlots);
1818 array->grow(numOutputSlots);
1819 outputBuffersChanged = true;
1820 }
1821 }
1822 }
1823 numOutputSlots = output->numSlots;
1824 }
1825 if (outputBuffersChanged) {
1826 mCCodecCallback->onOutputBuffersChanged();
1827 }
1828 if (needMaxDequeueBufferCountUpdate) {
Wonsik Kim84f439f2021-05-03 10:57:09 -07001829 int maxDequeueCount = 0;
Sungtak Leea714f112021-03-16 05:40:03 -07001830 {
1831 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1832 maxDequeueCount = output->maxDequeueBuffers =
1833 numOutputSlots + reorderDepth + kRenderingDepth;
1834 if (output->surface) {
1835 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1836 }
1837 }
1838 if (maxDequeueCount > 0) {
1839 mComponent->setOutputSurfaceMaxDequeueCount(maxDequeueCount);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001840 }
1841 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001842
Pawin Vongmasa36653902018-11-15 00:10:25 -08001843 int32_t flags = 0;
1844 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1845 flags |= MediaCodec::BUFFER_FLAG_EOS;
1846 ALOGV("[%s] onWorkDone: output EOS", mName);
1847 }
1848
Pawin Vongmasa36653902018-11-15 00:10:25 -08001849 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1850 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1851 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1852 // shall correspond to the client input timesamp (in customOrdinal). By using the
1853 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1854 // produces multiple output.
1855 c2_cntr64_t timestamp =
1856 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1857 - work->input.ordinal.timestamp;
Wonsik Kim95ba0162019-03-19 15:51:54 -07001858 if (mInputSurface != nullptr) {
1859 // When using input surface we need to restore the original input timestamp.
1860 timestamp = work->input.ordinal.customOrdinal;
1861 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001862 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1863 mName,
1864 work->input.ordinal.customOrdinal.peekll(),
1865 work->input.ordinal.timestamp.peekll(),
1866 worklet->output.ordinal.timestamp.peekll(),
1867 timestamp.peekll());
1868
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001869 // csd cannot be re-ordered and will always arrive first.
Pawin Vongmasa36653902018-11-15 00:10:25 -08001870 if (initData != nullptr) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001871 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001872 if (output->buffers && outputFormat) {
1873 output->buffers->updateSkipCutBuffer(outputFormat);
1874 output->buffers->setFormat(outputFormat);
1875 }
1876 if (!notifyClient) {
1877 return false;
1878 }
1879 size_t index;
1880 sp<MediaCodecBuffer> outBuffer;
Wonsik Kim936a89c2020-05-08 16:07:50 -07001881 if (output->buffers && output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001882 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1883 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1884 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1885
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001886 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001887 mCallback->onOutputBufferAvailable(index, outBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001888 } else {
1889 ALOGD("[%s] onWorkDone: unable to register csd", mName);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001890 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001891 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001892 return false;
1893 }
1894 }
1895
Wonsik Kimec585c32021-10-01 01:11:00 -07001896 bool drop = false;
1897 if (worklet->output.flags & C2FrameData::FLAG_DROP_FRAME) {
1898 ALOGV("[%s] onWorkDone: drop buffer but keep metadata", mName);
1899 drop = true;
1900 }
1901
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001902 if (notifyClient && !buffer && !flags) {
Wonsik Kimec585c32021-10-01 01:11:00 -07001903 if (mTunneled && drop && outputFormat) {
1904 ALOGV("[%s] onWorkDone: Keep tunneled, drop frame with format change (%lld)",
1905 mName, work->input.ordinal.frameIndex.peekull());
1906 } else {
1907 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
1908 mName, work->input.ordinal.frameIndex.peekull());
1909 notifyClient = false;
1910 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001911 }
1912
1913 if (buffer) {
1914 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1915 // TODO: properly translate these to metadata
1916 switch (info->coreIndex().coreIndex()) {
1917 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001918 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001919 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1920 }
1921 break;
1922 default:
1923 break;
1924 }
1925 }
1926 }
1927
1928 {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001929 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimc23cc402020-05-28 14:53:40 -07001930 if (!output->buffers) {
1931 return false;
1932 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001933 output->buffers->pushToStash(
1934 buffer,
1935 notifyClient,
1936 timestamp.peek(),
1937 flags,
1938 outputFormat,
1939 worklet->output.ordinal);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001940 }
1941 sendOutputBuffers();
1942 return true;
1943}
1944
1945void CCodecBufferChannel::sendOutputBuffers() {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001946 OutputBuffers::BufferAction action;
Wonsik Kima4e049d2020-04-28 19:42:23 +00001947 size_t index;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001948 sp<MediaCodecBuffer> outBuffer;
1949 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001950
1951 while (true) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001952 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001953 if (!output->buffers) {
1954 return;
1955 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001956 action = output->buffers->popFromStashAndRegister(
1957 &c2Buffer, &index, &outBuffer);
1958 switch (action) {
1959 case OutputBuffers::SKIP:
1960 return;
1961 case OutputBuffers::DISCARD:
1962 break;
1963 case OutputBuffers::NOTIFY_CLIENT:
Wonsik Kima4e049d2020-04-28 19:42:23 +00001964 output.unlock();
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001965 mCallback->onOutputBufferAvailable(index, outBuffer);
1966 break;
1967 case OutputBuffers::REALLOCATE:
1968 if (!output->buffers->isArrayMode()) {
1969 output->buffers =
1970 output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001971 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001972 static_cast<OutputBuffersArray*>(output->buffers.get())->
1973 realloc(c2Buffer);
1974 output.unlock();
1975 mCCodecCallback->onOutputBuffersChanged();
Wonsik Kim4ada73d2020-05-26 14:58:07 -07001976 break;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001977 case OutputBuffers::RETRY:
1978 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
1979 mName);
1980 return;
1981 default:
1982 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
1983 "corrupted BufferAction value (%d) "
1984 "returned from popFromStashAndRegister.",
1985 mName, int(action));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001986 return;
1987 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001988 }
1989}
1990
1991status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1992 static std::atomic_uint32_t surfaceGeneration{0};
1993 uint32_t generation = (getpid() << 10) |
1994 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1995 & ((1 << 10) - 1));
1996
1997 sp<IGraphicBufferProducer> producer;
Sungtak Leedb14cba2021-04-10 00:50:23 -07001998 int maxDequeueCount = mOutputSurface.lock()->maxDequeueBuffers;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001999 if (newSurface) {
2000 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Sungtak Leeab6f2f32019-02-15 14:43:51 -08002001 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
Sungtak Leedb14cba2021-04-10 00:50:23 -07002002 newSurface->setMaxDequeuedBufferCount(maxDequeueCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002003 producer = newSurface->getIGraphicBufferProducer();
2004 producer->setGenerationNumber(generation);
2005 } else {
2006 ALOGE("[%s] setting output surface to null", mName);
2007 return INVALID_OPERATION;
2008 }
2009
2010 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
2011 C2BlockPool::local_id_t outputPoolId;
2012 {
2013 Mutexed<BlockPools>::Locked pools(mBlockPools);
2014 outputPoolId = pools->outputPoolId;
2015 outputPoolIntf = pools->outputPoolIntf;
2016 }
2017
2018 if (outputPoolIntf) {
2019 if (mComponent->setOutputSurface(
2020 outputPoolId,
2021 producer,
Sungtak Leedb14cba2021-04-10 00:50:23 -07002022 generation,
2023 maxDequeueCount) != C2_OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002024 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
2025 return INVALID_OPERATION;
2026 }
2027 }
2028
2029 {
2030 Mutexed<OutputSurface>::Locked output(mOutputSurface);
2031 output->surface = newSurface;
2032 output->generation = generation;
2033 }
2034
2035 return OK;
2036}
2037
Wonsik Kimab34ed62019-01-31 15:28:46 -08002038PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08002039 // When client pushed EOS, we want all the work to be done quickly.
2040 // Otherwise, component may have stalled work due to input starvation up to
2041 // the sum of the delay in the pipeline.
Wonsik Kimf0e7d222019-06-28 12:33:16 -07002042 size_t n = 0;
2043 if (!mInputMetEos) {
2044 size_t outputDelay = mOutput.lock()->outputDelay;
2045 Mutexed<Input>::Locked input(mInput);
2046 n = input->inputDelay + input->pipelineDelay + outputDelay;
2047 }
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08002048 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
Wonsik Kimab34ed62019-01-31 15:28:46 -08002049}
2050
Pawin Vongmasa36653902018-11-15 00:10:25 -08002051void CCodecBufferChannel::setMetaMode(MetaMode mode) {
2052 mMetaMode = mode;
2053}
2054
Wonsik Kim596187e2019-10-25 12:44:10 -07002055void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002056 if (mCrypto != nullptr) {
2057 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
2058 mCrypto->unsetHeap(entry.second);
2059 }
2060 mHeapSeqNumMap.clear();
2061 if (mHeapSeqNum >= 0) {
2062 mCrypto->unsetHeap(mHeapSeqNum);
2063 mHeapSeqNum = -1;
2064 }
2065 }
Wonsik Kim596187e2019-10-25 12:44:10 -07002066 mCrypto = crypto;
2067}
2068
2069void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
2070 mDescrambler = descrambler;
2071}
2072
Pawin Vongmasa36653902018-11-15 00:10:25 -08002073status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
2074 // C2_OK is always translated to OK.
2075 if (c2s == C2_OK) {
2076 return OK;
2077 }
2078
2079 // Operation-dependent translation
2080 // TODO: Add as necessary
2081 switch (c2op) {
2082 case C2_OPERATION_Component_start:
2083 switch (c2s) {
2084 case C2_NO_MEMORY:
2085 return NO_MEMORY;
2086 default:
2087 return UNKNOWN_ERROR;
2088 }
2089 default:
2090 break;
2091 }
2092
2093 // Backup operation-agnostic translation
2094 switch (c2s) {
2095 case C2_BAD_INDEX:
2096 return BAD_INDEX;
2097 case C2_BAD_VALUE:
2098 return BAD_VALUE;
2099 case C2_BLOCKING:
2100 return WOULD_BLOCK;
2101 case C2_DUPLICATE:
2102 return ALREADY_EXISTS;
2103 case C2_NO_INIT:
2104 return NO_INIT;
2105 case C2_NO_MEMORY:
2106 return NO_MEMORY;
2107 case C2_NOT_FOUND:
2108 return NAME_NOT_FOUND;
2109 case C2_TIMED_OUT:
2110 return TIMED_OUT;
2111 case C2_BAD_STATE:
2112 case C2_CANCELED:
2113 case C2_CANNOT_DO:
2114 case C2_CORRUPTED:
2115 case C2_OMITTED:
2116 case C2_REFUSED:
2117 return UNKNOWN_ERROR;
2118 default:
2119 return -static_cast<status_t>(c2s);
2120 }
2121}
2122
2123} // namespace android