blob: 4d2700a7838a07c7519324b24534669c6533f1e3 [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
Pawin Vongmasae7bb8612020-06-04 06:15:22 -070021#include <algorithm>
22#include <list>
Pawin Vongmasa36653902018-11-15 00:10:25 -080023#include <numeric>
24
25#include <C2AllocatorGralloc.h>
26#include <C2PlatformSupport.h>
27#include <C2BlockInternal.h>
28#include <C2Config.h>
29#include <C2Debug.h>
30
31#include <android/hardware/cas/native/1.0/IDescrambler.h>
Robert Shih895fba92019-07-16 16:29:44 -070032#include <android/hardware/drm/1.0/types.h>
Josh Hou8eddf4b2021-02-02 16:26:53 +080033#include <android-base/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080034#include <android-base/stringprintf.h>
Wonsik Kimfb7a7672019-12-27 17:13:33 -080035#include <binder/MemoryBase.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080036#include <binder/MemoryDealer.h>
Ray Essick18ea0452019-08-27 16:07:27 -070037#include <cutils/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080038#include <gui/Surface.h>
Robert Shih895fba92019-07-16 16:29:44 -070039#include <hidlmemory/FrameworkUtils.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080040#include <media/openmax/OMX_Core.h>
41#include <media/stagefright/foundation/ABuffer.h>
42#include <media/stagefright/foundation/ALookup.h>
43#include <media/stagefright/foundation/AMessage.h>
44#include <media/stagefright/foundation/AUtils.h>
45#include <media/stagefright/foundation/hexdump.h>
46#include <media/stagefright/MediaCodec.h>
47#include <media/stagefright/MediaCodecConstants.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070048#include <media/stagefright/SkipCutBuffer.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080049#include <media/MediaCodecBuffer.h>
Wonsik Kim41d83432020-04-27 16:40:49 -070050#include <mediadrm/ICrypto.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080051#include <system/window.h>
52
53#include "CCodecBufferChannel.h"
54#include "Codec2Buffer.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080055
56namespace android {
57
58using android::base::StringPrintf;
59using hardware::hidl_handle;
60using hardware::hidl_string;
61using hardware::hidl_vec;
Robert Shih895fba92019-07-16 16:29:44 -070062using hardware::fromHeap;
63using hardware::HidlMemory;
64
Pawin Vongmasa36653902018-11-15 00:10:25 -080065using namespace hardware::cas::V1_0;
66using namespace hardware::cas::native::V1_0;
67
68using CasStatus = hardware::cas::V1_0::Status;
Robert Shih895fba92019-07-16 16:29:44 -070069using DrmBufferType = hardware::drm::V1_0::BufferType;
Pawin Vongmasa36653902018-11-15 00:10:25 -080070
Pawin Vongmasa36653902018-11-15 00:10:25 -080071namespace {
72
Wonsik Kim469c8342019-04-11 16:46:09 -070073constexpr size_t kSmoothnessFactor = 4;
74constexpr size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -080075
Sungtak Leeab6f2f32019-02-15 14:43:51 -080076// This is for keeping IGBP's buffer dropping logic in legacy mode other
77// than making it non-blocking. Do not change this value.
78const static size_t kDequeueTimeoutNs = 0;
79
Pawin Vongmasa36653902018-11-15 00:10:25 -080080} // namespace
81
82CCodecBufferChannel::QueueGuard::QueueGuard(
83 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
84 Mutex::Autolock l(mSync.mGuardLock);
85 // At this point it's guaranteed that mSync is not under state transition,
86 // as we are holding its mutex.
87
88 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
89 if (count->value == -1) {
90 mRunning = false;
91 } else {
92 ++count->value;
93 mRunning = true;
94 }
95}
96
97CCodecBufferChannel::QueueGuard::~QueueGuard() {
98 if (mRunning) {
99 // We are not holding mGuardLock at this point so that QueueSync::stop() can
100 // keep holding the lock until mCount reaches zero.
101 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
102 --count->value;
103 count->cond.broadcast();
104 }
105}
106
107void CCodecBufferChannel::QueueSync::start() {
108 Mutex::Autolock l(mGuardLock);
109 // If stopped, it goes to running state; otherwise no-op.
110 Mutexed<Counter>::Locked count(mCount);
111 if (count->value == -1) {
112 count->value = 0;
113 }
114}
115
116void CCodecBufferChannel::QueueSync::stop() {
117 Mutex::Autolock l(mGuardLock);
118 Mutexed<Counter>::Locked count(mCount);
119 if (count->value == -1) {
120 // no-op
121 return;
122 }
123 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
124 // mCount can only decrement. In other words, threads that acquired the lock
125 // are allowed to finish execution but additional threads trying to acquire
126 // the lock at this point will block, and then get QueueGuard at STOPPED
127 // state.
128 while (count->value != 0) {
129 count.waitForCondition(count->cond);
130 }
131 count->value = -1;
132}
133
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700134// Input
135
136CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
137
Pawin Vongmasa36653902018-11-15 00:10:25 -0800138// CCodecBufferChannel
139
140CCodecBufferChannel::CCodecBufferChannel(
141 const std::shared_ptr<CCodecCallback> &callback)
142 : mHeapSeqNum(-1),
143 mCCodecCallback(callback),
144 mFrameIndex(0u),
145 mFirstValidFrameIndex(0u),
146 mMetaMode(MODE_NONE),
Sungtak Lee04b30352020-07-27 13:57:25 -0700147 mInputMetEos(false),
148 mSendEncryptedInfoBuffer(false) {
Sungtak Leed7463d12019-09-04 16:01:00 -0700149 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700150 {
151 Mutexed<Input>::Locked input(mInput);
152 input->buffers.reset(new DummyInputBuffers(""));
153 input->extraBuffers.flush();
154 input->inputDelay = 0u;
155 input->pipelineDelay = 0u;
156 input->numSlots = kSmoothnessFactor;
157 input->numExtraSlots = 0u;
158 }
159 {
160 Mutexed<Output>::Locked output(mOutput);
161 output->outputDelay = 0u;
162 output->numSlots = kSmoothnessFactor;
163 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800164}
165
166CCodecBufferChannel::~CCodecBufferChannel() {
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800167 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800168 mCrypto->unsetHeap(mHeapSeqNum);
169 }
170}
171
172void CCodecBufferChannel::setComponent(
173 const std::shared_ptr<Codec2Client::Component> &component) {
174 mComponent = component;
175 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
176 mName = mComponentName.c_str();
177}
178
179status_t CCodecBufferChannel::setInputSurface(
180 const std::shared_ptr<InputSurfaceWrapper> &surface) {
181 ALOGV("[%s] setInputSurface", mName);
182 mInputSurface = surface;
183 return mInputSurface->connect(mComponent);
184}
185
186status_t CCodecBufferChannel::signalEndOfInputStream() {
187 if (mInputSurface == nullptr) {
188 return INVALID_OPERATION;
189 }
190 return mInputSurface->signalEndOfInputStream();
191}
192
Sungtak Lee04b30352020-07-27 13:57:25 -0700193status_t CCodecBufferChannel::queueInputBufferInternal(
194 sp<MediaCodecBuffer> buffer,
195 std::shared_ptr<C2LinearBlock> encryptedBlock,
196 size_t blockSize) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800197 int64_t timeUs;
198 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
199
200 if (mInputMetEos) {
201 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
202 return OK;
203 }
204
205 int32_t flags = 0;
206 int32_t tmp = 0;
207 bool eos = false;
208 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
209 eos = true;
210 mInputMetEos = true;
211 ALOGV("[%s] input EOS", mName);
212 }
213 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
214 flags |= C2FrameData::FLAG_CODEC_CONFIG;
215 }
216 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
Wonsik Kime1104ca2020-11-24 15:01:33 -0800217 std::list<std::unique_ptr<C2Work>> items;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800218 std::unique_ptr<C2Work> work(new C2Work);
219 work->input.ordinal.timestamp = timeUs;
220 work->input.ordinal.frameIndex = mFrameIndex++;
221 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
222 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
223 // Keep client timestamp in customOrdinal
224 work->input.ordinal.customOrdinal = timeUs;
225 work->input.buffers.clear();
226
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700227 sp<Codec2Buffer> copy;
Wonsik Kime1104ca2020-11-24 15:01:33 -0800228 bool usesFrameReassembler = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800229
Pawin Vongmasa36653902018-11-15 00:10:25 -0800230 if (buffer->size() > 0u) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700231 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800232 std::shared_ptr<C2Buffer> c2buffer;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700233 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800234 return -ENOENT;
235 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700236 // TODO: we want to delay copying buffers.
237 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
238 copy = input->buffers->cloneAndReleaseBuffer(buffer);
239 if (copy != nullptr) {
240 (void)input->extraBuffers.assignSlot(copy);
241 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
242 return UNKNOWN_ERROR;
243 }
244 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
245 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
246 mName, released ? "" : "not ");
247 buffer.clear();
248 } else {
249 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
250 "buffer starvation on component.", mName);
251 }
252 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800253 if (input->frameReassembler) {
254 usesFrameReassembler = true;
255 input->frameReassembler.process(buffer, &items);
256 } else {
Byeongjo Park25c3a3d2020-06-12 17:24:21 +0900257 int32_t cvo = 0;
258 if (buffer->meta()->findInt32("cvo", &cvo)) {
259 int32_t rotation = cvo % 360;
260 // change rotation to counter-clock wise.
261 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
262
263 Mutexed<OutputSurface>::Locked output(mOutputSurface);
264 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
265 output->rotation[frameIndex] = rotation;
266 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800267 work->input.buffers.push_back(c2buffer);
268 if (encryptedBlock) {
269 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
270 kParamIndexEncryptedBuffer,
271 encryptedBlock->share(0, blockSize, C2Fence())));
272 }
Sungtak Lee04b30352020-07-27 13:57:25 -0700273 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800274 } else if (eos) {
275 flags |= C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800276 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800277 if (usesFrameReassembler) {
278 if (!items.empty()) {
279 items.front()->input.configUpdate = std::move(mParamsToBeSet);
280 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
281 }
282 } else {
283 work->input.flags = (C2FrameData::flags_t)flags;
284 // TODO: fill info's
Pawin Vongmasa36653902018-11-15 00:10:25 -0800285
Wonsik Kime1104ca2020-11-24 15:01:33 -0800286 work->input.configUpdate = std::move(mParamsToBeSet);
287 work->worklets.clear();
288 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800289
Wonsik Kime1104ca2020-11-24 15:01:33 -0800290 items.push_back(std::move(work));
291
292 eos = eos && buffer->size() > 0u;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800293 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800294 if (eos) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800295 work.reset(new C2Work);
296 work->input.ordinal.timestamp = timeUs;
297 work->input.ordinal.frameIndex = mFrameIndex++;
298 // WORKAROUND: keep client timestamp in customOrdinal
299 work->input.ordinal.customOrdinal = timeUs;
300 work->input.buffers.clear();
301 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800302 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800303 items.push_back(std::move(work));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800304 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800305 c2_status_t err = C2_OK;
306 if (!items.empty()) {
307 {
308 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
309 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
310 for (const std::unique_ptr<C2Work> &work : items) {
311 watcher->onWorkQueued(
312 work->input.ordinal.frameIndex.peeku(),
313 std::vector(work->input.buffers),
314 now);
315 }
316 }
317 err = mComponent->queue(&items);
318 }
319 if (err != C2_OK) {
320 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
321 for (const std::unique_ptr<C2Work> &work : items) {
322 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
323 }
324 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700325 Mutexed<Input>::Locked input(mInput);
326 bool released = false;
327 if (buffer) {
328 released = input->buffers->releaseBuffer(buffer, nullptr, true);
329 } else if (copy) {
330 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
331 }
332 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
333 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800334 }
335
336 feedInputBufferIfAvailableInternal();
337 return err;
338}
339
340status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
341 QueueGuard guard(mSync);
342 if (!guard.isRunning()) {
343 ALOGD("[%s] setParameters is only supported in the running state.", mName);
344 return -ENOSYS;
345 }
346 mParamsToBeSet.insert(mParamsToBeSet.end(),
347 std::make_move_iterator(params.begin()),
348 std::make_move_iterator(params.end()));
349 params.clear();
350 return OK;
351}
352
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800353status_t CCodecBufferChannel::attachBuffer(
354 const std::shared_ptr<C2Buffer> &c2Buffer,
355 const sp<MediaCodecBuffer> &buffer) {
356 if (!buffer->copy(c2Buffer)) {
357 return -ENOSYS;
358 }
359 return OK;
360}
361
362void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
363 if (!mDecryptDestination || mDecryptDestination->size() < size) {
364 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
365 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
366 mCrypto->unsetHeap(mHeapSeqNum);
367 }
368 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
369 if (mCrypto) {
370 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
371 }
372 }
373}
374
375int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
376 CHECK(mCrypto);
377 auto it = mHeapSeqNumMap.find(memory);
378 int32_t heapSeqNum = -1;
379 if (it == mHeapSeqNumMap.end()) {
380 heapSeqNum = mCrypto->setHeap(memory);
381 mHeapSeqNumMap.emplace(memory, heapSeqNum);
382 } else {
383 heapSeqNum = it->second;
384 }
385 return heapSeqNum;
386}
387
388status_t CCodecBufferChannel::attachEncryptedBuffer(
389 const sp<hardware::HidlMemory> &memory,
390 bool secure,
391 const uint8_t *key,
392 const uint8_t *iv,
393 CryptoPlugin::Mode mode,
394 CryptoPlugin::Pattern pattern,
395 size_t offset,
396 const CryptoPlugin::SubSample *subSamples,
397 size_t numSubSamples,
398 const sp<MediaCodecBuffer> &buffer) {
399 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
400 static const C2MemoryUsage kDefaultReadWriteUsage{
401 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
402
403 size_t size = 0;
404 for (size_t i = 0; i < numSubSamples; ++i) {
405 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
406 }
407 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
408 std::shared_ptr<C2LinearBlock> block;
409 c2_status_t err = pool->fetchLinearBlock(
410 size,
411 secure ? kSecureUsage : kDefaultReadWriteUsage,
412 &block);
413 if (err != C2_OK) {
414 return NO_MEMORY;
415 }
416 if (!secure) {
417 ensureDecryptDestination(size);
418 }
419 ssize_t result = -1;
420 ssize_t codecDataOffset = 0;
421 if (mCrypto) {
422 AString errorDetailMsg;
423 int32_t heapSeqNum = getHeapSeqNum(memory);
424 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
425 hardware::drm::V1_0::DestinationBuffer dst;
426 if (secure) {
427 dst.type = DrmBufferType::NATIVE_HANDLE;
428 dst.secureMemory = hardware::hidl_handle(block->handle());
429 } else {
430 dst.type = DrmBufferType::SHARED_MEMORY;
431 IMemoryToSharedBuffer(
432 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
433 }
434 result = mCrypto->decrypt(
435 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
436 dst, &errorDetailMsg);
437 if (result < 0) {
438 return result;
439 }
440 if (dst.type == DrmBufferType::SHARED_MEMORY) {
441 C2WriteView view = block->map().get();
442 if (view.error() != C2_OK) {
443 return false;
444 }
445 if (view.size() < result) {
446 return false;
447 }
448 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
449 }
450 } else {
451 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
452 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
453 hidl_vec<SubSample> hidlSubSamples;
454 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
455
456 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
457 hardware::cas::native::V1_0::DestinationBuffer dst;
458 if (secure) {
459 dst.type = BufferType::NATIVE_HANDLE;
460 dst.secureMemory = hardware::hidl_handle(block->handle());
461 } else {
462 dst.type = BufferType::SHARED_MEMORY;
463 dst.nonsecureMemory = src;
464 }
465
466 CasStatus status = CasStatus::OK;
467 hidl_string detailedError;
468 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
469
470 if (key != nullptr) {
471 sctrl = (ScramblingControl)key[0];
472 // Adjust for the PES offset
473 codecDataOffset = key[2] | (key[3] << 8);
474 }
475
476 auto returnVoid = mDescrambler->descramble(
477 sctrl,
478 hidlSubSamples,
479 src,
480 0,
481 dst,
482 0,
483 [&status, &result, &detailedError] (
484 CasStatus _status, uint32_t _bytesWritten,
485 const hidl_string& _detailedError) {
486 status = _status;
487 result = (ssize_t)_bytesWritten;
488 detailedError = _detailedError;
489 });
490
491 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
492 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
493 mName, returnVoid.description().c_str(), status, result);
494 return UNKNOWN_ERROR;
495 }
496
497 if (result < codecDataOffset) {
498 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
499 return BAD_VALUE;
500 }
501 }
502 if (!secure) {
503 C2WriteView view = block->map().get();
504 if (view.error() != C2_OK) {
505 return UNKNOWN_ERROR;
506 }
507 if (view.size() < result) {
508 return UNKNOWN_ERROR;
509 }
510 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
511 }
512 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
513 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
514 if (!buffer->copy(c2Buffer)) {
515 return -ENOSYS;
516 }
517 return OK;
518}
519
Pawin Vongmasa36653902018-11-15 00:10:25 -0800520status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
521 QueueGuard guard(mSync);
522 if (!guard.isRunning()) {
523 ALOGD("[%s] No more buffers should be queued at current state.", mName);
524 return -ENOSYS;
525 }
526 return queueInputBufferInternal(buffer);
527}
528
529status_t CCodecBufferChannel::queueSecureInputBuffer(
530 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
531 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
532 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
533 AString *errorDetailMsg) {
534 QueueGuard guard(mSync);
535 if (!guard.isRunning()) {
536 ALOGD("[%s] No more buffers should be queued at current state.", mName);
537 return -ENOSYS;
538 }
539
540 if (!hasCryptoOrDescrambler()) {
541 return -ENOSYS;
542 }
543 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
544
Sungtak Lee04b30352020-07-27 13:57:25 -0700545 std::shared_ptr<C2LinearBlock> block;
546 size_t allocSize = buffer->size();
547 size_t bufferSize = 0;
548 c2_status_t blockRes = C2_OK;
549 bool copied = false;
550 if (mSendEncryptedInfoBuffer) {
551 static const C2MemoryUsage kDefaultReadWriteUsage{
552 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
553 constexpr int kAllocGranule0 = 1024 * 64;
554 constexpr int kAllocGranule1 = 1024 * 1024;
555 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
556 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
557 if (allocSize <= kAllocGranule1) {
558 bufferSize = align(allocSize, kAllocGranule0);
559 } else {
560 bufferSize = align(allocSize, kAllocGranule1);
561 }
562 blockRes = pool->fetchLinearBlock(
563 bufferSize, kDefaultReadWriteUsage, &block);
564
565 if (blockRes == C2_OK) {
566 C2WriteView view = block->map().get();
567 if (view.error() == C2_OK && view.size() == bufferSize) {
568 copied = true;
569 // TODO: only copy clear sections
570 memcpy(view.data(), buffer->data(), allocSize);
571 }
572 }
573 }
574
575 if (!copied) {
576 block.reset();
577 }
578
Pawin Vongmasa36653902018-11-15 00:10:25 -0800579 ssize_t result = -1;
580 ssize_t codecDataOffset = 0;
Wonsik Kim557c88c2020-03-13 11:03:52 -0700581 if (numSubSamples == 1
582 && subSamples[0].mNumBytesOfClearData == 0
583 && subSamples[0].mNumBytesOfEncryptedData == 0) {
584 // We don't need to go through crypto or descrambler if the input is empty.
585 result = 0;
586 } else if (mCrypto != nullptr) {
Robert Shih895fba92019-07-16 16:29:44 -0700587 hardware::drm::V1_0::DestinationBuffer destination;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800588 if (secure) {
Robert Shih895fba92019-07-16 16:29:44 -0700589 destination.type = DrmBufferType::NATIVE_HANDLE;
590 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800591 } else {
Robert Shih895fba92019-07-16 16:29:44 -0700592 destination.type = DrmBufferType::SHARED_MEMORY;
593 IMemoryToSharedBuffer(
594 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800595 }
Robert Shih895fba92019-07-16 16:29:44 -0700596 hardware::drm::V1_0::SharedBuffer source;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800597 encryptedBuffer->fillSourceBuffer(&source);
598 result = mCrypto->decrypt(
599 key, iv, mode, pattern, source, buffer->offset(),
600 subSamples, numSubSamples, destination, errorDetailMsg);
601 if (result < 0) {
Wonsik Kim557c88c2020-03-13 11:03:52 -0700602 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800603 return result;
604 }
Robert Shih895fba92019-07-16 16:29:44 -0700605 if (destination.type == DrmBufferType::SHARED_MEMORY) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800606 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
607 }
608 } else {
609 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
610 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
611 hidl_vec<SubSample> hidlSubSamples;
612 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
613
614 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
615 encryptedBuffer->fillSourceBuffer(&srcBuffer);
616
617 DestinationBuffer dstBuffer;
618 if (secure) {
619 dstBuffer.type = BufferType::NATIVE_HANDLE;
620 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
621 } else {
622 dstBuffer.type = BufferType::SHARED_MEMORY;
623 dstBuffer.nonsecureMemory = srcBuffer;
624 }
625
626 CasStatus status = CasStatus::OK;
627 hidl_string detailedError;
628 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
629
630 if (key != nullptr) {
631 sctrl = (ScramblingControl)key[0];
632 // Adjust for the PES offset
633 codecDataOffset = key[2] | (key[3] << 8);
634 }
635
636 auto returnVoid = mDescrambler->descramble(
637 sctrl,
638 hidlSubSamples,
639 srcBuffer,
640 0,
641 dstBuffer,
642 0,
643 [&status, &result, &detailedError] (
644 CasStatus _status, uint32_t _bytesWritten,
645 const hidl_string& _detailedError) {
646 status = _status;
647 result = (ssize_t)_bytesWritten;
648 detailedError = _detailedError;
649 });
650
651 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
652 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
653 mName, returnVoid.description().c_str(), status, result);
654 return UNKNOWN_ERROR;
655 }
656
657 if (result < codecDataOffset) {
658 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
659 return BAD_VALUE;
660 }
661
662 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
663
664 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
665 encryptedBuffer->copyDecryptedContentFromMemory(result);
666 }
667 }
668
669 buffer->setRange(codecDataOffset, result - codecDataOffset);
Sungtak Lee04b30352020-07-27 13:57:25 -0700670
671 return queueInputBufferInternal(buffer, block, bufferSize);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800672}
673
674void CCodecBufferChannel::feedInputBufferIfAvailable() {
675 QueueGuard guard(mSync);
676 if (!guard.isRunning()) {
677 ALOGV("[%s] We're not running --- no input buffer reported", mName);
678 return;
679 }
680 feedInputBufferIfAvailableInternal();
681}
682
683void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
Taehwan Kimda0517d2020-09-16 17:29:37 +0900684 if (mInputMetEos) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800685 return;
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700686 }
687 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700688 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700689 if (!output->buffers ||
690 output->buffers->hasPending() ||
Wonsik Kim0487b782020-10-28 11:45:50 -0700691 output->buffers->numActiveSlots() >= output->numSlots) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800692 return;
693 }
694 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700695 size_t numActiveSlots = 0;
696 while (!mPipelineWatcher.lock()->pipelineFull()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800697 sp<MediaCodecBuffer> inBuffer;
698 size_t index;
699 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700700 Mutexed<Input>::Locked input(mInput);
Wonsik Kim0487b782020-10-28 11:45:50 -0700701 numActiveSlots = input->buffers->numActiveSlots();
702 if (numActiveSlots >= input->numSlots) {
703 break;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800704 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700705 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800706 ALOGV("[%s] no new buffer available", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800707 break;
708 }
709 }
710 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
711 mCallback->onInputBufferAvailable(index, inBuffer);
712 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700713 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800714}
715
716status_t CCodecBufferChannel::renderOutputBuffer(
717 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800718 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800719 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800720 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800721 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700722 Mutexed<Output>::Locked output(mOutput);
723 if (output->buffers) {
724 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800725 }
726 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800727 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
728 // set to true.
729 sendOutputBuffers();
730 // input buffer feeding may have been gated by pending output buffers
731 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800732 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800733 if (released) {
Wonsik Kimf7529dd2019-04-18 17:35:53 -0700734 std::call_once(mRenderWarningFlag, [this] {
735 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
736 "timestamp or render=true with non-video buffers. Apps should "
737 "call releaseOutputBuffer() with render=false for those.",
738 mName);
739 });
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800740 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800741 return INVALID_OPERATION;
742 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800743
744#if 0
745 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
746 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
747 for (const std::shared_ptr<const C2Info> &info : infoParams) {
748 AString res;
749 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
750 if (ix) res.append(", ");
751 res.append(*((int32_t*)info.get() + (ix / 4)));
752 }
753 ALOGV(" [%s]", res.c_str());
754 }
755#endif
756 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
757 std::static_pointer_cast<const C2StreamRotationInfo::output>(
758 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
759 bool flip = rotation && (rotation->flip & 1);
760 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
Byeongjo Park25c3a3d2020-06-12 17:24:21 +0900761
762 {
763 Mutexed<OutputSurface>::Locked output(mOutputSurface);
764 if (output->surface == nullptr) {
765 ALOGI("[%s] cannot render buffer without surface", mName);
766 return OK;
767 }
768 int64_t frameIndex;
769 buffer->meta()->findInt64("frameIndex", &frameIndex);
770 if (output->rotation.count(frameIndex) != 0) {
771 auto it = output->rotation.find(frameIndex);
772 quarters = (it->second / 90) & 3;
773 output->rotation.erase(it);
774 }
775 }
776
Pawin Vongmasa36653902018-11-15 00:10:25 -0800777 uint32_t transform = 0;
778 switch (quarters) {
779 case 0: // no rotation
780 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
781 break;
782 case 1: // 90 degrees counter-clockwise
783 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
784 : HAL_TRANSFORM_ROT_270;
785 break;
786 case 2: // 180 degrees
787 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
788 break;
789 case 3: // 90 degrees clockwise
790 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
791 : HAL_TRANSFORM_ROT_90;
792 break;
793 }
794
795 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
796 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
797 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
798 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
799 if (surfaceScaling) {
800 videoScalingMode = surfaceScaling->value;
801 }
802
803 // Use dataspace from format as it has the default aspects already applied
804 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
805 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
806
807 // HDR static info
808 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
809 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
810 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
811
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800812 // HDR10 plus info
813 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
814 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
815 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
Yichi Chen54be23c2020-06-15 14:30:53 +0800816 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
817 hdr10PlusInfo.reset();
818 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800819
Pawin Vongmasa36653902018-11-15 00:10:25 -0800820 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
821 if (blocks.size() != 1u) {
822 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
823 return UNKNOWN_ERROR;
824 }
825 const C2ConstGraphicBlock &block = blocks.front();
826
827 // TODO: revisit this after C2Fence implementation.
828 android::IGraphicBufferProducer::QueueBufferInput qbi(
829 timestampNs,
830 false, // droppable
831 dataSpace,
832 Rect(blocks.front().crop().left,
833 blocks.front().crop().top,
834 blocks.front().crop().right(),
835 blocks.front().crop().bottom()),
836 videoScalingMode,
837 transform,
838 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800839 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800840 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800841 if (hdrStaticInfo) {
wenchangliuf3f92882020-05-14 00:02:01 +0800842 // If mastering max and min luminance fields are 0, do not use them.
843 // It indicates the value may not be present in the stream.
844 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
845 hdrStaticInfo->mastering.minLuminance > 0.0f) {
846 struct android_smpte2086_metadata smpte2086_meta = {
847 .displayPrimaryRed = {
848 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
849 },
850 .displayPrimaryGreen = {
851 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
852 },
853 .displayPrimaryBlue = {
854 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
855 },
856 .whitePoint = {
857 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
858 },
859 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
860 .minLuminance = hdrStaticInfo->mastering.minLuminance,
861 };
Yichi Chen54be23c2020-06-15 14:30:53 +0800862 hdr.validTypes |= HdrMetadata::SMPTE2086;
wenchangliuf3f92882020-05-14 00:02:01 +0800863 hdr.smpte2086 = smpte2086_meta;
864 }
Chong Zhang3bb2a7f2020-04-21 10:35:12 -0700865 // If the content light level fields are 0, do not use them, it
866 // indicates the value may not be present in the stream.
867 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
868 struct android_cta861_3_metadata cta861_meta = {
869 .maxContentLightLevel = hdrStaticInfo->maxCll,
870 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
871 };
872 hdr.validTypes |= HdrMetadata::CTA861_3;
873 hdr.cta8613 = cta861_meta;
874 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800875 }
876 if (hdr10PlusInfo) {
877 hdr.validTypes |= HdrMetadata::HDR10PLUS;
878 hdr.hdr10plus.assign(
879 hdr10PlusInfo->m.value,
880 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
881 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800882 qbi.setHdrMetadata(hdr);
883 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800884 // we don't have dirty regions
885 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800886 android::IGraphicBufferProducer::QueueBufferOutput qbo;
887 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
888 if (result != OK) {
889 ALOGI("[%s] queueBuffer failed: %d", mName, result);
Sungtak Lee47c018a2020-11-07 01:02:49 -0800890 if (result == NO_INIT) {
891 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
892 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800893 return result;
894 }
Josh Hou8eddf4b2021-02-02 16:26:53 +0800895
896 if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
897 ALOGD("[%s] queue buffer successful", mName);
898 } else {
899 ALOGV("[%s] queue buffer successful", mName);
900 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800901
902 int64_t mediaTimeUs = 0;
903 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
904 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
905
906 return OK;
907}
908
909status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
910 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
911 bool released = false;
912 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700913 Mutexed<Input>::Locked input(mInput);
914 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800915 released = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800916 }
917 }
918 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700919 Mutexed<Output>::Locked output(mOutput);
920 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800921 released = true;
922 }
923 }
924 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800925 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800926 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800927 } else {
928 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
929 }
930 return OK;
931}
932
933void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
934 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700935 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800936
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700937 if (!input->buffers->isArrayMode()) {
938 input->buffers = input->buffers->toArrayMode(input->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800939 }
940
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700941 input->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800942}
943
944void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
945 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700946 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800947
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700948 if (!output->buffers->isArrayMode()) {
949 output->buffers = output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800950 }
951
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700952 output->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800953}
954
955status_t CCodecBufferChannel::start(
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800956 const sp<AMessage> &inputFormat,
957 const sp<AMessage> &outputFormat,
958 bool buffersBoundToCodec) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800959 C2StreamBufferTypeSetting::input iStreamFormat(0u);
960 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kime1104ca2020-11-24 15:01:33 -0800961 C2ComponentKindSetting kind;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800962 C2PortReorderBufferDepthTuning::output reorderDepth;
963 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800964 C2PortActualDelayTuning::input inputDelay(0);
965 C2PortActualDelayTuning::output outputDelay(0);
966 C2ActualPipelineDelayTuning pipelineDelay(0);
Sungtak Lee04b30352020-07-27 13:57:25 -0700967 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
Wonsik Kim078b58e2019-01-09 15:08:06 -0800968
Pawin Vongmasa36653902018-11-15 00:10:25 -0800969 c2_status_t err = mComponent->query(
970 {
971 &iStreamFormat,
972 &oStreamFormat,
Wonsik Kime1104ca2020-11-24 15:01:33 -0800973 &kind,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800974 &reorderDepth,
975 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800976 &inputDelay,
977 &pipelineDelay,
978 &outputDelay,
Sungtak Lee04b30352020-07-27 13:57:25 -0700979 &secureMode,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800980 },
981 {},
982 C2_DONT_BLOCK,
983 nullptr);
984 if (err == C2_BAD_INDEX) {
Wonsik Kime1104ca2020-11-24 15:01:33 -0800985 if (!iStreamFormat || !oStreamFormat || !kind) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800986 return UNKNOWN_ERROR;
987 }
988 } else if (err != C2_OK) {
989 return UNKNOWN_ERROR;
990 }
991
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -0800992 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
993 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
994 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
995
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700996 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
997 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800998
Pawin Vongmasa36653902018-11-15 00:10:25 -0800999 // TODO: get this from input format
1000 bool secure = mComponent->getName().find(".secure") != std::string::npos;
1001
Sungtak Lee04b30352020-07-27 13:57:25 -07001002 // secure mode is a static parameter (shall not change in the executing state)
1003 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
1004
Pawin Vongmasa36653902018-11-15 00:10:25 -08001005 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001006 int poolMask = GetCodec2PoolMask();
1007 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001008
1009 if (inputFormat != nullptr) {
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001010 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
Wonsik Kime1104ca2020-11-24 15:01:33 -08001011 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001012 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1013 API_REFLECTION |
1014 API_VALUES |
1015 API_CURRENT_VALUES |
1016 API_DEPENDENCY |
1017 API_SAME_INPUT_BUFFER);
Wonsik Kime1104ca2020-11-24 15:01:33 -08001018 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1019 C2StreamSampleRateInfo::input sampleRate(0u);
1020 C2StreamChannelCountInfo::input channelCount(0u);
1021 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001022 std::shared_ptr<C2BlockPool> pool;
1023 {
1024 Mutexed<BlockPools>::Locked pools(mBlockPools);
1025
1026 // set default allocator ID.
1027 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001028 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001029
1030 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1031 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1032 std::vector<std::unique_ptr<C2Param>> params;
Wonsik Kimffb889a2020-05-28 11:32:25 -07001033 C2ApiFeaturesSetting featuresSetting{apiFeatures};
Wonsik Kime1104ca2020-11-24 15:01:33 -08001034 std::vector<C2Param *> stackParams({&featuresSetting});
1035 if (audioEncoder) {
1036 stackParams.push_back(&encoderFrameSize);
1037 stackParams.push_back(&sampleRate);
1038 stackParams.push_back(&channelCount);
1039 stackParams.push_back(&pcmEncoding);
1040 } else {
1041 encoderFrameSize.invalidate();
1042 sampleRate.invalidate();
1043 channelCount.invalidate();
1044 pcmEncoding.invalidate();
1045 }
1046 err = mComponent->query(stackParams,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001047 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1048 C2_DONT_BLOCK,
1049 &params);
1050 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1051 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1052 mName, params.size(), asString(err), err);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001053 } else if (params.size() == 1) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001054 C2PortAllocatorsTuning::input *inputAllocators =
1055 C2PortAllocatorsTuning::input::From(params[0].get());
1056 if (inputAllocators && inputAllocators->flexCount() > 0) {
1057 std::shared_ptr<C2Allocator> allocator;
1058 // verify allocator IDs and resolve default allocator
1059 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1060 if (allocator) {
1061 pools->inputAllocatorId = allocator->getId();
1062 } else {
1063 ALOGD("[%s] component requested invalid input allocator ID %u",
1064 mName, inputAllocators->m.values[0]);
1065 }
1066 }
1067 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001068 if (featuresSetting) {
1069 apiFeatures = featuresSetting.value;
1070 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001071
1072 // TODO: use C2Component wrapper to associate this pool with ourselves
1073 if ((poolMask >> pools->inputAllocatorId) & 1) {
1074 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1075 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1076 mName, pools->inputAllocatorId,
1077 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1078 asString(err), err);
1079 } else {
1080 err = C2_NOT_FOUND;
1081 }
1082 if (err != C2_OK) {
1083 C2BlockPool::local_id_t inputPoolId =
1084 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1085 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1086 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1087 mName, (unsigned long long)inputPoolId,
1088 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1089 asString(err), err);
1090 if (err != C2_OK) {
1091 return NO_MEMORY;
1092 }
1093 }
1094 pools->inputPool = pool;
1095 }
1096
Wonsik Kim51051262018-11-28 13:59:05 -08001097 bool forceArrayMode = false;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001098 Mutexed<Input>::Locked input(mInput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001099 input->inputDelay = inputDelayValue;
1100 input->pipelineDelay = pipelineDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001101 input->numSlots = numInputSlots;
1102 input->extraBuffers.flush();
1103 input->numExtraSlots = 0u;
Wonsik Kime1104ca2020-11-24 15:01:33 -08001104 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1105 input->frameReassembler.init(
1106 pool,
1107 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1108 encoderFrameSize.value,
1109 sampleRate.value,
1110 channelCount.value,
1111 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1112 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001113 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1114 // For encrypted content, framework decrypts source buffer (ashmem) into
1115 // C2Buffers. Thus non-conforming codecs can process these.
Wonsik Kime1104ca2020-11-24 15:01:33 -08001116 if (!buffersBoundToCodec
1117 && !input->frameReassembler
1118 && (hasCryptoOrDescrambler() || conforming)) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001119 input->buffers.reset(new SlotInputBuffers(mName));
1120 } else if (graphic) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001121 if (mInputSurface) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001122 input->buffers.reset(new DummyInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001123 } else if (mMetaMode == MODE_ANW) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001124 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
Wonsik Kim1221fd12019-07-12 12:52:05 -07001125 // This is to ensure buffers do not get released prematurely.
1126 // TODO: handle this without going into array mode
1127 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001128 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001129 input->buffers.reset(new GraphicInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001130 }
1131 } else {
1132 if (hasCryptoOrDescrambler()) {
1133 int32_t capacity = kLinearBufferSize;
1134 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1135 if ((size_t)capacity > kMaxLinearBufferSize) {
1136 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1137 capacity = kMaxLinearBufferSize;
1138 }
1139 if (mDealer == nullptr) {
1140 mDealer = new MemoryDealer(
1141 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001142 * (numInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001143 "EncryptedLinearInputBuffers");
1144 mDecryptDestination = mDealer->allocate((size_t)capacity);
1145 }
1146 if (mCrypto != nullptr && mHeapSeqNum < 0) {
Robert Shih895fba92019-07-16 16:29:44 -07001147 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1148 mHeapSeqNum = mCrypto->setHeap(heap);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001149 } else {
1150 mHeapSeqNum = -1;
1151 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001152 input->buffers.reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001153 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001154 numInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08001155 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001156 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001157 input->buffers.reset(new LinearInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001158 }
1159 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001160 input->buffers->setFormat(inputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001161
1162 if (err == C2_OK) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001163 input->buffers->setPool(pool);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001164 } else {
1165 // TODO: error
1166 }
Wonsik Kim51051262018-11-28 13:59:05 -08001167
1168 if (forceArrayMode) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001169 input->buffers = input->buffers->toArrayMode(numInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08001170 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001171 }
1172
1173 if (outputFormat != nullptr) {
1174 sp<IGraphicBufferProducer> outputSurface;
1175 uint32_t outputGeneration;
1176 {
1177 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Sungtak Leed7463d12019-09-04 16:01:00 -07001178 output->maxDequeueBuffers = numOutputSlots +
Sungtak Lee7a7b7422019-07-16 17:40:40 -07001179 reorderDepth.value + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001180 outputSurface = output->surface ?
1181 output->surface->getIGraphicBufferProducer() : nullptr;
Wonsik Kimf5e5c832019-02-21 11:36:05 -08001182 if (outputSurface) {
1183 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1184 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001185 outputGeneration = output->generation;
1186 }
1187
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001188 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001189 C2BlockPool::local_id_t outputPoolId_;
1190
1191 {
1192 Mutexed<BlockPools>::Locked pools(mBlockPools);
1193
1194 // set default allocator ID.
1195 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001196 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001197
1198 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1199 // unsuccessful.
1200 std::vector<std::unique_ptr<C2Param>> params;
1201 err = mComponent->query({ },
1202 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1203 C2_DONT_BLOCK,
1204 &params);
1205 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1206 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1207 mName, params.size(), asString(err), err);
1208 } else if (err == C2_OK && params.size() == 1) {
1209 C2PortAllocatorsTuning::output *outputAllocators =
1210 C2PortAllocatorsTuning::output::From(params[0].get());
1211 if (outputAllocators && outputAllocators->flexCount() > 0) {
1212 std::shared_ptr<C2Allocator> allocator;
1213 // verify allocator IDs and resolve default allocator
1214 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1215 if (allocator) {
1216 pools->outputAllocatorId = allocator->getId();
1217 } else {
1218 ALOGD("[%s] component requested invalid output allocator ID %u",
1219 mName, outputAllocators->m.values[0]);
1220 }
1221 }
1222 }
1223
1224 // use bufferqueue if outputting to a surface.
1225 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1226 // if unsuccessful.
1227 if (outputSurface) {
1228 params.clear();
1229 err = mComponent->query({ },
1230 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1231 C2_DONT_BLOCK,
1232 &params);
1233 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1234 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1235 mName, params.size(), asString(err), err);
1236 } else if (err == C2_OK && params.size() == 1) {
1237 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1238 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1239 if (surfaceAllocator) {
1240 std::shared_ptr<C2Allocator> allocator;
1241 // verify allocator IDs and resolve default allocator
1242 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1243 if (allocator) {
1244 pools->outputAllocatorId = allocator->getId();
1245 } else {
1246 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1247 mName, surfaceAllocator->value);
1248 err = C2_BAD_VALUE;
1249 }
1250 }
1251 }
1252 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1253 && err != C2_OK
1254 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1255 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1256 }
1257 }
1258
1259 if ((poolMask >> pools->outputAllocatorId) & 1) {
1260 err = mComponent->createBlockPool(
1261 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1262 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1263 mName, pools->outputAllocatorId,
1264 (unsigned long long)pools->outputPoolId,
1265 asString(err));
1266 } else {
1267 err = C2_NOT_FOUND;
1268 }
1269 if (err != C2_OK) {
1270 // use basic pool instead
1271 pools->outputPoolId =
1272 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1273 }
1274
1275 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1276 // component.
1277 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1278 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1279
1280 std::vector<std::unique_ptr<C2SettingResult>> failures;
1281 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1282 ALOGD("[%s] Configured output block pool ids %llu => %s",
1283 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1284 outputPoolId_ = pools->outputPoolId;
1285 }
1286
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001287 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001288 output->outputDelay = outputDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001289 output->numSlots = numOutputSlots;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001290 if (graphic) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001291 if (outputSurface || !buffersBoundToCodec) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001292 output->buffers.reset(new GraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001293 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001294 output->buffers.reset(new RawGraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001295 }
1296 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001297 output->buffers.reset(new LinearOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001298 }
Wonsik Kime4716c02020-02-28 10:42:21 -08001299 output->buffers->setFormat(outputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001300
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001301 output->buffers->clearStash();
1302 if (reorderDepth) {
1303 output->buffers->setReorderDepth(reorderDepth.value);
1304 }
1305 if (reorderKey) {
1306 output->buffers->setReorderKey(reorderKey.value);
1307 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001308
1309 // Try to set output surface to created block pool if given.
1310 if (outputSurface) {
1311 mComponent->setOutputSurface(
1312 outputPoolId_,
1313 outputSurface,
1314 outputGeneration);
1315 }
1316
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001317 if (oStreamFormat.value == C2BufferData::LINEAR) {
Wonsik Kim58713302020-01-29 22:25:23 -08001318 if (buffersBoundToCodec) {
1319 // WORKAROUND: if we're using early CSD workaround we convert to
1320 // array mode, to appease apps assuming the output
1321 // buffers to be of the same size.
1322 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1323 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001324
1325 int32_t channelCount;
1326 int32_t sampleRate;
1327 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1328 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1329 int32_t delay = 0;
1330 int32_t padding = 0;;
1331 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1332 delay = 0;
1333 }
1334 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1335 padding = 0;
1336 }
1337 if (delay || padding) {
1338 // We need write access to the buffers, and we're already in
1339 // array mode.
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001340 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001341 }
1342 }
1343 }
1344 }
1345
1346 // Set up pipeline control. This has to be done after mInputBuffers and
1347 // mOutputBuffers are initialized to make sure that lingering callbacks
1348 // about buffers from the previous generation do not interfere with the
1349 // newly initialized pipeline capacity.
1350
Wonsik Kimab34ed62019-01-31 15:28:46 -08001351 {
1352 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001353 watcher->inputDelay(inputDelayValue)
1354 .pipelineDelay(pipelineDelayValue)
1355 .outputDelay(outputDelayValue)
Wonsik Kimab34ed62019-01-31 15:28:46 -08001356 .smoothnessFactor(kSmoothnessFactor);
1357 watcher->flush();
1358 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001359
1360 mInputMetEos = false;
1361 mSync.start();
1362 return OK;
1363}
1364
1365status_t CCodecBufferChannel::requestInitialInputBuffers() {
1366 if (mInputSurface) {
1367 return OK;
1368 }
1369
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001370 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001371 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1372 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1373 if (err != C2_OK && err != C2_BAD_INDEX) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001374 return UNKNOWN_ERROR;
1375 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001376 size_t numInputSlots = mInput.lock()->numSlots;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001377
1378 struct ClientInputBuffer {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001379 size_t index;
1380 sp<MediaCodecBuffer> buffer;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001381 size_t capacity;
1382 };
1383 std::list<ClientInputBuffer> clientInputBuffers;
1384
1385 {
1386 Mutexed<Input>::Locked input(mInput);
1387 while (clientInputBuffers.size() < numInputSlots) {
1388 ClientInputBuffer clientInputBuffer;
1389 if (!input->buffers->requestNewBuffer(&clientInputBuffer.index,
1390 &clientInputBuffer.buffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001391 break;
1392 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001393 clientInputBuffer.capacity = clientInputBuffer.buffer->capacity();
1394 clientInputBuffers.emplace_back(std::move(clientInputBuffer));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001395 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001396 }
1397 if (clientInputBuffers.empty()) {
1398 ALOGW("[%s] start: cannot allocate memory at all", mName);
1399 return NO_MEMORY;
1400 } else if (clientInputBuffers.size() < numInputSlots) {
1401 ALOGD("[%s] start: cannot allocate memory for all slots, "
1402 "only %zu buffers allocated",
1403 mName, clientInputBuffers.size());
1404 } else {
1405 ALOGV("[%s] %zu initial input buffers available",
1406 mName, clientInputBuffers.size());
1407 }
1408 // Sort input buffers by their capacities in increasing order.
1409 clientInputBuffers.sort(
1410 [](const ClientInputBuffer& a, const ClientInputBuffer& b) {
1411 return a.capacity < b.capacity;
1412 });
1413
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001414 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1415 mFlushedConfigs.lock()->swap(flushedConfigs);
1416 if (!flushedConfigs.empty()) {
1417 err = mComponent->queue(&flushedConfigs);
1418 if (err != C2_OK) {
1419 ALOGW("[%s] Error while queueing a flushed config", mName);
1420 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001421 }
1422 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001423 if (oStreamFormat.value == C2BufferData::LINEAR &&
1424 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE)) {
1425 sp<MediaCodecBuffer> buffer = clientInputBuffers.front().buffer;
1426 // WORKAROUND: Some apps expect CSD available without queueing
1427 // any input. Queue an empty buffer to get the CSD.
1428 buffer->setRange(0, 0);
1429 buffer->meta()->clear();
1430 buffer->meta()->setInt64("timeUs", 0);
1431 if (queueInputBufferInternal(buffer) != OK) {
1432 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1433 mName);
1434 return UNKNOWN_ERROR;
1435 }
1436 clientInputBuffers.pop_front();
1437 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001438
1439 for (const ClientInputBuffer& clientInputBuffer: clientInputBuffers) {
1440 mCallback->onInputBufferAvailable(
1441 clientInputBuffer.index,
1442 clientInputBuffer.buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001443 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001444
Pawin Vongmasa36653902018-11-15 00:10:25 -08001445 return OK;
1446}
1447
1448void CCodecBufferChannel::stop() {
1449 mSync.stop();
1450 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
1451 if (mInputSurface != nullptr) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001452 mInputSurface.reset();
1453 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001454 mPipelineWatcher.lock()->flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001455}
1456
Wonsik Kim936a89c2020-05-08 16:07:50 -07001457void CCodecBufferChannel::reset() {
1458 stop();
1459 {
1460 Mutexed<Input>::Locked input(mInput);
1461 input->buffers.reset(new DummyInputBuffers(""));
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001462 input->extraBuffers.flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001463 }
1464 {
1465 Mutexed<Output>::Locked output(mOutput);
1466 output->buffers.reset();
1467 }
1468}
1469
1470void CCodecBufferChannel::release() {
1471 mComponent.reset();
1472 mInputAllocator.reset();
1473 mOutputSurface.lock()->surface.clear();
1474 {
1475 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1476 blockPools->inputPool.reset();
1477 blockPools->outputPoolIntf.reset();
1478 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001479 setCrypto(nullptr);
1480 setDescrambler(nullptr);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001481}
1482
1483
Pawin Vongmasa36653902018-11-15 00:10:25 -08001484void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1485 ALOGV("[%s] flush", mName);
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001486 std::list<std::unique_ptr<C2Work>> configs;
1487 for (const std::unique_ptr<C2Work> &work : flushedWork) {
1488 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1489 continue;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001490 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001491 if (work->input.buffers.empty()
1492 || work->input.buffers.front() == nullptr
1493 || work->input.buffers.front()->data().linearBlocks().empty()) {
1494 ALOGD("[%s] no linear codec config data found", mName);
1495 continue;
1496 }
1497 std::unique_ptr<C2Work> copy(new C2Work);
1498 copy->input.flags = C2FrameData::flags_t(work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1499 copy->input.ordinal = work->input.ordinal;
1500 copy->input.buffers.insert(
1501 copy->input.buffers.begin(),
1502 work->input.buffers.begin(),
1503 work->input.buffers.end());
1504 for (const std::unique_ptr<C2Param> &param : work->input.configUpdate) {
1505 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1506 }
1507 copy->input.infoBuffers.insert(
1508 copy->input.infoBuffers.begin(),
1509 work->input.infoBuffers.begin(),
1510 work->input.infoBuffers.end());
1511 copy->worklets.emplace_back(new C2Worklet);
1512 configs.push_back(std::move(copy));
1513 ALOGV("[%s] stashed flushed codec config data", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001514 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001515 mFlushedConfigs.lock()->swap(configs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001516 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001517 Mutexed<Input>::Locked input(mInput);
1518 input->buffers->flush();
1519 input->extraBuffers.flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001520 }
1521 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001522 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001523 if (output->buffers) {
1524 output->buffers->flush(flushedWork);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001525 output->buffers->flushStash();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001526 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001527 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001528 mPipelineWatcher.lock()->flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001529}
1530
1531void CCodecBufferChannel::onWorkDone(
1532 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
Wonsik Kimab34ed62019-01-31 15:28:46 -08001533 const C2StreamInitDataInfo::output *initData) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001534 if (handleWork(std::move(work), outputFormat, initData)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001535 feedInputBufferIfAvailable();
1536 }
1537}
1538
1539void CCodecBufferChannel::onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -08001540 uint64_t frameIndex, size_t arrayIndex) {
Pawin Vongmasa8e2cfb52019-05-15 05:20:52 -07001541 if (mInputSurface) {
1542 return;
1543 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001544 std::shared_ptr<C2Buffer> buffer =
1545 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001546 bool newInputSlotAvailable;
1547 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001548 Mutexed<Input>::Locked input(mInput);
1549 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1550 if (!newInputSlotAvailable) {
1551 (void)input->extraBuffers.expireComponentBuffer(buffer);
1552 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001553 }
1554 if (newInputSlotAvailable) {
1555 feedInputBufferIfAvailable();
1556 }
1557}
1558
1559bool CCodecBufferChannel::handleWork(
1560 std::unique_ptr<C2Work> work,
1561 const sp<AMessage> &outputFormat,
1562 const C2StreamInitDataInfo::output *initData) {
Wonsik Kim936a89c2020-05-08 16:07:50 -07001563 {
Wonsik Kima4e049d2020-04-28 19:42:23 +00001564 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001565 if (!output->buffers) {
1566 return false;
1567 }
Wonsik Kime75a5da2020-02-14 17:29:03 -08001568 }
1569
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001570 // Whether the output buffer should be reported to the client or not.
1571 bool notifyClient = false;
1572
1573 if (work->result == C2_OK){
1574 notifyClient = true;
1575 } else if (work->result == C2_NOT_FOUND) {
1576 ALOGD("[%s] flushed work; ignored.", mName);
1577 } else {
1578 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1579 // the config update.
1580 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1581 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1582 return false;
1583 }
1584
1585 if ((work->input.ordinal.frameIndex -
1586 mFirstValidFrameIndex.load()).peek() < 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001587 // Discard frames from previous generation.
1588 ALOGD("[%s] Discard frames from previous generation.", mName);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001589 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001590 }
1591
Wonsik Kim524b0582019-03-12 11:28:57 -07001592 if (mInputSurface == nullptr && (work->worklets.size() != 1u
Pawin Vongmasa36653902018-11-15 00:10:25 -08001593 || !work->worklets.front()
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001594 || !(work->worklets.front()->output.flags &
1595 C2FrameData::FLAG_INCOMPLETE))) {
1596 mPipelineWatcher.lock()->onWorkDone(
1597 work->input.ordinal.frameIndex.peeku());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001598 }
1599
1600 // NOTE: MediaCodec usage supposedly have only one worklet
1601 if (work->worklets.size() != 1u) {
1602 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1603 mName, work->worklets.size());
1604 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1605 return false;
1606 }
1607
1608 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1609
1610 std::shared_ptr<C2Buffer> buffer;
1611 // NOTE: MediaCodec usage supposedly have only one output stream.
1612 if (worklet->output.buffers.size() > 1u) {
1613 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1614 mName, worklet->output.buffers.size());
1615 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1616 return false;
1617 } else if (worklet->output.buffers.size() == 1u) {
1618 buffer = worklet->output.buffers[0];
1619 if (!buffer) {
1620 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1621 }
1622 }
1623
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001624 std::optional<uint32_t> newInputDelay, newPipelineDelay;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001625 bool needMaxDequeueBufferCountUpdate = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001626 while (!worklet->output.configUpdate.empty()) {
1627 std::unique_ptr<C2Param> param;
1628 worklet->output.configUpdate.back().swap(param);
1629 worklet->output.configUpdate.pop_back();
1630 switch (param->coreIndex().coreIndex()) {
1631 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1632 C2PortReorderBufferDepthTuning::output reorderDepth;
1633 if (reorderDepth.updateFrom(*param)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001634 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1635 mName, reorderDepth.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001636 mOutput.lock()->buffers->setReorderDepth(reorderDepth.value);
1637 needMaxDequeueBufferCountUpdate = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001638 } else {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001639 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1640 mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001641 }
1642 break;
1643 }
1644 case C2PortReorderKeySetting::CORE_INDEX: {
1645 C2PortReorderKeySetting::output reorderKey;
1646 if (reorderKey.updateFrom(*param)) {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001647 mOutput.lock()->buffers->setReorderKey(reorderKey.value);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001648 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1649 mName, reorderKey.value);
1650 } else {
1651 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1652 }
1653 break;
1654 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001655 case C2PortActualDelayTuning::CORE_INDEX: {
1656 if (param->isGlobal()) {
1657 C2ActualPipelineDelayTuning pipelineDelay;
1658 if (pipelineDelay.updateFrom(*param)) {
1659 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1660 mName, pipelineDelay.value);
1661 newPipelineDelay = pipelineDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001662 (void)mPipelineWatcher.lock()->pipelineDelay(
1663 pipelineDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001664 }
1665 }
1666 if (param->forInput()) {
1667 C2PortActualDelayTuning::input inputDelay;
1668 if (inputDelay.updateFrom(*param)) {
1669 ALOGV("[%s] onWorkDone: updating input delay %u",
1670 mName, inputDelay.value);
1671 newInputDelay = inputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001672 (void)mPipelineWatcher.lock()->inputDelay(
1673 inputDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001674 }
1675 }
1676 if (param->forOutput()) {
1677 C2PortActualDelayTuning::output outputDelay;
1678 if (outputDelay.updateFrom(*param)) {
1679 ALOGV("[%s] onWorkDone: updating output delay %u",
1680 mName, outputDelay.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001681 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
1682 needMaxDequeueBufferCountUpdate = true;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001683
1684 bool outputBuffersChanged = false;
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001685 size_t numOutputSlots = 0;
1686 {
1687 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001688 if (!output->buffers) {
1689 return false;
1690 }
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001691 output->outputDelay = outputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001692 numOutputSlots = outputDelay.value +
1693 kSmoothnessFactor;
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001694 if (output->numSlots < numOutputSlots) {
1695 output->numSlots = numOutputSlots;
1696 if (output->buffers->isArrayMode()) {
1697 OutputBuffersArray *array =
1698 (OutputBuffersArray *)output->buffers.get();
1699 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1700 mName, numOutputSlots);
1701 array->grow(numOutputSlots);
1702 outputBuffersChanged = true;
1703 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001704 }
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001705 numOutputSlots = output->numSlots;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001706 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001707
1708 if (outputBuffersChanged) {
1709 mCCodecCallback->onOutputBuffersChanged();
1710 }
1711 }
1712 }
1713 break;
1714 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001715 default:
1716 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1717 mName, param->index());
1718 break;
1719 }
1720 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001721 if (newInputDelay || newPipelineDelay) {
1722 Mutexed<Input>::Locked input(mInput);
1723 size_t newNumSlots =
1724 newInputDelay.value_or(input->inputDelay) +
1725 newPipelineDelay.value_or(input->pipelineDelay) +
1726 kSmoothnessFactor;
1727 if (input->buffers->isArrayMode()) {
1728 if (input->numSlots >= newNumSlots) {
1729 input->numExtraSlots = 0;
1730 } else {
1731 input->numExtraSlots = newNumSlots - input->numSlots;
1732 }
1733 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1734 mName, input->numExtraSlots);
1735 } else {
1736 input->numSlots = newNumSlots;
1737 }
1738 }
Wonsik Kim315e40a2020-09-09 14:11:50 -07001739 if (needMaxDequeueBufferCountUpdate) {
1740 size_t numOutputSlots = 0;
1741 uint32_t reorderDepth = 0;
1742 {
1743 Mutexed<Output>::Locked output(mOutput);
1744 numOutputSlots = output->numSlots;
1745 reorderDepth = output->buffers->getReorderDepth();
1746 }
1747 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1748 output->maxDequeueBuffers = numOutputSlots + reorderDepth + kRenderingDepth;
1749 if (output->surface) {
1750 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1751 }
1752 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001753
Pawin Vongmasa36653902018-11-15 00:10:25 -08001754 int32_t flags = 0;
1755 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1756 flags |= MediaCodec::BUFFER_FLAG_EOS;
1757 ALOGV("[%s] onWorkDone: output EOS", mName);
1758 }
1759
Pawin Vongmasa36653902018-11-15 00:10:25 -08001760 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1761 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1762 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1763 // shall correspond to the client input timesamp (in customOrdinal). By using the
1764 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1765 // produces multiple output.
1766 c2_cntr64_t timestamp =
1767 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1768 - work->input.ordinal.timestamp;
Wonsik Kim95ba0162019-03-19 15:51:54 -07001769 if (mInputSurface != nullptr) {
1770 // When using input surface we need to restore the original input timestamp.
1771 timestamp = work->input.ordinal.customOrdinal;
1772 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001773 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1774 mName,
1775 work->input.ordinal.customOrdinal.peekll(),
1776 work->input.ordinal.timestamp.peekll(),
1777 worklet->output.ordinal.timestamp.peekll(),
1778 timestamp.peekll());
1779
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001780 // csd cannot be re-ordered and will always arrive first.
Pawin Vongmasa36653902018-11-15 00:10:25 -08001781 if (initData != nullptr) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001782 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001783 if (output->buffers && outputFormat) {
1784 output->buffers->updateSkipCutBuffer(outputFormat);
1785 output->buffers->setFormat(outputFormat);
1786 }
1787 if (!notifyClient) {
1788 return false;
1789 }
1790 size_t index;
1791 sp<MediaCodecBuffer> outBuffer;
Wonsik Kim936a89c2020-05-08 16:07:50 -07001792 if (output->buffers && output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001793 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1794 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1795 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1796
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001797 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001798 mCallback->onOutputBufferAvailable(index, outBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001799 } else {
1800 ALOGD("[%s] onWorkDone: unable to register csd", mName);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001801 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001802 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001803 return false;
1804 }
1805 }
1806
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001807 if (notifyClient && !buffer && !flags) {
Wonsik Kim35bf5732020-05-14 17:40:29 +00001808 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
Pawin Vongmasa36653902018-11-15 00:10:25 -08001809 mName, work->input.ordinal.frameIndex.peekull());
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001810 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001811 }
1812
1813 if (buffer) {
1814 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1815 // TODO: properly translate these to metadata
1816 switch (info->coreIndex().coreIndex()) {
1817 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001818 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001819 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1820 }
1821 break;
1822 default:
1823 break;
1824 }
1825 }
1826 }
1827
1828 {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001829 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimc23cc402020-05-28 14:53:40 -07001830 if (!output->buffers) {
1831 return false;
1832 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001833 output->buffers->pushToStash(
1834 buffer,
1835 notifyClient,
1836 timestamp.peek(),
1837 flags,
1838 outputFormat,
1839 worklet->output.ordinal);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001840 }
1841 sendOutputBuffers();
1842 return true;
1843}
1844
1845void CCodecBufferChannel::sendOutputBuffers() {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001846 OutputBuffers::BufferAction action;
Wonsik Kima4e049d2020-04-28 19:42:23 +00001847 size_t index;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001848 sp<MediaCodecBuffer> outBuffer;
1849 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001850
1851 while (true) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001852 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001853 if (!output->buffers) {
1854 return;
1855 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001856 action = output->buffers->popFromStashAndRegister(
1857 &c2Buffer, &index, &outBuffer);
1858 switch (action) {
1859 case OutputBuffers::SKIP:
1860 return;
1861 case OutputBuffers::DISCARD:
1862 break;
1863 case OutputBuffers::NOTIFY_CLIENT:
Wonsik Kima4e049d2020-04-28 19:42:23 +00001864 output.unlock();
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001865 mCallback->onOutputBufferAvailable(index, outBuffer);
1866 break;
1867 case OutputBuffers::REALLOCATE:
1868 if (!output->buffers->isArrayMode()) {
1869 output->buffers =
1870 output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001871 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001872 static_cast<OutputBuffersArray*>(output->buffers.get())->
1873 realloc(c2Buffer);
1874 output.unlock();
1875 mCCodecCallback->onOutputBuffersChanged();
Wonsik Kim4ada73d2020-05-26 14:58:07 -07001876 break;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001877 case OutputBuffers::RETRY:
1878 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
1879 mName);
1880 return;
1881 default:
1882 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
1883 "corrupted BufferAction value (%d) "
1884 "returned from popFromStashAndRegister.",
1885 mName, int(action));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001886 return;
1887 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001888 }
1889}
1890
1891status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1892 static std::atomic_uint32_t surfaceGeneration{0};
1893 uint32_t generation = (getpid() << 10) |
1894 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1895 & ((1 << 10) - 1));
1896
1897 sp<IGraphicBufferProducer> producer;
1898 if (newSurface) {
1899 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Sungtak Leeab6f2f32019-02-15 14:43:51 -08001900 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
Sungtak Lee08515812019-06-05 11:16:32 -07001901 newSurface->setMaxDequeuedBufferCount(mOutputSurface.lock()->maxDequeueBuffers);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001902 producer = newSurface->getIGraphicBufferProducer();
1903 producer->setGenerationNumber(generation);
1904 } else {
1905 ALOGE("[%s] setting output surface to null", mName);
1906 return INVALID_OPERATION;
1907 }
1908
1909 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
1910 C2BlockPool::local_id_t outputPoolId;
1911 {
1912 Mutexed<BlockPools>::Locked pools(mBlockPools);
1913 outputPoolId = pools->outputPoolId;
1914 outputPoolIntf = pools->outputPoolIntf;
1915 }
1916
1917 if (outputPoolIntf) {
1918 if (mComponent->setOutputSurface(
1919 outputPoolId,
1920 producer,
1921 generation) != C2_OK) {
1922 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
1923 return INVALID_OPERATION;
1924 }
1925 }
1926
1927 {
1928 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1929 output->surface = newSurface;
1930 output->generation = generation;
1931 }
1932
1933 return OK;
1934}
1935
Wonsik Kimab34ed62019-01-31 15:28:46 -08001936PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001937 // When client pushed EOS, we want all the work to be done quickly.
1938 // Otherwise, component may have stalled work due to input starvation up to
1939 // the sum of the delay in the pipeline.
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001940 size_t n = 0;
1941 if (!mInputMetEos) {
1942 size_t outputDelay = mOutput.lock()->outputDelay;
1943 Mutexed<Input>::Locked input(mInput);
1944 n = input->inputDelay + input->pipelineDelay + outputDelay;
1945 }
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001946 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
Wonsik Kimab34ed62019-01-31 15:28:46 -08001947}
1948
Pawin Vongmasa36653902018-11-15 00:10:25 -08001949void CCodecBufferChannel::setMetaMode(MetaMode mode) {
1950 mMetaMode = mode;
1951}
1952
Wonsik Kim596187e2019-10-25 12:44:10 -07001953void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001954 if (mCrypto != nullptr) {
1955 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
1956 mCrypto->unsetHeap(entry.second);
1957 }
1958 mHeapSeqNumMap.clear();
1959 if (mHeapSeqNum >= 0) {
1960 mCrypto->unsetHeap(mHeapSeqNum);
1961 mHeapSeqNum = -1;
1962 }
1963 }
Wonsik Kim596187e2019-10-25 12:44:10 -07001964 mCrypto = crypto;
1965}
1966
1967void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
1968 mDescrambler = descrambler;
1969}
1970
Pawin Vongmasa36653902018-11-15 00:10:25 -08001971status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
1972 // C2_OK is always translated to OK.
1973 if (c2s == C2_OK) {
1974 return OK;
1975 }
1976
1977 // Operation-dependent translation
1978 // TODO: Add as necessary
1979 switch (c2op) {
1980 case C2_OPERATION_Component_start:
1981 switch (c2s) {
1982 case C2_NO_MEMORY:
1983 return NO_MEMORY;
1984 default:
1985 return UNKNOWN_ERROR;
1986 }
1987 default:
1988 break;
1989 }
1990
1991 // Backup operation-agnostic translation
1992 switch (c2s) {
1993 case C2_BAD_INDEX:
1994 return BAD_INDEX;
1995 case C2_BAD_VALUE:
1996 return BAD_VALUE;
1997 case C2_BLOCKING:
1998 return WOULD_BLOCK;
1999 case C2_DUPLICATE:
2000 return ALREADY_EXISTS;
2001 case C2_NO_INIT:
2002 return NO_INIT;
2003 case C2_NO_MEMORY:
2004 return NO_MEMORY;
2005 case C2_NOT_FOUND:
2006 return NAME_NOT_FOUND;
2007 case C2_TIMED_OUT:
2008 return TIMED_OUT;
2009 case C2_BAD_STATE:
2010 case C2_CANCELED:
2011 case C2_CANNOT_DO:
2012 case C2_CORRUPTED:
2013 case C2_OMITTED:
2014 case C2_REFUSED:
2015 return UNKNOWN_ERROR;
2016 default:
2017 return -static_cast<status_t>(c2s);
2018 }
2019}
2020
2021} // namespace android