blob: 7386c7db687ceae8e097455d2c05d916f2f4a499 [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright 2017, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodecBufferChannel"
19#include <utils/Log.h>
20
Pawin Vongmasae7bb8612020-06-04 06:15:22 -070021#include <algorithm>
22#include <list>
Pawin Vongmasa36653902018-11-15 00:10:25 -080023#include <numeric>
24
25#include <C2AllocatorGralloc.h>
26#include <C2PlatformSupport.h>
27#include <C2BlockInternal.h>
28#include <C2Config.h>
29#include <C2Debug.h>
30
31#include <android/hardware/cas/native/1.0/IDescrambler.h>
Robert Shih895fba92019-07-16 16:29:44 -070032#include <android/hardware/drm/1.0/types.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080033#include <android-base/stringprintf.h>
Wonsik Kimfb7a7672019-12-27 17:13:33 -080034#include <binder/MemoryBase.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080035#include <binder/MemoryDealer.h>
Ray Essick18ea0452019-08-27 16:07:27 -070036#include <cutils/properties.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080037#include <gui/Surface.h>
Robert Shih895fba92019-07-16 16:29:44 -070038#include <hidlmemory/FrameworkUtils.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080039#include <media/openmax/OMX_Core.h>
40#include <media/stagefright/foundation/ABuffer.h>
41#include <media/stagefright/foundation/ALookup.h>
42#include <media/stagefright/foundation/AMessage.h>
43#include <media/stagefright/foundation/AUtils.h>
44#include <media/stagefright/foundation/hexdump.h>
45#include <media/stagefright/MediaCodec.h>
46#include <media/stagefright/MediaCodecConstants.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070047#include <media/stagefright/SkipCutBuffer.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080048#include <media/MediaCodecBuffer.h>
Wonsik Kim41d83432020-04-27 16:40:49 -070049#include <mediadrm/ICrypto.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080050#include <system/window.h>
51
52#include "CCodecBufferChannel.h"
53#include "Codec2Buffer.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080054
55namespace android {
56
57using android::base::StringPrintf;
58using hardware::hidl_handle;
59using hardware::hidl_string;
60using hardware::hidl_vec;
Robert Shih895fba92019-07-16 16:29:44 -070061using hardware::fromHeap;
62using hardware::HidlMemory;
63
Pawin Vongmasa36653902018-11-15 00:10:25 -080064using namespace hardware::cas::V1_0;
65using namespace hardware::cas::native::V1_0;
66
67using CasStatus = hardware::cas::V1_0::Status;
Robert Shih895fba92019-07-16 16:29:44 -070068using DrmBufferType = hardware::drm::V1_0::BufferType;
Pawin Vongmasa36653902018-11-15 00:10:25 -080069
Pawin Vongmasa36653902018-11-15 00:10:25 -080070namespace {
71
Wonsik Kim469c8342019-04-11 16:46:09 -070072constexpr size_t kSmoothnessFactor = 4;
73constexpr size_t kRenderingDepth = 3;
Pawin Vongmasa36653902018-11-15 00:10:25 -080074
Sungtak Leeab6f2f32019-02-15 14:43:51 -080075// This is for keeping IGBP's buffer dropping logic in legacy mode other
76// than making it non-blocking. Do not change this value.
77const static size_t kDequeueTimeoutNs = 0;
78
Pawin Vongmasa36653902018-11-15 00:10:25 -080079} // namespace
80
81CCodecBufferChannel::QueueGuard::QueueGuard(
82 CCodecBufferChannel::QueueSync &sync) : mSync(sync) {
83 Mutex::Autolock l(mSync.mGuardLock);
84 // At this point it's guaranteed that mSync is not under state transition,
85 // as we are holding its mutex.
86
87 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
88 if (count->value == -1) {
89 mRunning = false;
90 } else {
91 ++count->value;
92 mRunning = true;
93 }
94}
95
96CCodecBufferChannel::QueueGuard::~QueueGuard() {
97 if (mRunning) {
98 // We are not holding mGuardLock at this point so that QueueSync::stop() can
99 // keep holding the lock until mCount reaches zero.
100 Mutexed<CCodecBufferChannel::QueueSync::Counter>::Locked count(mSync.mCount);
101 --count->value;
102 count->cond.broadcast();
103 }
104}
105
106void CCodecBufferChannel::QueueSync::start() {
107 Mutex::Autolock l(mGuardLock);
108 // If stopped, it goes to running state; otherwise no-op.
109 Mutexed<Counter>::Locked count(mCount);
110 if (count->value == -1) {
111 count->value = 0;
112 }
113}
114
115void CCodecBufferChannel::QueueSync::stop() {
116 Mutex::Autolock l(mGuardLock);
117 Mutexed<Counter>::Locked count(mCount);
118 if (count->value == -1) {
119 // no-op
120 return;
121 }
122 // Holding mGuardLock here blocks creation of additional QueueGuard objects, so
123 // mCount can only decrement. In other words, threads that acquired the lock
124 // are allowed to finish execution but additional threads trying to acquire
125 // the lock at this point will block, and then get QueueGuard at STOPPED
126 // state.
127 while (count->value != 0) {
128 count.waitForCondition(count->cond);
129 }
130 count->value = -1;
131}
132
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700133// Input
134
135CCodecBufferChannel::Input::Input() : extraBuffers("extra") {}
136
Pawin Vongmasa36653902018-11-15 00:10:25 -0800137// CCodecBufferChannel
138
139CCodecBufferChannel::CCodecBufferChannel(
140 const std::shared_ptr<CCodecCallback> &callback)
141 : mHeapSeqNum(-1),
142 mCCodecCallback(callback),
143 mFrameIndex(0u),
144 mFirstValidFrameIndex(0u),
145 mMetaMode(MODE_NONE),
Sungtak Lee04b30352020-07-27 13:57:25 -0700146 mInputMetEos(false),
147 mSendEncryptedInfoBuffer(false) {
Sungtak Leed7463d12019-09-04 16:01:00 -0700148 mOutputSurface.lock()->maxDequeueBuffers = kSmoothnessFactor + kRenderingDepth;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700149 {
150 Mutexed<Input>::Locked input(mInput);
151 input->buffers.reset(new DummyInputBuffers(""));
152 input->extraBuffers.flush();
153 input->inputDelay = 0u;
154 input->pipelineDelay = 0u;
155 input->numSlots = kSmoothnessFactor;
156 input->numExtraSlots = 0u;
157 }
158 {
159 Mutexed<Output>::Locked output(mOutput);
160 output->outputDelay = 0u;
161 output->numSlots = kSmoothnessFactor;
162 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800163}
164
165CCodecBufferChannel::~CCodecBufferChannel() {
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800166 if (mCrypto != nullptr && mHeapSeqNum >= 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800167 mCrypto->unsetHeap(mHeapSeqNum);
168 }
169}
170
171void CCodecBufferChannel::setComponent(
172 const std::shared_ptr<Codec2Client::Component> &component) {
173 mComponent = component;
174 mComponentName = component->getName() + StringPrintf("#%d", int(uintptr_t(component.get()) % 997));
175 mName = mComponentName.c_str();
176}
177
178status_t CCodecBufferChannel::setInputSurface(
179 const std::shared_ptr<InputSurfaceWrapper> &surface) {
180 ALOGV("[%s] setInputSurface", mName);
181 mInputSurface = surface;
182 return mInputSurface->connect(mComponent);
183}
184
185status_t CCodecBufferChannel::signalEndOfInputStream() {
186 if (mInputSurface == nullptr) {
187 return INVALID_OPERATION;
188 }
189 return mInputSurface->signalEndOfInputStream();
190}
191
Sungtak Lee04b30352020-07-27 13:57:25 -0700192status_t CCodecBufferChannel::queueInputBufferInternal(
193 sp<MediaCodecBuffer> buffer,
194 std::shared_ptr<C2LinearBlock> encryptedBlock,
195 size_t blockSize) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800196 int64_t timeUs;
197 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
198
199 if (mInputMetEos) {
200 ALOGD("[%s] buffers after EOS ignored (%lld us)", mName, (long long)timeUs);
201 return OK;
202 }
203
204 int32_t flags = 0;
205 int32_t tmp = 0;
206 bool eos = false;
207 if (buffer->meta()->findInt32("eos", &tmp) && tmp) {
208 eos = true;
209 mInputMetEos = true;
210 ALOGV("[%s] input EOS", mName);
211 }
212 if (buffer->meta()->findInt32("csd", &tmp) && tmp) {
213 flags |= C2FrameData::FLAG_CODEC_CONFIG;
214 }
215 ALOGV("[%s] queueInputBuffer: buffer->size() = %zu", mName, buffer->size());
Wonsik Kime1104ca2020-11-24 15:01:33 -0800216 std::list<std::unique_ptr<C2Work>> items;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800217 std::unique_ptr<C2Work> work(new C2Work);
218 work->input.ordinal.timestamp = timeUs;
219 work->input.ordinal.frameIndex = mFrameIndex++;
220 // WORKAROUND: until codecs support handling work after EOS and max output sizing, use timestamp
221 // manipulation to achieve image encoding via video codec, and to constrain encoded output.
222 // Keep client timestamp in customOrdinal
223 work->input.ordinal.customOrdinal = timeUs;
224 work->input.buffers.clear();
225
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700226 sp<Codec2Buffer> copy;
Wonsik Kime1104ca2020-11-24 15:01:33 -0800227 bool usesFrameReassembler = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800228
Pawin Vongmasa36653902018-11-15 00:10:25 -0800229 if (buffer->size() > 0u) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700230 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800231 std::shared_ptr<C2Buffer> c2buffer;
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700232 if (!input->buffers->releaseBuffer(buffer, &c2buffer, false)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800233 return -ENOENT;
234 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700235 // TODO: we want to delay copying buffers.
236 if (input->extraBuffers.numComponentBuffers() < input->numExtraSlots) {
237 copy = input->buffers->cloneAndReleaseBuffer(buffer);
238 if (copy != nullptr) {
239 (void)input->extraBuffers.assignSlot(copy);
240 if (!input->extraBuffers.releaseSlot(copy, &c2buffer, false)) {
241 return UNKNOWN_ERROR;
242 }
243 bool released = input->buffers->releaseBuffer(buffer, nullptr, true);
244 ALOGV("[%s] queueInputBuffer: buffer copied; %sreleased",
245 mName, released ? "" : "not ");
246 buffer.clear();
247 } else {
248 ALOGW("[%s] queueInputBuffer: failed to copy a buffer; this may cause input "
249 "buffer starvation on component.", mName);
250 }
251 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800252 if (input->frameReassembler) {
253 usesFrameReassembler = true;
254 input->frameReassembler.process(buffer, &items);
255 } else {
Byeongjo Park25c3a3d2020-06-12 17:24:21 +0900256 int32_t cvo = 0;
257 if (buffer->meta()->findInt32("cvo", &cvo)) {
258 int32_t rotation = cvo % 360;
259 // change rotation to counter-clock wise.
260 rotation = ((rotation <= 0) ? 0 : 360) - rotation;
261
262 Mutexed<OutputSurface>::Locked output(mOutputSurface);
263 uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
264 output->rotation[frameIndex] = rotation;
265 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800266 work->input.buffers.push_back(c2buffer);
267 if (encryptedBlock) {
268 work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
269 kParamIndexEncryptedBuffer,
270 encryptedBlock->share(0, blockSize, C2Fence())));
271 }
Sungtak Lee04b30352020-07-27 13:57:25 -0700272 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800273 } else if (eos) {
274 flags |= C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800275 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800276 if (usesFrameReassembler) {
277 if (!items.empty()) {
278 items.front()->input.configUpdate = std::move(mParamsToBeSet);
279 mFrameIndex = (items.back()->input.ordinal.frameIndex + 1).peek();
280 }
281 } else {
282 work->input.flags = (C2FrameData::flags_t)flags;
283 // TODO: fill info's
Pawin Vongmasa36653902018-11-15 00:10:25 -0800284
Wonsik Kime1104ca2020-11-24 15:01:33 -0800285 work->input.configUpdate = std::move(mParamsToBeSet);
286 work->worklets.clear();
287 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800288
Wonsik Kime1104ca2020-11-24 15:01:33 -0800289 items.push_back(std::move(work));
290
291 eos = eos && buffer->size() > 0u;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800292 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800293 if (eos) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800294 work.reset(new C2Work);
295 work->input.ordinal.timestamp = timeUs;
296 work->input.ordinal.frameIndex = mFrameIndex++;
297 // WORKAROUND: keep client timestamp in customOrdinal
298 work->input.ordinal.customOrdinal = timeUs;
299 work->input.buffers.clear();
300 work->input.flags = C2FrameData::FLAG_END_OF_STREAM;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800301 work->worklets.emplace_back(new C2Worklet);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800302 items.push_back(std::move(work));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800303 }
Wonsik Kime1104ca2020-11-24 15:01:33 -0800304 c2_status_t err = C2_OK;
305 if (!items.empty()) {
306 {
307 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
308 PipelineWatcher::Clock::time_point now = PipelineWatcher::Clock::now();
309 for (const std::unique_ptr<C2Work> &work : items) {
310 watcher->onWorkQueued(
311 work->input.ordinal.frameIndex.peeku(),
312 std::vector(work->input.buffers),
313 now);
314 }
315 }
316 err = mComponent->queue(&items);
317 }
318 if (err != C2_OK) {
319 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
320 for (const std::unique_ptr<C2Work> &work : items) {
321 watcher->onWorkDone(work->input.ordinal.frameIndex.peeku());
322 }
323 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700324 Mutexed<Input>::Locked input(mInput);
325 bool released = false;
326 if (buffer) {
327 released = input->buffers->releaseBuffer(buffer, nullptr, true);
328 } else if (copy) {
329 released = input->extraBuffers.releaseSlot(copy, nullptr, true);
330 }
331 ALOGV("[%s] queueInputBuffer: buffer%s %sreleased",
332 mName, (buffer == nullptr) ? "(copy)" : "", released ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800333 }
334
335 feedInputBufferIfAvailableInternal();
336 return err;
337}
338
339status_t CCodecBufferChannel::setParameters(std::vector<std::unique_ptr<C2Param>> &params) {
340 QueueGuard guard(mSync);
341 if (!guard.isRunning()) {
342 ALOGD("[%s] setParameters is only supported in the running state.", mName);
343 return -ENOSYS;
344 }
345 mParamsToBeSet.insert(mParamsToBeSet.end(),
346 std::make_move_iterator(params.begin()),
347 std::make_move_iterator(params.end()));
348 params.clear();
349 return OK;
350}
351
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800352status_t CCodecBufferChannel::attachBuffer(
353 const std::shared_ptr<C2Buffer> &c2Buffer,
354 const sp<MediaCodecBuffer> &buffer) {
355 if (!buffer->copy(c2Buffer)) {
356 return -ENOSYS;
357 }
358 return OK;
359}
360
361void CCodecBufferChannel::ensureDecryptDestination(size_t size) {
362 if (!mDecryptDestination || mDecryptDestination->size() < size) {
363 sp<IMemoryHeap> heap{new MemoryHeapBase(size * 2)};
364 if (mDecryptDestination && mCrypto && mHeapSeqNum >= 0) {
365 mCrypto->unsetHeap(mHeapSeqNum);
366 }
367 mDecryptDestination = new MemoryBase(heap, 0, size * 2);
368 if (mCrypto) {
369 mHeapSeqNum = mCrypto->setHeap(hardware::fromHeap(heap));
370 }
371 }
372}
373
374int32_t CCodecBufferChannel::getHeapSeqNum(const sp<HidlMemory> &memory) {
375 CHECK(mCrypto);
376 auto it = mHeapSeqNumMap.find(memory);
377 int32_t heapSeqNum = -1;
378 if (it == mHeapSeqNumMap.end()) {
379 heapSeqNum = mCrypto->setHeap(memory);
380 mHeapSeqNumMap.emplace(memory, heapSeqNum);
381 } else {
382 heapSeqNum = it->second;
383 }
384 return heapSeqNum;
385}
386
387status_t CCodecBufferChannel::attachEncryptedBuffer(
388 const sp<hardware::HidlMemory> &memory,
389 bool secure,
390 const uint8_t *key,
391 const uint8_t *iv,
392 CryptoPlugin::Mode mode,
393 CryptoPlugin::Pattern pattern,
394 size_t offset,
395 const CryptoPlugin::SubSample *subSamples,
396 size_t numSubSamples,
397 const sp<MediaCodecBuffer> &buffer) {
398 static const C2MemoryUsage kSecureUsage{C2MemoryUsage::READ_PROTECTED, 0};
399 static const C2MemoryUsage kDefaultReadWriteUsage{
400 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
401
402 size_t size = 0;
403 for (size_t i = 0; i < numSubSamples; ++i) {
404 size += subSamples[i].mNumBytesOfClearData + subSamples[i].mNumBytesOfEncryptedData;
405 }
406 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
407 std::shared_ptr<C2LinearBlock> block;
408 c2_status_t err = pool->fetchLinearBlock(
409 size,
410 secure ? kSecureUsage : kDefaultReadWriteUsage,
411 &block);
412 if (err != C2_OK) {
413 return NO_MEMORY;
414 }
415 if (!secure) {
416 ensureDecryptDestination(size);
417 }
418 ssize_t result = -1;
419 ssize_t codecDataOffset = 0;
420 if (mCrypto) {
421 AString errorDetailMsg;
422 int32_t heapSeqNum = getHeapSeqNum(memory);
423 hardware::drm::V1_0::SharedBuffer src{(uint32_t)heapSeqNum, offset, size};
424 hardware::drm::V1_0::DestinationBuffer dst;
425 if (secure) {
426 dst.type = DrmBufferType::NATIVE_HANDLE;
427 dst.secureMemory = hardware::hidl_handle(block->handle());
428 } else {
429 dst.type = DrmBufferType::SHARED_MEMORY;
430 IMemoryToSharedBuffer(
431 mDecryptDestination, mHeapSeqNum, &dst.nonsecureMemory);
432 }
433 result = mCrypto->decrypt(
434 key, iv, mode, pattern, src, 0, subSamples, numSubSamples,
435 dst, &errorDetailMsg);
436 if (result < 0) {
437 return result;
438 }
439 if (dst.type == DrmBufferType::SHARED_MEMORY) {
440 C2WriteView view = block->map().get();
441 if (view.error() != C2_OK) {
442 return false;
443 }
444 if (view.size() < result) {
445 return false;
446 }
447 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
448 }
449 } else {
450 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
451 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
452 hidl_vec<SubSample> hidlSubSamples;
453 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
454
455 hardware::cas::native::V1_0::SharedBuffer src{*memory, offset, size};
456 hardware::cas::native::V1_0::DestinationBuffer dst;
457 if (secure) {
458 dst.type = BufferType::NATIVE_HANDLE;
459 dst.secureMemory = hardware::hidl_handle(block->handle());
460 } else {
461 dst.type = BufferType::SHARED_MEMORY;
462 dst.nonsecureMemory = src;
463 }
464
465 CasStatus status = CasStatus::OK;
466 hidl_string detailedError;
467 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
468
469 if (key != nullptr) {
470 sctrl = (ScramblingControl)key[0];
471 // Adjust for the PES offset
472 codecDataOffset = key[2] | (key[3] << 8);
473 }
474
475 auto returnVoid = mDescrambler->descramble(
476 sctrl,
477 hidlSubSamples,
478 src,
479 0,
480 dst,
481 0,
482 [&status, &result, &detailedError] (
483 CasStatus _status, uint32_t _bytesWritten,
484 const hidl_string& _detailedError) {
485 status = _status;
486 result = (ssize_t)_bytesWritten;
487 detailedError = _detailedError;
488 });
489
490 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
491 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
492 mName, returnVoid.description().c_str(), status, result);
493 return UNKNOWN_ERROR;
494 }
495
496 if (result < codecDataOffset) {
497 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
498 return BAD_VALUE;
499 }
500 }
501 if (!secure) {
502 C2WriteView view = block->map().get();
503 if (view.error() != C2_OK) {
504 return UNKNOWN_ERROR;
505 }
506 if (view.size() < result) {
507 return UNKNOWN_ERROR;
508 }
509 memcpy(view.data(), mDecryptDestination->unsecurePointer(), result);
510 }
511 std::shared_ptr<C2Buffer> c2Buffer{C2Buffer::CreateLinearBuffer(
512 block->share(codecDataOffset, result - codecDataOffset, C2Fence{}))};
513 if (!buffer->copy(c2Buffer)) {
514 return -ENOSYS;
515 }
516 return OK;
517}
518
Pawin Vongmasa36653902018-11-15 00:10:25 -0800519status_t CCodecBufferChannel::queueInputBuffer(const sp<MediaCodecBuffer> &buffer) {
520 QueueGuard guard(mSync);
521 if (!guard.isRunning()) {
522 ALOGD("[%s] No more buffers should be queued at current state.", mName);
523 return -ENOSYS;
524 }
525 return queueInputBufferInternal(buffer);
526}
527
528status_t CCodecBufferChannel::queueSecureInputBuffer(
529 const sp<MediaCodecBuffer> &buffer, bool secure, const uint8_t *key,
530 const uint8_t *iv, CryptoPlugin::Mode mode, CryptoPlugin::Pattern pattern,
531 const CryptoPlugin::SubSample *subSamples, size_t numSubSamples,
532 AString *errorDetailMsg) {
533 QueueGuard guard(mSync);
534 if (!guard.isRunning()) {
535 ALOGD("[%s] No more buffers should be queued at current state.", mName);
536 return -ENOSYS;
537 }
538
539 if (!hasCryptoOrDescrambler()) {
540 return -ENOSYS;
541 }
542 sp<EncryptedLinearBlockBuffer> encryptedBuffer((EncryptedLinearBlockBuffer *)buffer.get());
543
Sungtak Lee04b30352020-07-27 13:57:25 -0700544 std::shared_ptr<C2LinearBlock> block;
545 size_t allocSize = buffer->size();
546 size_t bufferSize = 0;
547 c2_status_t blockRes = C2_OK;
548 bool copied = false;
549 if (mSendEncryptedInfoBuffer) {
550 static const C2MemoryUsage kDefaultReadWriteUsage{
551 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
552 constexpr int kAllocGranule0 = 1024 * 64;
553 constexpr int kAllocGranule1 = 1024 * 1024;
554 std::shared_ptr<C2BlockPool> pool = mBlockPools.lock()->inputPool;
555 // round up encrypted sizes to limit fragmentation and encourage buffer reuse
556 if (allocSize <= kAllocGranule1) {
557 bufferSize = align(allocSize, kAllocGranule0);
558 } else {
559 bufferSize = align(allocSize, kAllocGranule1);
560 }
561 blockRes = pool->fetchLinearBlock(
562 bufferSize, kDefaultReadWriteUsage, &block);
563
564 if (blockRes == C2_OK) {
565 C2WriteView view = block->map().get();
566 if (view.error() == C2_OK && view.size() == bufferSize) {
567 copied = true;
568 // TODO: only copy clear sections
569 memcpy(view.data(), buffer->data(), allocSize);
570 }
571 }
572 }
573
574 if (!copied) {
575 block.reset();
576 }
577
Pawin Vongmasa36653902018-11-15 00:10:25 -0800578 ssize_t result = -1;
579 ssize_t codecDataOffset = 0;
Wonsik Kim557c88c2020-03-13 11:03:52 -0700580 if (numSubSamples == 1
581 && subSamples[0].mNumBytesOfClearData == 0
582 && subSamples[0].mNumBytesOfEncryptedData == 0) {
583 // We don't need to go through crypto or descrambler if the input is empty.
584 result = 0;
585 } else if (mCrypto != nullptr) {
Robert Shih895fba92019-07-16 16:29:44 -0700586 hardware::drm::V1_0::DestinationBuffer destination;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800587 if (secure) {
Robert Shih895fba92019-07-16 16:29:44 -0700588 destination.type = DrmBufferType::NATIVE_HANDLE;
589 destination.secureMemory = hidl_handle(encryptedBuffer->handle());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800590 } else {
Robert Shih895fba92019-07-16 16:29:44 -0700591 destination.type = DrmBufferType::SHARED_MEMORY;
592 IMemoryToSharedBuffer(
593 mDecryptDestination, mHeapSeqNum, &destination.nonsecureMemory);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800594 }
Robert Shih895fba92019-07-16 16:29:44 -0700595 hardware::drm::V1_0::SharedBuffer source;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800596 encryptedBuffer->fillSourceBuffer(&source);
597 result = mCrypto->decrypt(
598 key, iv, mode, pattern, source, buffer->offset(),
599 subSamples, numSubSamples, destination, errorDetailMsg);
600 if (result < 0) {
Wonsik Kim557c88c2020-03-13 11:03:52 -0700601 ALOGI("[%s] decrypt failed: result=%zd", mName, result);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800602 return result;
603 }
Robert Shih895fba92019-07-16 16:29:44 -0700604 if (destination.type == DrmBufferType::SHARED_MEMORY) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800605 encryptedBuffer->copyDecryptedContent(mDecryptDestination, result);
606 }
607 } else {
608 // Here we cast CryptoPlugin::SubSample to hardware::cas::native::V1_0::SubSample
609 // directly, the structure definitions should match as checked in DescramblerImpl.cpp.
610 hidl_vec<SubSample> hidlSubSamples;
611 hidlSubSamples.setToExternal((SubSample *)subSamples, numSubSamples, false /*own*/);
612
613 hardware::cas::native::V1_0::SharedBuffer srcBuffer;
614 encryptedBuffer->fillSourceBuffer(&srcBuffer);
615
616 DestinationBuffer dstBuffer;
617 if (secure) {
618 dstBuffer.type = BufferType::NATIVE_HANDLE;
619 dstBuffer.secureMemory = hidl_handle(encryptedBuffer->handle());
620 } else {
621 dstBuffer.type = BufferType::SHARED_MEMORY;
622 dstBuffer.nonsecureMemory = srcBuffer;
623 }
624
625 CasStatus status = CasStatus::OK;
626 hidl_string detailedError;
627 ScramblingControl sctrl = ScramblingControl::UNSCRAMBLED;
628
629 if (key != nullptr) {
630 sctrl = (ScramblingControl)key[0];
631 // Adjust for the PES offset
632 codecDataOffset = key[2] | (key[3] << 8);
633 }
634
635 auto returnVoid = mDescrambler->descramble(
636 sctrl,
637 hidlSubSamples,
638 srcBuffer,
639 0,
640 dstBuffer,
641 0,
642 [&status, &result, &detailedError] (
643 CasStatus _status, uint32_t _bytesWritten,
644 const hidl_string& _detailedError) {
645 status = _status;
646 result = (ssize_t)_bytesWritten;
647 detailedError = _detailedError;
648 });
649
650 if (!returnVoid.isOk() || status != CasStatus::OK || result < 0) {
651 ALOGI("[%s] descramble failed, trans=%s, status=%d, result=%zd",
652 mName, returnVoid.description().c_str(), status, result);
653 return UNKNOWN_ERROR;
654 }
655
656 if (result < codecDataOffset) {
657 ALOGD("invalid codec data offset: %zd, result %zd", codecDataOffset, result);
658 return BAD_VALUE;
659 }
660
661 ALOGV("[%s] descramble succeeded, %zd bytes", mName, result);
662
663 if (dstBuffer.type == BufferType::SHARED_MEMORY) {
664 encryptedBuffer->copyDecryptedContentFromMemory(result);
665 }
666 }
667
668 buffer->setRange(codecDataOffset, result - codecDataOffset);
Sungtak Lee04b30352020-07-27 13:57:25 -0700669
670 return queueInputBufferInternal(buffer, block, bufferSize);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800671}
672
673void CCodecBufferChannel::feedInputBufferIfAvailable() {
674 QueueGuard guard(mSync);
675 if (!guard.isRunning()) {
676 ALOGV("[%s] We're not running --- no input buffer reported", mName);
677 return;
678 }
679 feedInputBufferIfAvailableInternal();
680}
681
682void CCodecBufferChannel::feedInputBufferIfAvailableInternal() {
Taehwan Kimda0517d2020-09-16 17:29:37 +0900683 if (mInputMetEos) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800684 return;
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700685 }
686 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700687 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasac3c536d2020-06-12 04:00:04 -0700688 if (!output->buffers ||
689 output->buffers->hasPending() ||
Wonsik Kim0487b782020-10-28 11:45:50 -0700690 output->buffers->numActiveSlots() >= output->numSlots) {
Wonsik Kimdf5dd142019-02-06 10:15:46 -0800691 return;
692 }
693 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700694 size_t numActiveSlots = 0;
695 while (!mPipelineWatcher.lock()->pipelineFull()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800696 sp<MediaCodecBuffer> inBuffer;
697 size_t index;
698 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700699 Mutexed<Input>::Locked input(mInput);
Wonsik Kim0487b782020-10-28 11:45:50 -0700700 numActiveSlots = input->buffers->numActiveSlots();
701 if (numActiveSlots >= input->numSlots) {
702 break;
Wonsik Kimab34ed62019-01-31 15:28:46 -0800703 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700704 if (!input->buffers->requestNewBuffer(&index, &inBuffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800705 ALOGV("[%s] no new buffer available", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800706 break;
707 }
708 }
709 ALOGV("[%s] new input index = %zu [%p]", mName, index, inBuffer.get());
710 mCallback->onInputBufferAvailable(index, inBuffer);
711 }
Wonsik Kim0487b782020-10-28 11:45:50 -0700712 ALOGV("[%s] # active slots after feedInputBufferIfAvailable = %zu", mName, numActiveSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800713}
714
715status_t CCodecBufferChannel::renderOutputBuffer(
716 const sp<MediaCodecBuffer> &buffer, int64_t timestampNs) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800717 ALOGV("[%s] renderOutputBuffer: %p", mName, buffer.get());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800718 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800719 bool released = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800720 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700721 Mutexed<Output>::Locked output(mOutput);
722 if (output->buffers) {
723 released = output->buffers->releaseBuffer(buffer, &c2Buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800724 }
725 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800726 // NOTE: some apps try to releaseOutputBuffer() with timestamp and/or render
727 // set to true.
728 sendOutputBuffers();
729 // input buffer feeding may have been gated by pending output buffers
730 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800731 if (!c2Buffer) {
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800732 if (released) {
Wonsik Kimf7529dd2019-04-18 17:35:53 -0700733 std::call_once(mRenderWarningFlag, [this] {
734 ALOGW("[%s] The app is calling releaseOutputBuffer() with "
735 "timestamp or render=true with non-video buffers. Apps should "
736 "call releaseOutputBuffer() with render=false for those.",
737 mName);
738 });
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800739 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800740 return INVALID_OPERATION;
741 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800742
743#if 0
744 const std::vector<std::shared_ptr<const C2Info>> infoParams = c2Buffer->info();
745 ALOGV("[%s] queuing gfx buffer with %zu infos", mName, infoParams.size());
746 for (const std::shared_ptr<const C2Info> &info : infoParams) {
747 AString res;
748 for (size_t ix = 0; ix + 3 < info->size(); ix += 4) {
749 if (ix) res.append(", ");
750 res.append(*((int32_t*)info.get() + (ix / 4)));
751 }
752 ALOGV(" [%s]", res.c_str());
753 }
754#endif
755 std::shared_ptr<const C2StreamRotationInfo::output> rotation =
756 std::static_pointer_cast<const C2StreamRotationInfo::output>(
757 c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
758 bool flip = rotation && (rotation->flip & 1);
759 uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
Byeongjo Park25c3a3d2020-06-12 17:24:21 +0900760
761 {
762 Mutexed<OutputSurface>::Locked output(mOutputSurface);
763 if (output->surface == nullptr) {
764 ALOGI("[%s] cannot render buffer without surface", mName);
765 return OK;
766 }
767 int64_t frameIndex;
768 buffer->meta()->findInt64("frameIndex", &frameIndex);
769 if (output->rotation.count(frameIndex) != 0) {
770 auto it = output->rotation.find(frameIndex);
771 quarters = (it->second / 90) & 3;
772 output->rotation.erase(it);
773 }
774 }
775
Pawin Vongmasa36653902018-11-15 00:10:25 -0800776 uint32_t transform = 0;
777 switch (quarters) {
778 case 0: // no rotation
779 transform = flip ? HAL_TRANSFORM_FLIP_H : 0;
780 break;
781 case 1: // 90 degrees counter-clockwise
782 transform = flip ? (HAL_TRANSFORM_FLIP_V | HAL_TRANSFORM_ROT_90)
783 : HAL_TRANSFORM_ROT_270;
784 break;
785 case 2: // 180 degrees
786 transform = flip ? HAL_TRANSFORM_FLIP_V : HAL_TRANSFORM_ROT_180;
787 break;
788 case 3: // 90 degrees clockwise
789 transform = flip ? (HAL_TRANSFORM_FLIP_H | HAL_TRANSFORM_ROT_90)
790 : HAL_TRANSFORM_ROT_90;
791 break;
792 }
793
794 std::shared_ptr<const C2StreamSurfaceScalingInfo::output> surfaceScaling =
795 std::static_pointer_cast<const C2StreamSurfaceScalingInfo::output>(
796 c2Buffer->getInfo(C2StreamSurfaceScalingInfo::output::PARAM_TYPE));
797 uint32_t videoScalingMode = NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW;
798 if (surfaceScaling) {
799 videoScalingMode = surfaceScaling->value;
800 }
801
802 // Use dataspace from format as it has the default aspects already applied
803 android_dataspace_t dataSpace = HAL_DATASPACE_UNKNOWN; // this is 0
804 (void)buffer->format()->findInt32("android._dataspace", (int32_t *)&dataSpace);
805
806 // HDR static info
807 std::shared_ptr<const C2StreamHdrStaticInfo::output> hdrStaticInfo =
808 std::static_pointer_cast<const C2StreamHdrStaticInfo::output>(
809 c2Buffer->getInfo(C2StreamHdrStaticInfo::output::PARAM_TYPE));
810
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800811 // HDR10 plus info
812 std::shared_ptr<const C2StreamHdr10PlusInfo::output> hdr10PlusInfo =
813 std::static_pointer_cast<const C2StreamHdr10PlusInfo::output>(
814 c2Buffer->getInfo(C2StreamHdr10PlusInfo::output::PARAM_TYPE));
Yichi Chen54be23c2020-06-15 14:30:53 +0800815 if (hdr10PlusInfo && hdr10PlusInfo->flexCount() == 0) {
816 hdr10PlusInfo.reset();
817 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800818
Pawin Vongmasa36653902018-11-15 00:10:25 -0800819 std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
820 if (blocks.size() != 1u) {
821 ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
822 return UNKNOWN_ERROR;
823 }
824 const C2ConstGraphicBlock &block = blocks.front();
825
826 // TODO: revisit this after C2Fence implementation.
827 android::IGraphicBufferProducer::QueueBufferInput qbi(
828 timestampNs,
829 false, // droppable
830 dataSpace,
831 Rect(blocks.front().crop().left,
832 blocks.front().crop().top,
833 blocks.front().crop().right(),
834 blocks.front().crop().bottom()),
835 videoScalingMode,
836 transform,
837 Fence::NO_FENCE, 0);
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800838 if (hdrStaticInfo || hdr10PlusInfo) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800839 HdrMetadata hdr;
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800840 if (hdrStaticInfo) {
wenchangliuf3f92882020-05-14 00:02:01 +0800841 // If mastering max and min luminance fields are 0, do not use them.
842 // It indicates the value may not be present in the stream.
843 if (hdrStaticInfo->mastering.maxLuminance > 0.0f &&
844 hdrStaticInfo->mastering.minLuminance > 0.0f) {
845 struct android_smpte2086_metadata smpte2086_meta = {
846 .displayPrimaryRed = {
847 hdrStaticInfo->mastering.red.x, hdrStaticInfo->mastering.red.y
848 },
849 .displayPrimaryGreen = {
850 hdrStaticInfo->mastering.green.x, hdrStaticInfo->mastering.green.y
851 },
852 .displayPrimaryBlue = {
853 hdrStaticInfo->mastering.blue.x, hdrStaticInfo->mastering.blue.y
854 },
855 .whitePoint = {
856 hdrStaticInfo->mastering.white.x, hdrStaticInfo->mastering.white.y
857 },
858 .maxLuminance = hdrStaticInfo->mastering.maxLuminance,
859 .minLuminance = hdrStaticInfo->mastering.minLuminance,
860 };
Yichi Chen54be23c2020-06-15 14:30:53 +0800861 hdr.validTypes |= HdrMetadata::SMPTE2086;
wenchangliuf3f92882020-05-14 00:02:01 +0800862 hdr.smpte2086 = smpte2086_meta;
863 }
Chong Zhang3bb2a7f2020-04-21 10:35:12 -0700864 // If the content light level fields are 0, do not use them, it
865 // indicates the value may not be present in the stream.
866 if (hdrStaticInfo->maxCll > 0.0f && hdrStaticInfo->maxFall > 0.0f) {
867 struct android_cta861_3_metadata cta861_meta = {
868 .maxContentLightLevel = hdrStaticInfo->maxCll,
869 .maxFrameAverageLightLevel = hdrStaticInfo->maxFall,
870 };
871 hdr.validTypes |= HdrMetadata::CTA861_3;
872 hdr.cta8613 = cta861_meta;
873 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800874 }
875 if (hdr10PlusInfo) {
876 hdr.validTypes |= HdrMetadata::HDR10PLUS;
877 hdr.hdr10plus.assign(
878 hdr10PlusInfo->m.value,
879 hdr10PlusInfo->m.value + hdr10PlusInfo->flexCount());
880 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800881 qbi.setHdrMetadata(hdr);
882 }
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800883 // we don't have dirty regions
884 qbi.setSurfaceDamage(Region::INVALID_REGION);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800885 android::IGraphicBufferProducer::QueueBufferOutput qbo;
886 status_t result = mComponent->queueToOutputSurface(block, qbi, &qbo);
887 if (result != OK) {
888 ALOGI("[%s] queueBuffer failed: %d", mName, result);
Sungtak Lee47c018a2020-11-07 01:02:49 -0800889 if (result == NO_INIT) {
890 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
891 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800892 return result;
893 }
894 ALOGV("[%s] queue buffer successful", mName);
895
896 int64_t mediaTimeUs = 0;
897 (void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
898 mCCodecCallback->onOutputFramesRendered(mediaTimeUs, timestampNs);
899
900 return OK;
901}
902
903status_t CCodecBufferChannel::discardBuffer(const sp<MediaCodecBuffer> &buffer) {
904 ALOGV("[%s] discardBuffer: %p", mName, buffer.get());
905 bool released = false;
906 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700907 Mutexed<Input>::Locked input(mInput);
908 if (input->buffers && input->buffers->releaseBuffer(buffer, nullptr, true)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800909 released = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800910 }
911 }
912 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700913 Mutexed<Output>::Locked output(mOutput);
914 if (output->buffers && output->buffers->releaseBuffer(buffer, nullptr)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800915 released = true;
916 }
917 }
918 if (released) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800919 sendOutputBuffers();
Pawin Vongmasa8be93112018-12-11 14:01:42 -0800920 feedInputBufferIfAvailable();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800921 } else {
922 ALOGD("[%s] MediaCodec discarded an unknown buffer", mName);
923 }
924 return OK;
925}
926
927void CCodecBufferChannel::getInputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
928 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700929 Mutexed<Input>::Locked input(mInput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800930
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700931 if (!input->buffers->isArrayMode()) {
932 input->buffers = input->buffers->toArrayMode(input->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800933 }
934
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700935 input->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800936}
937
938void CCodecBufferChannel::getOutputBufferArray(Vector<sp<MediaCodecBuffer>> *array) {
939 array->clear();
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700940 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800941
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700942 if (!output->buffers->isArrayMode()) {
943 output->buffers = output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800944 }
945
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700946 output->buffers->getArray(array);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800947}
948
949status_t CCodecBufferChannel::start(
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800950 const sp<AMessage> &inputFormat,
951 const sp<AMessage> &outputFormat,
952 bool buffersBoundToCodec) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800953 C2StreamBufferTypeSetting::input iStreamFormat(0u);
954 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kime1104ca2020-11-24 15:01:33 -0800955 C2ComponentKindSetting kind;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800956 C2PortReorderBufferDepthTuning::output reorderDepth;
957 C2PortReorderKeySetting::output reorderKey;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800958 C2PortActualDelayTuning::input inputDelay(0);
959 C2PortActualDelayTuning::output outputDelay(0);
960 C2ActualPipelineDelayTuning pipelineDelay(0);
Sungtak Lee04b30352020-07-27 13:57:25 -0700961 C2SecureModeTuning secureMode(C2Config::SM_UNPROTECTED);
Wonsik Kim078b58e2019-01-09 15:08:06 -0800962
Pawin Vongmasa36653902018-11-15 00:10:25 -0800963 c2_status_t err = mComponent->query(
964 {
965 &iStreamFormat,
966 &oStreamFormat,
Wonsik Kime1104ca2020-11-24 15:01:33 -0800967 &kind,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800968 &reorderDepth,
969 &reorderKey,
Wonsik Kim078b58e2019-01-09 15:08:06 -0800970 &inputDelay,
971 &pipelineDelay,
972 &outputDelay,
Sungtak Lee04b30352020-07-27 13:57:25 -0700973 &secureMode,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800974 },
975 {},
976 C2_DONT_BLOCK,
977 nullptr);
978 if (err == C2_BAD_INDEX) {
Wonsik Kime1104ca2020-11-24 15:01:33 -0800979 if (!iStreamFormat || !oStreamFormat || !kind) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800980 return UNKNOWN_ERROR;
981 }
982 } else if (err != C2_OK) {
983 return UNKNOWN_ERROR;
984 }
985
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -0800986 uint32_t inputDelayValue = inputDelay ? inputDelay.value : 0;
987 uint32_t pipelineDelayValue = pipelineDelay ? pipelineDelay.value : 0;
988 uint32_t outputDelayValue = outputDelay ? outputDelay.value : 0;
989
Wonsik Kim5ecf3832019-04-18 10:28:58 -0700990 size_t numInputSlots = inputDelayValue + pipelineDelayValue + kSmoothnessFactor;
991 size_t numOutputSlots = outputDelayValue + kSmoothnessFactor;
Wonsik Kim078b58e2019-01-09 15:08:06 -0800992
Pawin Vongmasa36653902018-11-15 00:10:25 -0800993 // TODO: get this from input format
994 bool secure = mComponent->getName().find(".secure") != std::string::npos;
995
Sungtak Lee04b30352020-07-27 13:57:25 -0700996 // secure mode is a static parameter (shall not change in the executing state)
997 mSendEncryptedInfoBuffer = secureMode.value == C2Config::SM_READ_PROTECTED_WITH_ENCRYPTED;
998
Pawin Vongmasa36653902018-11-15 00:10:25 -0800999 std::shared_ptr<C2AllocatorStore> allocatorStore = GetCodec2PlatformAllocatorStore();
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001000 int poolMask = GetCodec2PoolMask();
1001 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001002
1003 if (inputFormat != nullptr) {
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001004 bool graphic = (iStreamFormat.value == C2BufferData::GRAPHIC);
Wonsik Kime1104ca2020-11-24 15:01:33 -08001005 bool audioEncoder = !graphic && (kind.value == C2Component::KIND_ENCODER);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001006 C2Config::api_feature_t apiFeatures = C2Config::api_feature_t(
1007 API_REFLECTION |
1008 API_VALUES |
1009 API_CURRENT_VALUES |
1010 API_DEPENDENCY |
1011 API_SAME_INPUT_BUFFER);
Wonsik Kime1104ca2020-11-24 15:01:33 -08001012 C2StreamAudioFrameSizeInfo::input encoderFrameSize(0u);
1013 C2StreamSampleRateInfo::input sampleRate(0u);
1014 C2StreamChannelCountInfo::input channelCount(0u);
1015 C2StreamPcmEncodingInfo::input pcmEncoding(0u);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001016 std::shared_ptr<C2BlockPool> pool;
1017 {
1018 Mutexed<BlockPools>::Locked pools(mBlockPools);
1019
1020 // set default allocator ID.
1021 pools->inputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001022 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001023
1024 // query C2PortAllocatorsTuning::input from component. If an allocator ID is obtained
1025 // from component, create the input block pool with given ID. Otherwise, use default IDs.
1026 std::vector<std::unique_ptr<C2Param>> params;
Wonsik Kimffb889a2020-05-28 11:32:25 -07001027 C2ApiFeaturesSetting featuresSetting{apiFeatures};
Wonsik Kime1104ca2020-11-24 15:01:33 -08001028 std::vector<C2Param *> stackParams({&featuresSetting});
1029 if (audioEncoder) {
1030 stackParams.push_back(&encoderFrameSize);
1031 stackParams.push_back(&sampleRate);
1032 stackParams.push_back(&channelCount);
1033 stackParams.push_back(&pcmEncoding);
1034 } else {
1035 encoderFrameSize.invalidate();
1036 sampleRate.invalidate();
1037 channelCount.invalidate();
1038 pcmEncoding.invalidate();
1039 }
1040 err = mComponent->query(stackParams,
Pawin Vongmasa36653902018-11-15 00:10:25 -08001041 { C2PortAllocatorsTuning::input::PARAM_TYPE },
1042 C2_DONT_BLOCK,
1043 &params);
1044 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1045 ALOGD("[%s] Query input allocators returned %zu params => %s (%u)",
1046 mName, params.size(), asString(err), err);
Wonsik Kimffb889a2020-05-28 11:32:25 -07001047 } else if (params.size() == 1) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001048 C2PortAllocatorsTuning::input *inputAllocators =
1049 C2PortAllocatorsTuning::input::From(params[0].get());
1050 if (inputAllocators && inputAllocators->flexCount() > 0) {
1051 std::shared_ptr<C2Allocator> allocator;
1052 // verify allocator IDs and resolve default allocator
1053 allocatorStore->fetchAllocator(inputAllocators->m.values[0], &allocator);
1054 if (allocator) {
1055 pools->inputAllocatorId = allocator->getId();
1056 } else {
1057 ALOGD("[%s] component requested invalid input allocator ID %u",
1058 mName, inputAllocators->m.values[0]);
1059 }
1060 }
1061 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001062 if (featuresSetting) {
1063 apiFeatures = featuresSetting.value;
1064 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001065
1066 // TODO: use C2Component wrapper to associate this pool with ourselves
1067 if ((poolMask >> pools->inputAllocatorId) & 1) {
1068 err = CreateCodec2BlockPool(pools->inputAllocatorId, nullptr, &pool);
1069 ALOGD("[%s] Created input block pool with allocatorID %u => poolID %llu - %s (%d)",
1070 mName, pools->inputAllocatorId,
1071 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1072 asString(err), err);
1073 } else {
1074 err = C2_NOT_FOUND;
1075 }
1076 if (err != C2_OK) {
1077 C2BlockPool::local_id_t inputPoolId =
1078 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1079 err = GetCodec2BlockPool(inputPoolId, nullptr, &pool);
1080 ALOGD("[%s] Using basic input block pool with poolID %llu => got %llu - %s (%d)",
1081 mName, (unsigned long long)inputPoolId,
1082 (unsigned long long)(pool ? pool->getLocalId() : 111000111),
1083 asString(err), err);
1084 if (err != C2_OK) {
1085 return NO_MEMORY;
1086 }
1087 }
1088 pools->inputPool = pool;
1089 }
1090
Wonsik Kim51051262018-11-28 13:59:05 -08001091 bool forceArrayMode = false;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001092 Mutexed<Input>::Locked input(mInput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001093 input->inputDelay = inputDelayValue;
1094 input->pipelineDelay = pipelineDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001095 input->numSlots = numInputSlots;
1096 input->extraBuffers.flush();
1097 input->numExtraSlots = 0u;
Wonsik Kime1104ca2020-11-24 15:01:33 -08001098 if (audioEncoder && encoderFrameSize && sampleRate && channelCount) {
1099 input->frameReassembler.init(
1100 pool,
1101 {C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE},
1102 encoderFrameSize.value,
1103 sampleRate.value,
1104 channelCount.value,
1105 pcmEncoding ? pcmEncoding.value : C2Config::PCM_16);
1106 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07001107 bool conforming = (apiFeatures & API_SAME_INPUT_BUFFER);
1108 // For encrypted content, framework decrypts source buffer (ashmem) into
1109 // C2Buffers. Thus non-conforming codecs can process these.
Wonsik Kime1104ca2020-11-24 15:01:33 -08001110 if (!buffersBoundToCodec
1111 && !input->frameReassembler
1112 && (hasCryptoOrDescrambler() || conforming)) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001113 input->buffers.reset(new SlotInputBuffers(mName));
1114 } else if (graphic) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001115 if (mInputSurface) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001116 input->buffers.reset(new DummyInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001117 } else if (mMetaMode == MODE_ANW) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001118 input->buffers.reset(new GraphicMetadataInputBuffers(mName));
Wonsik Kim1221fd12019-07-12 12:52:05 -07001119 // This is to ensure buffers do not get released prematurely.
1120 // TODO: handle this without going into array mode
1121 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001122 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001123 input->buffers.reset(new GraphicInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001124 }
1125 } else {
1126 if (hasCryptoOrDescrambler()) {
1127 int32_t capacity = kLinearBufferSize;
1128 (void)inputFormat->findInt32(KEY_MAX_INPUT_SIZE, &capacity);
1129 if ((size_t)capacity > kMaxLinearBufferSize) {
1130 ALOGD("client requested %d, capped to %zu", capacity, kMaxLinearBufferSize);
1131 capacity = kMaxLinearBufferSize;
1132 }
1133 if (mDealer == nullptr) {
1134 mDealer = new MemoryDealer(
1135 align(capacity, MemoryDealer::getAllocationAlignment())
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001136 * (numInputSlots + 1),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001137 "EncryptedLinearInputBuffers");
1138 mDecryptDestination = mDealer->allocate((size_t)capacity);
1139 }
1140 if (mCrypto != nullptr && mHeapSeqNum < 0) {
Robert Shih895fba92019-07-16 16:29:44 -07001141 sp<HidlMemory> heap = fromHeap(mDealer->getMemoryHeap());
1142 mHeapSeqNum = mCrypto->setHeap(heap);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001143 } else {
1144 mHeapSeqNum = -1;
1145 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001146 input->buffers.reset(new EncryptedLinearInputBuffers(
Wonsik Kim078b58e2019-01-09 15:08:06 -08001147 secure, mDealer, mCrypto, mHeapSeqNum, (size_t)capacity,
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001148 numInputSlots, mName));
Wonsik Kim51051262018-11-28 13:59:05 -08001149 forceArrayMode = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001150 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001151 input->buffers.reset(new LinearInputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001152 }
1153 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001154 input->buffers->setFormat(inputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001155
1156 if (err == C2_OK) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001157 input->buffers->setPool(pool);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001158 } else {
1159 // TODO: error
1160 }
Wonsik Kim51051262018-11-28 13:59:05 -08001161
1162 if (forceArrayMode) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001163 input->buffers = input->buffers->toArrayMode(numInputSlots);
Wonsik Kim51051262018-11-28 13:59:05 -08001164 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001165 }
1166
1167 if (outputFormat != nullptr) {
1168 sp<IGraphicBufferProducer> outputSurface;
1169 uint32_t outputGeneration;
1170 {
1171 Mutexed<OutputSurface>::Locked output(mOutputSurface);
Sungtak Leed7463d12019-09-04 16:01:00 -07001172 output->maxDequeueBuffers = numOutputSlots +
Sungtak Lee7a7b7422019-07-16 17:40:40 -07001173 reorderDepth.value + kRenderingDepth;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001174 outputSurface = output->surface ?
1175 output->surface->getIGraphicBufferProducer() : nullptr;
Wonsik Kimf5e5c832019-02-21 11:36:05 -08001176 if (outputSurface) {
1177 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1178 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001179 outputGeneration = output->generation;
1180 }
1181
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001182 bool graphic = (oStreamFormat.value == C2BufferData::GRAPHIC);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001183 C2BlockPool::local_id_t outputPoolId_;
1184
1185 {
1186 Mutexed<BlockPools>::Locked pools(mBlockPools);
1187
1188 // set default allocator ID.
1189 pools->outputAllocatorId = (graphic) ? C2PlatformAllocatorStore::GRALLOC
Pin-chih Linaa18ea52019-11-19 18:48:50 +08001190 : preferredLinearId;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001191
1192 // query C2PortAllocatorsTuning::output from component, or use default allocator if
1193 // unsuccessful.
1194 std::vector<std::unique_ptr<C2Param>> params;
1195 err = mComponent->query({ },
1196 { C2PortAllocatorsTuning::output::PARAM_TYPE },
1197 C2_DONT_BLOCK,
1198 &params);
1199 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1200 ALOGD("[%s] Query output allocators returned %zu params => %s (%u)",
1201 mName, params.size(), asString(err), err);
1202 } else if (err == C2_OK && params.size() == 1) {
1203 C2PortAllocatorsTuning::output *outputAllocators =
1204 C2PortAllocatorsTuning::output::From(params[0].get());
1205 if (outputAllocators && outputAllocators->flexCount() > 0) {
1206 std::shared_ptr<C2Allocator> allocator;
1207 // verify allocator IDs and resolve default allocator
1208 allocatorStore->fetchAllocator(outputAllocators->m.values[0], &allocator);
1209 if (allocator) {
1210 pools->outputAllocatorId = allocator->getId();
1211 } else {
1212 ALOGD("[%s] component requested invalid output allocator ID %u",
1213 mName, outputAllocators->m.values[0]);
1214 }
1215 }
1216 }
1217
1218 // use bufferqueue if outputting to a surface.
1219 // query C2PortSurfaceAllocatorTuning::output from component, or use default allocator
1220 // if unsuccessful.
1221 if (outputSurface) {
1222 params.clear();
1223 err = mComponent->query({ },
1224 { C2PortSurfaceAllocatorTuning::output::PARAM_TYPE },
1225 C2_DONT_BLOCK,
1226 &params);
1227 if ((err != C2_OK && err != C2_BAD_INDEX) || params.size() != 1) {
1228 ALOGD("[%s] Query output surface allocator returned %zu params => %s (%u)",
1229 mName, params.size(), asString(err), err);
1230 } else if (err == C2_OK && params.size() == 1) {
1231 C2PortSurfaceAllocatorTuning::output *surfaceAllocator =
1232 C2PortSurfaceAllocatorTuning::output::From(params[0].get());
1233 if (surfaceAllocator) {
1234 std::shared_ptr<C2Allocator> allocator;
1235 // verify allocator IDs and resolve default allocator
1236 allocatorStore->fetchAllocator(surfaceAllocator->value, &allocator);
1237 if (allocator) {
1238 pools->outputAllocatorId = allocator->getId();
1239 } else {
1240 ALOGD("[%s] component requested invalid surface output allocator ID %u",
1241 mName, surfaceAllocator->value);
1242 err = C2_BAD_VALUE;
1243 }
1244 }
1245 }
1246 if (pools->outputAllocatorId == C2PlatformAllocatorStore::GRALLOC
1247 && err != C2_OK
1248 && ((poolMask >> C2PlatformAllocatorStore::BUFFERQUEUE) & 1)) {
1249 pools->outputAllocatorId = C2PlatformAllocatorStore::BUFFERQUEUE;
1250 }
1251 }
1252
1253 if ((poolMask >> pools->outputAllocatorId) & 1) {
1254 err = mComponent->createBlockPool(
1255 pools->outputAllocatorId, &pools->outputPoolId, &pools->outputPoolIntf);
1256 ALOGI("[%s] Created output block pool with allocatorID %u => poolID %llu - %s",
1257 mName, pools->outputAllocatorId,
1258 (unsigned long long)pools->outputPoolId,
1259 asString(err));
1260 } else {
1261 err = C2_NOT_FOUND;
1262 }
1263 if (err != C2_OK) {
1264 // use basic pool instead
1265 pools->outputPoolId =
1266 graphic ? C2BlockPool::BASIC_GRAPHIC : C2BlockPool::BASIC_LINEAR;
1267 }
1268
1269 // Configure output block pool ID as parameter C2PortBlockPoolsTuning::output to
1270 // component.
1271 std::unique_ptr<C2PortBlockPoolsTuning::output> poolIdsTuning =
1272 C2PortBlockPoolsTuning::output::AllocUnique({ pools->outputPoolId });
1273
1274 std::vector<std::unique_ptr<C2SettingResult>> failures;
1275 err = mComponent->config({ poolIdsTuning.get() }, C2_MAY_BLOCK, &failures);
1276 ALOGD("[%s] Configured output block pool ids %llu => %s",
1277 mName, (unsigned long long)poolIdsTuning->m.values[0], asString(err));
1278 outputPoolId_ = pools->outputPoolId;
1279 }
1280
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001281 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimbdffead2019-07-01 12:00:07 -07001282 output->outputDelay = outputDelayValue;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001283 output->numSlots = numOutputSlots;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001284 if (graphic) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001285 if (outputSurface || !buffersBoundToCodec) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001286 output->buffers.reset(new GraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001287 } else {
Wonsik Kim41d83432020-04-27 16:40:49 -07001288 output->buffers.reset(new RawGraphicOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001289 }
1290 } else {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001291 output->buffers.reset(new LinearOutputBuffers(mName));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001292 }
Wonsik Kime4716c02020-02-28 10:42:21 -08001293 output->buffers->setFormat(outputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001294
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001295 output->buffers->clearStash();
1296 if (reorderDepth) {
1297 output->buffers->setReorderDepth(reorderDepth.value);
1298 }
1299 if (reorderKey) {
1300 output->buffers->setReorderKey(reorderKey.value);
1301 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001302
1303 // Try to set output surface to created block pool if given.
1304 if (outputSurface) {
1305 mComponent->setOutputSurface(
1306 outputPoolId_,
1307 outputSurface,
1308 outputGeneration);
1309 }
1310
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001311 if (oStreamFormat.value == C2BufferData::LINEAR) {
Wonsik Kim58713302020-01-29 22:25:23 -08001312 if (buffersBoundToCodec) {
1313 // WORKAROUND: if we're using early CSD workaround we convert to
1314 // array mode, to appease apps assuming the output
1315 // buffers to be of the same size.
1316 output->buffers = output->buffers->toArrayMode(numOutputSlots);
1317 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001318
1319 int32_t channelCount;
1320 int32_t sampleRate;
1321 if (outputFormat->findInt32(KEY_CHANNEL_COUNT, &channelCount)
1322 && outputFormat->findInt32(KEY_SAMPLE_RATE, &sampleRate)) {
1323 int32_t delay = 0;
1324 int32_t padding = 0;;
1325 if (!outputFormat->findInt32("encoder-delay", &delay)) {
1326 delay = 0;
1327 }
1328 if (!outputFormat->findInt32("encoder-padding", &padding)) {
1329 padding = 0;
1330 }
1331 if (delay || padding) {
1332 // We need write access to the buffers, and we're already in
1333 // array mode.
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001334 output->buffers->initSkipCutBuffer(delay, padding, sampleRate, channelCount);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001335 }
1336 }
1337 }
1338 }
1339
1340 // Set up pipeline control. This has to be done after mInputBuffers and
1341 // mOutputBuffers are initialized to make sure that lingering callbacks
1342 // about buffers from the previous generation do not interfere with the
1343 // newly initialized pipeline capacity.
1344
Wonsik Kimab34ed62019-01-31 15:28:46 -08001345 {
1346 Mutexed<PipelineWatcher>::Locked watcher(mPipelineWatcher);
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001347 watcher->inputDelay(inputDelayValue)
1348 .pipelineDelay(pipelineDelayValue)
1349 .outputDelay(outputDelayValue)
Wonsik Kimab34ed62019-01-31 15:28:46 -08001350 .smoothnessFactor(kSmoothnessFactor);
1351 watcher->flush();
1352 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001353
1354 mInputMetEos = false;
1355 mSync.start();
1356 return OK;
1357}
1358
1359status_t CCodecBufferChannel::requestInitialInputBuffers() {
1360 if (mInputSurface) {
1361 return OK;
1362 }
1363
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001364 C2StreamBufferTypeSetting::output oStreamFormat(0u);
Wonsik Kim8ab25aa2019-06-24 16:37:37 -07001365 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
1366 c2_status_t err = mComponent->query({ &oStreamFormat, &prepend }, {}, C2_DONT_BLOCK, nullptr);
1367 if (err != C2_OK && err != C2_BAD_INDEX) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001368 return UNKNOWN_ERROR;
1369 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001370 size_t numInputSlots = mInput.lock()->numSlots;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001371
1372 struct ClientInputBuffer {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001373 size_t index;
1374 sp<MediaCodecBuffer> buffer;
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001375 size_t capacity;
1376 };
1377 std::list<ClientInputBuffer> clientInputBuffers;
1378
1379 {
1380 Mutexed<Input>::Locked input(mInput);
1381 while (clientInputBuffers.size() < numInputSlots) {
1382 ClientInputBuffer clientInputBuffer;
1383 if (!input->buffers->requestNewBuffer(&clientInputBuffer.index,
1384 &clientInputBuffer.buffer)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001385 break;
1386 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001387 clientInputBuffer.capacity = clientInputBuffer.buffer->capacity();
1388 clientInputBuffers.emplace_back(std::move(clientInputBuffer));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001389 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001390 }
1391 if (clientInputBuffers.empty()) {
1392 ALOGW("[%s] start: cannot allocate memory at all", mName);
1393 return NO_MEMORY;
1394 } else if (clientInputBuffers.size() < numInputSlots) {
1395 ALOGD("[%s] start: cannot allocate memory for all slots, "
1396 "only %zu buffers allocated",
1397 mName, clientInputBuffers.size());
1398 } else {
1399 ALOGV("[%s] %zu initial input buffers available",
1400 mName, clientInputBuffers.size());
1401 }
1402 // Sort input buffers by their capacities in increasing order.
1403 clientInputBuffers.sort(
1404 [](const ClientInputBuffer& a, const ClientInputBuffer& b) {
1405 return a.capacity < b.capacity;
1406 });
1407
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001408 std::list<std::unique_ptr<C2Work>> flushedConfigs;
1409 mFlushedConfigs.lock()->swap(flushedConfigs);
1410 if (!flushedConfigs.empty()) {
1411 err = mComponent->queue(&flushedConfigs);
1412 if (err != C2_OK) {
1413 ALOGW("[%s] Error while queueing a flushed config", mName);
1414 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001415 }
1416 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001417 if (oStreamFormat.value == C2BufferData::LINEAR &&
1418 (!prepend || prepend.value == PREPEND_HEADER_TO_NONE)) {
1419 sp<MediaCodecBuffer> buffer = clientInputBuffers.front().buffer;
1420 // WORKAROUND: Some apps expect CSD available without queueing
1421 // any input. Queue an empty buffer to get the CSD.
1422 buffer->setRange(0, 0);
1423 buffer->meta()->clear();
1424 buffer->meta()->setInt64("timeUs", 0);
1425 if (queueInputBufferInternal(buffer) != OK) {
1426 ALOGW("[%s] Error while queueing an empty buffer to get CSD",
1427 mName);
1428 return UNKNOWN_ERROR;
1429 }
1430 clientInputBuffers.pop_front();
1431 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001432
1433 for (const ClientInputBuffer& clientInputBuffer: clientInputBuffers) {
1434 mCallback->onInputBufferAvailable(
1435 clientInputBuffer.index,
1436 clientInputBuffer.buffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001437 }
Pawin Vongmasae7bb8612020-06-04 06:15:22 -07001438
Pawin Vongmasa36653902018-11-15 00:10:25 -08001439 return OK;
1440}
1441
1442void CCodecBufferChannel::stop() {
1443 mSync.stop();
1444 mFirstValidFrameIndex = mFrameIndex.load(std::memory_order_relaxed);
1445 if (mInputSurface != nullptr) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001446 mInputSurface.reset();
1447 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001448 mPipelineWatcher.lock()->flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001449}
1450
Wonsik Kim936a89c2020-05-08 16:07:50 -07001451void CCodecBufferChannel::reset() {
1452 stop();
1453 {
1454 Mutexed<Input>::Locked input(mInput);
1455 input->buffers.reset(new DummyInputBuffers(""));
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001456 input->extraBuffers.flush();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001457 }
1458 {
1459 Mutexed<Output>::Locked output(mOutput);
1460 output->buffers.reset();
1461 }
1462}
1463
1464void CCodecBufferChannel::release() {
1465 mComponent.reset();
1466 mInputAllocator.reset();
1467 mOutputSurface.lock()->surface.clear();
1468 {
1469 Mutexed<BlockPools>::Locked blockPools{mBlockPools};
1470 blockPools->inputPool.reset();
1471 blockPools->outputPoolIntf.reset();
1472 }
Wonsik Kima2e3cdd2020-05-20 15:14:42 -07001473 setCrypto(nullptr);
1474 setDescrambler(nullptr);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001475}
1476
1477
Pawin Vongmasa36653902018-11-15 00:10:25 -08001478void CCodecBufferChannel::flush(const std::list<std::unique_ptr<C2Work>> &flushedWork) {
1479 ALOGV("[%s] flush", mName);
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001480 std::list<std::unique_ptr<C2Work>> configs;
1481 for (const std::unique_ptr<C2Work> &work : flushedWork) {
1482 if (!(work->input.flags & C2FrameData::FLAG_CODEC_CONFIG)) {
1483 continue;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001484 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001485 if (work->input.buffers.empty()
1486 || work->input.buffers.front() == nullptr
1487 || work->input.buffers.front()->data().linearBlocks().empty()) {
1488 ALOGD("[%s] no linear codec config data found", mName);
1489 continue;
1490 }
1491 std::unique_ptr<C2Work> copy(new C2Work);
1492 copy->input.flags = C2FrameData::flags_t(work->input.flags | C2FrameData::FLAG_DROP_FRAME);
1493 copy->input.ordinal = work->input.ordinal;
1494 copy->input.buffers.insert(
1495 copy->input.buffers.begin(),
1496 work->input.buffers.begin(),
1497 work->input.buffers.end());
1498 for (const std::unique_ptr<C2Param> &param : work->input.configUpdate) {
1499 copy->input.configUpdate.push_back(C2Param::Copy(*param));
1500 }
1501 copy->input.infoBuffers.insert(
1502 copy->input.infoBuffers.begin(),
1503 work->input.infoBuffers.begin(),
1504 work->input.infoBuffers.end());
1505 copy->worklets.emplace_back(new C2Worklet);
1506 configs.push_back(std::move(copy));
1507 ALOGV("[%s] stashed flushed codec config data", mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001508 }
Wonsik Kim5ebfcb22021-01-05 18:58:15 -08001509 mFlushedConfigs.lock()->swap(configs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001510 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001511 Mutexed<Input>::Locked input(mInput);
1512 input->buffers->flush();
1513 input->extraBuffers.flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001514 }
1515 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001516 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001517 if (output->buffers) {
1518 output->buffers->flush(flushedWork);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001519 output->buffers->flushStash();
Wonsik Kim936a89c2020-05-08 16:07:50 -07001520 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001521 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001522 mPipelineWatcher.lock()->flush();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001523}
1524
1525void CCodecBufferChannel::onWorkDone(
1526 std::unique_ptr<C2Work> work, const sp<AMessage> &outputFormat,
Wonsik Kimab34ed62019-01-31 15:28:46 -08001527 const C2StreamInitDataInfo::output *initData) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001528 if (handleWork(std::move(work), outputFormat, initData)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001529 feedInputBufferIfAvailable();
1530 }
1531}
1532
1533void CCodecBufferChannel::onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -08001534 uint64_t frameIndex, size_t arrayIndex) {
Pawin Vongmasa8e2cfb52019-05-15 05:20:52 -07001535 if (mInputSurface) {
1536 return;
1537 }
Wonsik Kimab34ed62019-01-31 15:28:46 -08001538 std::shared_ptr<C2Buffer> buffer =
1539 mPipelineWatcher.lock()->onInputBufferReleased(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001540 bool newInputSlotAvailable;
1541 {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001542 Mutexed<Input>::Locked input(mInput);
1543 newInputSlotAvailable = input->buffers->expireComponentBuffer(buffer);
1544 if (!newInputSlotAvailable) {
1545 (void)input->extraBuffers.expireComponentBuffer(buffer);
1546 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001547 }
1548 if (newInputSlotAvailable) {
1549 feedInputBufferIfAvailable();
1550 }
1551}
1552
1553bool CCodecBufferChannel::handleWork(
1554 std::unique_ptr<C2Work> work,
1555 const sp<AMessage> &outputFormat,
1556 const C2StreamInitDataInfo::output *initData) {
Wonsik Kim936a89c2020-05-08 16:07:50 -07001557 {
Wonsik Kima4e049d2020-04-28 19:42:23 +00001558 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001559 if (!output->buffers) {
1560 return false;
1561 }
Wonsik Kime75a5da2020-02-14 17:29:03 -08001562 }
1563
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001564 // Whether the output buffer should be reported to the client or not.
1565 bool notifyClient = false;
1566
1567 if (work->result == C2_OK){
1568 notifyClient = true;
1569 } else if (work->result == C2_NOT_FOUND) {
1570 ALOGD("[%s] flushed work; ignored.", mName);
1571 } else {
1572 // C2_OK and C2_NOT_FOUND are the only results that we accept for processing
1573 // the config update.
1574 ALOGD("[%s] work failed to complete: %d", mName, work->result);
1575 mCCodecCallback->onError(work->result, ACTION_CODE_FATAL);
1576 return false;
1577 }
1578
1579 if ((work->input.ordinal.frameIndex -
1580 mFirstValidFrameIndex.load()).peek() < 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001581 // Discard frames from previous generation.
1582 ALOGD("[%s] Discard frames from previous generation.", mName);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001583 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001584 }
1585
Wonsik Kim524b0582019-03-12 11:28:57 -07001586 if (mInputSurface == nullptr && (work->worklets.size() != 1u
Pawin Vongmasa36653902018-11-15 00:10:25 -08001587 || !work->worklets.front()
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001588 || !(work->worklets.front()->output.flags &
1589 C2FrameData::FLAG_INCOMPLETE))) {
1590 mPipelineWatcher.lock()->onWorkDone(
1591 work->input.ordinal.frameIndex.peeku());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001592 }
1593
1594 // NOTE: MediaCodec usage supposedly have only one worklet
1595 if (work->worklets.size() != 1u) {
1596 ALOGI("[%s] onWorkDone: incorrect number of worklets: %zu",
1597 mName, work->worklets.size());
1598 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1599 return false;
1600 }
1601
1602 const std::unique_ptr<C2Worklet> &worklet = work->worklets.front();
1603
1604 std::shared_ptr<C2Buffer> buffer;
1605 // NOTE: MediaCodec usage supposedly have only one output stream.
1606 if (worklet->output.buffers.size() > 1u) {
1607 ALOGI("[%s] onWorkDone: incorrect number of output buffers: %zu",
1608 mName, worklet->output.buffers.size());
1609 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1610 return false;
1611 } else if (worklet->output.buffers.size() == 1u) {
1612 buffer = worklet->output.buffers[0];
1613 if (!buffer) {
1614 ALOGD("[%s] onWorkDone: nullptr found in buffers; ignored.", mName);
1615 }
1616 }
1617
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001618 std::optional<uint32_t> newInputDelay, newPipelineDelay;
Wonsik Kim315e40a2020-09-09 14:11:50 -07001619 bool needMaxDequeueBufferCountUpdate = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001620 while (!worklet->output.configUpdate.empty()) {
1621 std::unique_ptr<C2Param> param;
1622 worklet->output.configUpdate.back().swap(param);
1623 worklet->output.configUpdate.pop_back();
1624 switch (param->coreIndex().coreIndex()) {
1625 case C2PortReorderBufferDepthTuning::CORE_INDEX: {
1626 C2PortReorderBufferDepthTuning::output reorderDepth;
1627 if (reorderDepth.updateFrom(*param)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001628 ALOGV("[%s] onWorkDone: updated reorder depth to %u",
1629 mName, reorderDepth.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001630 mOutput.lock()->buffers->setReorderDepth(reorderDepth.value);
1631 needMaxDequeueBufferCountUpdate = true;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001632 } else {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001633 ALOGD("[%s] onWorkDone: failed to read reorder depth",
1634 mName);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001635 }
1636 break;
1637 }
1638 case C2PortReorderKeySetting::CORE_INDEX: {
1639 C2PortReorderKeySetting::output reorderKey;
1640 if (reorderKey.updateFrom(*param)) {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001641 mOutput.lock()->buffers->setReorderKey(reorderKey.value);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001642 ALOGV("[%s] onWorkDone: updated reorder key to %u",
1643 mName, reorderKey.value);
1644 } else {
1645 ALOGD("[%s] onWorkDone: failed to read reorder key", mName);
1646 }
1647 break;
1648 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001649 case C2PortActualDelayTuning::CORE_INDEX: {
1650 if (param->isGlobal()) {
1651 C2ActualPipelineDelayTuning pipelineDelay;
1652 if (pipelineDelay.updateFrom(*param)) {
1653 ALOGV("[%s] onWorkDone: updating pipeline delay %u",
1654 mName, pipelineDelay.value);
1655 newPipelineDelay = pipelineDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001656 (void)mPipelineWatcher.lock()->pipelineDelay(
1657 pipelineDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001658 }
1659 }
1660 if (param->forInput()) {
1661 C2PortActualDelayTuning::input inputDelay;
1662 if (inputDelay.updateFrom(*param)) {
1663 ALOGV("[%s] onWorkDone: updating input delay %u",
1664 mName, inputDelay.value);
1665 newInputDelay = inputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001666 (void)mPipelineWatcher.lock()->inputDelay(
1667 inputDelay.value);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001668 }
1669 }
1670 if (param->forOutput()) {
1671 C2PortActualDelayTuning::output outputDelay;
1672 if (outputDelay.updateFrom(*param)) {
1673 ALOGV("[%s] onWorkDone: updating output delay %u",
1674 mName, outputDelay.value);
Wonsik Kim315e40a2020-09-09 14:11:50 -07001675 (void)mPipelineWatcher.lock()->outputDelay(outputDelay.value);
1676 needMaxDequeueBufferCountUpdate = true;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001677
1678 bool outputBuffersChanged = false;
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001679 size_t numOutputSlots = 0;
1680 {
1681 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001682 if (!output->buffers) {
1683 return false;
1684 }
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001685 output->outputDelay = outputDelay.value;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001686 numOutputSlots = outputDelay.value +
1687 kSmoothnessFactor;
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001688 if (output->numSlots < numOutputSlots) {
1689 output->numSlots = numOutputSlots;
1690 if (output->buffers->isArrayMode()) {
1691 OutputBuffersArray *array =
1692 (OutputBuffersArray *)output->buffers.get();
1693 ALOGV("[%s] onWorkDone: growing output buffer array to %zu",
1694 mName, numOutputSlots);
1695 array->grow(numOutputSlots);
1696 outputBuffersChanged = true;
1697 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001698 }
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001699 numOutputSlots = output->numSlots;
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001700 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001701
1702 if (outputBuffersChanged) {
1703 mCCodecCallback->onOutputBuffersChanged();
1704 }
1705 }
1706 }
1707 break;
1708 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001709 default:
1710 ALOGV("[%s] onWorkDone: unrecognized config update (%08X)",
1711 mName, param->index());
1712 break;
1713 }
1714 }
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001715 if (newInputDelay || newPipelineDelay) {
1716 Mutexed<Input>::Locked input(mInput);
1717 size_t newNumSlots =
1718 newInputDelay.value_or(input->inputDelay) +
1719 newPipelineDelay.value_or(input->pipelineDelay) +
1720 kSmoothnessFactor;
1721 if (input->buffers->isArrayMode()) {
1722 if (input->numSlots >= newNumSlots) {
1723 input->numExtraSlots = 0;
1724 } else {
1725 input->numExtraSlots = newNumSlots - input->numSlots;
1726 }
1727 ALOGV("[%s] onWorkDone: updated number of extra slots to %zu (input array mode)",
1728 mName, input->numExtraSlots);
1729 } else {
1730 input->numSlots = newNumSlots;
1731 }
1732 }
Wonsik Kim315e40a2020-09-09 14:11:50 -07001733 if (needMaxDequeueBufferCountUpdate) {
1734 size_t numOutputSlots = 0;
1735 uint32_t reorderDepth = 0;
1736 {
1737 Mutexed<Output>::Locked output(mOutput);
1738 numOutputSlots = output->numSlots;
1739 reorderDepth = output->buffers->getReorderDepth();
1740 }
1741 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1742 output->maxDequeueBuffers = numOutputSlots + reorderDepth + kRenderingDepth;
1743 if (output->surface) {
1744 output->surface->setMaxDequeuedBufferCount(output->maxDequeueBuffers);
1745 }
1746 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001747
Pawin Vongmasa36653902018-11-15 00:10:25 -08001748 int32_t flags = 0;
1749 if (worklet->output.flags & C2FrameData::FLAG_END_OF_STREAM) {
1750 flags |= MediaCodec::BUFFER_FLAG_EOS;
1751 ALOGV("[%s] onWorkDone: output EOS", mName);
1752 }
1753
Pawin Vongmasa36653902018-11-15 00:10:25 -08001754 // WORKAROUND: adjust output timestamp based on client input timestamp and codec
1755 // input timestamp. Codec output timestamp (in the timestamp field) shall correspond to
1756 // the codec input timestamp, but client output timestamp should (reported in timeUs)
1757 // shall correspond to the client input timesamp (in customOrdinal). By using the
1758 // delta between the two, this allows for some timestamp deviation - e.g. if one input
1759 // produces multiple output.
1760 c2_cntr64_t timestamp =
1761 worklet->output.ordinal.timestamp + work->input.ordinal.customOrdinal
1762 - work->input.ordinal.timestamp;
Wonsik Kim95ba0162019-03-19 15:51:54 -07001763 if (mInputSurface != nullptr) {
1764 // When using input surface we need to restore the original input timestamp.
1765 timestamp = work->input.ordinal.customOrdinal;
1766 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001767 ALOGV("[%s] onWorkDone: input %lld, codec %lld => output %lld => %lld",
1768 mName,
1769 work->input.ordinal.customOrdinal.peekll(),
1770 work->input.ordinal.timestamp.peekll(),
1771 worklet->output.ordinal.timestamp.peekll(),
1772 timestamp.peekll());
1773
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001774 // csd cannot be re-ordered and will always arrive first.
Pawin Vongmasa36653902018-11-15 00:10:25 -08001775 if (initData != nullptr) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001776 Mutexed<Output>::Locked output(mOutput);
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001777 if (output->buffers && outputFormat) {
1778 output->buffers->updateSkipCutBuffer(outputFormat);
1779 output->buffers->setFormat(outputFormat);
1780 }
1781 if (!notifyClient) {
1782 return false;
1783 }
1784 size_t index;
1785 sp<MediaCodecBuffer> outBuffer;
Wonsik Kim936a89c2020-05-08 16:07:50 -07001786 if (output->buffers && output->buffers->registerCsd(initData, &index, &outBuffer) == OK) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001787 outBuffer->meta()->setInt64("timeUs", timestamp.peek());
1788 outBuffer->meta()->setInt32("flags", MediaCodec::BUFFER_FLAG_CODECCONFIG);
1789 ALOGV("[%s] onWorkDone: csd index = %zu [%p]", mName, index, outBuffer.get());
1790
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001791 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001792 mCallback->onOutputBufferAvailable(index, outBuffer);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001793 } else {
1794 ALOGD("[%s] onWorkDone: unable to register csd", mName);
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001795 output.unlock();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001796 mCCodecCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001797 return false;
1798 }
1799 }
1800
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001801 if (notifyClient && !buffer && !flags) {
Wonsik Kim35bf5732020-05-14 17:40:29 +00001802 ALOGV("[%s] onWorkDone: Not reporting output buffer (%lld)",
Pawin Vongmasa36653902018-11-15 00:10:25 -08001803 mName, work->input.ordinal.frameIndex.peekull());
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001804 notifyClient = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001805 }
1806
1807 if (buffer) {
1808 for (const std::shared_ptr<const C2Info> &info : buffer->info()) {
1809 // TODO: properly translate these to metadata
1810 switch (info->coreIndex().coreIndex()) {
1811 case C2StreamPictureTypeMaskInfo::CORE_INDEX:
Lajos Molnar3bb81cd2019-02-20 15:10:30 -08001812 if (((C2StreamPictureTypeMaskInfo *)info.get())->value & C2Config::SYNC_FRAME) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001813 flags |= MediaCodec::BUFFER_FLAG_SYNCFRAME;
1814 }
1815 break;
1816 default:
1817 break;
1818 }
1819 }
1820 }
1821
1822 {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001823 Mutexed<Output>::Locked output(mOutput);
Wonsik Kimc23cc402020-05-28 14:53:40 -07001824 if (!output->buffers) {
1825 return false;
1826 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001827 output->buffers->pushToStash(
1828 buffer,
1829 notifyClient,
1830 timestamp.peek(),
1831 flags,
1832 outputFormat,
1833 worklet->output.ordinal);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001834 }
1835 sendOutputBuffers();
1836 return true;
1837}
1838
1839void CCodecBufferChannel::sendOutputBuffers() {
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001840 OutputBuffers::BufferAction action;
Wonsik Kima4e049d2020-04-28 19:42:23 +00001841 size_t index;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001842 sp<MediaCodecBuffer> outBuffer;
1843 std::shared_ptr<C2Buffer> c2Buffer;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001844
1845 while (true) {
Wonsik Kim5ecf3832019-04-18 10:28:58 -07001846 Mutexed<Output>::Locked output(mOutput);
Wonsik Kim936a89c2020-05-08 16:07:50 -07001847 if (!output->buffers) {
1848 return;
1849 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001850 action = output->buffers->popFromStashAndRegister(
1851 &c2Buffer, &index, &outBuffer);
1852 switch (action) {
1853 case OutputBuffers::SKIP:
1854 return;
1855 case OutputBuffers::DISCARD:
1856 break;
1857 case OutputBuffers::NOTIFY_CLIENT:
Wonsik Kima4e049d2020-04-28 19:42:23 +00001858 output.unlock();
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001859 mCallback->onOutputBufferAvailable(index, outBuffer);
1860 break;
1861 case OutputBuffers::REALLOCATE:
1862 if (!output->buffers->isArrayMode()) {
1863 output->buffers =
1864 output->buffers->toArrayMode(output->numSlots);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001865 }
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001866 static_cast<OutputBuffersArray*>(output->buffers.get())->
1867 realloc(c2Buffer);
1868 output.unlock();
1869 mCCodecCallback->onOutputBuffersChanged();
Wonsik Kim4ada73d2020-05-26 14:58:07 -07001870 break;
Pawin Vongmasa9b906982020-04-11 05:07:15 -07001871 case OutputBuffers::RETRY:
1872 ALOGV("[%s] sendOutputBuffers: unable to register output buffer",
1873 mName);
1874 return;
1875 default:
1876 LOG_ALWAYS_FATAL("[%s] sendOutputBuffers: "
1877 "corrupted BufferAction value (%d) "
1878 "returned from popFromStashAndRegister.",
1879 mName, int(action));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001880 return;
1881 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001882 }
1883}
1884
1885status_t CCodecBufferChannel::setSurface(const sp<Surface> &newSurface) {
1886 static std::atomic_uint32_t surfaceGeneration{0};
1887 uint32_t generation = (getpid() << 10) |
1888 ((surfaceGeneration.fetch_add(1, std::memory_order_relaxed) + 1)
1889 & ((1 << 10) - 1));
1890
1891 sp<IGraphicBufferProducer> producer;
1892 if (newSurface) {
1893 newSurface->setScalingMode(NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW);
Sungtak Leeab6f2f32019-02-15 14:43:51 -08001894 newSurface->setDequeueTimeout(kDequeueTimeoutNs);
Sungtak Lee08515812019-06-05 11:16:32 -07001895 newSurface->setMaxDequeuedBufferCount(mOutputSurface.lock()->maxDequeueBuffers);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001896 producer = newSurface->getIGraphicBufferProducer();
1897 producer->setGenerationNumber(generation);
1898 } else {
1899 ALOGE("[%s] setting output surface to null", mName);
1900 return INVALID_OPERATION;
1901 }
1902
1903 std::shared_ptr<Codec2Client::Configurable> outputPoolIntf;
1904 C2BlockPool::local_id_t outputPoolId;
1905 {
1906 Mutexed<BlockPools>::Locked pools(mBlockPools);
1907 outputPoolId = pools->outputPoolId;
1908 outputPoolIntf = pools->outputPoolIntf;
1909 }
1910
1911 if (outputPoolIntf) {
1912 if (mComponent->setOutputSurface(
1913 outputPoolId,
1914 producer,
1915 generation) != C2_OK) {
1916 ALOGI("[%s] setSurface: component setOutputSurface failed", mName);
1917 return INVALID_OPERATION;
1918 }
1919 }
1920
1921 {
1922 Mutexed<OutputSurface>::Locked output(mOutputSurface);
1923 output->surface = newSurface;
1924 output->generation = generation;
1925 }
1926
1927 return OK;
1928}
1929
Wonsik Kimab34ed62019-01-31 15:28:46 -08001930PipelineWatcher::Clock::duration CCodecBufferChannel::elapsed() {
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001931 // When client pushed EOS, we want all the work to be done quickly.
1932 // Otherwise, component may have stalled work due to input starvation up to
1933 // the sum of the delay in the pipeline.
Wonsik Kimf0e7d222019-06-28 12:33:16 -07001934 size_t n = 0;
1935 if (!mInputMetEos) {
1936 size_t outputDelay = mOutput.lock()->outputDelay;
1937 Mutexed<Input>::Locked input(mInput);
1938 n = input->inputDelay + input->pipelineDelay + outputDelay;
1939 }
Wonsik Kim4fa4f2b2019-02-13 11:02:58 -08001940 return mPipelineWatcher.lock()->elapsed(PipelineWatcher::Clock::now(), n);
Wonsik Kimab34ed62019-01-31 15:28:46 -08001941}
1942
Pawin Vongmasa36653902018-11-15 00:10:25 -08001943void CCodecBufferChannel::setMetaMode(MetaMode mode) {
1944 mMetaMode = mode;
1945}
1946
Wonsik Kim596187e2019-10-25 12:44:10 -07001947void CCodecBufferChannel::setCrypto(const sp<ICrypto> &crypto) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001948 if (mCrypto != nullptr) {
1949 for (std::pair<wp<HidlMemory>, int32_t> entry : mHeapSeqNumMap) {
1950 mCrypto->unsetHeap(entry.second);
1951 }
1952 mHeapSeqNumMap.clear();
1953 if (mHeapSeqNum >= 0) {
1954 mCrypto->unsetHeap(mHeapSeqNum);
1955 mHeapSeqNum = -1;
1956 }
1957 }
Wonsik Kim596187e2019-10-25 12:44:10 -07001958 mCrypto = crypto;
1959}
1960
1961void CCodecBufferChannel::setDescrambler(const sp<IDescrambler> &descrambler) {
1962 mDescrambler = descrambler;
1963}
1964
Pawin Vongmasa36653902018-11-15 00:10:25 -08001965status_t toStatusT(c2_status_t c2s, c2_operation_t c2op) {
1966 // C2_OK is always translated to OK.
1967 if (c2s == C2_OK) {
1968 return OK;
1969 }
1970
1971 // Operation-dependent translation
1972 // TODO: Add as necessary
1973 switch (c2op) {
1974 case C2_OPERATION_Component_start:
1975 switch (c2s) {
1976 case C2_NO_MEMORY:
1977 return NO_MEMORY;
1978 default:
1979 return UNKNOWN_ERROR;
1980 }
1981 default:
1982 break;
1983 }
1984
1985 // Backup operation-agnostic translation
1986 switch (c2s) {
1987 case C2_BAD_INDEX:
1988 return BAD_INDEX;
1989 case C2_BAD_VALUE:
1990 return BAD_VALUE;
1991 case C2_BLOCKING:
1992 return WOULD_BLOCK;
1993 case C2_DUPLICATE:
1994 return ALREADY_EXISTS;
1995 case C2_NO_INIT:
1996 return NO_INIT;
1997 case C2_NO_MEMORY:
1998 return NO_MEMORY;
1999 case C2_NOT_FOUND:
2000 return NAME_NOT_FOUND;
2001 case C2_TIMED_OUT:
2002 return TIMED_OUT;
2003 case C2_BAD_STATE:
2004 case C2_CANCELED:
2005 case C2_CANNOT_DO:
2006 case C2_CORRUPTED:
2007 case C2_OMITTED:
2008 case C2_REFUSED:
2009 return UNKNOWN_ERROR;
2010 default:
2011 return -static_cast<status_t>(c2s);
2012 }
2013}
2014
2015} // namespace android