blob: 0a895b0ae607c09413ff226cd06b39e54e4ee29e [file] [log] [blame]
Pawin Vongmasa36653902018-11-15 00:10:25 -08001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "CCodec"
19#include <utils/Log.h>
20
21#include <sstream>
22#include <thread>
23
24#include <C2Config.h>
25#include <C2Debug.h>
26#include <C2ParamInternal.h>
27#include <C2PlatformSupport.h>
28
Pawin Vongmasa36653902018-11-15 00:10:25 -080029#include <android/IOMXBufferSource.h>
Pawin Vongmasabf69de92019-10-29 06:21:27 -070030#include <android/hardware/media/c2/1.0/IInputSurface.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080031#include <android/hardware/media/omx/1.0/IGraphicBufferSource.h>
32#include <android/hardware/media/omx/1.0/IOmx.h>
33#include <android-base/stringprintf.h>
34#include <cutils/properties.h>
35#include <gui/IGraphicBufferProducer.h>
36#include <gui/Surface.h>
37#include <gui/bufferqueue/1.0/H2BGraphicBufferProducer.h>
Wonsik Kim9917d4a2019-10-24 12:56:38 -070038#include <media/omx/1.0/WOmxNode.h>
39#include <media/openmax/OMX_Core.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080040#include <media/openmax/OMX_IndexExt.h>
Wonsik Kim9917d4a2019-10-24 12:56:38 -070041#include <media/stagefright/omx/1.0/WGraphicBufferSource.h>
42#include <media/stagefright/omx/OmxGraphicBufferSource.h>
Wonsik Kim155d5cb2019-10-09 12:49:49 -070043#include <media/stagefright/CCodec.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080044#include <media/stagefright/BufferProducerWrapper.h>
45#include <media/stagefright/MediaCodecConstants.h>
46#include <media/stagefright/PersistentSurface.h>
ted.sun765db4d2020-06-23 14:03:41 +080047#include <utils/NativeHandle.h>
Pawin Vongmasa36653902018-11-15 00:10:25 -080048
49#include "C2OMXNode.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080050#include "CCodecBufferChannel.h"
Wonsik Kim155d5cb2019-10-09 12:49:49 -070051#include "CCodecConfig.h"
Wonsik Kimfb7a7672019-12-27 17:13:33 -080052#include "Codec2Mapper.h"
Pawin Vongmasa36653902018-11-15 00:10:25 -080053#include "InputSurfaceWrapper.h"
54
55extern "C" android::PersistentSurface *CreateInputSurface();
56
57namespace android {
58
59using namespace std::chrono_literals;
60using ::android::hardware::graphics::bufferqueue::V1_0::utils::H2BGraphicBufferProducer;
61using android::base::StringPrintf;
Pawin Vongmasad0f0e142018-11-15 03:36:28 -080062using ::android::hardware::media::c2::V1_0::IInputSurface;
Pawin Vongmasa36653902018-11-15 00:10:25 -080063
Wonsik Kim9917d4a2019-10-24 12:56:38 -070064typedef hardware::media::omx::V1_0::IGraphicBufferSource HGraphicBufferSource;
Wonsik Kim155d5cb2019-10-09 12:49:49 -070065typedef CCodecConfig Config;
Wonsik Kim9917d4a2019-10-24 12:56:38 -070066
Pawin Vongmasa36653902018-11-15 00:10:25 -080067namespace {
68
69class CCodecWatchdog : public AHandler {
70private:
71 enum {
72 kWhatWatch,
73 };
74 constexpr static int64_t kWatchIntervalUs = 3300000; // 3.3 secs
75
76public:
77 static sp<CCodecWatchdog> getInstance() {
78 static sp<CCodecWatchdog> instance(new CCodecWatchdog);
79 static std::once_flag flag;
80 // Call Init() only once.
81 std::call_once(flag, Init, instance);
82 return instance;
83 }
84
85 ~CCodecWatchdog() = default;
86
87 void watch(sp<CCodec> codec) {
88 bool shouldPost = false;
89 {
90 Mutexed<std::set<wp<CCodec>>>::Locked codecs(mCodecsToWatch);
91 // If a watch message is in flight, piggy-back this instance as well.
92 // Otherwise, post a new watch message.
93 shouldPost = codecs->empty();
94 codecs->emplace(codec);
95 }
96 if (shouldPost) {
97 ALOGV("posting watch message");
98 (new AMessage(kWhatWatch, this))->post(kWatchIntervalUs);
99 }
100 }
101
102protected:
103 void onMessageReceived(const sp<AMessage> &msg) {
104 switch (msg->what()) {
105 case kWhatWatch: {
106 Mutexed<std::set<wp<CCodec>>>::Locked codecs(mCodecsToWatch);
107 ALOGV("watch for %zu codecs", codecs->size());
108 for (auto it = codecs->begin(); it != codecs->end(); ++it) {
109 sp<CCodec> codec = it->promote();
110 if (codec == nullptr) {
111 continue;
112 }
113 codec->initiateReleaseIfStuck();
114 }
115 codecs->clear();
116 break;
117 }
118
119 default: {
120 TRESPASS("CCodecWatchdog: unrecognized message");
121 }
122 }
123 }
124
125private:
126 CCodecWatchdog() : mLooper(new ALooper) {}
127
128 static void Init(const sp<CCodecWatchdog> &thiz) {
129 ALOGV("Init");
130 thiz->mLooper->setName("CCodecWatchdog");
131 thiz->mLooper->registerHandler(thiz);
132 thiz->mLooper->start();
133 }
134
135 sp<ALooper> mLooper;
136
137 Mutexed<std::set<wp<CCodec>>> mCodecsToWatch;
138};
139
140class C2InputSurfaceWrapper : public InputSurfaceWrapper {
141public:
142 explicit C2InputSurfaceWrapper(
143 const std::shared_ptr<Codec2Client::InputSurface> &surface) :
144 mSurface(surface) {
145 }
146
147 ~C2InputSurfaceWrapper() override = default;
148
149 status_t connect(const std::shared_ptr<Codec2Client::Component> &comp) override {
150 if (mConnection != nullptr) {
151 return ALREADY_EXISTS;
152 }
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800153 return toStatusT(comp->connectToInputSurface(mSurface, &mConnection));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800154 }
155
156 void disconnect() override {
157 if (mConnection != nullptr) {
158 mConnection->disconnect();
159 mConnection = nullptr;
160 }
161 }
162
163 status_t start() override {
164 // InputSurface does not distinguish started state
165 return OK;
166 }
167
168 status_t signalEndOfInputStream() override {
169 C2InputSurfaceEosTuning eos(true);
170 std::vector<std::unique_ptr<C2SettingResult>> failures;
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800171 c2_status_t err = mSurface->config({&eos}, C2_MAY_BLOCK, &failures);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800172 if (err != C2_OK) {
173 return UNKNOWN_ERROR;
174 }
175 return OK;
176 }
177
178 status_t configure(Config &config __unused) {
179 // TODO
180 return OK;
181 }
182
183private:
184 std::shared_ptr<Codec2Client::InputSurface> mSurface;
185 std::shared_ptr<Codec2Client::InputSurfaceConnection> mConnection;
186};
187
188class GraphicBufferSourceWrapper : public InputSurfaceWrapper {
189public:
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700190 typedef hardware::media::omx::V1_0::Status OmxStatus;
191
Pawin Vongmasa36653902018-11-15 00:10:25 -0800192 GraphicBufferSourceWrapper(
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700193 const sp<HGraphicBufferSource> &source,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800194 uint32_t width,
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700195 uint32_t height,
196 uint64_t usage)
Pawin Vongmasa36653902018-11-15 00:10:25 -0800197 : mSource(source), mWidth(width), mHeight(height) {
198 mDataSpace = HAL_DATASPACE_BT709;
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700199 mConfig.mUsage = usage;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800200 }
201 ~GraphicBufferSourceWrapper() override = default;
202
203 status_t connect(const std::shared_ptr<Codec2Client::Component> &comp) override {
204 mNode = new C2OMXNode(comp);
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700205 mOmxNode = new hardware::media::omx::V1_0::utils::TWOmxNode(mNode);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800206 mNode->setFrameSize(mWidth, mHeight);
207
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700208 // Usage is queried during configure(), so setting it beforehand.
209 OMX_U32 usage = mConfig.mUsage & 0xFFFFFFFF;
210 (void)mNode->setParameter(
211 (OMX_INDEXTYPE)OMX_IndexParamConsumerUsageBits,
212 &usage, sizeof(usage));
213
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700214 mSource->configure(
215 mOmxNode, static_cast<hardware::graphics::common::V1_0::Dataspace>(mDataSpace));
Pawin Vongmasa36653902018-11-15 00:10:25 -0800216 return OK;
217 }
218
219 void disconnect() override {
220 if (mNode == nullptr) {
221 return;
222 }
223 sp<IOMXBufferSource> source = mNode->getSource();
224 if (source == nullptr) {
225 ALOGD("GBSWrapper::disconnect: node is not configured with OMXBufferSource.");
226 return;
227 }
228 source->onOmxIdle();
229 source->onOmxLoaded();
230 mNode.clear();
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700231 mOmxNode.clear();
Pawin Vongmasa36653902018-11-15 00:10:25 -0800232 }
233
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700234 status_t GetStatus(hardware::Return<OmxStatus> &&status) {
235 if (status.isOk()) {
236 return static_cast<status_t>(status.withDefault(OmxStatus::UNKNOWN_ERROR));
237 } else if (status.isDeadObject()) {
238 return DEAD_OBJECT;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800239 }
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700240 return UNKNOWN_ERROR;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800241 }
242
243 status_t start() override {
244 sp<IOMXBufferSource> source = mNode->getSource();
245 if (source == nullptr) {
246 return NO_INIT;
247 }
Taehwan Kim8b3bcdd2020-11-26 22:40:40 +0900248
Wonsik Kim0f6b61d2021-01-05 18:55:22 -0800249 size_t numSlots = 16;
Wonsik Kim34d66012021-03-01 16:40:33 -0800250 constexpr OMX_U32 kPortIndexInput = 0;
Taehwan Kim8b3bcdd2020-11-26 22:40:40 +0900251
Wonsik Kim34d66012021-03-01 16:40:33 -0800252 OMX_PARAM_PORTDEFINITIONTYPE param;
253 param.nPortIndex = kPortIndexInput;
254 status_t err = mNode->getParameter(OMX_IndexParamPortDefinition,
255 &param, sizeof(param));
256 if (err == OK) {
257 numSlots = param.nBufferCountActual;
Taehwan Kim8b3bcdd2020-11-26 22:40:40 +0900258 }
259
260 for (size_t i = 0; i < numSlots; ++i) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800261 source->onInputBufferAdded(i);
262 }
263
264 source->onOmxExecuting();
265 return OK;
266 }
267
268 status_t signalEndOfInputStream() override {
269 return GetStatus(mSource->signalEndOfInputStream());
270 }
271
272 status_t configure(Config &config) {
273 std::stringstream status;
274 status_t err = OK;
275
276 // handle each configuration granually, in case we need to handle part of the configuration
277 // elsewhere
278
279 // TRICKY: we do not unset frame delay repeating
280 if (config.mMinFps > 0 && config.mMinFps != mConfig.mMinFps) {
281 int64_t us = 1e6 / config.mMinFps + 0.5;
282 status_t res = GetStatus(mSource->setRepeatPreviousFrameDelayUs(us));
283 status << " minFps=" << config.mMinFps << " => repeatDelayUs=" << us;
284 if (res != OK) {
285 status << " (=> " << asString(res) << ")";
286 err = res;
287 }
288 mConfig.mMinFps = config.mMinFps;
289 }
290
291 // pts gap
292 if (config.mMinAdjustedFps > 0 || config.mFixedAdjustedFps > 0) {
293 if (mNode != nullptr) {
294 OMX_PARAM_U32TYPE ptrGapParam = {};
295 ptrGapParam.nSize = sizeof(OMX_PARAM_U32TYPE);
Wonsik Kim95ba0162019-03-19 15:51:54 -0700296 float gap = (config.mMinAdjustedFps > 0)
Pawin Vongmasa36653902018-11-15 00:10:25 -0800297 ? c2_min(INT32_MAX + 0., 1e6 / config.mMinAdjustedFps + 0.5)
298 : c2_max(0. - INT32_MAX, -1e6 / config.mFixedAdjustedFps - 0.5);
Wonsik Kim95ba0162019-03-19 15:51:54 -0700299 // float -> uint32_t is undefined if the value is negative.
300 // First convert to int32_t to ensure the expected behavior.
301 ptrGapParam.nU32 = int32_t(gap);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800302 (void)mNode->setParameter(
303 (OMX_INDEXTYPE)OMX_IndexParamMaxFrameDurationForBitrateControl,
304 &ptrGapParam, sizeof(ptrGapParam));
305 }
306 }
307
308 // max fps
309 // TRICKY: we do not unset max fps to 0 unless using fixed fps
Wonsik Kim95ba0162019-03-19 15:51:54 -0700310 if ((config.mMaxFps > 0 || (config.mFixedAdjustedFps > 0 && config.mMaxFps == -1))
Pawin Vongmasa36653902018-11-15 00:10:25 -0800311 && config.mMaxFps != mConfig.mMaxFps) {
312 status_t res = GetStatus(mSource->setMaxFps(config.mMaxFps));
313 status << " maxFps=" << config.mMaxFps;
314 if (res != OK) {
315 status << " (=> " << asString(res) << ")";
316 err = res;
317 }
318 mConfig.mMaxFps = config.mMaxFps;
319 }
320
321 if (config.mTimeOffsetUs != mConfig.mTimeOffsetUs) {
322 status_t res = GetStatus(mSource->setTimeOffsetUs(config.mTimeOffsetUs));
323 status << " timeOffset " << config.mTimeOffsetUs << "us";
324 if (res != OK) {
325 status << " (=> " << asString(res) << ")";
326 err = res;
327 }
328 mConfig.mTimeOffsetUs = config.mTimeOffsetUs;
329 }
330
331 if (config.mCaptureFps != mConfig.mCaptureFps || config.mCodedFps != mConfig.mCodedFps) {
332 status_t res =
333 GetStatus(mSource->setTimeLapseConfig(config.mCodedFps, config.mCaptureFps));
334 status << " timeLapse " << config.mCaptureFps << "fps as " << config.mCodedFps << "fps";
335 if (res != OK) {
336 status << " (=> " << asString(res) << ")";
337 err = res;
338 }
339 mConfig.mCaptureFps = config.mCaptureFps;
340 mConfig.mCodedFps = config.mCodedFps;
341 }
342
343 if (config.mStartAtUs != mConfig.mStartAtUs
344 || (config.mStopped != mConfig.mStopped && !config.mStopped)) {
345 status_t res = GetStatus(mSource->setStartTimeUs(config.mStartAtUs));
346 status << " start at " << config.mStartAtUs << "us";
347 if (res != OK) {
348 status << " (=> " << asString(res) << ")";
349 err = res;
350 }
351 mConfig.mStartAtUs = config.mStartAtUs;
352 mConfig.mStopped = config.mStopped;
353 }
354
355 // suspend-resume
356 if (config.mSuspended != mConfig.mSuspended) {
357 status_t res = GetStatus(mSource->setSuspend(config.mSuspended, config.mSuspendAtUs));
358 status << " " << (config.mSuspended ? "suspend" : "resume")
359 << " at " << config.mSuspendAtUs << "us";
360 if (res != OK) {
361 status << " (=> " << asString(res) << ")";
362 err = res;
363 }
364 mConfig.mSuspended = config.mSuspended;
365 mConfig.mSuspendAtUs = config.mSuspendAtUs;
366 }
367
368 if (config.mStopped != mConfig.mStopped && config.mStopped) {
369 status_t res = GetStatus(mSource->setStopTimeUs(config.mStopAtUs));
370 status << " stop at " << config.mStopAtUs << "us";
371 if (res != OK) {
372 status << " (=> " << asString(res) << ")";
373 err = res;
374 } else {
375 status << " delayUs";
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700376 hardware::Return<void> trans = mSource->getStopTimeOffsetUs(
377 [&res, &delayUs = config.mInputDelayUs](
378 auto status, auto stopTimeOffsetUs) {
379 res = static_cast<status_t>(status);
380 delayUs = stopTimeOffsetUs;
381 });
382 if (!trans.isOk()) {
383 res = trans.isDeadObject() ? DEAD_OBJECT : UNKNOWN_ERROR;
384 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800385 if (res != OK) {
386 status << " (=> " << asString(res) << ")";
387 } else {
388 status << "=" << config.mInputDelayUs << "us";
389 }
390 mConfig.mInputDelayUs = config.mInputDelayUs;
391 }
392 mConfig.mStopAtUs = config.mStopAtUs;
393 mConfig.mStopped = config.mStopped;
394 }
395
396 // color aspects (android._color-aspects)
397
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700398 // consumer usage is queried earlier.
399
Wonsik Kima1335e12021-04-22 16:28:29 -0700400 // priority
401 if (mConfig.mPriority != config.mPriority) {
402 if (config.mPriority != INT_MAX) {
403 mNode->setPriority(config.mPriority);
404 }
405 mConfig.mPriority = config.mPriority;
406 }
407
Wonsik Kimbd557932019-07-02 15:51:20 -0700408 if (status.str().empty()) {
409 ALOGD("ISConfig not changed");
410 } else {
411 ALOGD("ISConfig%s", status.str().c_str());
412 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800413 return err;
414 }
415
Wonsik Kim4f3314d2019-03-26 17:00:34 -0700416 void onInputBufferDone(c2_cntr64_t index) override {
417 mNode->onInputBufferDone(index);
418 }
419
Wonsik Kim673dd192021-01-29 14:58:12 -0800420 android_dataspace getDataspace() override {
421 return mNode->getDataspace();
422 }
423
Pawin Vongmasa36653902018-11-15 00:10:25 -0800424private:
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700425 sp<HGraphicBufferSource> mSource;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800426 sp<C2OMXNode> mNode;
Wonsik Kim9917d4a2019-10-24 12:56:38 -0700427 sp<hardware::media::omx::V1_0::IOmxNode> mOmxNode;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800428 uint32_t mWidth;
429 uint32_t mHeight;
430 Config mConfig;
431};
432
433class Codec2ClientInterfaceWrapper : public C2ComponentStore {
434 std::shared_ptr<Codec2Client> mClient;
435
436public:
437 Codec2ClientInterfaceWrapper(std::shared_ptr<Codec2Client> client)
438 : mClient(client) { }
439
440 virtual ~Codec2ClientInterfaceWrapper() = default;
441
442 virtual c2_status_t config_sm(
443 const std::vector<C2Param *> &params,
444 std::vector<std::unique_ptr<C2SettingResult>> *const failures) {
445 return mClient->config(params, C2_MAY_BLOCK, failures);
446 };
447
448 virtual c2_status_t copyBuffer(
449 std::shared_ptr<C2GraphicBuffer>,
450 std::shared_ptr<C2GraphicBuffer>) {
451 return C2_OMITTED;
452 }
453
454 virtual c2_status_t createComponent(
455 C2String, std::shared_ptr<C2Component> *const component) {
456 component->reset();
457 return C2_OMITTED;
458 }
459
460 virtual c2_status_t createInterface(
461 C2String, std::shared_ptr<C2ComponentInterface> *const interface) {
462 interface->reset();
463 return C2_OMITTED;
464 }
465
466 virtual c2_status_t query_sm(
467 const std::vector<C2Param *> &stackParams,
468 const std::vector<C2Param::Index> &heapParamIndices,
469 std::vector<std::unique_ptr<C2Param>> *const heapParams) const {
470 return mClient->query(stackParams, heapParamIndices, C2_MAY_BLOCK, heapParams);
471 }
472
473 virtual c2_status_t querySupportedParams_nb(
474 std::vector<std::shared_ptr<C2ParamDescriptor>> *const params) const {
475 return mClient->querySupportedParams(params);
476 }
477
478 virtual c2_status_t querySupportedValues_sm(
479 std::vector<C2FieldSupportedValuesQuery> &fields) const {
480 return mClient->querySupportedValues(fields, C2_MAY_BLOCK);
481 }
482
483 virtual C2String getName() const {
484 return mClient->getName();
485 }
486
487 virtual std::shared_ptr<C2ParamReflector> getParamReflector() const {
488 return mClient->getParamReflector();
489 }
490
491 virtual std::vector<std::shared_ptr<const C2Component::Traits>> listComponents() {
492 return std::vector<std::shared_ptr<const C2Component::Traits>>();
493 }
494};
495
Wonsik Kim3b4349a2020-11-10 11:54:15 -0800496void RevertOutputFormatIfNeeded(
497 const sp<AMessage> &oldFormat, sp<AMessage> &currentFormat) {
498 // We used to not report changes to these keys to the client.
499 const static std::set<std::string> sIgnoredKeys({
500 KEY_BIT_RATE,
Harish Mahendrakar8c537502021-02-23 21:20:22 -0800501 KEY_FRAME_RATE,
Wonsik Kim3b4349a2020-11-10 11:54:15 -0800502 KEY_MAX_BIT_RATE,
Harish Mahendrakar8c537502021-02-23 21:20:22 -0800503 KEY_MAX_WIDTH,
504 KEY_MAX_HEIGHT,
Wonsik Kim3b4349a2020-11-10 11:54:15 -0800505 "csd-0",
506 "csd-1",
507 "csd-2",
508 });
509 if (currentFormat == oldFormat) {
510 return;
511 }
512 sp<AMessage> diff = currentFormat->changesFrom(oldFormat);
513 AMessage::Type type;
514 for (size_t i = diff->countEntries(); i > 0; --i) {
515 if (sIgnoredKeys.count(diff->getEntryNameAt(i - 1, &type)) > 0) {
516 diff->removeEntryAt(i - 1);
517 }
518 }
519 if (diff->countEntries() == 0) {
520 currentFormat = oldFormat;
521 }
522}
523
Pawin Vongmasa36653902018-11-15 00:10:25 -0800524} // namespace
525
526// CCodec::ClientListener
527
528struct CCodec::ClientListener : public Codec2Client::Listener {
529
530 explicit ClientListener(const wp<CCodec> &codec) : mCodec(codec) {}
531
532 virtual void onWorkDone(
533 const std::weak_ptr<Codec2Client::Component>& component,
Wonsik Kimab34ed62019-01-31 15:28:46 -0800534 std::list<std::unique_ptr<C2Work>>& workItems) override {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800535 (void)component;
536 sp<CCodec> codec(mCodec.promote());
537 if (!codec) {
538 return;
539 }
Wonsik Kimab34ed62019-01-31 15:28:46 -0800540 codec->onWorkDone(workItems);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800541 }
542
543 virtual void onTripped(
544 const std::weak_ptr<Codec2Client::Component>& component,
545 const std::vector<std::shared_ptr<C2SettingResult>>& settingResult
546 ) override {
547 // TODO
548 (void)component;
549 (void)settingResult;
550 }
551
552 virtual void onError(
553 const std::weak_ptr<Codec2Client::Component>& component,
554 uint32_t errorCode) override {
Praveen Chavan72eff012020-11-20 23:20:28 -0800555 {
556 // Component is only used for reporting as we use a separate listener for each instance
557 std::shared_ptr<Codec2Client::Component> comp = component.lock();
558 if (!comp) {
559 ALOGD("Component died with error: 0x%x", errorCode);
560 } else {
561 ALOGD("Component \"%s\" returned error: 0x%x", comp->getName().c_str(), errorCode);
562 }
563 }
564
565 // Report to MediaCodec
Wonsik Kim10f33c02021-03-04 15:04:14 -0800566 // Note: for now we do not propagate the error code to MediaCodec
567 // except for C2_NO_MEMORY, as we would need to translate to a MediaCodec error.
Praveen Chavan72eff012020-11-20 23:20:28 -0800568 sp<CCodec> codec(mCodec.promote());
569 if (!codec || !codec->mCallback) {
570 return;
571 }
Wonsik Kim10f33c02021-03-04 15:04:14 -0800572 codec->mCallback->onError(
573 errorCode == C2_NO_MEMORY ? NO_MEMORY : UNKNOWN_ERROR,
574 ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800575 }
576
577 virtual void onDeath(
578 const std::weak_ptr<Codec2Client::Component>& component) override {
579 { // Log the death of the component.
580 std::shared_ptr<Codec2Client::Component> comp = component.lock();
581 if (!comp) {
582 ALOGE("Codec2 component died.");
583 } else {
584 ALOGE("Codec2 component \"%s\" died.", comp->getName().c_str());
585 }
586 }
587
588 // Report to MediaCodec.
589 sp<CCodec> codec(mCodec.promote());
590 if (!codec || !codec->mCallback) {
591 return;
592 }
593 codec->mCallback->onError(DEAD_OBJECT, ACTION_CODE_FATAL);
594 }
595
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800596 virtual void onFrameRendered(uint64_t bufferQueueId,
597 int32_t slotId,
598 int64_t timestampNs) override {
599 // TODO: implement
600 (void)bufferQueueId;
601 (void)slotId;
602 (void)timestampNs;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800603 }
604
605 virtual void onInputBufferDone(
Wonsik Kimab34ed62019-01-31 15:28:46 -0800606 uint64_t frameIndex, size_t arrayIndex) override {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800607 sp<CCodec> codec(mCodec.promote());
608 if (codec) {
Wonsik Kimab34ed62019-01-31 15:28:46 -0800609 codec->onInputBufferDone(frameIndex, arrayIndex);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800610 }
611 }
612
613private:
614 wp<CCodec> mCodec;
615};
616
617// CCodecCallbackImpl
618
619class CCodecCallbackImpl : public CCodecCallback {
620public:
621 explicit CCodecCallbackImpl(CCodec *codec) : mCodec(codec) {}
622 ~CCodecCallbackImpl() override = default;
623
624 void onError(status_t err, enum ActionCode actionCode) override {
625 mCodec->mCallback->onError(err, actionCode);
626 }
627
628 void onOutputFramesRendered(int64_t mediaTimeUs, nsecs_t renderTimeNs) override {
629 mCodec->mCallback->onOutputFramesRendered(
630 {RenderedFrameInfo(mediaTimeUs, renderTimeNs)});
631 }
632
Pawin Vongmasa36653902018-11-15 00:10:25 -0800633 void onOutputBuffersChanged() override {
634 mCodec->mCallback->onOutputBuffersChanged();
635 }
636
637private:
638 CCodec *mCodec;
639};
640
641// CCodec
642
643CCodec::CCodec()
Wonsik Kim155d5cb2019-10-09 12:49:49 -0700644 : mChannel(new CCodecBufferChannel(std::make_shared<CCodecCallbackImpl>(this))),
645 mConfig(new CCodecConfig) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800646}
647
648CCodec::~CCodec() {
649}
650
651std::shared_ptr<BufferChannelBase> CCodec::getBufferChannel() {
652 return mChannel;
653}
654
655status_t CCodec::tryAndReportOnError(std::function<status_t()> job) {
656 status_t err = job();
657 if (err != C2_OK) {
658 mCallback->onError(err, ACTION_CODE_FATAL);
659 }
660 return err;
661}
662
663void CCodec::initiateAllocateComponent(const sp<AMessage> &msg) {
664 auto setAllocating = [this] {
665 Mutexed<State>::Locked state(mState);
666 if (state->get() != RELEASED) {
667 return INVALID_OPERATION;
668 }
669 state->set(ALLOCATING);
670 return OK;
671 };
672 if (tryAndReportOnError(setAllocating) != OK) {
673 return;
674 }
675
676 sp<RefBase> codecInfo;
677 CHECK(msg->findObject("codecInfo", &codecInfo));
678 // For Codec 2.0 components, componentName == codecInfo->getCodecName().
679
680 sp<AMessage> allocMsg(new AMessage(kWhatAllocate, this));
681 allocMsg->setObject("codecInfo", codecInfo);
682 allocMsg->post();
683}
684
685void CCodec::allocate(const sp<MediaCodecInfo> &codecInfo) {
686 if (codecInfo == nullptr) {
687 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
688 return;
689 }
690 ALOGD("allocate(%s)", codecInfo->getCodecName());
691 mClientListener.reset(new ClientListener(this));
692
693 AString componentName = codecInfo->getCodecName();
694 std::shared_ptr<Codec2Client> client;
695
696 // set up preferred component store to access vendor store parameters
Pawin Vongmasa892c81d2019-03-12 00:56:50 -0700697 client = Codec2Client::CreateFromService("default");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800698 if (client) {
Pawin Vongmasa1c75a232019-01-09 04:41:52 -0800699 ALOGI("setting up '%s' as default (vendor) store", client->getServiceName().c_str());
Pawin Vongmasa36653902018-11-15 00:10:25 -0800700 SetPreferredCodec2ComponentStore(
701 std::make_shared<Codec2ClientInterfaceWrapper>(client));
702 }
703
Chih-Yu Huangb8fe0792020-12-07 17:14:55 +0900704 std::shared_ptr<Codec2Client::Component> comp;
705 c2_status_t status = Codec2Client::CreateComponentByName(
Pawin Vongmasa36653902018-11-15 00:10:25 -0800706 componentName.c_str(),
707 mClientListener,
Chih-Yu Huangb8fe0792020-12-07 17:14:55 +0900708 &comp,
Pawin Vongmasa36653902018-11-15 00:10:25 -0800709 &client);
Chih-Yu Huangb8fe0792020-12-07 17:14:55 +0900710 if (status != C2_OK) {
711 ALOGE("Failed Create component: %s, error=%d", componentName.c_str(), status);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800712 Mutexed<State>::Locked state(mState);
713 state->set(RELEASED);
714 state.unlock();
Chih-Yu Huangb8fe0792020-12-07 17:14:55 +0900715 mCallback->onError((status == C2_NO_MEMORY ? NO_MEMORY : UNKNOWN_ERROR), ACTION_CODE_FATAL);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800716 state.lock();
717 return;
718 }
719 ALOGI("Created component [%s]", componentName.c_str());
720 mChannel->setComponent(comp);
721 auto setAllocated = [this, comp, client] {
722 Mutexed<State>::Locked state(mState);
723 if (state->get() != ALLOCATING) {
724 state->set(RELEASED);
725 return UNKNOWN_ERROR;
726 }
727 state->set(ALLOCATED);
728 state->comp = comp;
729 mClient = client;
730 return OK;
731 };
732 if (tryAndReportOnError(setAllocated) != OK) {
733 return;
734 }
735
736 // initialize config here in case setParameters is called prior to configure
Wonsik Kim155d5cb2019-10-09 12:49:49 -0700737 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
738 const std::unique_ptr<Config> &config = *configLocked;
Wonsik Kim8a6ed372019-12-03 16:05:51 -0800739 status_t err = config->initialize(mClient->getParamReflector(), comp);
Pawin Vongmasa36653902018-11-15 00:10:25 -0800740 if (err != OK) {
741 ALOGW("Failed to initialize configuration support");
742 // TODO: report error once we complete implementation.
743 }
744 config->queryConfiguration(comp);
745
746 mCallback->onComponentAllocated(componentName.c_str());
747}
748
749void CCodec::initiateConfigureComponent(const sp<AMessage> &format) {
750 auto checkAllocated = [this] {
751 Mutexed<State>::Locked state(mState);
752 return (state->get() != ALLOCATED) ? UNKNOWN_ERROR : OK;
753 };
754 if (tryAndReportOnError(checkAllocated) != OK) {
755 return;
756 }
757
758 sp<AMessage> msg(new AMessage(kWhatConfigure, this));
759 msg->setMessage("format", format);
760 msg->post();
761}
762
763void CCodec::configure(const sp<AMessage> &msg) {
764 std::shared_ptr<Codec2Client::Component> comp;
765 auto checkAllocated = [this, &comp] {
766 Mutexed<State>::Locked state(mState);
767 if (state->get() != ALLOCATED) {
768 state->set(RELEASED);
769 return UNKNOWN_ERROR;
770 }
771 comp = state->comp;
772 return OK;
773 };
774 if (tryAndReportOnError(checkAllocated) != OK) {
775 return;
776 }
777
778 auto doConfig = [msg, comp, this]() -> status_t {
779 AString mime;
780 if (!msg->findString("mime", &mime)) {
781 return BAD_VALUE;
782 }
783
784 int32_t encoder;
785 if (!msg->findInt32("encoder", &encoder)) {
786 encoder = false;
787 }
788
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800789 int32_t flags;
790 if (!msg->findInt32("flags", &flags)) {
791 return BAD_VALUE;
792 }
793
Pawin Vongmasa36653902018-11-15 00:10:25 -0800794 // TODO: read from intf()
795 if ((!encoder) != (comp->getName().find("encoder") == std::string::npos)) {
796 return UNKNOWN_ERROR;
797 }
798
799 int32_t storeMeta;
800 if (encoder
801 && msg->findInt32("android._input-metadata-buffer-type", &storeMeta)
802 && storeMeta != kMetadataBufferTypeInvalid) {
803 if (storeMeta != kMetadataBufferTypeANWBuffer) {
804 ALOGD("Only ANW buffers are supported for legacy metadata mode");
805 return BAD_VALUE;
806 }
807 mChannel->setMetaMode(CCodecBufferChannel::MODE_ANW);
808 }
809
ted.sun765db4d2020-06-23 14:03:41 +0800810 status_t err = OK;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800811 sp<RefBase> obj;
812 sp<Surface> surface;
813 if (msg->findObject("native-window", &obj)) {
814 surface = static_cast<Surface *>(obj.get());
ted.sun765db4d2020-06-23 14:03:41 +0800815 // setup tunneled playback
816 if (surface != nullptr) {
817 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
818 const std::unique_ptr<Config> &config = *configLocked;
819 if ((config->mDomain & Config::IS_DECODER)
820 && (config->mDomain & Config::IS_VIDEO)) {
821 int32_t tunneled;
822 if (msg->findInt32("feature-tunneled-playback", &tunneled) && tunneled != 0) {
823 ALOGI("Configuring TUNNELED video playback.");
824
825 err = configureTunneledVideoPlayback(comp, &config->mSidebandHandle, msg);
826 if (err != OK) {
827 ALOGE("configureTunneledVideoPlayback failed!");
828 return err;
829 }
830 config->mTunneled = true;
831 }
832 }
833 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800834 setSurface(surface);
835 }
836
Wonsik Kim155d5cb2019-10-09 12:49:49 -0700837 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
838 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800839 config->mUsingSurface = surface != nullptr;
Wonsik Kimfb7a7672019-12-27 17:13:33 -0800840 config->mBuffersBoundToCodec = ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) == 0);
841 ALOGD("[%s] buffers are %sbound to CCodec for this session",
842 comp->getName().c_str(), config->mBuffersBoundToCodec ? "" : "not ");
Pawin Vongmasa36653902018-11-15 00:10:25 -0800843
Wonsik Kim1114eea2019-02-25 14:35:24 -0800844 // Enforce required parameters
845 int32_t i32;
846 float flt;
847 if (config->mDomain & Config::IS_AUDIO) {
848 if (!msg->findInt32(KEY_SAMPLE_RATE, &i32)) {
849 ALOGD("sample rate is missing, which is required for audio components.");
850 return BAD_VALUE;
851 }
852 if (!msg->findInt32(KEY_CHANNEL_COUNT, &i32)) {
853 ALOGD("channel count is missing, which is required for audio components.");
854 return BAD_VALUE;
855 }
856 if ((config->mDomain & Config::IS_ENCODER)
857 && !mime.equalsIgnoreCase(MEDIA_MIMETYPE_AUDIO_FLAC)
858 && !msg->findInt32(KEY_BIT_RATE, &i32)
859 && !msg->findFloat(KEY_BIT_RATE, &flt)) {
860 ALOGD("bitrate is missing, which is required for audio encoders.");
861 return BAD_VALUE;
862 }
863 }
Wonsik Kimd91f3fb2021-02-24 12:35:31 -0800864 int32_t width = 0;
865 int32_t height = 0;
Wonsik Kim1114eea2019-02-25 14:35:24 -0800866 if (config->mDomain & (Config::IS_IMAGE | Config::IS_VIDEO)) {
Wonsik Kimd91f3fb2021-02-24 12:35:31 -0800867 if (!msg->findInt32(KEY_WIDTH, &width)) {
Wonsik Kim1114eea2019-02-25 14:35:24 -0800868 ALOGD("width is missing, which is required for image/video components.");
869 return BAD_VALUE;
870 }
Wonsik Kimd91f3fb2021-02-24 12:35:31 -0800871 if (!msg->findInt32(KEY_HEIGHT, &height)) {
Wonsik Kim1114eea2019-02-25 14:35:24 -0800872 ALOGD("height is missing, which is required for image/video components.");
873 return BAD_VALUE;
874 }
875 if ((config->mDomain & Config::IS_ENCODER) && (config->mDomain & Config::IS_VIDEO)) {
Harish Mahendrakar71cbb9d2019-05-21 11:21:27 -0700876 int32_t mode = BITRATE_MODE_VBR;
877 if (msg->findInt32(KEY_BITRATE_MODE, &mode) && mode == BITRATE_MODE_CQ) {
Harish Mahendrakar817d3182019-03-11 16:37:47 -0700878 if (!msg->findInt32(KEY_QUALITY, &i32)) {
879 ALOGD("quality is missing, which is required for video encoders in CQ.");
880 return BAD_VALUE;
881 }
882 } else {
883 if (!msg->findInt32(KEY_BIT_RATE, &i32)
884 && !msg->findFloat(KEY_BIT_RATE, &flt)) {
885 ALOGD("bitrate is missing, which is required for video encoders.");
886 return BAD_VALUE;
887 }
Wonsik Kim1114eea2019-02-25 14:35:24 -0800888 }
889 if (!msg->findInt32(KEY_I_FRAME_INTERVAL, &i32)
890 && !msg->findFloat(KEY_I_FRAME_INTERVAL, &flt)) {
891 ALOGD("I frame interval is missing, which is required for video encoders.");
892 return BAD_VALUE;
893 }
Wonsik Kimaab2eea2019-05-22 10:37:58 -0700894 if (!msg->findInt32(KEY_FRAME_RATE, &i32)
895 && !msg->findFloat(KEY_FRAME_RATE, &flt)) {
896 ALOGD("frame rate is missing, which is required for video encoders.");
897 return BAD_VALUE;
898 }
Wonsik Kim1114eea2019-02-25 14:35:24 -0800899 }
900 }
901
Pawin Vongmasa36653902018-11-15 00:10:25 -0800902 /*
903 * Handle input surface configuration
904 */
905 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))
906 && (config->mDomain & Config::IS_ENCODER)) {
907 config->mISConfig.reset(new InputSurfaceWrapper::Config{});
908 {
909 config->mISConfig->mMinFps = 0;
910 int64_t value;
Chong Zhang038e8f82019-02-06 19:05:14 -0800911 if (msg->findInt64(KEY_REPEAT_PREVIOUS_FRAME_AFTER, &value) && value > 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800912 config->mISConfig->mMinFps = 1e6 / value;
913 }
Wonsik Kim95ba0162019-03-19 15:51:54 -0700914 if (!msg->findFloat(
915 KEY_MAX_FPS_TO_ENCODER, &config->mISConfig->mMaxFps)) {
916 config->mISConfig->mMaxFps = -1;
917 }
Pawin Vongmasa36653902018-11-15 00:10:25 -0800918 config->mISConfig->mMinAdjustedFps = 0;
919 config->mISConfig->mFixedAdjustedFps = 0;
Chong Zhang038e8f82019-02-06 19:05:14 -0800920 if (msg->findInt64(KEY_MAX_PTS_GAP_TO_ENCODER, &value)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800921 if (value < 0 && value >= INT32_MIN) {
922 config->mISConfig->mFixedAdjustedFps = -1e6 / value;
Wonsik Kim95ba0162019-03-19 15:51:54 -0700923 config->mISConfig->mMaxFps = -1;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800924 } else if (value > 0 && value <= INT32_MAX) {
925 config->mISConfig->mMinAdjustedFps = 1e6 / value;
926 }
927 }
928 }
929
930 {
Wonsik Kim8e55f3a2019-09-03 14:10:37 -0700931 bool captureFpsFound = false;
932 double timeLapseFps;
933 float captureRate;
934 if (msg->findDouble("time-lapse-fps", &timeLapseFps)) {
935 config->mISConfig->mCaptureFps = timeLapseFps;
936 captureFpsFound = true;
937 } else if (msg->findAsFloat(KEY_CAPTURE_RATE, &captureRate)) {
938 config->mISConfig->mCaptureFps = captureRate;
939 captureFpsFound = true;
940 }
941 if (captureFpsFound) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800942 (void)msg->findAsFloat(KEY_FRAME_RATE, &config->mISConfig->mCodedFps);
943 }
944 }
945
946 {
947 config->mISConfig->mSuspended = false;
948 config->mISConfig->mSuspendAtUs = -1;
949 int32_t value;
Chong Zhang038e8f82019-02-06 19:05:14 -0800950 if (msg->findInt32(KEY_CREATE_INPUT_SURFACE_SUSPENDED, &value) && value) {
Pawin Vongmasa36653902018-11-15 00:10:25 -0800951 config->mISConfig->mSuspended = true;
952 }
953 }
Wonsik Kim9eac4d12019-05-23 12:58:48 -0700954 config->mISConfig->mUsage = 0;
Wonsik Kima1335e12021-04-22 16:28:29 -0700955 config->mISConfig->mPriority = INT_MAX;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800956 }
957
958 /*
959 * Handle desired color format.
960 */
Wonsik Kimd79ee1f2020-08-27 17:41:56 -0700961 int32_t defaultColorFormat = COLOR_FormatYUV420Flexible;
Pawin Vongmasa36653902018-11-15 00:10:25 -0800962 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
Wonsik Kimd79ee1f2020-08-27 17:41:56 -0700963 int32_t format = 0;
964 // Query vendor format for Flexible YUV
965 std::vector<std::unique_ptr<C2Param>> heapParams;
966 C2StoreFlexiblePixelFormatDescriptorsInfo *pixelFormatInfo = nullptr;
967 if (mClient->query(
968 {},
969 {C2StoreFlexiblePixelFormatDescriptorsInfo::PARAM_TYPE},
970 C2_MAY_BLOCK,
971 &heapParams) == C2_OK
972 && heapParams.size() == 1u) {
973 pixelFormatInfo = C2StoreFlexiblePixelFormatDescriptorsInfo::From(
974 heapParams[0].get());
975 } else {
976 pixelFormatInfo = nullptr;
977 }
978 std::optional<uint32_t> flexPixelFormat{};
979 std::optional<uint32_t> flexPlanarPixelFormat{};
980 std::optional<uint32_t> flexSemiPlanarPixelFormat{};
981 if (pixelFormatInfo && *pixelFormatInfo) {
982 for (size_t i = 0; i < pixelFormatInfo->flexCount(); ++i) {
983 const C2FlexiblePixelFormatDescriptorStruct &desc =
984 pixelFormatInfo->m.values[i];
985 if (desc.bitDepth != 8
986 || desc.subsampling != C2Color::YUV_420
987 // TODO(b/180076105): some device report wrong layout
988 // || desc.layout == C2Color::INTERLEAVED_PACKED
989 // || desc.layout == C2Color::INTERLEAVED_ALIGNED
990 || desc.layout == C2Color::UNKNOWN_LAYOUT) {
991 continue;
992 }
993 if (!flexPixelFormat) {
994 flexPixelFormat = desc.pixelFormat;
995 }
996 if (desc.layout == C2Color::PLANAR_PACKED && !flexPlanarPixelFormat) {
997 flexPlanarPixelFormat = desc.pixelFormat;
998 }
999 if (desc.layout == C2Color::SEMIPLANAR_PACKED && !flexSemiPlanarPixelFormat) {
1000 flexSemiPlanarPixelFormat = desc.pixelFormat;
1001 }
1002 }
1003 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001004 if (!msg->findInt32(KEY_COLOR_FORMAT, &format)) {
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001005 // Also handle default color format (encoders require color format, so this is only
1006 // needed for decoders.
Pawin Vongmasa36653902018-11-15 00:10:25 -08001007 if (!(config->mDomain & Config::IS_ENCODER)) {
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001008 if (surface == nullptr) {
Wonsik Kim1eb88a92021-03-29 20:44:04 -07001009 const char *prefix = "";
1010 if (flexSemiPlanarPixelFormat) {
1011 format = COLOR_FormatYUV420SemiPlanar;
1012 prefix = "semi-";
1013 } else {
1014 format = COLOR_FormatYUV420Planar;
1015 }
1016 ALOGD("Client requested ByteBuffer mode decoder w/o color format set: "
1017 "using default %splanar color format", prefix);
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001018 } else {
1019 format = COLOR_FormatSurface;
1020 }
1021 defaultColorFormat = format;
1022 }
1023 } else {
1024 if ((config->mDomain & Config::IS_ENCODER) || !surface) {
1025 switch (format) {
1026 case COLOR_FormatYUV420Flexible:
1027 format = flexPixelFormat.value_or(COLOR_FormatYUV420Planar);
1028 break;
1029 case COLOR_FormatYUV420Planar:
1030 case COLOR_FormatYUV420PackedPlanar:
1031 format = flexPlanarPixelFormat.value_or(
1032 flexPixelFormat.value_or(format));
1033 break;
1034 case COLOR_FormatYUV420SemiPlanar:
1035 case COLOR_FormatYUV420PackedSemiPlanar:
1036 format = flexSemiPlanarPixelFormat.value_or(
1037 flexPixelFormat.value_or(format));
1038 break;
1039 default:
1040 // No-op
1041 break;
1042 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001043 }
1044 }
1045
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001046 if (format != 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001047 msg->setInt32("android._color-format", format);
1048 }
1049 }
1050
Wonsik Kim77e97c72021-01-20 10:33:22 -08001051 /*
1052 * Handle dataspace
1053 */
1054 int32_t usingRecorder;
1055 if (msg->findInt32("android._using-recorder", &usingRecorder) && usingRecorder) {
1056 android_dataspace dataSpace = HAL_DATASPACE_BT709;
1057 int32_t width, height;
1058 if (msg->findInt32("width", &width)
1059 && msg->findInt32("height", &height)) {
Wonsik Kim4f13d112021-03-17 04:37:46 +00001060 ColorAspects aspects;
1061 getColorAspectsFromFormat(msg, aspects);
1062 setDefaultCodecColorAspectsIfNeeded(aspects, width, height);
Wonsik Kim77e97c72021-01-20 10:33:22 -08001063 // TODO: read dataspace / color aspect from the component
Wonsik Kim4f13d112021-03-17 04:37:46 +00001064 setColorAspectsIntoFormat(aspects, const_cast<sp<AMessage> &>(msg));
1065 dataSpace = getDataSpaceForColorAspects(aspects, true /* mayexpand */);
Wonsik Kim77e97c72021-01-20 10:33:22 -08001066 }
1067 msg->setInt32("android._dataspace", (int32_t)dataSpace);
1068 ALOGD("setting dataspace to %x", dataSpace);
1069 }
1070
Wonsik Kim8a6ed372019-12-03 16:05:51 -08001071 int32_t subscribeToAllVendorParams;
1072 if (msg->findInt32("x-*", &subscribeToAllVendorParams) && subscribeToAllVendorParams) {
1073 if (config->subscribeToAllVendorParams(comp, C2_MAY_BLOCK) != OK) {
1074 ALOGD("[%s] Failed to subscribe to all vendor params", comp->getName().c_str());
1075 }
1076 }
1077
Pawin Vongmasa36653902018-11-15 00:10:25 -08001078 std::vector<std::unique_ptr<C2Param>> configUpdate;
Wonsik Kimaa484ac2019-02-13 16:54:02 -08001079 // NOTE: We used to ignore "video-bitrate" at configure; replicate
1080 // the behavior here.
1081 sp<AMessage> sdkParams = msg;
1082 int32_t videoBitrate;
1083 if (sdkParams->findInt32(PARAMETER_KEY_VIDEO_BITRATE, &videoBitrate)) {
1084 sdkParams = msg->dup();
1085 sdkParams->removeEntryAt(sdkParams->findEntryByName(PARAMETER_KEY_VIDEO_BITRATE));
1086 }
ted.sun765db4d2020-06-23 14:03:41 +08001087 err = config->getConfigUpdateFromSdkParams(
Wonsik Kimaa484ac2019-02-13 16:54:02 -08001088 comp, sdkParams, Config::IS_CONFIG, C2_DONT_BLOCK, &configUpdate);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001089 if (err != OK) {
1090 ALOGW("failed to convert configuration to c2 params");
1091 }
Wonsik Kimaab2eea2019-05-22 10:37:58 -07001092
1093 int32_t maxBframes = 0;
1094 if ((config->mDomain & Config::IS_ENCODER)
1095 && (config->mDomain & Config::IS_VIDEO)
1096 && sdkParams->findInt32(KEY_MAX_B_FRAMES, &maxBframes)
1097 && maxBframes > 0) {
1098 std::unique_ptr<C2StreamGopTuning::output> gop =
1099 C2StreamGopTuning::output::AllocUnique(2 /* flexCount */, 0u /* stream */);
1100 gop->m.values[0] = { P_FRAME, UINT32_MAX };
1101 gop->m.values[1] = {
1102 C2Config::picture_type_t(P_FRAME | B_FRAME),
1103 uint32_t(maxBframes)
1104 };
1105 configUpdate.push_back(std::move(gop));
1106 }
1107
Ray Essicka0ae6972021-03-10 19:40:01 -08001108 if ((config->mDomain & Config::IS_ENCODER)
1109 && (config->mDomain & Config::IS_VIDEO)) {
1110 // we may not use all 3 of these entries
1111 std::unique_ptr<C2StreamPictureQuantizationTuning::output> qp =
1112 C2StreamPictureQuantizationTuning::output::AllocUnique(3 /* flexCount */,
1113 0u /* stream */);
1114
1115 int ix = 0;
1116
1117 int32_t iMax = INT32_MAX;
1118 int32_t iMin = INT32_MIN;
1119 (void) sdkParams->findInt32(KEY_VIDEO_QP_I_MAX, &iMax);
1120 (void) sdkParams->findInt32(KEY_VIDEO_QP_I_MIN, &iMin);
1121 if (iMax != INT32_MAX || iMin != INT32_MIN) {
1122 qp->m.values[ix++] = {I_FRAME, iMin, iMax};
1123 }
1124
1125 int32_t pMax = INT32_MAX;
1126 int32_t pMin = INT32_MIN;
1127 (void) sdkParams->findInt32(KEY_VIDEO_QP_P_MAX, &pMax);
1128 (void) sdkParams->findInt32(KEY_VIDEO_QP_P_MIN, &pMin);
1129 if (pMax != INT32_MAX || pMin != INT32_MIN) {
1130 qp->m.values[ix++] = {P_FRAME, pMin, pMax};
1131 }
1132
1133 int32_t bMax = INT32_MAX;
1134 int32_t bMin = INT32_MIN;
1135 (void) sdkParams->findInt32(KEY_VIDEO_QP_B_MAX, &bMax);
1136 (void) sdkParams->findInt32(KEY_VIDEO_QP_B_MIN, &bMin);
1137 if (bMax != INT32_MAX || bMin != INT32_MIN) {
1138 qp->m.values[ix++] = {B_FRAME, bMin, bMax};
1139 }
1140
1141 // adjust to reflect actual use.
1142 qp->setFlexCount(ix);
1143
1144 configUpdate.push_back(std::move(qp));
1145 }
1146
Wonsik Kima1335e12021-04-22 16:28:29 -07001147 int32_t background = 0;
1148 if ((config->mDomain & Config::IS_VIDEO)
1149 && msg->findInt32("android._background-mode", &background)
1150 && background) {
1151 androidSetThreadPriority(gettid(), ANDROID_PRIORITY_BACKGROUND);
1152 if (config->mISConfig) {
1153 config->mISConfig->mPriority = ANDROID_PRIORITY_BACKGROUND;
1154 }
1155 }
1156
Pawin Vongmasa36653902018-11-15 00:10:25 -08001157 err = config->setParameters(comp, configUpdate, C2_DONT_BLOCK);
1158 if (err != OK) {
1159 ALOGW("failed to configure c2 params");
1160 return err;
1161 }
1162
1163 std::vector<std::unique_ptr<C2Param>> params;
1164 C2StreamUsageTuning::input usage(0u, 0u);
1165 C2StreamMaxBufferSizeInfo::input maxInputSize(0u, 0u);
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001166 C2PrependHeaderModeSetting prepend(PREPEND_HEADER_TO_NONE);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001167
Wonsik Kim3baecda2021-02-07 22:19:56 -08001168 C2Param::Index colorAspectsRequestIndex =
1169 C2StreamColorAspectsInfo::output::PARAM_TYPE | C2Param::CoreIndex::IS_REQUEST_FLAG;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001170 std::initializer_list<C2Param::Index> indices {
Wonsik Kim3baecda2021-02-07 22:19:56 -08001171 colorAspectsRequestIndex.withStream(0u),
Pawin Vongmasa36653902018-11-15 00:10:25 -08001172 };
1173 c2_status_t c2err = comp->query(
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001174 { &usage, &maxInputSize, &prepend },
Pawin Vongmasa36653902018-11-15 00:10:25 -08001175 indices,
1176 C2_DONT_BLOCK,
1177 &params);
1178 if (c2err != C2_OK && c2err != C2_BAD_INDEX) {
1179 ALOGE("Failed to query component interface: %d", c2err);
1180 return UNKNOWN_ERROR;
1181 }
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001182 if (usage) {
1183 if (usage.value & C2MemoryUsage::CPU_READ) {
1184 config->mInputFormat->setInt32("using-sw-read-often", true);
1185 }
1186 if (config->mISConfig) {
1187 C2AndroidMemoryUsage androidUsage(C2MemoryUsage(usage.value));
1188 config->mISConfig->mUsage = androidUsage.asGrallocUsage();
1189 }
Wonsik Kim666604a2020-05-14 16:57:49 -07001190 config->mInputFormat->setInt64("android._C2MemoryUsage", usage.value);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001191 }
1192
1193 // NOTE: we don't blindly use client specified input size if specified as clients
1194 // at times specify too small size. Instead, mimic the behavior from OMX, where the
1195 // client specified size is only used to ask for bigger buffers than component suggested
1196 // size.
1197 int32_t clientInputSize = 0;
1198 bool clientSpecifiedInputSize =
1199 msg->findInt32(KEY_MAX_INPUT_SIZE, &clientInputSize) && clientInputSize > 0;
1200 // TEMP: enforce minimum buffer size of 1MB for video decoders
1201 // and 16K / 4K for audio encoders/decoders
1202 if (maxInputSize.value == 0) {
1203 if (config->mDomain & Config::IS_AUDIO) {
1204 maxInputSize.value = encoder ? 16384 : 4096;
1205 } else if (!encoder) {
1206 maxInputSize.value = 1048576u;
1207 }
1208 }
1209
1210 // verify that CSD fits into this size (if defined)
1211 if ((config->mDomain & Config::IS_DECODER) && maxInputSize.value > 0) {
1212 sp<ABuffer> csd;
1213 for (size_t ix = 0; msg->findBuffer(StringPrintf("csd-%zu", ix).c_str(), &csd); ++ix) {
1214 if (csd && csd->size() > maxInputSize.value) {
1215 maxInputSize.value = csd->size();
1216 }
1217 }
1218 }
1219
1220 // TODO: do this based on component requiring linear allocator for input
1221 if ((config->mDomain & Config::IS_DECODER) || (config->mDomain & Config::IS_AUDIO)) {
1222 if (clientSpecifiedInputSize) {
1223 // Warn that we're overriding client's max input size if necessary.
1224 if ((uint32_t)clientInputSize < maxInputSize.value) {
1225 ALOGD("client requested max input size %d, which is smaller than "
1226 "what component recommended (%u); overriding with component "
1227 "recommendation.", clientInputSize, maxInputSize.value);
1228 ALOGW("This behavior is subject to change. It is recommended that "
1229 "app developers double check whether the requested "
1230 "max input size is in reasonable range.");
1231 } else {
1232 maxInputSize.value = clientInputSize;
1233 }
1234 }
1235 // Pass max input size on input format to the buffer channel (if supplied by the
1236 // component or by a default)
1237 if (maxInputSize.value) {
1238 config->mInputFormat->setInt32(
1239 KEY_MAX_INPUT_SIZE,
1240 (int32_t)(c2_min(maxInputSize.value, uint32_t(INT32_MAX))));
1241 }
1242 }
1243
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001244 int32_t clientPrepend;
1245 if ((config->mDomain & Config::IS_VIDEO)
1246 && (config->mDomain & Config::IS_ENCODER)
Lajos Molnara2b5f5a2020-10-14 16:36:18 -07001247 && msg->findInt32(KEY_PREPEND_HEADER_TO_SYNC_FRAMES, &clientPrepend)
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001248 && clientPrepend
1249 && (!prepend || prepend.value != PREPEND_HEADER_TO_ALL_SYNC)) {
Lajos Molnara2b5f5a2020-10-14 16:36:18 -07001250 ALOGE("Failed to set KEY_PREPEND_HEADER_TO_SYNC_FRAMES");
Wonsik Kim9ca01d32019-04-01 14:45:47 -07001251 return BAD_VALUE;
1252 }
1253
Wonsik Kimd91f3fb2021-02-24 12:35:31 -08001254 int32_t componentColorFormat = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001255 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
1256 // propagate HDR static info to output format for both encoders and decoders
1257 // if component supports this info, we will update from component, but only the raw port,
1258 // so don't propagate if component already filled it in.
1259 sp<ABuffer> hdrInfo;
1260 if (msg->findBuffer(KEY_HDR_STATIC_INFO, &hdrInfo)
1261 && !config->mOutputFormat->findBuffer(KEY_HDR_STATIC_INFO, &hdrInfo)) {
1262 config->mOutputFormat->setBuffer(KEY_HDR_STATIC_INFO, hdrInfo);
1263 }
1264
1265 // Set desired color format from configuration parameter
1266 int32_t format;
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001267 if (!msg->findInt32(KEY_COLOR_FORMAT, &format)) {
1268 format = defaultColorFormat;
1269 }
1270 if (config->mDomain & Config::IS_ENCODER) {
1271 config->mInputFormat->setInt32(KEY_COLOR_FORMAT, format);
Wonsik Kimd91f3fb2021-02-24 12:35:31 -08001272 if (msg->findInt32("android._color-format", &componentColorFormat)) {
1273 config->mInputFormat->setInt32("android._color-format", componentColorFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001274 }
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07001275 } else {
1276 config->mOutputFormat->setInt32(KEY_COLOR_FORMAT, format);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001277 }
1278 }
1279
1280 // propagate encoder delay and padding to output format
1281 if ((config->mDomain & Config::IS_DECODER) && (config->mDomain & Config::IS_AUDIO)) {
1282 int delay = 0;
1283 if (msg->findInt32("encoder-delay", &delay)) {
1284 config->mOutputFormat->setInt32("encoder-delay", delay);
1285 }
1286 int padding = 0;
1287 if (msg->findInt32("encoder-padding", &padding)) {
1288 config->mOutputFormat->setInt32("encoder-padding", padding);
1289 }
1290 }
1291
1292 // set channel-mask
1293 if (config->mDomain & Config::IS_AUDIO) {
1294 int32_t mask;
1295 if (msg->findInt32(KEY_CHANNEL_MASK, &mask)) {
1296 if (config->mDomain & Config::IS_ENCODER) {
1297 config->mInputFormat->setInt32(KEY_CHANNEL_MASK, mask);
1298 } else {
1299 config->mOutputFormat->setInt32(KEY_CHANNEL_MASK, mask);
1300 }
1301 }
1302 }
1303
Wonsik Kim3baecda2021-02-07 22:19:56 -08001304 std::unique_ptr<C2Param> colorTransferRequestParam;
1305 for (std::unique_ptr<C2Param> &param : params) {
1306 if (param->index() == colorAspectsRequestIndex.withStream(0u)) {
1307 ALOGI("found color transfer request param");
1308 colorTransferRequestParam = std::move(param);
1309 }
1310 }
1311 int32_t colorTransferRequest = 0;
1312 if (config->mDomain & (Config::IS_IMAGE | Config::IS_VIDEO)
1313 && !sdkParams->findInt32("color-transfer-request", &colorTransferRequest)) {
1314 colorTransferRequest = 0;
1315 }
1316
1317 if (colorTransferRequest != 0) {
1318 if (colorTransferRequestParam && *colorTransferRequestParam) {
1319 C2StreamColorAspectsInfo::output *info =
1320 static_cast<C2StreamColorAspectsInfo::output *>(
1321 colorTransferRequestParam.get());
1322 if (!C2Mapper::map(info->transfer, &colorTransferRequest)) {
1323 colorTransferRequest = 0;
1324 }
1325 } else {
1326 colorTransferRequest = 0;
1327 }
1328 config->mInputFormat->setInt32("color-transfer-request", colorTransferRequest);
1329 }
1330
Wonsik Kimd91f3fb2021-02-24 12:35:31 -08001331 if (componentColorFormat != 0 && componentColorFormat != COLOR_FormatSurface) {
1332 // Need to get stride/vstride
1333 uint32_t pixelFormat = PIXEL_FORMAT_UNKNOWN;
1334 if (C2Mapper::mapPixelFormatFrameworkToCodec(componentColorFormat, &pixelFormat)) {
1335 // TODO: retrieve these values without allocating a buffer.
1336 // Currently allocating a buffer is necessary to retrieve the layout.
1337 int64_t blockUsage =
1338 usage.value | C2MemoryUsage::CPU_READ | C2MemoryUsage::CPU_WRITE;
1339 std::shared_ptr<C2GraphicBlock> block = FetchGraphicBlock(
1340 width, height, pixelFormat, blockUsage, {comp->getName()});
1341 sp<GraphicBlockBuffer> buffer;
1342 if (block) {
1343 buffer = GraphicBlockBuffer::Allocate(
1344 config->mInputFormat,
1345 block,
1346 [](size_t size) -> sp<ABuffer> { return new ABuffer(size); });
1347 } else {
1348 ALOGD("Failed to allocate a graphic block "
1349 "(width=%d height=%d pixelFormat=%u usage=%llx)",
1350 width, height, pixelFormat, (long long)blockUsage);
1351 // This means that byte buffer mode is not supported in this configuration
1352 // anyway. Skip setting stride/vstride to input format.
1353 }
1354 if (buffer) {
1355 sp<ABuffer> imageData = buffer->getImageData();
1356 MediaImage2 *img = nullptr;
1357 if (imageData && imageData->data()
1358 && imageData->size() >= sizeof(MediaImage2)) {
1359 img = (MediaImage2*)imageData->data();
1360 }
1361 if (img && img->mNumPlanes > 0 && img->mType != img->MEDIA_IMAGE_TYPE_UNKNOWN) {
1362 int32_t stride = img->mPlane[0].mRowInc;
1363 config->mInputFormat->setInt32(KEY_STRIDE, stride);
1364 if (img->mNumPlanes > 1 && stride > 0) {
1365 int64_t offsetDelta =
1366 (int64_t)img->mPlane[1].mOffset - (int64_t)img->mPlane[0].mOffset;
1367 if (offsetDelta % stride == 0) {
1368 int32_t vstride = int32_t(offsetDelta / stride);
1369 config->mInputFormat->setInt32(KEY_SLICE_HEIGHT, vstride);
1370 } else {
1371 ALOGD("Cannot report accurate slice height: "
1372 "offsetDelta = %lld stride = %d",
1373 (long long)offsetDelta, stride);
1374 }
1375 }
1376 }
1377 }
1378 }
1379 }
1380
1381 ALOGD("setup formats input: %s",
1382 config->mInputFormat->debugString().c_str());
1383 ALOGD("setup formats output: %s",
Pawin Vongmasa36653902018-11-15 00:10:25 -08001384 config->mOutputFormat->debugString().c_str());
1385 return OK;
1386 };
1387 if (tryAndReportOnError(doConfig) != OK) {
1388 return;
1389 }
1390
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001391 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1392 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001393
Houxiang Dai183ca3d2021-02-04 14:20:11 +08001394 config->queryConfiguration(comp);
1395
Pawin Vongmasa36653902018-11-15 00:10:25 -08001396 mCallback->onComponentConfigured(config->mInputFormat, config->mOutputFormat);
1397}
1398
1399void CCodec::initiateCreateInputSurface() {
1400 status_t err = [this] {
1401 Mutexed<State>::Locked state(mState);
1402 if (state->get() != ALLOCATED) {
1403 return UNKNOWN_ERROR;
1404 }
1405 // TODO: read it from intf() properly.
1406 if (state->comp->getName().find("encoder") == std::string::npos) {
1407 return INVALID_OPERATION;
1408 }
1409 return OK;
1410 }();
1411 if (err != OK) {
1412 mCallback->onInputSurfaceCreationFailed(err);
1413 return;
1414 }
1415
1416 (new AMessage(kWhatCreateInputSurface, this))->post();
1417}
1418
Lajos Molnar47118272019-01-31 16:28:04 -08001419sp<PersistentSurface> CCodec::CreateOmxInputSurface() {
1420 using namespace android::hardware::media::omx::V1_0;
1421 using namespace android::hardware::media::omx::V1_0::utils;
1422 using namespace android::hardware::graphics::bufferqueue::V1_0::utils;
1423 typedef android::hardware::media::omx::V1_0::Status OmxStatus;
1424 android::sp<IOmx> omx = IOmx::getService();
1425 typedef android::hardware::graphics::bufferqueue::V1_0::
1426 IGraphicBufferProducer HGraphicBufferProducer;
1427 typedef android::hardware::media::omx::V1_0::
1428 IGraphicBufferSource HGraphicBufferSource;
1429 OmxStatus s;
1430 android::sp<HGraphicBufferProducer> gbp;
1431 android::sp<HGraphicBufferSource> gbs;
Pawin Vongmasa18588322019-05-18 01:52:13 -07001432
Chong Zhangc8ce1d82019-03-27 10:18:38 -07001433 using ::android::hardware::Return;
1434 Return<void> transStatus = omx->createInputSurface(
Lajos Molnar47118272019-01-31 16:28:04 -08001435 [&s, &gbp, &gbs](
1436 OmxStatus status,
1437 const android::sp<HGraphicBufferProducer>& producer,
1438 const android::sp<HGraphicBufferSource>& source) {
1439 s = status;
1440 gbp = producer;
1441 gbs = source;
1442 });
1443 if (transStatus.isOk() && s == OmxStatus::OK) {
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001444 return new PersistentSurface(new H2BGraphicBufferProducer(gbp), gbs);
Lajos Molnar47118272019-01-31 16:28:04 -08001445 }
1446
1447 return nullptr;
1448}
1449
1450sp<PersistentSurface> CCodec::CreateCompatibleInputSurface() {
1451 sp<PersistentSurface> surface(CreateInputSurface());
1452
1453 if (surface == nullptr) {
1454 surface = CreateOmxInputSurface();
1455 }
1456
1457 return surface;
1458}
1459
Pawin Vongmasa36653902018-11-15 00:10:25 -08001460void CCodec::createInputSurface() {
1461 status_t err;
1462 sp<IGraphicBufferProducer> bufferProducer;
1463
1464 sp<AMessage> inputFormat;
1465 sp<AMessage> outputFormat;
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001466 uint64_t usage = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001467 {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001468 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1469 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001470 inputFormat = config->mInputFormat;
1471 outputFormat = config->mOutputFormat;
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001472 usage = config->mISConfig ? config->mISConfig->mUsage : 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001473 }
1474
Lajos Molnar47118272019-01-31 16:28:04 -08001475 sp<PersistentSurface> persistentSurface = CreateCompatibleInputSurface();
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001476 sp<hidl::base::V1_0::IBase> hidlTarget = persistentSurface->getHidlTarget();
1477 sp<IInputSurface> hidlInputSurface = IInputSurface::castFrom(hidlTarget);
1478 sp<HGraphicBufferSource> gbs = HGraphicBufferSource::castFrom(hidlTarget);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001479
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001480 if (hidlInputSurface) {
Pawin Vongmasa1c75a232019-01-09 04:41:52 -08001481 std::shared_ptr<Codec2Client::InputSurface> inputSurface =
1482 std::make_shared<Codec2Client::InputSurface>(hidlInputSurface);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001483 err = setupInputSurface(std::make_shared<C2InputSurfaceWrapper>(
Pawin Vongmasa1c75a232019-01-09 04:41:52 -08001484 inputSurface));
1485 bufferProducer = inputSurface->getGraphicBufferProducer();
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001486 } else if (gbs) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001487 int32_t width = 0;
1488 (void)outputFormat->findInt32("width", &width);
1489 int32_t height = 0;
1490 (void)outputFormat->findInt32("height", &height);
1491 err = setupInputSurface(std::make_shared<GraphicBufferSourceWrapper>(
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001492 gbs, width, height, usage));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001493 bufferProducer = persistentSurface->getBufferProducer();
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001494 } else {
1495 ALOGE("Corrupted input surface");
1496 mCallback->onInputSurfaceCreationFailed(UNKNOWN_ERROR);
1497 return;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001498 }
1499
1500 if (err != OK) {
1501 ALOGE("Failed to set up input surface: %d", err);
1502 mCallback->onInputSurfaceCreationFailed(err);
1503 return;
1504 }
1505
1506 mCallback->onInputSurfaceCreated(
1507 inputFormat,
1508 outputFormat,
1509 new BufferProducerWrapper(bufferProducer));
1510}
1511
1512status_t CCodec::setupInputSurface(const std::shared_ptr<InputSurfaceWrapper> &surface) {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001513 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1514 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001515 config->mUsingSurface = true;
1516
1517 // we are now using surface - apply default color aspects to input format - as well as
1518 // get dataspace
Wonsik Kim8a6ed372019-12-03 16:05:51 -08001519 bool inputFormatChanged = config->updateFormats(Config::IS_INPUT);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001520 ALOGD("input format %s to %s",
1521 inputFormatChanged ? "changed" : "unchanged",
1522 config->mInputFormat->debugString().c_str());
1523
1524 // configure dataspace
1525 static_assert(sizeof(int32_t) == sizeof(android_dataspace), "dataspace size mismatch");
1526 android_dataspace dataSpace = HAL_DATASPACE_UNKNOWN;
1527 (void)config->mInputFormat->findInt32("android._dataspace", (int32_t*)&dataSpace);
1528 surface->setDataSpace(dataSpace);
1529
1530 status_t err = mChannel->setInputSurface(surface);
1531 if (err != OK) {
1532 // undo input format update
1533 config->mUsingSurface = false;
Wonsik Kim8a6ed372019-12-03 16:05:51 -08001534 (void)config->updateFormats(Config::IS_INPUT);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001535 return err;
1536 }
1537 config->mInputSurface = surface;
1538
1539 if (config->mISConfig) {
1540 surface->configure(*config->mISConfig);
1541 } else {
1542 ALOGD("ISConfig: no configuration");
1543 }
1544
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001545 return OK;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001546}
1547
1548void CCodec::initiateSetInputSurface(const sp<PersistentSurface> &surface) {
1549 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
1550 msg->setObject("surface", surface);
1551 msg->post();
1552}
1553
1554void CCodec::setInputSurface(const sp<PersistentSurface> &surface) {
1555 sp<AMessage> inputFormat;
1556 sp<AMessage> outputFormat;
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001557 uint64_t usage = 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001558 {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001559 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1560 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001561 inputFormat = config->mInputFormat;
1562 outputFormat = config->mOutputFormat;
Wonsik Kim9eac4d12019-05-23 12:58:48 -07001563 usage = config->mISConfig ? config->mISConfig->mUsage : 0;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001564 }
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001565 sp<hidl::base::V1_0::IBase> hidlTarget = surface->getHidlTarget();
1566 sp<IInputSurface> inputSurface = IInputSurface::castFrom(hidlTarget);
1567 sp<HGraphicBufferSource> gbs = HGraphicBufferSource::castFrom(hidlTarget);
1568 if (inputSurface) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001569 status_t err = setupInputSurface(std::make_shared<C2InputSurfaceWrapper>(
1570 std::make_shared<Codec2Client::InputSurface>(inputSurface)));
1571 if (err != OK) {
1572 ALOGE("Failed to set up input surface: %d", err);
1573 mCallback->onInputSurfaceDeclined(err);
1574 return;
1575 }
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001576 } else if (gbs) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001577 int32_t width = 0;
1578 (void)outputFormat->findInt32("width", &width);
1579 int32_t height = 0;
1580 (void)outputFormat->findInt32("height", &height);
1581 status_t err = setupInputSurface(std::make_shared<GraphicBufferSourceWrapper>(
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001582 gbs, width, height, usage));
Pawin Vongmasa36653902018-11-15 00:10:25 -08001583 if (err != OK) {
1584 ALOGE("Failed to set up input surface: %d", err);
1585 mCallback->onInputSurfaceDeclined(err);
1586 return;
1587 }
Wonsik Kim9917d4a2019-10-24 12:56:38 -07001588 } else {
1589 ALOGE("Failed to set input surface: Corrupted surface.");
1590 mCallback->onInputSurfaceDeclined(UNKNOWN_ERROR);
1591 return;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001592 }
1593 mCallback->onInputSurfaceAccepted(inputFormat, outputFormat);
1594}
1595
1596void CCodec::initiateStart() {
1597 auto setStarting = [this] {
1598 Mutexed<State>::Locked state(mState);
1599 if (state->get() != ALLOCATED) {
1600 return UNKNOWN_ERROR;
1601 }
1602 state->set(STARTING);
1603 return OK;
1604 };
1605 if (tryAndReportOnError(setStarting) != OK) {
1606 return;
1607 }
1608
1609 (new AMessage(kWhatStart, this))->post();
1610}
1611
1612void CCodec::start() {
1613 std::shared_ptr<Codec2Client::Component> comp;
1614 auto checkStarting = [this, &comp] {
1615 Mutexed<State>::Locked state(mState);
1616 if (state->get() != STARTING) {
1617 return UNKNOWN_ERROR;
1618 }
1619 comp = state->comp;
1620 return OK;
1621 };
1622 if (tryAndReportOnError(checkStarting) != OK) {
1623 return;
1624 }
1625
1626 c2_status_t err = comp->start();
1627 if (err != C2_OK) {
1628 mCallback->onError(toStatusT(err, C2_OPERATION_Component_start),
1629 ACTION_CODE_FATAL);
1630 return;
1631 }
1632 sp<AMessage> inputFormat;
1633 sp<AMessage> outputFormat;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001634 status_t err2 = OK;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001635 bool buffersBoundToCodec = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001636 {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001637 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1638 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001639 inputFormat = config->mInputFormat;
Wonsik Kim274c8322020-02-28 10:42:21 -08001640 // start triggers format dup
1641 outputFormat = config->mOutputFormat = config->mOutputFormat->dup();
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001642 if (config->mInputSurface) {
1643 err2 = config->mInputSurface->start();
Wonsik Kim673dd192021-01-29 14:58:12 -08001644 config->mInputSurfaceDataspace = config->mInputSurface->getDataspace();
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001645 }
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001646 buffersBoundToCodec = config->mBuffersBoundToCodec;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001647 }
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001648 if (err2 != OK) {
1649 mCallback->onError(err2, ACTION_CODE_FATAL);
1650 return;
1651 }
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001652 err2 = mChannel->start(inputFormat, outputFormat, buffersBoundToCodec);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001653 if (err2 != OK) {
1654 mCallback->onError(err2, ACTION_CODE_FATAL);
1655 return;
1656 }
1657
1658 auto setRunning = [this] {
1659 Mutexed<State>::Locked state(mState);
1660 if (state->get() != STARTING) {
1661 return UNKNOWN_ERROR;
1662 }
1663 state->set(RUNNING);
1664 return OK;
1665 };
1666 if (tryAndReportOnError(setRunning) != OK) {
1667 return;
1668 }
1669 mCallback->onStartCompleted();
1670
1671 (void)mChannel->requestInitialInputBuffers();
1672}
1673
1674void CCodec::initiateShutdown(bool keepComponentAllocated) {
1675 if (keepComponentAllocated) {
1676 initiateStop();
1677 } else {
1678 initiateRelease();
1679 }
1680}
1681
1682void CCodec::initiateStop() {
1683 {
1684 Mutexed<State>::Locked state(mState);
1685 if (state->get() == ALLOCATED
1686 || state->get() == RELEASED
1687 || state->get() == STOPPING
1688 || state->get() == RELEASING) {
1689 // We're already stopped, released, or doing it right now.
1690 state.unlock();
1691 mCallback->onStopCompleted();
1692 state.lock();
1693 return;
1694 }
1695 state->set(STOPPING);
1696 }
1697
Wonsik Kim936a89c2020-05-08 16:07:50 -07001698 mChannel->reset();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001699 (new AMessage(kWhatStop, this))->post();
1700}
1701
1702void CCodec::stop() {
1703 std::shared_ptr<Codec2Client::Component> comp;
1704 {
1705 Mutexed<State>::Locked state(mState);
1706 if (state->get() == RELEASING) {
1707 state.unlock();
1708 // We're already stopped or release is in progress.
1709 mCallback->onStopCompleted();
1710 state.lock();
1711 return;
1712 } else if (state->get() != STOPPING) {
1713 state.unlock();
1714 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1715 state.lock();
1716 return;
1717 }
1718 comp = state->comp;
1719 }
1720 status_t err = comp->stop();
1721 if (err != C2_OK) {
1722 // TODO: convert err into status_t
1723 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1724 }
1725
1726 {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001727 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1728 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001729 if (config->mInputSurface) {
1730 config->mInputSurface->disconnect();
1731 config->mInputSurface = nullptr;
Wonsik Kim673dd192021-01-29 14:58:12 -08001732 config->mInputSurfaceDataspace = HAL_DATASPACE_UNKNOWN;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001733 }
1734 }
1735 {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001736 Mutexed<State>::Locked state(mState);
1737 if (state->get() == STOPPING) {
1738 state->set(ALLOCATED);
1739 }
1740 }
1741 mCallback->onStopCompleted();
1742}
1743
1744void CCodec::initiateRelease(bool sendCallback /* = true */) {
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001745 bool clearInputSurfaceIfNeeded = false;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001746 {
1747 Mutexed<State>::Locked state(mState);
1748 if (state->get() == RELEASED || state->get() == RELEASING) {
1749 // We're already released or doing it right now.
1750 if (sendCallback) {
1751 state.unlock();
1752 mCallback->onReleaseCompleted();
1753 state.lock();
1754 }
1755 return;
1756 }
1757 if (state->get() == ALLOCATING) {
1758 state->set(RELEASING);
1759 // With the altered state allocate() would fail and clean up.
1760 if (sendCallback) {
1761 state.unlock();
1762 mCallback->onReleaseCompleted();
1763 state.lock();
1764 }
1765 return;
1766 }
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001767 if (state->get() == STARTING
1768 || state->get() == RUNNING
1769 || state->get() == STOPPING) {
1770 // Input surface may have been started, so clean up is needed.
1771 clearInputSurfaceIfNeeded = true;
1772 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001773 state->set(RELEASING);
1774 }
1775
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001776 if (clearInputSurfaceIfNeeded) {
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001777 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1778 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001779 if (config->mInputSurface) {
1780 config->mInputSurface->disconnect();
1781 config->mInputSurface = nullptr;
Wonsik Kim673dd192021-01-29 14:58:12 -08001782 config->mInputSurfaceDataspace = HAL_DATASPACE_UNKNOWN;
Pawin Vongmasa1f213362019-01-24 06:59:16 -08001783 }
1784 }
1785
Wonsik Kim936a89c2020-05-08 16:07:50 -07001786 mChannel->reset();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001787 // thiz holds strong ref to this while the thread is running.
1788 sp<CCodec> thiz(this);
1789 std::thread([thiz, sendCallback] { thiz->release(sendCallback); }).detach();
1790}
1791
1792void CCodec::release(bool sendCallback) {
1793 std::shared_ptr<Codec2Client::Component> comp;
1794 {
1795 Mutexed<State>::Locked state(mState);
1796 if (state->get() == RELEASED) {
1797 if (sendCallback) {
1798 state.unlock();
1799 mCallback->onReleaseCompleted();
1800 state.lock();
1801 }
1802 return;
1803 }
1804 comp = state->comp;
1805 }
1806 comp->release();
1807
1808 {
1809 Mutexed<State>::Locked state(mState);
1810 state->set(RELEASED);
1811 state->comp.reset();
1812 }
Wonsik Kim936a89c2020-05-08 16:07:50 -07001813 (new AMessage(kWhatRelease, this))->post();
Pawin Vongmasa36653902018-11-15 00:10:25 -08001814 if (sendCallback) {
1815 mCallback->onReleaseCompleted();
1816 }
1817}
1818
1819status_t CCodec::setSurface(const sp<Surface> &surface) {
Wonsik Kim75e22f42021-04-14 23:34:51 -07001820 {
1821 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1822 const std::unique_ptr<Config> &config = *configLocked;
1823 if (config->mTunneled && config->mSidebandHandle != nullptr) {
1824 sp<ANativeWindow> nativeWindow = static_cast<ANativeWindow *>(surface.get());
1825 status_t err = native_window_set_sideband_stream(
1826 nativeWindow.get(),
1827 const_cast<native_handle_t *>(config->mSidebandHandle->handle()));
1828 if (err != OK) {
1829 ALOGE("NativeWindow(%p) native_window_set_sideband_stream(%p) failed! (err %d).",
1830 nativeWindow.get(), config->mSidebandHandle->handle(), err);
1831 return err;
1832 }
ted.sun765db4d2020-06-23 14:03:41 +08001833 }
1834 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001835 return mChannel->setSurface(surface);
1836}
1837
1838void CCodec::signalFlush() {
1839 status_t err = [this] {
1840 Mutexed<State>::Locked state(mState);
1841 if (state->get() == FLUSHED) {
1842 return ALREADY_EXISTS;
1843 }
1844 if (state->get() != RUNNING) {
1845 return UNKNOWN_ERROR;
1846 }
1847 state->set(FLUSHING);
1848 return OK;
1849 }();
1850 switch (err) {
1851 case ALREADY_EXISTS:
1852 mCallback->onFlushCompleted();
1853 return;
1854 case OK:
1855 break;
1856 default:
1857 mCallback->onError(err, ACTION_CODE_FATAL);
1858 return;
1859 }
1860
1861 mChannel->stop();
1862 (new AMessage(kWhatFlush, this))->post();
1863}
1864
1865void CCodec::flush() {
1866 std::shared_ptr<Codec2Client::Component> comp;
1867 auto checkFlushing = [this, &comp] {
1868 Mutexed<State>::Locked state(mState);
1869 if (state->get() != FLUSHING) {
1870 return UNKNOWN_ERROR;
1871 }
1872 comp = state->comp;
1873 return OK;
1874 };
1875 if (tryAndReportOnError(checkFlushing) != OK) {
1876 return;
1877 }
1878
1879 std::list<std::unique_ptr<C2Work>> flushedWork;
1880 c2_status_t err = comp->flush(C2Component::FLUSH_COMPONENT, &flushedWork);
1881 {
1882 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
1883 flushedWork.splice(flushedWork.end(), *queue);
1884 }
1885 if (err != C2_OK) {
1886 // TODO: convert err into status_t
1887 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1888 }
1889
1890 mChannel->flush(flushedWork);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001891
1892 {
1893 Mutexed<State>::Locked state(mState);
Iris Changce521ee2019-07-05 16:18:54 +08001894 if (state->get() == FLUSHING) {
1895 state->set(FLUSHED);
1896 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08001897 }
1898 mCallback->onFlushCompleted();
1899}
1900
1901void CCodec::signalResume() {
Wonsik Kime75a5da2020-02-14 17:29:03 -08001902 std::shared_ptr<Codec2Client::Component> comp;
1903 auto setResuming = [this, &comp] {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001904 Mutexed<State>::Locked state(mState);
1905 if (state->get() != FLUSHED) {
1906 return UNKNOWN_ERROR;
1907 }
1908 state->set(RESUMING);
Wonsik Kime75a5da2020-02-14 17:29:03 -08001909 comp = state->comp;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001910 return OK;
1911 };
1912 if (tryAndReportOnError(setResuming) != OK) {
1913 return;
1914 }
1915
Wonsik Kime75a5da2020-02-14 17:29:03 -08001916 {
1917 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1918 const std::unique_ptr<Config> &config = *configLocked;
Harish Mahendrakar8c537502021-02-23 21:20:22 -08001919 sp<AMessage> outputFormat = config->mOutputFormat;
Wonsik Kime75a5da2020-02-14 17:29:03 -08001920 config->queryConfiguration(comp);
Harish Mahendrakar8c537502021-02-23 21:20:22 -08001921 RevertOutputFormatIfNeeded(outputFormat, config->mOutputFormat);
Wonsik Kime75a5da2020-02-14 17:29:03 -08001922 }
1923
Wonsik Kimfb7a7672019-12-27 17:13:33 -08001924 (void)mChannel->start(nullptr, nullptr, [&]{
1925 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1926 const std::unique_ptr<Config> &config = *configLocked;
1927 return config->mBuffersBoundToCodec;
1928 }());
Pawin Vongmasa36653902018-11-15 00:10:25 -08001929
1930 {
1931 Mutexed<State>::Locked state(mState);
1932 if (state->get() != RESUMING) {
1933 state.unlock();
1934 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
1935 state.lock();
1936 return;
1937 }
1938 state->set(RUNNING);
1939 }
1940
1941 (void)mChannel->requestInitialInputBuffers();
1942}
1943
Wonsik Kimaa484ac2019-02-13 16:54:02 -08001944void CCodec::signalSetParameters(const sp<AMessage> &msg) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001945 std::shared_ptr<Codec2Client::Component> comp;
1946 auto checkState = [this, &comp] {
1947 Mutexed<State>::Locked state(mState);
1948 if (state->get() == RELEASED) {
1949 return INVALID_OPERATION;
1950 }
1951 comp = state->comp;
1952 return OK;
1953 };
1954 if (tryAndReportOnError(checkState) != OK) {
1955 return;
1956 }
1957
Wonsik Kimaa484ac2019-02-13 16:54:02 -08001958 // NOTE: We used to ignore "bitrate" at setParameters; replicate
1959 // the behavior here.
1960 sp<AMessage> params = msg;
1961 int32_t bitrate;
1962 if (params->findInt32(KEY_BIT_RATE, &bitrate)) {
1963 params = msg->dup();
1964 params->removeEntryAt(params->findEntryByName(KEY_BIT_RATE));
1965 }
1966
Houxiang Dai5a97b472021-03-22 17:56:04 +08001967 int32_t syncId = 0;
1968 if (params->findInt32("audio-hw-sync", &syncId)
1969 || params->findInt32("hw-av-sync-id", &syncId)) {
1970 configureTunneledVideoPlayback(comp, nullptr, params);
1971 }
1972
Wonsik Kim155d5cb2019-10-09 12:49:49 -07001973 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
1974 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08001975
1976 /**
1977 * Handle input surface parameters
1978 */
1979 if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))
Wonsik Kim8a6ed372019-12-03 16:05:51 -08001980 && (config->mDomain & Config::IS_ENCODER)
1981 && config->mInputSurface && config->mISConfig) {
Chong Zhang038e8f82019-02-06 19:05:14 -08001982 (void)params->findInt64(PARAMETER_KEY_OFFSET_TIME, &config->mISConfig->mTimeOffsetUs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001983
1984 if (params->findInt64("skip-frames-before", &config->mISConfig->mStartAtUs)) {
1985 config->mISConfig->mStopped = false;
1986 } else if (params->findInt64("stop-time-us", &config->mISConfig->mStopAtUs)) {
1987 config->mISConfig->mStopped = true;
1988 }
1989
1990 int32_t value;
Chong Zhang038e8f82019-02-06 19:05:14 -08001991 if (params->findInt32(PARAMETER_KEY_SUSPEND, &value)) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08001992 config->mISConfig->mSuspended = value;
1993 config->mISConfig->mSuspendAtUs = -1;
Chong Zhang038e8f82019-02-06 19:05:14 -08001994 (void)params->findInt64(PARAMETER_KEY_SUSPEND_TIME, &config->mISConfig->mSuspendAtUs);
Pawin Vongmasa36653902018-11-15 00:10:25 -08001995 }
1996
1997 (void)config->mInputSurface->configure(*config->mISConfig);
1998 if (config->mISConfig->mStopped) {
1999 config->mInputFormat->setInt64(
2000 "android._stop-time-offset-us", config->mISConfig->mInputDelayUs);
2001 }
2002 }
2003
2004 std::vector<std::unique_ptr<C2Param>> configUpdate;
2005 (void)config->getConfigUpdateFromSdkParams(
2006 comp, params, Config::IS_PARAM, C2_MAY_BLOCK, &configUpdate);
2007 // Prefer to pass parameters to the buffer channel, so they can be synchronized with the frames.
2008 // Parameter synchronization is not defined when using input surface. For now, route
2009 // these directly to the component.
2010 if (config->mInputSurface == nullptr
2011 && (property_get_bool("debug.stagefright.ccodec_delayed_params", false)
2012 || comp->getName().find("c2.android.") == 0)) {
2013 mChannel->setParameters(configUpdate);
2014 } else {
Wonsik Kim3b4349a2020-11-10 11:54:15 -08002015 sp<AMessage> outputFormat = config->mOutputFormat;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002016 (void)config->setParameters(comp, configUpdate, C2_MAY_BLOCK);
Wonsik Kim3b4349a2020-11-10 11:54:15 -08002017 RevertOutputFormatIfNeeded(outputFormat, config->mOutputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002018 }
2019}
2020
2021void CCodec::signalEndOfInputStream() {
2022 mCallback->onSignaledInputEOS(mChannel->signalEndOfInputStream());
2023}
2024
2025void CCodec::signalRequestIDRFrame() {
2026 std::shared_ptr<Codec2Client::Component> comp;
2027 {
2028 Mutexed<State>::Locked state(mState);
2029 if (state->get() == RELEASED) {
2030 ALOGD("no IDR request sent since component is released");
2031 return;
2032 }
2033 comp = state->comp;
2034 }
2035 ALOGV("request IDR");
Wonsik Kim155d5cb2019-10-09 12:49:49 -07002036 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2037 const std::unique_ptr<Config> &config = *configLocked;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002038 std::vector<std::unique_ptr<C2Param>> params;
2039 params.push_back(
2040 std::make_unique<C2StreamRequestSyncFrameTuning::output>(0u, true));
2041 config->setParameters(comp, params, C2_MAY_BLOCK);
2042}
2043
Wonsik Kim874ad382021-03-12 09:59:36 -08002044status_t CCodec::querySupportedParameters(std::vector<std::string> *names) {
2045 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2046 const std::unique_ptr<Config> &config = *configLocked;
2047 return config->querySupportedParameters(names);
2048}
2049
2050status_t CCodec::describeParameter(
2051 const std::string &name, CodecParameterDescriptor *desc) {
2052 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2053 const std::unique_ptr<Config> &config = *configLocked;
2054 return config->describe(name, desc);
2055}
2056
2057status_t CCodec::subscribeToParameters(const std::vector<std::string> &names) {
2058 std::shared_ptr<Codec2Client::Component> comp = mState.lock()->comp;
2059 if (!comp) {
2060 return INVALID_OPERATION;
2061 }
2062 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2063 const std::unique_ptr<Config> &config = *configLocked;
2064 return config->subscribeToVendorConfigUpdate(comp, names);
2065}
2066
2067status_t CCodec::unsubscribeFromParameters(const std::vector<std::string> &names) {
2068 std::shared_ptr<Codec2Client::Component> comp = mState.lock()->comp;
2069 if (!comp) {
2070 return INVALID_OPERATION;
2071 }
2072 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2073 const std::unique_ptr<Config> &config = *configLocked;
2074 return config->unsubscribeFromVendorConfigUpdate(comp, names);
2075}
2076
Wonsik Kimab34ed62019-01-31 15:28:46 -08002077void CCodec::onWorkDone(std::list<std::unique_ptr<C2Work>> &workItems) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002078 if (!workItems.empty()) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08002079 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
2080 queue->splice(queue->end(), workItems);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002081 }
2082 (new AMessage(kWhatWorkDone, this))->post();
2083}
2084
Wonsik Kimab34ed62019-01-31 15:28:46 -08002085void CCodec::onInputBufferDone(uint64_t frameIndex, size_t arrayIndex) {
2086 mChannel->onInputBufferDone(frameIndex, arrayIndex);
Wonsik Kim4f3314d2019-03-26 17:00:34 -07002087 if (arrayIndex == 0) {
2088 // We always put no more than one buffer per work, if we use an input surface.
Wonsik Kim155d5cb2019-10-09 12:49:49 -07002089 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2090 const std::unique_ptr<Config> &config = *configLocked;
Wonsik Kim4f3314d2019-03-26 17:00:34 -07002091 if (config->mInputSurface) {
2092 config->mInputSurface->onInputBufferDone(frameIndex);
2093 }
2094 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002095}
2096
2097void CCodec::onMessageReceived(const sp<AMessage> &msg) {
2098 TimePoint now = std::chrono::steady_clock::now();
2099 CCodecWatchdog::getInstance()->watch(this);
2100 switch (msg->what()) {
2101 case kWhatAllocate: {
2102 // C2ComponentStore::createComponent() should return within 100ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002103 setDeadline(now, 1500ms, "allocate");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002104 sp<RefBase> obj;
2105 CHECK(msg->findObject("codecInfo", &obj));
2106 allocate((MediaCodecInfo *)obj.get());
2107 break;
2108 }
2109 case kWhatConfigure: {
2110 // C2Component::commit_sm() should return within 5ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002111 setDeadline(now, 1500ms, "configure");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002112 sp<AMessage> format;
2113 CHECK(msg->findMessage("format", &format));
2114 configure(format);
2115 break;
2116 }
2117 case kWhatStart: {
2118 // C2Component::start() should return within 500ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002119 setDeadline(now, 1500ms, "start");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002120 start();
2121 break;
2122 }
2123 case kWhatStop: {
2124 // C2Component::stop() should return within 500ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002125 setDeadline(now, 1500ms, "stop");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002126 stop();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002127 break;
2128 }
2129 case kWhatFlush: {
2130 // C2Component::flush_sm() should return within 5ms.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002131 setDeadline(now, 1500ms, "flush");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002132 flush();
2133 break;
2134 }
Wonsik Kim936a89c2020-05-08 16:07:50 -07002135 case kWhatRelease: {
2136 mChannel->release();
2137 mClient.reset();
2138 mClientListener.reset();
2139 break;
2140 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002141 case kWhatCreateInputSurface: {
2142 // Surface operations may be briefly blocking.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002143 setDeadline(now, 1500ms, "createInputSurface");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002144 createInputSurface();
2145 break;
2146 }
2147 case kWhatSetInputSurface: {
2148 // Surface operations may be briefly blocking.
Wonsik Kim9ee5a7c2019-06-17 11:33:24 -07002149 setDeadline(now, 1500ms, "setInputSurface");
Pawin Vongmasa36653902018-11-15 00:10:25 -08002150 sp<RefBase> obj;
2151 CHECK(msg->findObject("surface", &obj));
2152 sp<PersistentSurface> surface(static_cast<PersistentSurface *>(obj.get()));
2153 setInputSurface(surface);
2154 break;
2155 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002156 case kWhatWorkDone: {
2157 std::unique_ptr<C2Work> work;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002158 bool shouldPost = false;
2159 {
2160 Mutexed<std::list<std::unique_ptr<C2Work>>>::Locked queue(mWorkDoneQueue);
2161 if (queue->empty()) {
2162 break;
2163 }
2164 work.swap(queue->front());
2165 queue->pop_front();
2166 shouldPost = !queue->empty();
2167 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002168 if (shouldPost) {
2169 (new AMessage(kWhatWorkDone, this))->post();
2170 }
2171
Pawin Vongmasa36653902018-11-15 00:10:25 -08002172 // handle configuration changes in work done
Wonsik Kim75e22f42021-04-14 23:34:51 -07002173 std::unique_ptr<C2Param> initData;
2174 sp<AMessage> outputFormat = nullptr;
2175 {
2176 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2177 const std::unique_ptr<Config> &config = *configLocked;
2178 Config::Watcher<C2StreamInitDataInfo::output> initDataWatcher =
2179 config->watch<C2StreamInitDataInfo::output>();
2180 if (!work->worklets.empty()
2181 && (work->worklets.front()->output.flags
2182 & C2FrameData::FLAG_DISCARD_FRAME) == 0) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002183
Wonsik Kim75e22f42021-04-14 23:34:51 -07002184 // copy buffer info to config
2185 std::vector<std::unique_ptr<C2Param>> updates;
2186 for (const std::unique_ptr<C2Param> &param
2187 : work->worklets.front()->output.configUpdate) {
2188 updates.push_back(C2Param::Copy(*param));
2189 }
2190 unsigned stream = 0;
2191 std::vector<std::shared_ptr<C2Buffer>> &outputBuffers =
2192 work->worklets.front()->output.buffers;
2193 for (const std::shared_ptr<C2Buffer> &buf : outputBuffers) {
2194 for (const std::shared_ptr<const C2Info> &info : buf->info()) {
2195 // move all info into output-stream #0 domain
2196 updates.emplace_back(
2197 C2Param::CopyAsStream(*info, true /* output */, stream));
2198 }
2199
2200 const std::vector<C2ConstGraphicBlock> blocks = buf->data().graphicBlocks();
2201 // for now only do the first block
2202 if (!blocks.empty()) {
2203 // ALOGV("got output buffer with crop %u,%u+%u,%u and size %u,%u",
2204 // block.crop().left, block.crop().top,
2205 // block.crop().width, block.crop().height,
2206 // block.width(), block.height());
2207 const C2ConstGraphicBlock &block = blocks[0];
2208 updates.emplace_back(new C2StreamCropRectInfo::output(
2209 stream, block.crop()));
2210 updates.emplace_back(new C2StreamPictureSizeInfo::output(
2211 stream, block.crop().width, block.crop().height));
2212 }
2213 ++stream;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002214 }
George Burgess IVc813a592020-02-22 22:54:44 -08002215
Wonsik Kim75e22f42021-04-14 23:34:51 -07002216 sp<AMessage> oldFormat = config->mOutputFormat;
2217 config->updateConfiguration(updates, config->mOutputDomain);
2218 RevertOutputFormatIfNeeded(oldFormat, config->mOutputFormat);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002219
Wonsik Kim75e22f42021-04-14 23:34:51 -07002220 // copy standard infos to graphic buffers if not already present (otherwise, we
2221 // may overwrite the actual intermediate value with a final value)
2222 stream = 0;
2223 const static C2Param::Index stdGfxInfos[] = {
2224 C2StreamRotationInfo::output::PARAM_TYPE,
2225 C2StreamColorAspectsInfo::output::PARAM_TYPE,
2226 C2StreamDataSpaceInfo::output::PARAM_TYPE,
2227 C2StreamHdrStaticInfo::output::PARAM_TYPE,
2228 C2StreamHdr10PlusInfo::output::PARAM_TYPE,
2229 C2StreamPixelAspectRatioInfo::output::PARAM_TYPE,
2230 C2StreamSurfaceScalingInfo::output::PARAM_TYPE
2231 };
2232 for (const std::shared_ptr<C2Buffer> &buf : outputBuffers) {
2233 if (buf->data().graphicBlocks().size()) {
2234 for (C2Param::Index ix : stdGfxInfos) {
2235 if (!buf->hasInfo(ix)) {
2236 const C2Param *param =
2237 config->getConfigParameterValue(ix.withStream(stream));
2238 if (param) {
2239 std::shared_ptr<C2Param> info(C2Param::Copy(*param));
2240 buf->setInfo(std::static_pointer_cast<C2Info>(info));
2241 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002242 }
2243 }
2244 }
Wonsik Kim75e22f42021-04-14 23:34:51 -07002245 ++stream;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002246 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002247 }
Wonsik Kim75e22f42021-04-14 23:34:51 -07002248 if (config->mInputSurface) {
2249 config->mInputSurface->onInputBufferDone(work->input.ordinal.frameIndex);
2250 }
2251 if (initDataWatcher.hasChanged()) {
2252 initData = C2Param::Copy(*initDataWatcher.update().get());
2253 }
2254 outputFormat = config->mOutputFormat;
Wonsik Kim9c387412021-04-19 21:03:53 +00002255 }
2256 mChannel->onWorkDone(
Wonsik Kim75e22f42021-04-14 23:34:51 -07002257 std::move(work), outputFormat,
2258 initData ? (C2StreamInitDataInfo::output *)initData.get() : nullptr);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002259 break;
2260 }
2261 case kWhatWatch: {
2262 // watch message already posted; no-op.
2263 break;
2264 }
2265 default: {
2266 ALOGE("unrecognized message");
2267 break;
2268 }
2269 }
2270 setDeadline(TimePoint::max(), 0ms, "none");
2271}
2272
2273void CCodec::setDeadline(
2274 const TimePoint &now,
2275 const std::chrono::milliseconds &timeout,
2276 const char *name) {
2277 int32_t mult = std::max(1, property_get_int32("debug.stagefright.ccodec_timeout_mult", 1));
2278 Mutexed<NamedTimePoint>::Locked deadline(mDeadline);
2279 deadline->set(now + (timeout * mult), name);
2280}
2281
ted.sun765db4d2020-06-23 14:03:41 +08002282status_t CCodec::configureTunneledVideoPlayback(
2283 std::shared_ptr<Codec2Client::Component> comp,
2284 sp<NativeHandle> *sidebandHandle,
2285 const sp<AMessage> &msg) {
2286 std::vector<std::unique_ptr<C2SettingResult>> failures;
2287
2288 std::unique_ptr<C2PortTunneledModeTuning::output> tunneledPlayback =
2289 C2PortTunneledModeTuning::output::AllocUnique(
2290 1,
2291 C2PortTunneledModeTuning::Struct::SIDEBAND,
2292 C2PortTunneledModeTuning::Struct::REALTIME,
2293 0);
2294 // TODO: use KEY_AUDIO_HW_SYNC, KEY_HARDWARE_AV_SYNC_ID when they are in MediaCodecConstants.h
2295 if (msg->findInt32("audio-hw-sync", &tunneledPlayback->m.syncId[0])) {
2296 tunneledPlayback->m.syncType = C2PortTunneledModeTuning::Struct::sync_type_t::AUDIO_HW_SYNC;
2297 } else if (msg->findInt32("hw-av-sync-id", &tunneledPlayback->m.syncId[0])) {
2298 tunneledPlayback->m.syncType = C2PortTunneledModeTuning::Struct::sync_type_t::HW_AV_SYNC;
2299 } else {
2300 tunneledPlayback->m.syncType = C2PortTunneledModeTuning::Struct::sync_type_t::REALTIME;
2301 tunneledPlayback->setFlexCount(0);
2302 }
2303 c2_status_t c2err = comp->config({ tunneledPlayback.get() }, C2_MAY_BLOCK, &failures);
2304 if (c2err != C2_OK) {
2305 return UNKNOWN_ERROR;
2306 }
2307
Houxiang Dai5a97b472021-03-22 17:56:04 +08002308 if (sidebandHandle == nullptr) {
2309 return OK;
2310 }
2311
ted.sun765db4d2020-06-23 14:03:41 +08002312 std::vector<std::unique_ptr<C2Param>> params;
2313 c2err = comp->query({}, {C2PortTunnelHandleTuning::output::PARAM_TYPE}, C2_DONT_BLOCK, &params);
2314 if (c2err == C2_OK && params.size() == 1u) {
2315 C2PortTunnelHandleTuning::output *videoTunnelSideband =
2316 C2PortTunnelHandleTuning::output::From(params[0].get());
2317 // Currently, Codec2 only supports non-fd case for sideband native_handle.
2318 native_handle_t *handle = native_handle_create(0, videoTunnelSideband->flexCount());
2319 *sidebandHandle = NativeHandle::create(handle, true /* ownsHandle */);
2320 if (handle != nullptr && videoTunnelSideband->flexCount()) {
2321 memcpy(handle->data, videoTunnelSideband->m.values,
2322 sizeof(int32_t) * videoTunnelSideband->flexCount());
2323 return OK;
2324 } else {
2325 return NO_MEMORY;
2326 }
2327 }
2328 return UNKNOWN_ERROR;
2329}
2330
Pawin Vongmasa36653902018-11-15 00:10:25 -08002331void CCodec::initiateReleaseIfStuck() {
2332 std::string name;
2333 bool pendingDeadline = false;
Wonsik Kimab34ed62019-01-31 15:28:46 -08002334 {
2335 Mutexed<NamedTimePoint>::Locked deadline(mDeadline);
Pawin Vongmasa36653902018-11-15 00:10:25 -08002336 if (deadline->get() < std::chrono::steady_clock::now()) {
2337 name = deadline->getName();
Pawin Vongmasa36653902018-11-15 00:10:25 -08002338 }
2339 if (deadline->get() != TimePoint::max()) {
2340 pendingDeadline = true;
2341 }
2342 }
Wonsik Kim75e22f42021-04-14 23:34:51 -07002343 bool tunneled = false;
2344 {
2345 Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
2346 const std::unique_ptr<Config> &config = *configLocked;
2347 tunneled = config->mTunneled;
2348 }
2349 if (!tunneled && name.empty()) {
Wonsik Kimab34ed62019-01-31 15:28:46 -08002350 constexpr std::chrono::steady_clock::duration kWorkDurationThreshold = 3s;
2351 std::chrono::steady_clock::duration elapsed = mChannel->elapsed();
2352 if (elapsed >= kWorkDurationThreshold) {
2353 name = "queue";
2354 }
2355 if (elapsed > 0s) {
2356 pendingDeadline = true;
2357 }
2358 }
2359 if (name.empty()) {
Pawin Vongmasa36653902018-11-15 00:10:25 -08002360 // We're not stuck.
2361 if (pendingDeadline) {
2362 // If we are not stuck yet but still has deadline coming up,
2363 // post watch message to check back later.
2364 (new AMessage(kWhatWatch, this))->post();
2365 }
2366 return;
2367 }
2368
Chih-Yu Huang82e5ab32021-02-17 16:27:08 +09002369 C2String compName;
2370 {
2371 Mutexed<State>::Locked state(mState);
2372 compName = state->comp->getName();
2373 }
2374 ALOGW("[%s] previous call to %s exceeded timeout", compName.c_str(), name.c_str());
2375
Pawin Vongmasa36653902018-11-15 00:10:25 -08002376 initiateRelease(false);
2377 mCallback->onError(UNKNOWN_ERROR, ACTION_CODE_FATAL);
2378}
2379
Wonsik Kim155d5cb2019-10-09 12:49:49 -07002380// static
2381PersistentSurface *CCodec::CreateInputSurface() {
Pawin Vongmasa18588322019-05-18 01:52:13 -07002382 using namespace android;
Wonsik Kim9917d4a2019-10-24 12:56:38 -07002383 using ::android::hardware::media::omx::V1_0::implementation::TWGraphicBufferSource;
Pawin Vongmasa36653902018-11-15 00:10:25 -08002384 // Attempt to create a Codec2's input surface.
Pawin Vongmasa18588322019-05-18 01:52:13 -07002385 std::shared_ptr<Codec2Client::InputSurface> inputSurface =
2386 Codec2Client::CreateInputSurface();
Lajos Molnar47118272019-01-31 16:28:04 -08002387 if (!inputSurface) {
Pawin Vongmasa18588322019-05-18 01:52:13 -07002388 if (property_get_int32("debug.stagefright.c2inputsurface", 0) == -1) {
2389 sp<IGraphicBufferProducer> gbp;
2390 sp<OmxGraphicBufferSource> gbs = new OmxGraphicBufferSource();
2391 status_t err = gbs->initCheck();
2392 if (err != OK) {
2393 ALOGE("Failed to create persistent input surface: error %d", err);
2394 return nullptr;
2395 }
2396 return new PersistentSurface(
Wonsik Kim9917d4a2019-10-24 12:56:38 -07002397 gbs->getIGraphicBufferProducer(), new TWGraphicBufferSource(gbs));
Pawin Vongmasa18588322019-05-18 01:52:13 -07002398 } else {
2399 return nullptr;
2400 }
Pawin Vongmasa36653902018-11-15 00:10:25 -08002401 }
Pawin Vongmasa18588322019-05-18 01:52:13 -07002402 return new PersistentSurface(
Lajos Molnar47118272019-01-31 16:28:04 -08002403 inputSurface->getGraphicBufferProducer(),
Pawin Vongmasa18588322019-05-18 01:52:13 -07002404 static_cast<sp<android::hidl::base::V1_0::IBase>>(
Lajos Molnar47118272019-01-31 16:28:04 -08002405 inputSurface->getHalInterface()));
Pawin Vongmasa36653902018-11-15 00:10:25 -08002406}
2407
Wonsik Kimffb889a2020-05-28 11:32:25 -07002408class IntfCache {
2409public:
2410 IntfCache() = default;
2411
2412 status_t init(const std::string &name) {
2413 std::shared_ptr<Codec2Client::Interface> intf{
2414 Codec2Client::CreateInterfaceByName(name.c_str())};
2415 if (!intf) {
2416 ALOGW("IntfCache [%s]: Unrecognized interface name", name.c_str());
2417 mInitStatus = NO_INIT;
2418 return NO_INIT;
2419 }
2420 const static C2StreamUsageTuning::input sUsage{0u /* stream id */};
2421 mFields.push_back(C2FieldSupportedValuesQuery::Possible(
2422 C2ParamField{&sUsage, &sUsage.value}));
2423 c2_status_t err = intf->querySupportedValues(mFields, C2_MAY_BLOCK);
2424 if (err != C2_OK) {
2425 ALOGW("IntfCache [%s]: failed to query usage supported value (err=%d)",
2426 name.c_str(), err);
2427 mFields[0].status = err;
2428 }
2429 std::vector<std::unique_ptr<C2Param>> params;
2430 err = intf->query(
2431 {&mApiFeatures},
2432 {C2PortAllocatorsTuning::input::PARAM_TYPE},
2433 C2_MAY_BLOCK,
2434 &params);
2435 if (err != C2_OK && err != C2_BAD_INDEX) {
2436 ALOGW("IntfCache [%s]: failed to query api features (err=%d)",
2437 name.c_str(), err);
2438 }
2439 while (!params.empty()) {
2440 C2Param *param = params.back().release();
2441 params.pop_back();
2442 if (!param) {
2443 continue;
2444 }
2445 if (param->type() == C2PortAllocatorsTuning::input::PARAM_TYPE) {
2446 mInputAllocators.reset(
Wonsik Kimd79ee1f2020-08-27 17:41:56 -07002447 C2PortAllocatorsTuning::input::From(param));
Wonsik Kimffb889a2020-05-28 11:32:25 -07002448 }
2449 }
2450 mInitStatus = OK;
2451 return OK;
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002452 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002453
2454 status_t initCheck() const { return mInitStatus; }
2455
2456 const C2FieldSupportedValuesQuery &getUsageSupportedValues() const {
2457 CHECK_EQ(1u, mFields.size());
2458 return mFields[0];
2459 }
2460
2461 const C2ApiFeaturesSetting &getApiFeatures() const {
2462 return mApiFeatures;
2463 }
2464
2465 const C2PortAllocatorsTuning::input &getInputAllocators() const {
2466 static std::unique_ptr<C2PortAllocatorsTuning::input> sInvalidated = []{
2467 std::unique_ptr<C2PortAllocatorsTuning::input> param =
2468 C2PortAllocatorsTuning::input::AllocUnique(0);
2469 param->invalidate();
2470 return param;
2471 }();
2472 return mInputAllocators ? *mInputAllocators : *sInvalidated;
2473 }
2474
2475private:
2476 status_t mInitStatus{NO_INIT};
2477
2478 std::vector<C2FieldSupportedValuesQuery> mFields;
2479 C2ApiFeaturesSetting mApiFeatures;
2480 std::unique_ptr<C2PortAllocatorsTuning::input> mInputAllocators;
2481};
2482
2483static const IntfCache &GetIntfCache(const std::string &name) {
2484 static IntfCache sNullIntfCache;
2485 static std::mutex sMutex;
2486 static std::map<std::string, IntfCache> sCache;
2487 std::unique_lock<std::mutex> lock{sMutex};
2488 auto it = sCache.find(name);
2489 if (it == sCache.end()) {
2490 lock.unlock();
2491 IntfCache intfCache;
2492 status_t err = intfCache.init(name);
2493 if (err != OK) {
2494 return sNullIntfCache;
2495 }
2496 lock.lock();
2497 it = sCache.insert({name, std::move(intfCache)}).first;
2498 }
2499 return it->second;
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002500}
2501
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002502static status_t GetCommonAllocatorIds(
2503 const std::vector<std::string> &names,
2504 C2Allocator::type_t type,
2505 std::set<C2Allocator::id_t> *ids) {
2506 int poolMask = GetCodec2PoolMask();
2507 C2PlatformAllocatorStore::id_t preferredLinearId = GetPreferredLinearAllocatorId(poolMask);
2508 C2Allocator::id_t defaultAllocatorId =
2509 (type == C2Allocator::LINEAR) ? preferredLinearId : C2PlatformAllocatorStore::GRALLOC;
2510
2511 ids->clear();
2512 if (names.empty()) {
2513 return OK;
2514 }
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002515 bool firstIteration = true;
2516 for (const std::string &name : names) {
Wonsik Kimffb889a2020-05-28 11:32:25 -07002517 const IntfCache &intfCache = GetIntfCache(name);
2518 if (intfCache.initCheck() != OK) {
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002519 continue;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002520 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002521 const C2PortAllocatorsTuning::input &allocators = intfCache.getInputAllocators();
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002522 if (firstIteration) {
2523 firstIteration = false;
Wonsik Kimffb889a2020-05-28 11:32:25 -07002524 if (allocators && allocators.flexCount() > 0) {
2525 ids->insert(allocators.m.values,
2526 allocators.m.values + allocators.flexCount());
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002527 }
2528 if (ids->empty()) {
2529 // The component does not advertise allocators. Use default.
2530 ids->insert(defaultAllocatorId);
2531 }
2532 continue;
2533 }
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002534 bool filtered = false;
Wonsik Kimffb889a2020-05-28 11:32:25 -07002535 if (allocators && allocators.flexCount() > 0) {
2536 filtered = true;
2537 for (auto it = ids->begin(); it != ids->end(); ) {
2538 bool found = false;
2539 for (size_t j = 0; j < allocators.flexCount(); ++j) {
2540 if (allocators.m.values[j] == *it) {
2541 found = true;
2542 break;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002543 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002544 }
2545 if (found) {
2546 ++it;
2547 } else {
2548 it = ids->erase(it);
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002549 }
2550 }
2551 }
2552 if (!filtered) {
2553 // The component does not advertise supported allocators. Use default.
2554 bool containsDefault = (ids->count(defaultAllocatorId) > 0u);
2555 if (ids->size() != (containsDefault ? 1 : 0)) {
2556 ids->clear();
2557 if (containsDefault) {
2558 ids->insert(defaultAllocatorId);
2559 }
2560 }
2561 }
2562 }
2563 // Finally, filter with pool masks
2564 for (auto it = ids->begin(); it != ids->end(); ) {
2565 if ((poolMask >> *it) & 1) {
2566 ++it;
2567 } else {
2568 it = ids->erase(it);
2569 }
2570 }
2571 return OK;
2572}
2573
2574static status_t CalculateMinMaxUsage(
2575 const std::vector<std::string> &names, uint64_t *minUsage, uint64_t *maxUsage) {
2576 static C2StreamUsageTuning::input sUsage{0u /* stream id */};
2577 *minUsage = 0;
2578 *maxUsage = ~0ull;
2579 for (const std::string &name : names) {
Wonsik Kimffb889a2020-05-28 11:32:25 -07002580 const IntfCache &intfCache = GetIntfCache(name);
2581 if (intfCache.initCheck() != OK) {
Wonsik Kimfcf46c32020-04-22 13:50:45 -07002582 continue;
2583 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002584 const C2FieldSupportedValuesQuery &usageSupportedValues =
2585 intfCache.getUsageSupportedValues();
2586 if (usageSupportedValues.status != C2_OK) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002587 continue;
2588 }
Wonsik Kimffb889a2020-05-28 11:32:25 -07002589 const C2FieldSupportedValues &supported = usageSupportedValues.values;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002590 if (supported.type != C2FieldSupportedValues::FLAGS) {
2591 continue;
2592 }
2593 if (supported.values.empty()) {
2594 *maxUsage = 0;
2595 continue;
2596 }
2597 *minUsage |= supported.values[0].u64;
2598 int64_t currentMaxUsage = 0;
2599 for (const C2Value::Primitive &flags : supported.values) {
2600 currentMaxUsage |= flags.u64;
2601 }
2602 *maxUsage &= currentMaxUsage;
2603 }
2604 return OK;
2605}
2606
2607// static
2608status_t CCodec::CanFetchLinearBlock(
2609 const std::vector<std::string> &names, const C2MemoryUsage &usage, bool *isCompatible) {
Wonsik Kimffb889a2020-05-28 11:32:25 -07002610 for (const std::string &name : names) {
2611 const IntfCache &intfCache = GetIntfCache(name);
2612 if (intfCache.initCheck() != OK) {
2613 continue;
2614 }
2615 const C2ApiFeaturesSetting &features = intfCache.getApiFeatures();
2616 if (features && !(features.value & API_SAME_INPUT_BUFFER)) {
2617 *isCompatible = false;
2618 return OK;
2619 }
2620 }
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002621 std::set<C2Allocator::id_t> allocators;
2622 GetCommonAllocatorIds(names, C2Allocator::LINEAR, &allocators);
2623 if (allocators.empty()) {
2624 *isCompatible = false;
2625 return OK;
2626 }
Chih-Yu Huang990b5622020-10-13 14:53:37 +09002627
2628 uint64_t minUsage = 0;
2629 uint64_t maxUsage = ~0ull;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002630 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
Chih-Yu Huang990b5622020-10-13 14:53:37 +09002631 minUsage |= usage.expected;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002632 *isCompatible = ((maxUsage & minUsage) == minUsage);
2633 return OK;
2634}
2635
2636static std::shared_ptr<C2BlockPool> GetPool(C2Allocator::id_t allocId) {
2637 static std::mutex sMutex{};
2638 static std::map<C2Allocator::id_t, std::shared_ptr<C2BlockPool>> sPools;
2639 std::unique_lock<std::mutex> lock{sMutex};
2640 std::shared_ptr<C2BlockPool> pool;
2641 auto it = sPools.find(allocId);
2642 if (it == sPools.end()) {
2643 c2_status_t err = CreateCodec2BlockPool(allocId, nullptr, &pool);
2644 if (err == OK) {
2645 sPools.emplace(allocId, pool);
2646 } else {
2647 pool.reset();
2648 }
2649 } else {
2650 pool = it->second;
2651 }
2652 return pool;
2653}
2654
2655// static
2656std::shared_ptr<C2LinearBlock> CCodec::FetchLinearBlock(
2657 size_t capacity, const C2MemoryUsage &usage, const std::vector<std::string> &names) {
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002658 std::set<C2Allocator::id_t> allocators;
2659 GetCommonAllocatorIds(names, C2Allocator::LINEAR, &allocators);
2660 if (allocators.empty()) {
2661 allocators.insert(C2PlatformAllocatorStore::DEFAULT_LINEAR);
2662 }
Chih-Yu Huang990b5622020-10-13 14:53:37 +09002663
2664 uint64_t minUsage = 0;
2665 uint64_t maxUsage = ~0ull;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002666 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
Chih-Yu Huang990b5622020-10-13 14:53:37 +09002667 minUsage |= usage.expected;
Wonsik Kimfb7a7672019-12-27 17:13:33 -08002668 if ((maxUsage & minUsage) != minUsage) {
2669 allocators.clear();
2670 allocators.insert(C2PlatformAllocatorStore::DEFAULT_LINEAR);
2671 }
2672 std::shared_ptr<C2LinearBlock> block;
2673 for (C2Allocator::id_t allocId : allocators) {
2674 std::shared_ptr<C2BlockPool> pool = GetPool(allocId);
2675 if (!pool) {
2676 continue;
2677 }
2678 c2_status_t err = pool->fetchLinearBlock(capacity, C2MemoryUsage{minUsage}, &block);
2679 if (err != C2_OK || !block) {
2680 block.reset();
2681 continue;
2682 }
2683 break;
2684 }
2685 return block;
2686}
2687
2688// static
2689status_t CCodec::CanFetchGraphicBlock(
2690 const std::vector<std::string> &names, bool *isCompatible) {
2691 uint64_t minUsage = 0;
2692 uint64_t maxUsage = ~0ull;
2693 std::set<C2Allocator::id_t> allocators;
2694 GetCommonAllocatorIds(names, C2Allocator::GRAPHIC, &allocators);
2695 if (allocators.empty()) {
2696 *isCompatible = false;
2697 return OK;
2698 }
2699 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
2700 *isCompatible = ((maxUsage & minUsage) == minUsage);
2701 return OK;
2702}
2703
2704// static
2705std::shared_ptr<C2GraphicBlock> CCodec::FetchGraphicBlock(
2706 int32_t width,
2707 int32_t height,
2708 int32_t format,
2709 uint64_t usage,
2710 const std::vector<std::string> &names) {
2711 uint32_t halPixelFormat = HAL_PIXEL_FORMAT_YCBCR_420_888;
2712 if (!C2Mapper::mapPixelFormatFrameworkToCodec(format, &halPixelFormat)) {
2713 ALOGD("Unrecognized pixel format: %d", format);
2714 return nullptr;
2715 }
2716 uint64_t minUsage = 0;
2717 uint64_t maxUsage = ~0ull;
2718 std::set<C2Allocator::id_t> allocators;
2719 GetCommonAllocatorIds(names, C2Allocator::GRAPHIC, &allocators);
2720 if (allocators.empty()) {
2721 allocators.insert(C2PlatformAllocatorStore::DEFAULT_GRAPHIC);
2722 }
2723 CalculateMinMaxUsage(names, &minUsage, &maxUsage);
2724 minUsage |= usage;
2725 if ((maxUsage & minUsage) != minUsage) {
2726 allocators.clear();
2727 allocators.insert(C2PlatformAllocatorStore::DEFAULT_GRAPHIC);
2728 }
2729 std::shared_ptr<C2GraphicBlock> block;
2730 for (C2Allocator::id_t allocId : allocators) {
2731 std::shared_ptr<C2BlockPool> pool;
2732 c2_status_t err = CreateCodec2BlockPool(allocId, nullptr, &pool);
2733 if (err != C2_OK || !pool) {
2734 continue;
2735 }
2736 err = pool->fetchGraphicBlock(
2737 width, height, halPixelFormat, C2MemoryUsage{minUsage}, &block);
2738 if (err != C2_OK || !block) {
2739 block.reset();
2740 continue;
2741 }
2742 break;
2743 }
2744 return block;
2745}
2746
Wonsik Kim155d5cb2019-10-09 12:49:49 -07002747} // namespace android