blob: 273d91ccdef0c2234090d7eeafa302b492f8d453 [file] [log] [blame]
Chong Zhangea280cb2017-08-02 11:10:10 -07001/*
2 * Copyright (C) 2017 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17//#define LOG_NDEBUG 0
18#define LOG_TAG "HeifDecoderImpl"
19
20#include "HeifDecoderImpl.h"
21
22#include <stdio.h>
23
Marco Nelissendab79b32019-11-18 08:25:47 -080024#include <android/IDataSource.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070025#include <binder/IMemory.h>
Dongwon Kang49ce6712018-01-24 10:16:25 -080026#include <binder/MemoryDealer.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070027#include <drm/drm_framework_common.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070028#include <media/mediametadataretriever.h>
Marco Nelissen7291da62019-12-17 13:01:55 -080029#include <media/stagefright/MediaSource.h>
Chong Zhangd3e0d862017-10-03 13:17:13 -070030#include <media/stagefright/foundation/ADebug.h>
Chong Zhangea280cb2017-08-02 11:10:10 -070031#include <private/media/VideoFrame.h>
32#include <utils/Log.h>
33#include <utils/RefBase.h>
Chong Zhang8541d302019-07-12 11:19:47 -070034#include <vector>
Chong Zhangea280cb2017-08-02 11:10:10 -070035
36HeifDecoder* createHeifDecoder() {
37 return new android::HeifDecoderImpl();
38}
39
40namespace android {
41
Chong Zhang8541d302019-07-12 11:19:47 -070042void initFrameInfo(HeifFrameInfo *info, const VideoFrame *videoFrame) {
43 info->mWidth = videoFrame->mWidth;
44 info->mHeight = videoFrame->mHeight;
45 info->mRotationAngle = videoFrame->mRotationAngle;
46 info->mBytesPerPixel = videoFrame->mBytesPerPixel;
Chong Zhang0af4db62019-08-23 15:34:58 -070047 info->mDurationUs = videoFrame->mDurationUs;
Chong Zhang8541d302019-07-12 11:19:47 -070048 if (videoFrame->mIccSize > 0) {
49 info->mIccData.assign(
50 videoFrame->getFlattenedIccData(),
51 videoFrame->getFlattenedIccData() + videoFrame->mIccSize);
52 } else {
53 // clear old Icc data if there is no Icc data.
54 info->mIccData.clear();
55 }
56}
57
Chong Zhangea280cb2017-08-02 11:10:10 -070058/*
59 * HeifDataSource
60 *
61 * Proxies data requests over IDataSource interface from MediaMetadataRetriever
62 * to the HeifStream interface we received from the heif decoder client.
63 */
64class HeifDataSource : public BnDataSource {
65public:
66 /*
67 * Constructs HeifDataSource; will take ownership of |stream|.
68 */
69 HeifDataSource(HeifStream* stream)
Chong Zhang3f3ffab2017-08-23 13:51:59 -070070 : mStream(stream), mEOS(false),
71 mCachedOffset(0), mCachedSize(0), mCacheBufferSize(0) {}
Chong Zhangea280cb2017-08-02 11:10:10 -070072
73 ~HeifDataSource() override {}
74
75 /*
76 * Initializes internal resources.
77 */
78 bool init();
79
80 sp<IMemory> getIMemory() override { return mMemory; }
81 ssize_t readAt(off64_t offset, size_t size) override;
82 status_t getSize(off64_t* size) override ;
83 void close() {}
84 uint32_t getFlags() override { return 0; }
85 String8 toString() override { return String8("HeifDataSource"); }
Chong Zhangea280cb2017-08-02 11:10:10 -070086
87private:
Chong Zhangea280cb2017-08-02 11:10:10 -070088 enum {
Chong Zhang3f3ffab2017-08-23 13:51:59 -070089 /*
90 * Buffer size for passing the read data to mediaserver. Set to 64K
91 * (which is what MediaDataSource Java API's jni implementation uses).
92 */
Chong Zhangea280cb2017-08-02 11:10:10 -070093 kBufferSize = 64 * 1024,
Chong Zhang3f3ffab2017-08-23 13:51:59 -070094 /*
95 * Initial and max cache buffer size.
96 */
97 kInitialCacheBufferSize = 4 * 1024 * 1024,
98 kMaxCacheBufferSize = 64 * 1024 * 1024,
Chong Zhangea280cb2017-08-02 11:10:10 -070099 };
100 sp<IMemory> mMemory;
101 std::unique_ptr<HeifStream> mStream;
Chong Zhangea280cb2017-08-02 11:10:10 -0700102 bool mEOS;
Chong Zhangcd031922018-08-24 13:03:19 -0700103 std::unique_ptr<uint8_t[]> mCache;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700104 off64_t mCachedOffset;
105 size_t mCachedSize;
106 size_t mCacheBufferSize;
Chong Zhangea280cb2017-08-02 11:10:10 -0700107};
108
109bool HeifDataSource::init() {
110 sp<MemoryDealer> memoryDealer =
111 new MemoryDealer(kBufferSize, "HeifDataSource");
112 mMemory = memoryDealer->allocate(kBufferSize);
113 if (mMemory == nullptr) {
114 ALOGE("Failed to allocate shared memory!");
115 return false;
116 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700117 mCache.reset(new uint8_t[kInitialCacheBufferSize]);
118 if (mCache.get() == nullptr) {
119 ALOGE("mFailed to allocate cache!");
120 return false;
121 }
122 mCacheBufferSize = kInitialCacheBufferSize;
Chong Zhangea280cb2017-08-02 11:10:10 -0700123 return true;
124}
125
126ssize_t HeifDataSource::readAt(off64_t offset, size_t size) {
127 ALOGV("readAt: offset=%lld, size=%zu", (long long)offset, size);
128
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700129 if (offset < mCachedOffset) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700130 // try seek, then rewind/skip, fail if none worked
131 if (mStream->seek(offset)) {
132 ALOGV("readAt: seek to offset=%lld", (long long)offset);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700133 mCachedOffset = offset;
134 mCachedSize = 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700135 mEOS = false;
136 } else if (mStream->rewind()) {
137 ALOGV("readAt: rewind to offset=0");
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700138 mCachedOffset = 0;
139 mCachedSize = 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700140 mEOS = false;
141 } else {
142 ALOGE("readAt: couldn't seek or rewind!");
143 mEOS = true;
144 }
145 }
146
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700147 if (mEOS && (offset < mCachedOffset ||
148 offset >= (off64_t)(mCachedOffset + mCachedSize))) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700149 ALOGV("readAt: EOS");
150 return ERROR_END_OF_STREAM;
151 }
152
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700153 // at this point, offset must be >= mCachedOffset, other cases should
154 // have been caught above.
155 CHECK(offset >= mCachedOffset);
156
Sungtak Lee237f9032018-03-05 15:21:33 -0800157 off64_t resultOffset;
158 if (__builtin_add_overflow(offset, size, &resultOffset)) {
159 return ERROR_IO;
160 }
161
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700162 if (size == 0) {
163 return 0;
Chong Zhangea280cb2017-08-02 11:10:10 -0700164 }
165
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700166 // Can only read max of kBufferSize
Chong Zhangea280cb2017-08-02 11:10:10 -0700167 if (size > kBufferSize) {
168 size = kBufferSize;
169 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700170
171 // copy from cache if the request falls entirely in cache
172 if (offset + size <= mCachedOffset + mCachedSize) {
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700173 memcpy(mMemory->unsecurePointer(), mCache.get() + offset - mCachedOffset, size);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700174 return size;
175 }
176
177 // need to fetch more, check if we need to expand the cache buffer.
178 if ((off64_t)(offset + size) > mCachedOffset + kMaxCacheBufferSize) {
179 // it's reaching max cache buffer size, need to roll window, and possibly
180 // expand the cache buffer.
181 size_t newCacheBufferSize = mCacheBufferSize;
Chong Zhangcd031922018-08-24 13:03:19 -0700182 std::unique_ptr<uint8_t[]> newCache;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700183 uint8_t* dst = mCache.get();
184 if (newCacheBufferSize < kMaxCacheBufferSize) {
185 newCacheBufferSize = kMaxCacheBufferSize;
186 newCache.reset(new uint8_t[newCacheBufferSize]);
187 dst = newCache.get();
188 }
189
190 // when rolling the cache window, try to keep about half the old bytes
191 // in case that the client goes back.
192 off64_t newCachedOffset = offset - (off64_t)(newCacheBufferSize / 2);
193 if (newCachedOffset < mCachedOffset) {
194 newCachedOffset = mCachedOffset;
195 }
196
197 int64_t newCachedSize = (int64_t)(mCachedOffset + mCachedSize) - newCachedOffset;
198 if (newCachedSize > 0) {
199 // in this case, the new cache region partially overlop the old cache,
200 // move the portion of the cache we want to save to the beginning of
201 // the cache buffer.
202 memcpy(dst, mCache.get() + newCachedOffset - mCachedOffset, newCachedSize);
203 } else if (newCachedSize < 0){
204 // in this case, the new cache region is entirely out of the old cache,
205 // in order to guarantee sequential read, we need to skip a number of
206 // bytes before reading.
207 size_t bytesToSkip = -newCachedSize;
208 size_t bytesSkipped = mStream->read(nullptr, bytesToSkip);
209 if (bytesSkipped != bytesToSkip) {
210 // bytesSkipped is invalid, there is not enough bytes to reach
211 // the requested offset.
212 ALOGE("readAt: skip failed, EOS");
213
214 mEOS = true;
215 mCachedOffset = newCachedOffset;
216 mCachedSize = 0;
217 return ERROR_END_OF_STREAM;
218 }
219 // set cache size to 0, since we're not keeping any old cache
220 newCachedSize = 0;
221 }
222
223 if (newCache.get() != nullptr) {
224 mCache.reset(newCache.release());
225 mCacheBufferSize = newCacheBufferSize;
226 }
227 mCachedOffset = newCachedOffset;
228 mCachedSize = newCachedSize;
229
230 ALOGV("readAt: rolling cache window to (%lld, %zu), cache buffer size %zu",
231 (long long)mCachedOffset, mCachedSize, mCacheBufferSize);
232 } else {
233 // expand cache buffer, but no need to roll the window
234 size_t newCacheBufferSize = mCacheBufferSize;
235 while (offset + size > mCachedOffset + newCacheBufferSize) {
236 newCacheBufferSize *= 2;
237 }
238 CHECK(newCacheBufferSize <= kMaxCacheBufferSize);
239 if (mCacheBufferSize < newCacheBufferSize) {
240 uint8_t* newCache = new uint8_t[newCacheBufferSize];
241 memcpy(newCache, mCache.get(), mCachedSize);
242 mCache.reset(newCache);
243 mCacheBufferSize = newCacheBufferSize;
244
245 ALOGV("readAt: current cache window (%lld, %zu), new cache buffer size %zu",
246 (long long) mCachedOffset, mCachedSize, mCacheBufferSize);
247 }
248 }
249 size_t bytesToRead = offset + size - mCachedOffset - mCachedSize;
250 size_t bytesRead = mStream->read(mCache.get() + mCachedSize, bytesToRead);
251 if (bytesRead > bytesToRead || bytesRead == 0) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700252 // bytesRead is invalid
253 mEOS = true;
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700254 bytesRead = 0;
255 } else if (bytesRead < bytesToRead) {
256 // read some bytes but not all, set EOS
Chong Zhangea280cb2017-08-02 11:10:10 -0700257 mEOS = true;
258 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700259 mCachedSize += bytesRead;
260 ALOGV("readAt: current cache window (%lld, %zu)",
261 (long long) mCachedOffset, mCachedSize);
262
263 // here bytesAvailable could be negative if offset jumped past EOS.
264 int64_t bytesAvailable = mCachedOffset + mCachedSize - offset;
265 if (bytesAvailable <= 0) {
266 return ERROR_END_OF_STREAM;
267 }
268 if (bytesAvailable < (int64_t)size) {
269 size = bytesAvailable;
270 }
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700271 memcpy(mMemory->unsecurePointer(), mCache.get() + offset - mCachedOffset, size);
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700272 return size;
Chong Zhangea280cb2017-08-02 11:10:10 -0700273}
274
275status_t HeifDataSource::getSize(off64_t* size) {
276 if (!mStream->hasLength()) {
277 *size = -1;
278 ALOGE("getSize: not supported!");
279 return ERROR_UNSUPPORTED;
280 }
281 *size = mStream->getLength();
282 ALOGV("getSize: size=%lld", (long long)*size);
283 return OK;
284}
285
286/////////////////////////////////////////////////////////////////////////
287
Chong Zhang0c1407f2018-05-02 17:09:05 -0700288struct HeifDecoderImpl::DecodeThread : public Thread {
289 explicit DecodeThread(HeifDecoderImpl *decoder) : mDecoder(decoder) {}
290
291private:
292 HeifDecoderImpl* mDecoder;
293
294 bool threadLoop();
295
296 DISALLOW_EVIL_CONSTRUCTORS(DecodeThread);
297};
298
299bool HeifDecoderImpl::DecodeThread::threadLoop() {
300 return mDecoder->decodeAsync();
301}
302
303/////////////////////////////////////////////////////////////////////////
304
Chong Zhangea280cb2017-08-02 11:10:10 -0700305HeifDecoderImpl::HeifDecoderImpl() :
306 // output color format should always be set via setOutputColor(), in case
307 // it's not, default to HAL_PIXEL_FORMAT_RGB_565.
308 mOutputColor(HAL_PIXEL_FORMAT_RGB_565),
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700309 mCurScanline(0),
Chong Zhang8541d302019-07-12 11:19:47 -0700310 mTotalScanline(0),
Chong Zhangd3e0d862017-10-03 13:17:13 -0700311 mFrameDecoded(false),
312 mHasImage(false),
Chong Zhang0c1407f2018-05-02 17:09:05 -0700313 mHasVideo(false),
Chong Zhang8541d302019-07-12 11:19:47 -0700314 mSequenceLength(0),
Chong Zhang0c1407f2018-05-02 17:09:05 -0700315 mAvailableLines(0),
316 mNumSlices(1),
317 mSliceHeight(0),
318 mAsyncDecodeDone(false) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700319}
320
321HeifDecoderImpl::~HeifDecoderImpl() {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700322 if (mThread != nullptr) {
323 mThread->join();
324 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700325}
326
327bool HeifDecoderImpl::init(HeifStream* stream, HeifFrameInfo* frameInfo) {
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700328 mFrameDecoded = false;
Chong Zhangd3e0d862017-10-03 13:17:13 -0700329 mFrameMemory.clear();
330
Chong Zhangea280cb2017-08-02 11:10:10 -0700331 sp<HeifDataSource> dataSource = new HeifDataSource(stream);
332 if (!dataSource->init()) {
333 return false;
334 }
335 mDataSource = dataSource;
336
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500337 return reinit(frameInfo);
338}
339
340bool HeifDecoderImpl::reinit(HeifFrameInfo* frameInfo) {
341 mFrameDecoded = false;
342 mFrameMemory.clear();
343
Chong Zhangea280cb2017-08-02 11:10:10 -0700344 mRetriever = new MediaMetadataRetriever();
Chong Zhangd3e0d862017-10-03 13:17:13 -0700345 status_t err = mRetriever->setDataSource(mDataSource, "image/heif");
Chong Zhangea280cb2017-08-02 11:10:10 -0700346 if (err != OK) {
347 ALOGE("failed to set data source!");
348
349 mRetriever.clear();
350 mDataSource.clear();
351 return false;
352 }
353 ALOGV("successfully set data source.");
354
Chong Zhangd3e0d862017-10-03 13:17:13 -0700355 const char* hasImage = mRetriever->extractMetadata(METADATA_KEY_HAS_IMAGE);
Chong Zhangea280cb2017-08-02 11:10:10 -0700356 const char* hasVideo = mRetriever->extractMetadata(METADATA_KEY_HAS_VIDEO);
Chong Zhangd3e0d862017-10-03 13:17:13 -0700357
358 mHasImage = hasImage && !strcasecmp(hasImage, "yes");
359 mHasVideo = hasVideo && !strcasecmp(hasVideo, "yes");
Chong Zhang8541d302019-07-12 11:19:47 -0700360
361 HeifFrameInfo* defaultInfo = nullptr;
Chong Zhangd3e0d862017-10-03 13:17:13 -0700362 if (mHasImage) {
363 // image index < 0 to retrieve primary image
Chong Zhang8541d302019-07-12 11:19:47 -0700364 sp<IMemory> sharedMem = mRetriever->getImageAtIndex(
Chong Zhangd3e0d862017-10-03 13:17:13 -0700365 -1, mOutputColor, true /*metaOnly*/);
Chong Zhang8541d302019-07-12 11:19:47 -0700366
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700367 if (sharedMem == nullptr || sharedMem->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700368 ALOGE("init: videoFrame is a nullptr");
369 return false;
370 }
371
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700372 // TODO: Using unsecurePointer() has some associated security pitfalls
373 // (see declaration for details).
374 // Either document why it is safe in this case or address the
375 // issue (e.g. by copying).
376 VideoFrame* videoFrame = static_cast<VideoFrame*>(sharedMem->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700377
378 ALOGV("Image dimension %dx%d, display %dx%d, angle %d, iccSize %d",
379 videoFrame->mWidth,
380 videoFrame->mHeight,
381 videoFrame->mDisplayWidth,
382 videoFrame->mDisplayHeight,
383 videoFrame->mRotationAngle,
384 videoFrame->mIccSize);
385
386 initFrameInfo(&mImageInfo, videoFrame);
387
388 if (videoFrame->mTileHeight >= 512) {
389 // Try decoding in slices only if the image has tiles and is big enough.
390 mSliceHeight = videoFrame->mTileHeight;
391 ALOGV("mSliceHeight %u", mSliceHeight);
392 }
393
394 defaultInfo = &mImageInfo;
Chong Zhangea280cb2017-08-02 11:10:10 -0700395 }
396
Chong Zhang8541d302019-07-12 11:19:47 -0700397 if (mHasVideo) {
398 sp<IMemory> sharedMem = mRetriever->getFrameAtTime(0,
399 MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC,
400 mOutputColor, true /*metaOnly*/);
401
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700402 if (sharedMem == nullptr || sharedMem->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700403 ALOGE("init: videoFrame is a nullptr");
404 return false;
405 }
406
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700407 // TODO: Using unsecurePointer() has some associated security pitfalls
408 // (see declaration for details).
409 // Either document why it is safe in this case or address the
410 // issue (e.g. by copying).
411 VideoFrame* videoFrame = static_cast<VideoFrame*>(
412 sharedMem->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700413
414 ALOGV("Sequence dimension %dx%d, display %dx%d, angle %d, iccSize %d",
415 videoFrame->mWidth,
416 videoFrame->mHeight,
417 videoFrame->mDisplayWidth,
418 videoFrame->mDisplayHeight,
419 videoFrame->mRotationAngle,
420 videoFrame->mIccSize);
421
422 initFrameInfo(&mSequenceInfo, videoFrame);
423
424 mSequenceLength = atoi(mRetriever->extractMetadata(METADATA_KEY_VIDEO_FRAME_COUNT));
425
426 if (defaultInfo == nullptr) {
427 defaultInfo = &mSequenceInfo;
428 }
429 }
430
431 if (defaultInfo == nullptr) {
432 ALOGD("No valid image or sequence available");
Chong Zhangea280cb2017-08-02 11:10:10 -0700433 return false;
434 }
435
Chong Zhangea280cb2017-08-02 11:10:10 -0700436 if (frameInfo != nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700437 *frameInfo = *defaultInfo;
Chong Zhangea280cb2017-08-02 11:10:10 -0700438 }
Chong Zhang8541d302019-07-12 11:19:47 -0700439
440 // default total scanline, this might change if decodeSequence() is used
441 mTotalScanline = defaultInfo->mHeight;
442
443 return true;
444}
445
446bool HeifDecoderImpl::getSequenceInfo(
447 HeifFrameInfo* frameInfo, size_t *frameCount) {
448 ALOGV("%s", __FUNCTION__);
449 if (!mHasVideo) {
450 return false;
451 }
452 if (frameInfo != nullptr) {
453 *frameInfo = mSequenceInfo;
454 }
455 if (frameCount != nullptr) {
456 *frameCount = mSequenceLength;
Chong Zhang0c1407f2018-05-02 17:09:05 -0700457 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700458 return true;
459}
460
461bool HeifDecoderImpl::getEncodedColor(HeifEncodedColor* /*outColor*/) const {
462 ALOGW("getEncodedColor: not implemented!");
463 return false;
464}
465
466bool HeifDecoderImpl::setOutputColor(HeifColorFormat heifColor) {
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500467 if (heifColor == mOutputColor) {
468 return true;
469 }
470
Chong Zhangea280cb2017-08-02 11:10:10 -0700471 switch(heifColor) {
472 case kHeifColorFormat_RGB565:
473 {
474 mOutputColor = HAL_PIXEL_FORMAT_RGB_565;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500475 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700476 }
477 case kHeifColorFormat_RGBA_8888:
478 {
479 mOutputColor = HAL_PIXEL_FORMAT_RGBA_8888;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500480 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700481 }
482 case kHeifColorFormat_BGRA_8888:
483 {
484 mOutputColor = HAL_PIXEL_FORMAT_BGRA_8888;
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500485 break;
Chong Zhangea280cb2017-08-02 11:10:10 -0700486 }
487 default:
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500488 ALOGE("Unsupported output color format %d", heifColor);
489 return false;
Chong Zhangea280cb2017-08-02 11:10:10 -0700490 }
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500491
492 if (mFrameDecoded) {
493 return reinit(nullptr);
494 }
495 return true;
Chong Zhangea280cb2017-08-02 11:10:10 -0700496}
497
Chong Zhang0c1407f2018-05-02 17:09:05 -0700498bool HeifDecoderImpl::decodeAsync() {
499 for (size_t i = 1; i < mNumSlices; i++) {
500 ALOGV("decodeAsync(): decoding slice %zu", i);
501 size_t top = i * mSliceHeight;
502 size_t bottom = (i + 1) * mSliceHeight;
Chong Zhang8541d302019-07-12 11:19:47 -0700503 if (bottom > mImageInfo.mHeight) {
504 bottom = mImageInfo.mHeight;
Chong Zhang0c1407f2018-05-02 17:09:05 -0700505 }
506 sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
Chong Zhang8541d302019-07-12 11:19:47 -0700507 -1, mOutputColor, 0, top, mImageInfo.mWidth, bottom);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700508 {
509 Mutex::Autolock autolock(mLock);
510
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700511 if (frameMemory == nullptr || frameMemory->unsecurePointer() == nullptr) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700512 mAsyncDecodeDone = true;
513 mScanlineReady.signal();
514 break;
515 }
516 mFrameMemory = frameMemory;
517 mAvailableLines = bottom;
518 ALOGV("decodeAsync(): available lines %zu", mAvailableLines);
519 mScanlineReady.signal();
520 }
521 }
522 // Aggressive clear to avoid holding on to resources
523 mRetriever.clear();
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500524
525 // Hold on to mDataSource in case the client wants to redecode.
Chong Zhang0c1407f2018-05-02 17:09:05 -0700526 return false;
527}
528
Chong Zhangea280cb2017-08-02 11:10:10 -0700529bool HeifDecoderImpl::decode(HeifFrameInfo* frameInfo) {
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700530 // reset scanline pointer
531 mCurScanline = 0;
532
533 if (mFrameDecoded) {
534 return true;
535 }
536
Chong Zhang0c1407f2018-05-02 17:09:05 -0700537 // See if we want to decode in slices to allow client to start
538 // scanline processing in parallel with decode. If this fails
539 // we fallback to decoding the full frame.
Chong Zhang8541d302019-07-12 11:19:47 -0700540 if (mHasImage) {
541 if (mSliceHeight >= 512 &&
542 mImageInfo.mWidth >= 3000 &&
543 mImageInfo.mHeight >= 2000 ) {
544 // Try decoding in slices only if the image has tiles and is big enough.
545 mNumSlices = (mImageInfo.mHeight + mSliceHeight - 1) / mSliceHeight;
546 ALOGV("mSliceHeight %u, mNumSlices %zu", mSliceHeight, mNumSlices);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700547 }
548
Chong Zhang8541d302019-07-12 11:19:47 -0700549 if (mNumSlices > 1) {
550 // get first slice and metadata
551 sp<IMemory> frameMemory = mRetriever->getImageRectAtIndex(
552 -1, mOutputColor, 0, 0, mImageInfo.mWidth, mSliceHeight);
Chong Zhang0c1407f2018-05-02 17:09:05 -0700553
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700554 if (frameMemory == nullptr || frameMemory->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700555 ALOGE("decode: metadata is a nullptr");
556 return false;
557 }
558
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700559 // TODO: Using unsecurePointer() has some associated security pitfalls
560 // (see declaration for details).
561 // Either document why it is safe in this case or address the
562 // issue (e.g. by copying).
563 VideoFrame* videoFrame = static_cast<VideoFrame*>(frameMemory->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700564
565 if (frameInfo != nullptr) {
566 initFrameInfo(frameInfo, videoFrame);
567 }
568 mFrameMemory = frameMemory;
569 mAvailableLines = mSliceHeight;
570 mThread = new DecodeThread(this);
571 if (mThread->run("HeifDecode", ANDROID_PRIORITY_FOREGROUND) == OK) {
572 mFrameDecoded = true;
573 return true;
574 }
575 // Fallback to decode without slicing
576 mThread.clear();
577 mNumSlices = 1;
578 mSliceHeight = 0;
579 mAvailableLines = 0;
580 mFrameMemory.clear();
Chong Zhang0c1407f2018-05-02 17:09:05 -0700581 }
Chong Zhang0c1407f2018-05-02 17:09:05 -0700582 }
583
Chong Zhangd3e0d862017-10-03 13:17:13 -0700584 if (mHasImage) {
585 // image index < 0 to retrieve primary image
586 mFrameMemory = mRetriever->getImageAtIndex(-1, mOutputColor);
587 } else if (mHasVideo) {
588 mFrameMemory = mRetriever->getFrameAtTime(0,
589 MediaSource::ReadOptions::SEEK_PREVIOUS_SYNC, mOutputColor);
590 }
591
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700592 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700593 ALOGE("decode: videoFrame is a nullptr");
Chong Zhangea280cb2017-08-02 11:10:10 -0700594 return false;
595 }
596
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700597 // TODO: Using unsecurePointer() has some associated security pitfalls
598 // (see declaration for details).
599 // Either document why it is safe in this case or address the
600 // issue (e.g. by copying).
601 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700602 if (videoFrame->mSize == 0 ||
603 mFrameMemory->size() < videoFrame->getFlattenedSize()) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700604 ALOGE("decode: videoFrame size is invalid");
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700605 return false;
606 }
607
Chong Zhangea280cb2017-08-02 11:10:10 -0700608 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
609 videoFrame->mWidth,
610 videoFrame->mHeight,
611 videoFrame->mDisplayWidth,
612 videoFrame->mDisplayHeight,
613 videoFrame->mRotationAngle,
614 videoFrame->mRowBytes,
615 videoFrame->mSize);
616
617 if (frameInfo != nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700618 initFrameInfo(frameInfo, videoFrame);
619
Chong Zhangea280cb2017-08-02 11:10:10 -0700620 }
Chong Zhang3f3ffab2017-08-23 13:51:59 -0700621 mFrameDecoded = true;
Chong Zhang4c6398b2017-09-07 17:43:19 -0700622
Chong Zhang0c1407f2018-05-02 17:09:05 -0700623 // Aggressively clear to avoid holding on to resources
Chong Zhang4c6398b2017-09-07 17:43:19 -0700624 mRetriever.clear();
Leon Scroggins III7ca678b2020-01-22 14:09:55 -0500625
626 // Hold on to mDataSource in case the client wants to redecode.
Chong Zhangea280cb2017-08-02 11:10:10 -0700627 return true;
628}
629
Chong Zhang8541d302019-07-12 11:19:47 -0700630bool HeifDecoderImpl::decodeSequence(int frameIndex, HeifFrameInfo* frameInfo) {
631 ALOGV("%s: frame index %d", __FUNCTION__, frameIndex);
632 if (!mHasVideo) {
633 return false;
634 }
635
636 if (frameIndex < 0 || frameIndex >= mSequenceLength) {
637 ALOGE("invalid frame index: %d, total frames %zu", frameIndex, mSequenceLength);
638 return false;
639 }
640
641 mCurScanline = 0;
642
643 // set total scanline to sequence height now
644 mTotalScanline = mSequenceInfo.mHeight;
645
646 mFrameMemory = mRetriever->getFrameAtIndex(frameIndex, mOutputColor);
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700647 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhang8541d302019-07-12 11:19:47 -0700648 ALOGE("decode: videoFrame is a nullptr");
649 return false;
650 }
651
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700652 // TODO: Using unsecurePointer() has some associated security pitfalls
653 // (see declaration for details).
654 // Either document why it is safe in this case or address the
655 // issue (e.g. by copying).
656 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhang8541d302019-07-12 11:19:47 -0700657 if (videoFrame->mSize == 0 ||
658 mFrameMemory->size() < videoFrame->getFlattenedSize()) {
659 ALOGE("decode: videoFrame size is invalid");
660 return false;
661 }
662
663 ALOGV("Decoded dimension %dx%d, display %dx%d, angle %d, rowbytes %d, size %d",
664 videoFrame->mWidth,
665 videoFrame->mHeight,
666 videoFrame->mDisplayWidth,
667 videoFrame->mDisplayHeight,
668 videoFrame->mRotationAngle,
669 videoFrame->mRowBytes,
670 videoFrame->mSize);
671
672 if (frameInfo != nullptr) {
673 initFrameInfo(frameInfo, videoFrame);
674 }
675 return true;
676}
677
Chong Zhang0c1407f2018-05-02 17:09:05 -0700678bool HeifDecoderImpl::getScanlineInner(uint8_t* dst) {
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700679 if (mFrameMemory == nullptr || mFrameMemory->unsecurePointer() == nullptr) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700680 return false;
681 }
Ytai Ben-Tsvi7dd39722019-09-05 15:14:30 -0700682 // TODO: Using unsecurePointer() has some associated security pitfalls
683 // (see declaration for details).
684 // Either document why it is safe in this case or address the
685 // issue (e.g. by copying).
686 VideoFrame* videoFrame = static_cast<VideoFrame*>(mFrameMemory->unsecurePointer());
Chong Zhangea280cb2017-08-02 11:10:10 -0700687 uint8_t* src = videoFrame->getFlattenedData() + videoFrame->mRowBytes * mCurScanline++;
Chong Zhang70b0afd2018-02-27 12:43:06 -0800688 memcpy(dst, src, videoFrame->mBytesPerPixel * videoFrame->mWidth);
Chong Zhangea280cb2017-08-02 11:10:10 -0700689 return true;
690}
691
Chong Zhang0c1407f2018-05-02 17:09:05 -0700692bool HeifDecoderImpl::getScanline(uint8_t* dst) {
Chong Zhang8541d302019-07-12 11:19:47 -0700693 if (mCurScanline >= mTotalScanline) {
Chong Zhang0c1407f2018-05-02 17:09:05 -0700694 ALOGE("no more scanline available");
695 return false;
Chong Zhangea280cb2017-08-02 11:10:10 -0700696 }
Chong Zhangea280cb2017-08-02 11:10:10 -0700697
Chong Zhang0c1407f2018-05-02 17:09:05 -0700698 if (mNumSlices > 1) {
699 Mutex::Autolock autolock(mLock);
700
701 while (!mAsyncDecodeDone && mCurScanline >= mAvailableLines) {
702 mScanlineReady.wait(mLock);
703 }
704 return (mCurScanline < mAvailableLines) ? getScanlineInner(dst) : false;
705 }
706
707 return getScanlineInner(dst);
708}
709
710size_t HeifDecoderImpl::skipScanlines(size_t count) {
Chong Zhangea280cb2017-08-02 11:10:10 -0700711 uint32_t oldScanline = mCurScanline;
712 mCurScanline += count;
Chong Zhang8541d302019-07-12 11:19:47 -0700713 if (mCurScanline > mTotalScanline) {
714 mCurScanline = mTotalScanline;
Chong Zhangea280cb2017-08-02 11:10:10 -0700715 }
716 return (mCurScanline > oldScanline) ? (mCurScanline - oldScanline) : 0;
717}
718
719} // namespace android