media, mainline: Add basic AVIF decoding support
This CL contains files that are part of mainline.
Parse the AVIF boxes and pass the data along to a compatible AV1
decoder (similar to how HEIC frames are passed to the HEVC
decoder).
Test: BitmapFactory.decodeByteArray() is now able to decode valid AVIF images.
Test: HEIF related CTS tests still pass.
CTS tests to come in a different CL.
Bug: 141654151
Change-Id: I84a01db859281a1b14746fbb97c247f427d440f8
diff --git a/media/extractors/mp4/ItemTable.cpp b/media/extractors/mp4/ItemTable.cpp
index 2599c2c..0b1cfa0 100644
--- a/media/extractors/mp4/ItemTable.cpp
+++ b/media/extractors/mp4/ItemTable.cpp
@@ -76,6 +76,7 @@
size_t size;
sp<ABuffer> hvcc;
sp<ABuffer> icc;
+ sp<ABuffer> av1c;
Vector<uint32_t> thumbnails;
Vector<uint32_t> dimgRefs;
@@ -764,6 +765,39 @@
return OK;
}
+struct Av1cBox : public Box, public ItemProperty {
+ Av1cBox(DataSourceHelper *source) :
+ Box(source, FOURCC("av1C")) {}
+
+ status_t parse(off64_t offset, size_t size) override;
+
+ void attachTo(ImageItem &image) const override {
+ image.av1c = mAv1c;
+ }
+
+private:
+ sp<ABuffer> mAv1c;
+};
+
+status_t Av1cBox::parse(off64_t offset, size_t size) {
+ ALOGV("%s: offset %lld, size %zu", __FUNCTION__, (long long)offset, size);
+
+ mAv1c = new ABuffer(size);
+
+ if (mAv1c->data() == NULL) {
+ ALOGE("b/28471206");
+ return NO_MEMORY;
+ }
+
+ if (source()->readAt(offset, mAv1c->data(), size) < (ssize_t)size) {
+ return ERROR_IO;
+ }
+
+ ALOGV("property av1C");
+
+ return OK;
+}
+
struct IrotBox : public Box, public ItemProperty {
IrotBox(DataSourceHelper *source) :
Box(source, FOURCC("irot")), mAngle(0) {}
@@ -957,6 +991,11 @@
itemProperty = new ColrBox(source());
break;
}
+ case FOURCC("av1C"):
+ {
+ itemProperty = new Av1cBox(source());
+ break;
+ }
default:
{
// push dummy to maintain correct item property index
@@ -1203,8 +1242,9 @@
//////////////////////////////////////////////////////////////////
-ItemTable::ItemTable(DataSourceHelper *source)
+ItemTable::ItemTable(DataSourceHelper *source, bool isHeif)
: mDataSource(source),
+ mIsHeif(isHeif),
mPrimaryItemId(0),
mIdatOffset(0),
mIdatSize(0),
@@ -1363,7 +1403,8 @@
// 'Exif': EXIF metadata
if (info.itemType != FOURCC("grid") &&
info.itemType != FOURCC("hvc1") &&
- info.itemType != FOURCC("Exif")) {
+ info.itemType != FOURCC("Exif") &&
+ info.itemType != FOURCC("av01")) {
continue;
}
@@ -1509,7 +1550,9 @@
}
AMediaFormat *meta = AMediaFormat_new();
- AMediaFormat_setString(meta, AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);
+ AMediaFormat_setString(
+ meta, AMEDIAFORMAT_KEY_MIME,
+ mIsHeif ? MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC : MEDIA_MIMETYPE_IMAGE_AVIF);
if (image->itemId == mPrimaryItemId) {
AMediaFormat_setInt32(meta, AMEDIAFORMAT_KEY_IS_DEFAULT, 1);
@@ -1539,7 +1582,7 @@
ssize_t thumbItemIndex = mItemIdToItemMap.indexOfKey(image->thumbnails[0]);
if (thumbItemIndex >= 0) {
const ImageItem &thumbnail = mItemIdToItemMap[thumbItemIndex];
-
+ // TODO(vigneshv): Handle thumbnail for AVIF.
if (thumbnail.hvcc != NULL) {
AMediaFormat_setInt32(meta,
AMEDIAFORMAT_KEY_THUMBNAIL_WIDTH, thumbnail.width);
@@ -1574,12 +1617,21 @@
AMEDIAFORMAT_KEY_MAX_INPUT_SIZE, image->width * image->height * 3 / 2);
}
- if (image->hvcc == NULL) {
- ALOGE("%s: hvcc is missing for image[%u]!", __FUNCTION__, imageIndex);
- return NULL;
+ if (mIsHeif) {
+ if (image->hvcc == NULL) {
+ ALOGE("%s: hvcc is missing for image[%u]!", __FUNCTION__, imageIndex);
+ return NULL;
+ }
+ AMediaFormat_setBuffer(meta,
+ AMEDIAFORMAT_KEY_CSD_HEVC, image->hvcc->data(), image->hvcc->size());
+ } else {
+ if (image->av1c == NULL) {
+ ALOGE("%s: av1c is missing for image[%u]!", __FUNCTION__, imageIndex);
+ return NULL;
+ }
+ AMediaFormat_setBuffer(meta,
+ AMEDIAFORMAT_KEY_CSD_0, image->av1c->data(), image->av1c->size());
}
- AMediaFormat_setBuffer(meta,
- AMEDIAFORMAT_KEY_CSD_HEVC, image->hvcc->data(), image->hvcc->size());
if (image->icc != NULL) {
AMediaFormat_setBuffer(meta,
diff --git a/media/extractors/mp4/ItemTable.h b/media/extractors/mp4/ItemTable.h
index be81b59..b19dc18 100644
--- a/media/extractors/mp4/ItemTable.h
+++ b/media/extractors/mp4/ItemTable.h
@@ -42,12 +42,12 @@
/*
* ItemTable keeps track of all image items (including coded images, grids and
- * tiles) inside a HEIF still image (ISO/IEC FDIS 23008-12.2:2017(E)).
+ * tiles) inside a HEIF/AVIF still image (ISO/IEC FDIS 23008-12.2:2017(E)).
*/
class ItemTable : public RefBase {
public:
- explicit ItemTable(DataSourceHelper *source);
+ ItemTable(DataSourceHelper *source, bool isHeif);
status_t parse(uint32_t type, off64_t offset, size_t size);
@@ -65,6 +65,8 @@
private:
DataSourceHelper *mDataSource;
+ // If this is true, then this item table is for a HEIF image. Otherwise it is for an AVIF image.
+ bool mIsHeif;
KeyedVector<uint32_t, ItemLoc> mItemLocs;
Vector<ItemInfo> mItemInfos;
diff --git a/media/extractors/mp4/MPEG4Extractor.cpp b/media/extractors/mp4/MPEG4Extractor.cpp
index bd36403..42bfc39 100644
--- a/media/extractors/mp4/MPEG4Extractor.cpp
+++ b/media/extractors/mp4/MPEG4Extractor.cpp
@@ -149,6 +149,7 @@
uint8_t *mSrcBuffer;
bool mIsHeif;
+ bool mIsAvif;
bool mIsAudio;
bool mIsUsac = false;
sp<ItemTable> mItemTable;
@@ -414,6 +415,7 @@
mIsHeif(false),
mHasMoovBox(false),
mPreferHeif(mime != NULL && !strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_HEIF)),
+ mIsAvif(false),
mFirstTrack(NULL),
mLastTrack(NULL) {
ALOGV("mime=%s, mPreferHeif=%d", mime, mPreferHeif);
@@ -670,7 +672,7 @@
}
}
- if (mIsHeif && (mItemTable != NULL) && (mItemTable->countImages() > 0)) {
+ if ((mIsAvif || mIsHeif) && (mItemTable != NULL) && (mItemTable->countImages() > 0)) {
off64_t exifOffset;
size_t exifSize;
if (mItemTable->getExifOffsetAndSize(&exifOffset, &exifSize) == OK) {
@@ -696,7 +698,7 @@
}
mInitCheck = OK;
- ALOGV("adding HEIF image track %u", imageIndex);
+ ALOGV("adding %s image track %u", mIsHeif ? "HEIF" : "AVIF", imageIndex);
Track *track = new Track;
if (mLastTrack != NULL) {
mLastTrack->next = track;
@@ -722,6 +724,10 @@
MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC) != NULL) {
AMediaFormat_setString(mFileMetaData,
AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_CONTAINER_HEIF);
+ } else if (findTrackByMimePrefix(
+ MEDIA_MIMETYPE_IMAGE_AVIF) != NULL) {
+ AMediaFormat_setString(mFileMetaData,
+ AMEDIAFORMAT_KEY_MIME, MEDIA_MIMETYPE_IMAGE_AVIF);
} else {
AMediaFormat_setString(mFileMetaData,
AMEDIAFORMAT_KEY_MIME, "application/octet-stream");
@@ -2576,9 +2582,9 @@
case FOURCC("iref"):
case FOURCC("ipro"):
{
- if (mIsHeif) {
+ if (mIsHeif || mIsAvif) {
if (mItemTable == NULL) {
- mItemTable = new ItemTable(mDataSource);
+ mItemTable = new ItemTable(mDataSource, mIsHeif);
}
status_t err = mItemTable->parse(
chunk_type, data_offset, chunk_data_size);
@@ -3019,14 +3025,20 @@
mIsHeif = true;
brandSet.erase(FOURCC("mif1"));
brandSet.erase(FOURCC("heic"));
+ } else if (brandSet.count(FOURCC("avif")) > 0 ||
+ brandSet.count(FOURCC("avis")) > 0) {
+ ALOGV("identified AVIF image");
+ mIsAvif = true;
+ brandSet.erase(FOURCC("avif"));
+ brandSet.erase(FOURCC("avis"));
}
if (!brandSet.empty()) {
// This means that the file should have moov box.
// It could be any iso files (mp4, heifs, etc.)
mHasMoovBox = true;
- if (mIsHeif) {
- ALOGV("identified HEIF image with other tracks");
+ if (mIsHeif || mIsAvif) {
+ ALOGV("identified %s image with other tracks", mIsHeif ? "HEIF" : "AVIF");
}
}
}
@@ -4364,7 +4376,8 @@
if (size != 24 || ((ptr[0] != 1 || ptr[1] != 0) && (ptr[0] != 2 || ptr[1] != 1))) {
return NULL;
}
- } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)) {
+ } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AV1)
+ || !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF)) {
void *data;
size_t size;
if (!AMediaFormat_getBuffer(track->meta, AMEDIAFORMAT_KEY_CSD_0, &data, &size)) {
@@ -4376,6 +4389,9 @@
if (size < 5 || ptr[0] != 0x81) { // configurationVersion == 1
return NULL;
}
+ if (!strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF)) {
+ itemTable = mItemTable;
+ }
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_VP9)) {
void *data;
size_t size;
@@ -4937,7 +4953,6 @@
mStarted(false),
mBuffer(NULL),
mSrcBuffer(NULL),
- mIsHeif(itemTable != NULL),
mItemTable(itemTable),
mElstShiftStartTicks(elstShiftStartTicks),
mElstInitialEmptyEditTicks(elstInitialEmptyEditTicks) {
@@ -4972,6 +4987,8 @@
!strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);
mIsAC4 = !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AC4);
mIsDolbyVision = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_DOLBY_VISION);
+ mIsHeif = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC) && mItemTable != NULL;
+ mIsAvif = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_AVIF) && mItemTable != NULL;
if (mIsAVC) {
void *data;
@@ -5966,7 +5983,7 @@
if (options && options->getSeekTo(&seekTimeUs, &mode)) {
ALOGV("seekTimeUs:%" PRId64, seekTimeUs);
- if (mIsHeif) {
+ if (mIsHeif || mIsAvif) {
CHECK(mSampleTable == NULL);
CHECK(mItemTable != NULL);
int32_t imageIndex;
@@ -6111,7 +6128,7 @@
newBuffer = true;
status_t err;
- if (!mIsHeif) {
+ if (!mIsHeif && !mIsAvif) {
err = mSampleTable->getMetaDataForSample(mCurrentSampleIndex, &offset, &size,
(uint64_t*)&cts, &isSyncSample, &stts);
if(err == OK) {
@@ -6750,7 +6767,8 @@
|| !memcmp(header, "ftypM4A ", 8) || !memcmp(header, "ftypf4v ", 8)
|| !memcmp(header, "ftypkddi", 8) || !memcmp(header, "ftypM4VP", 8)
|| !memcmp(header, "ftypmif1", 8) || !memcmp(header, "ftypheic", 8)
- || !memcmp(header, "ftypmsf1", 8) || !memcmp(header, "ftyphevc", 8)) {
+ || !memcmp(header, "ftypmsf1", 8) || !memcmp(header, "ftyphevc", 8)
+ || !memcmp(header, "ftypavif", 8) || !memcmp(header, "ftypavis", 8)) {
*confidence = 0.4;
return true;
@@ -6786,6 +6804,8 @@
FOURCC("heic"), // HEIF image
FOURCC("msf1"), // HEIF image sequence
FOURCC("hevc"), // HEIF image sequence
+ FOURCC("avif"), // AVIF image
+ FOURCC("avis"), // AVIF image sequence
};
for (size_t i = 0;
diff --git a/media/extractors/mp4/MPEG4Extractor.h b/media/extractors/mp4/MPEG4Extractor.h
index bafc7f5..542a3e6 100644
--- a/media/extractors/mp4/MPEG4Extractor.h
+++ b/media/extractors/mp4/MPEG4Extractor.h
@@ -144,6 +144,7 @@
bool mIsHeif;
bool mHasMoovBox;
bool mPreferHeif;
+ bool mIsAvif;
Track *mFirstTrack, *mLastTrack;