heif: add muxer support for adding still images
This change adds basic support for adding still images to a
heif container.
bug: 63633199
test:
Locally built test app that adds more than one still images
and video sequences to a heif container. Verified the muxed
file can be played by locally built heif viewer app that
uses MediaExtractor/MediaMetadataRetriever/MediaPlayer apis.
Tested media post-submit CTS on Pixel2XL.
Change-Id: I185bbbbed3dbb4e76d64bcf7323ee44a42cc8623
diff --git a/media/extractors/mp4/ItemTable.cpp b/media/extractors/mp4/ItemTable.cpp
index 9a6cb64..85c66b2 100644
--- a/media/extractors/mp4/ItemTable.cpp
+++ b/media/extractors/mp4/ItemTable.cpp
@@ -1425,7 +1425,7 @@
meta->setCString(kKeyMIMEType, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);
if (image->itemId == mPrimaryItemId) {
- meta->setInt32(kKeyIsPrimaryImage, 1);
+ meta->setInt32(kKeyTrackIsDefault, 1);
}
ALOGV("image[%u]: size %dx%d", imageIndex, image->width, image->height);
diff --git a/media/extractors/mp4/MPEG4Extractor.cpp b/media/extractors/mp4/MPEG4Extractor.cpp
index 6671956..b411125 100644
--- a/media/extractors/mp4/MPEG4Extractor.cpp
+++ b/media/extractors/mp4/MPEG4Extractor.cpp
@@ -347,7 +347,7 @@
mHeaderTimescale(0),
mIsQT(false),
mIsHeif(false),
- mIsHeifSequence(false),
+ mHasMoovBox(false),
mPreferHeif(mime != NULL && !strcasecmp(mime, MEDIA_MIMETYPE_CONTAINER_HEIF)),
mFirstTrack(NULL),
mLastTrack(NULL),
@@ -563,9 +563,9 @@
status_t err;
bool sawMoovOrSidx = false;
- while (!((!mIsHeif && sawMoovOrSidx && (mMdatFound || mMoofFound)) ||
- (mIsHeif && (mPreferHeif || !mIsHeifSequence)
- && (mItemTable != NULL) && mItemTable->isValid()))) {
+ while (!((mHasMoovBox && sawMoovOrSidx && (mMdatFound || mMoofFound)) ||
+ (mIsHeif && (mPreferHeif || !mHasMoovBox) &&
+ (mItemTable != NULL) && mItemTable->isValid()))) {
off64_t orig_offset = offset;
err = parseChunk(&offset, 0);
@@ -582,34 +582,30 @@
}
}
- if (mIsHeif) {
- uint32_t imageCount = mItemTable->countImages();
- if (imageCount == 0) {
- ALOGE("found no image in heif!");
- } else {
- for (uint32_t imageIndex = 0; imageIndex < imageCount; imageIndex++) {
- sp<MetaData> meta = mItemTable->getImageMeta(imageIndex);
- if (meta == NULL) {
- ALOGE("heif image %u has no meta!", imageIndex);
- continue;
- }
-
- ALOGV("adding HEIF image track %u", imageIndex);
- Track *track = new Track;
- track->next = NULL;
- if (mLastTrack != NULL) {
- mLastTrack->next = track;
- } else {
- mFirstTrack = track;
- }
- mLastTrack = track;
-
- track->meta = meta;
- track->meta->setInt32(kKeyTrackID, imageIndex);
- track->includes_expensive_metadata = false;
- track->skipTrack = false;
- track->timescale = 0;
+ if (mIsHeif && (mItemTable != NULL) && (mItemTable->countImages() > 0)) {
+ for (uint32_t imageIndex = 0;
+ imageIndex < mItemTable->countImages(); imageIndex++) {
+ sp<MetaData> meta = mItemTable->getImageMeta(imageIndex);
+ if (meta == NULL) {
+ ALOGE("heif image %u has no meta!", imageIndex);
+ continue;
}
+
+ ALOGV("adding HEIF image track %u", imageIndex);
+ Track *track = new Track;
+ track->next = NULL;
+ if (mLastTrack != NULL) {
+ mLastTrack->next = track;
+ } else {
+ mFirstTrack = track;
+ }
+ mLastTrack = track;
+
+ track->meta = meta;
+ track->meta->setInt32(kKeyTrackID, imageIndex);
+ track->includes_expensive_metadata = false;
+ track->skipTrack = false;
+ track->timescale = 0;
}
}
@@ -2512,13 +2508,18 @@
} else {
if (brandSet.count(FOURCC('m', 'i', 'f', '1')) > 0
&& brandSet.count(FOURCC('h', 'e', 'i', 'c')) > 0) {
- mIsHeif = true;
ALOGV("identified HEIF image");
+
+ mIsHeif = true;
+ brandSet.erase(FOURCC('m', 'i', 'f', '1'));
+ brandSet.erase(FOURCC('h', 'e', 'i', 'c'));
}
- if (brandSet.count(FOURCC('m', 's', 'f', '1')) > 0
- && brandSet.count(FOURCC('h', 'e', 'v', 'c')) > 0) {
- mIsHeifSequence = true;
- ALOGV("identified HEIF image sequence");
+
+ if (!brandSet.empty()) {
+ // This means that the file should have moov box.
+ // It could be any iso files (mp4, heifs, etc.)
+ mHasMoovBox = true;
+ ALOGV("identified HEIF image with other tracks");
}
}
diff --git a/media/extractors/mp4/MPEG4Extractor.h b/media/extractors/mp4/MPEG4Extractor.h
index d4f17e3..76b549d 100644
--- a/media/extractors/mp4/MPEG4Extractor.h
+++ b/media/extractors/mp4/MPEG4Extractor.h
@@ -104,7 +104,7 @@
uint32_t mHeaderTimescale;
bool mIsQT;
bool mIsHeif;
- bool mIsHeifSequence;
+ bool mHasMoovBox;
bool mPreferHeif;
Track *mFirstTrack, *mLastTrack;
diff --git a/media/libmedia/include/media/mediarecorder.h b/media/libmedia/include/media/mediarecorder.h
index 071e7a1..b9717ea 100644
--- a/media/libmedia/include/media/mediarecorder.h
+++ b/media/libmedia/include/media/mediarecorder.h
@@ -77,6 +77,9 @@
/* VP8/VORBIS data in a WEBM container */
OUTPUT_FORMAT_WEBM = 9,
+ /* HEIC data in a HEIF container */
+ OUTPUT_FORMAT_HEIF = 10,
+
OUTPUT_FORMAT_LIST_END // must be last - used to validate format type
};
diff --git a/media/libstagefright/MPEG4Writer.cpp b/media/libstagefright/MPEG4Writer.cpp
index a132873..1828e69 100644
--- a/media/libstagefright/MPEG4Writer.cpp
+++ b/media/libstagefright/MPEG4Writer.cpp
@@ -112,14 +112,18 @@
int64_t getDurationUs() const;
int64_t getEstimatedTrackSizeBytes() const;
+ int32_t getMetaSizeIncrease() const;
void writeTrackHeader(bool use32BitOffset = true);
int64_t getMinCttsOffsetTimeUs();
void bufferChunk(int64_t timestampUs);
bool isAvc() const { return mIsAvc; }
bool isHevc() const { return mIsHevc; }
+ bool isHeic() const { return mIsHeic; }
bool isAudio() const { return mIsAudio; }
bool isMPEG4() const { return mIsMPEG4; }
+ bool usePrefix() const { return mIsAvc || mIsHevc || mIsHeic; }
void addChunkOffset(off64_t offset);
+ void addItemOffsetAndSize(off64_t offset, size_t size);
int32_t getTrackId() const { return mTrackId; }
status_t dump(int fd, const Vector<String16>& args) const;
static const char *getFourCCForMime(const char *mime);
@@ -281,6 +285,7 @@
bool mIsHevc;
bool mIsAudio;
bool mIsVideo;
+ bool mIsHeic;
bool mIsMPEG4;
bool mGotStartKeyFrame;
bool mIsMalformed;
@@ -347,6 +352,16 @@
int64_t mPreviousTrackTimeUs;
int64_t mTrackEveryTimeDurationUs;
+ int32_t mRotation;
+
+ Vector<uint16_t> mProperties;
+ Vector<uint16_t> mDimgRefs;
+ int32_t mIsPrimary;
+ int32_t mWidth, mHeight;
+ int32_t mGridWidth, mGridHeight;
+ int32_t mGridRows, mGridCols;
+ size_t mNumTiles, mTileIndex;
+
// Update the audio track's drift information.
void updateDriftTime(const sp<MetaData>& meta);
@@ -386,7 +401,6 @@
// Simple validation on the codec specific data
status_t checkCodecSpecificData() const;
- int32_t mRotation;
void updateTrackSizeEstimate();
void addOneStscTableEntry(size_t chunkId, size_t sampleId);
@@ -481,6 +495,11 @@
mStreamableFile = false;
mEstimatedMoovBoxSize = 0;
mTimeScale = -1;
+ mHasFileLevelMeta = false;
+ mHasMoovBox = false;
+ mPrimaryItemId = 0;
+ mAssociationEntryCount = 0;
+ mNumGrids = 0;
// Following variables only need to be set for the first recording session.
// And they will stay the same for all the recording sessions.
@@ -567,6 +586,8 @@
}
} else if (!strncasecmp(mime, "application/", 12)) {
return "mett";
+ } else if (!strcasecmp(MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC, mime)) {
+ return "heic";
} else {
ALOGE("Track (%s) other than video/audio/metadata is not supported", mime);
}
@@ -595,6 +616,9 @@
Track *track = new Track(this, source, 1 + mTracks.size());
mTracks.push_back(track);
+ mHasMoovBox |= !track->isHeic();
+ mHasFileLevelMeta |= track->isHeic();
+
return OK;
}
@@ -656,6 +680,32 @@
#endif
}
+int64_t MPEG4Writer::estimateFileLevelMetaSize() {
+ // base meta size
+ int64_t metaSize = 12 // meta fullbox header
+ + 33 // hdlr box
+ + 14 // pitm box
+ + 16 // iloc box (fixed size portion)
+ + 14 // iinf box (fixed size portion)
+ + 32 // iprp box (fixed size protion)
+ + 8 // idat box (when empty)
+ + 12 // iref box (when empty)
+ ;
+
+ for (List<Track *>::iterator it = mTracks.begin();
+ it != mTracks.end(); ++it) {
+ if ((*it)->isHeic()) {
+ metaSize += (*it)->getMetaSizeIncrease();
+ }
+ }
+
+ ALOGV("estimated meta size: %lld", (long long) metaSize);
+
+ // Need at least 8-byte padding at the end, otherwise the left-over
+ // freebox may become malformed
+ return metaSize + 8;
+}
+
int64_t MPEG4Writer::estimateMoovBoxSize(int32_t bitRate) {
// This implementation is highly experimental/heurisitic.
//
@@ -715,7 +765,11 @@
ALOGI("limits: %" PRId64 "/%" PRId64 " bytes/us, bit rate: %d bps and the"
" estimated moov size %" PRId64 " bytes",
mMaxFileSizeLimitBytes, mMaxFileDurationLimitUs, bitRate, size);
- return factor * size;
+
+ int64_t estimatedSize = factor * size;
+ CHECK_GE(estimatedSize, 8);
+
+ return estimatedSize;
}
status_t MPEG4Writer::start(MetaData *param) {
@@ -836,18 +890,26 @@
mMoovBoxBuffer = NULL;
mMoovBoxBufferOffset = 0;
+
+ ALOGV("muxer starting: mHasMoovBox %d, mHasFileLevelMeta %d",
+ mHasMoovBox, mHasFileLevelMeta);
+
writeFtypBox(param);
mFreeBoxOffset = mOffset;
if (mEstimatedMoovBoxSize == 0) {
int32_t bitRate = -1;
- if (param) {
- param->findInt32(kKeyBitRate, &bitRate);
+ if (mHasFileLevelMeta) {
+ mEstimatedMoovBoxSize += estimateFileLevelMetaSize();
}
- mEstimatedMoovBoxSize = estimateMoovBoxSize(bitRate);
+ if (mHasMoovBox) {
+ if (param) {
+ param->findInt32(kKeyBitRate, &bitRate);
+ }
+ mEstimatedMoovBoxSize += estimateMoovBoxSize(bitRate);
+ }
}
- CHECK_GE(mEstimatedMoovBoxSize, 8);
if (mStreamableFile) {
// Reserve a 'free' box only for streamable file
lseek64(mFd, mFreeBoxOffset, SEEK_SET);
@@ -1009,13 +1071,18 @@
status_t err = OK;
int64_t maxDurationUs = 0;
int64_t minDurationUs = 0x7fffffffffffffffLL;
+ int32_t nonImageTrackCount = 0;
for (List<Track *>::iterator it = mTracks.begin();
- it != mTracks.end(); ++it) {
+ it != mTracks.end(); ++it) {
status_t status = (*it)->stop(stopSource);
if (err == OK && status != OK) {
err = status;
}
+ // skip image tracks
+ if ((*it)->isHeic()) continue;
+ nonImageTrackCount++;
+
int64_t durationUs = (*it)->getDurationUs();
if (durationUs > maxDurationUs) {
maxDurationUs = durationUs;
@@ -1025,7 +1092,7 @@
}
}
- if (mTracks.size() > 1) {
+ if (nonImageTrackCount > 1) {
ALOGD("Duration from tracks range is [%" PRId64 ", %" PRId64 "] us",
minDurationUs, maxDurationUs);
}
@@ -1061,30 +1128,28 @@
mMoovBoxBuffer = (uint8_t *) malloc(mEstimatedMoovBoxSize);
CHECK(mMoovBoxBuffer != NULL);
}
- writeMoovBox(maxDurationUs);
- // mWriteMoovBoxToMemory could be set to false in
- // MPEG4Writer::write() method
- if (mWriteMoovBoxToMemory) {
- mWriteMoovBoxToMemory = false;
- // Content of the moov box is saved in the cache, and the in-memory
- // moov box needs to be written to the file in a single shot.
-
- CHECK_LE(mMoovBoxBufferOffset + 8, mEstimatedMoovBoxSize);
-
- // Moov box
- lseek64(mFd, mFreeBoxOffset, SEEK_SET);
- mOffset = mFreeBoxOffset;
- write(mMoovBoxBuffer, 1, mMoovBoxBufferOffset);
-
- // Free box
- lseek64(mFd, mOffset, SEEK_SET);
- writeInt32(mEstimatedMoovBoxSize - mMoovBoxBufferOffset);
- write("free", 4);
- } else {
- ALOGI("The mp4 file will not be streamable.");
+ if (mHasFileLevelMeta) {
+ writeFileLevelMetaBox();
+ if (mWriteMoovBoxToMemory) {
+ writeCachedBoxToFile("meta");
+ } else {
+ ALOGI("The file meta box is written at the end.");
+ }
}
+ if (mHasMoovBox) {
+ writeMoovBox(maxDurationUs);
+ // mWriteMoovBoxToMemory could be set to false in
+ // MPEG4Writer::write() method
+ if (mWriteMoovBoxToMemory) {
+ writeCachedBoxToFile("moov");
+ } else {
+ ALOGI("The mp4 file will not be streamable.");
+ }
+ }
+ mWriteMoovBoxToMemory = false;
+
// Free in-memory cache for moov box
if (mMoovBoxBuffer != NULL) {
free(mMoovBoxBuffer);
@@ -1098,6 +1163,42 @@
return err;
}
+/*
+ * Writes currently cached box into file.
+ *
+ * Must be called while mWriteMoovBoxToMemory is true, and will not modify
+ * mWriteMoovBoxToMemory. After the call, remaining cache size will be
+ * reduced and buffer offset will be set to the beginning of the cache.
+ */
+void MPEG4Writer::writeCachedBoxToFile(const char *type) {
+ CHECK(mWriteMoovBoxToMemory);
+
+ mWriteMoovBoxToMemory = false;
+ // Content of the moov box is saved in the cache, and the in-memory
+ // moov box needs to be written to the file in a single shot.
+
+ CHECK_LE(mMoovBoxBufferOffset + 8, mEstimatedMoovBoxSize);
+
+ // Cached box
+ lseek64(mFd, mFreeBoxOffset, SEEK_SET);
+ mOffset = mFreeBoxOffset;
+ write(mMoovBoxBuffer, 1, mMoovBoxBufferOffset);
+
+ // Free box
+ lseek64(mFd, mOffset, SEEK_SET);
+ mFreeBoxOffset = mOffset;
+ writeInt32(mEstimatedMoovBoxSize - mMoovBoxBufferOffset);
+ write("free", 4);
+
+ // Rewind buffering to the beginning, and restore mWriteMoovBoxToMemory flag
+ mEstimatedMoovBoxSize -= mMoovBoxBufferOffset;
+ mMoovBoxBufferOffset = 0;
+ mWriteMoovBoxToMemory = true;
+
+ ALOGV("dumped out %s box, estimated size remaining %lld",
+ type, (long long)mEstimatedMoovBoxSize);
+}
+
uint32_t MPEG4Writer::getMpeg4Time() {
time_t now = time(NULL);
// MP4 file uses time counting seconds since midnight, Jan. 1, 1904
@@ -1142,14 +1243,16 @@
if (mAreGeoTagsAvailable) {
writeUdtaBox();
}
- writeMetaBox();
+ writeMoovLevelMetaBox();
// Loop through all the tracks to get the global time offset if there is
// any ctts table appears in a video track.
int64_t minCttsOffsetTimeUs = kMaxCttsOffsetTimeUs;
for (List<Track *>::iterator it = mTracks.begin();
it != mTracks.end(); ++it) {
- minCttsOffsetTimeUs =
- std::min(minCttsOffsetTimeUs, (*it)->getMinCttsOffsetTimeUs());
+ if (!(*it)->isHeic()) {
+ minCttsOffsetTimeUs =
+ std::min(minCttsOffsetTimeUs, (*it)->getMinCttsOffsetTimeUs());
+ }
}
ALOGI("Ajust the moov start time from %lld us -> %lld us",
(long long)mStartTimestampUs,
@@ -1159,7 +1262,9 @@
for (List<Track *>::iterator it = mTracks.begin();
it != mTracks.end(); ++it) {
- (*it)->writeTrackHeader(mUse32BitOffset);
+ if (!(*it)->isHeic()) {
+ (*it)->writeTrackHeader(mUse32BitOffset);
+ }
}
endBox(); // moov
}
@@ -1168,17 +1273,31 @@
beginBox("ftyp");
int32_t fileType;
- if (param && param->findInt32(kKeyFileType, &fileType) &&
- fileType != OUTPUT_FORMAT_MPEG_4) {
+ if (!param || !param->findInt32(kKeyFileType, &fileType)) {
+ fileType = OUTPUT_FORMAT_MPEG_4;
+ }
+ if (fileType != OUTPUT_FORMAT_MPEG_4 && fileType != OUTPUT_FORMAT_HEIF) {
writeFourcc("3gp4");
writeInt32(0);
writeFourcc("isom");
writeFourcc("3gp4");
} else {
- writeFourcc("mp42");
+ // Only write "heic" as major brand if the client specified HEIF
+ // AND we indeed receive some image heic tracks.
+ if (fileType == OUTPUT_FORMAT_HEIF && mHasFileLevelMeta) {
+ writeFourcc("heic");
+ } else {
+ writeFourcc("mp42");
+ }
writeInt32(0);
- writeFourcc("isom");
- writeFourcc("mp42");
+ if (mHasFileLevelMeta) {
+ writeFourcc("mif1");
+ writeFourcc("heic");
+ }
+ if (mHasMoovBox) {
+ writeFourcc("isom");
+ writeFourcc("mp42");
+ }
}
endBox();
@@ -1225,15 +1344,21 @@
mLock.unlock();
}
-off64_t MPEG4Writer::addSample_l(MediaBuffer *buffer) {
+off64_t MPEG4Writer::addSample_l(
+ MediaBuffer *buffer, bool usePrefix, size_t *bytesWritten) {
off64_t old_offset = mOffset;
- ::write(mFd,
- (const uint8_t *)buffer->data() + buffer->range_offset(),
- buffer->range_length());
+ if (usePrefix) {
+ addMultipleLengthPrefixedSamples_l(buffer);
+ } else {
+ ::write(mFd,
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ buffer->range_length());
- mOffset += buffer->range_length();
+ mOffset += buffer->range_length();
+ }
+ *bytesWritten = mOffset - old_offset;
return old_offset;
}
@@ -1251,9 +1376,7 @@
}
}
-off64_t MPEG4Writer::addMultipleLengthPrefixedSamples_l(MediaBuffer *buffer) {
- off64_t old_offset = mOffset;
-
+void MPEG4Writer::addMultipleLengthPrefixedSamples_l(MediaBuffer *buffer) {
const size_t kExtensionNALSearchRange = 64; // bytes to look for non-VCL NALUs
const uint8_t *dataStart = (const uint8_t *)buffer->data() + buffer->range_offset();
@@ -1278,13 +1401,9 @@
buffer->set_range(buffer->range_offset() + currentNalOffset,
buffer->range_length() - currentNalOffset);
addLengthPrefixedSample_l(buffer);
-
- return old_offset;
}
-off64_t MPEG4Writer::addLengthPrefixedSample_l(MediaBuffer *buffer) {
- off64_t old_offset = mOffset;
-
+void MPEG4Writer::addLengthPrefixedSample_l(MediaBuffer *buffer) {
size_t length = buffer->range_length();
if (mUse4ByteNalLength) {
@@ -1312,8 +1431,6 @@
::write(mFd, (const uint8_t *)buffer->data() + buffer->range_offset(), length);
mOffset += length + 2;
}
-
- return old_offset;
}
size_t MPEG4Writer::write(
@@ -1379,8 +1496,8 @@
mBoxes.erase(--mBoxes.end());
if (mWriteMoovBoxToMemory) {
- int32_t x = htonl(mMoovBoxBufferOffset - offset);
- memcpy(mMoovBoxBuffer + offset, &x, 4);
+ int32_t x = htonl(mMoovBoxBufferOffset - offset);
+ memcpy(mMoovBoxBuffer + offset, &x, 4);
} else {
lseek64(mFd, offset, SEEK_SET);
writeInt32(mOffset - offset);
@@ -1584,7 +1701,7 @@
for (List<Track *>::iterator it = mTracks.begin();
it != mTracks.end(); ++it) {
- if ((*it)->getDurationUs() >= mMaxFileDurationLimitUs) {
+ if (!(*it)->isHeic() && (*it)->getDurationUs() >= mMaxFileDurationLimitUs) {
return true;
}
}
@@ -1656,7 +1773,16 @@
mGotAllCodecSpecificData(false),
mReachedEOS(false),
mStartTimestampUs(-1),
- mRotation(0) {
+ mRotation(0),
+ mIsPrimary(0),
+ mWidth(0),
+ mHeight(0),
+ mGridWidth(0),
+ mGridHeight(0),
+ mGridRows(0),
+ mGridCols(0),
+ mNumTiles(1),
+ mTileIndex(0) {
getCodecSpecificDataFromInputFormatIfPossible();
const char *mime;
@@ -1665,6 +1791,7 @@
mIsHevc = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC);
mIsAudio = !strncasecmp(mime, "audio/", 6);
mIsVideo = !strncasecmp(mime, "video/", 6);
+ mIsHeic = !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC);
mIsMPEG4 = !strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_MPEG4) ||
!strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC);
@@ -1676,7 +1803,27 @@
}
}
- setTimeScale();
+ if (!mIsHeic) {
+ setTimeScale();
+ } else {
+ CHECK(mMeta->findInt32(kKeyWidth, &mWidth) && (mWidth > 0));
+ CHECK(mMeta->findInt32(kKeyHeight, &mHeight) && (mHeight > 0));
+
+ int32_t gridWidth, gridHeight, gridRows, gridCols;
+ if (mMeta->findInt32(kKeyGridWidth, &gridWidth) && (gridWidth > 0) &&
+ mMeta->findInt32(kKeyGridHeight, &gridHeight) && (gridHeight > 0) &&
+ mMeta->findInt32(kKeyGridRows, &gridRows) && (gridRows > 0) &&
+ mMeta->findInt32(kKeyGridCols, &gridCols) && (gridCols > 0)) {
+ mGridWidth = gridWidth;
+ mGridHeight = gridHeight;
+ mGridRows = gridRows;
+ mGridCols = gridCols;
+ mNumTiles = gridRows * gridCols;
+ }
+ if (!mMeta->findInt32(kKeyTrackIsDefault, &mIsPrimary)) {
+ mIsPrimary = false;
+ }
+ }
}
// Clear all the internal states except the CSD data.
@@ -1724,15 +1871,15 @@
}
void MPEG4Writer::Track::updateTrackSizeEstimate() {
-
- uint32_t stcoBoxCount = (mOwner->use32BitFileOffset()
- ? mStcoTableEntries->count()
- : mCo64TableEntries->count());
- int64_t stcoBoxSizeBytes = stcoBoxCount * 4;
- int64_t stszBoxSizeBytes = mSamplesHaveSameSize? 4: (mStszTableEntries->count() * 4);
-
mEstimatedTrackSizeBytes = mMdatSizeBytes; // media data size
- if (!mOwner->isFileStreamable()) {
+
+ if (!isHeic() && !mOwner->isFileStreamable()) {
+ uint32_t stcoBoxCount = (mOwner->use32BitFileOffset()
+ ? mStcoTableEntries->count()
+ : mCo64TableEntries->count());
+ int64_t stcoBoxSizeBytes = stcoBoxCount * 4;
+ int64_t stszBoxSizeBytes = mSamplesHaveSameSize? 4: (mStszTableEntries->count() * 4);
+
// Reserved free space is not large enough to hold
// all meta data and thus wasted.
mEstimatedTrackSizeBytes += mStscTableEntries->count() * 12 + // stsc box size
@@ -1746,10 +1893,9 @@
void MPEG4Writer::Track::addOneStscTableEntry(
size_t chunkId, size_t sampleId) {
-
- mStscTableEntries->add(htonl(chunkId));
- mStscTableEntries->add(htonl(sampleId));
- mStscTableEntries->add(htonl(1));
+ mStscTableEntries->add(htonl(chunkId));
+ mStscTableEntries->add(htonl(sampleId));
+ mStscTableEntries->add(htonl(1));
}
void MPEG4Writer::Track::addOneStssTableEntry(size_t sampleId) {
@@ -1795,6 +1941,7 @@
}
void MPEG4Writer::Track::addChunkOffset(off64_t offset) {
+ CHECK(!mIsHeic);
if (mOwner->use32BitFileOffset()) {
uint32_t value = offset;
mStcoTableEntries->add(htonl(value));
@@ -1803,6 +1950,70 @@
}
}
+void MPEG4Writer::Track::addItemOffsetAndSize(off64_t offset, size_t size) {
+ CHECK(mIsHeic);
+
+ if (offset > UINT32_MAX || size > UINT32_MAX) {
+ ALOGE("offset or size is out of range: %lld, %lld",
+ (long long) offset, (long long) size);
+ mIsMalformed = true;
+ }
+ if (mIsMalformed) {
+ return;
+ }
+ if (mTileIndex >= mNumTiles) {
+ ALOGW("Ignoring excess tiles!");
+ return;
+ }
+
+ if (mProperties.empty()) {
+ mProperties.push_back(mOwner->addProperty_l({
+ .type = FOURCC('h', 'v', 'c', 'C'),
+ .hvcc = ABuffer::CreateAsCopy(mCodecSpecificData, mCodecSpecificDataSize)
+ }));
+
+ mProperties.push_back(mOwner->addProperty_l({
+ .type = FOURCC('i', 's', 'p', 'e'),
+ .width = (mNumTiles > 1) ? mGridWidth : mWidth,
+ .height = (mNumTiles > 1) ? mGridHeight : mHeight,
+ }));
+ }
+
+ uint16_t itemId = mOwner->addItem_l({
+ .itemType = "hvc1",
+ .isPrimary = (mNumTiles > 1) ? false : (mIsPrimary != 0),
+ .isHidden = (mNumTiles > 1),
+ .offset = (uint32_t)offset,
+ .size = (uint32_t)size,
+ .properties = mProperties,
+ });
+
+ mTileIndex++;
+ if (mNumTiles > 1) {
+ mDimgRefs.push_back(itemId);
+
+ if (mTileIndex == mNumTiles) {
+ mProperties.clear();
+ mProperties.push_back(mOwner->addProperty_l({
+ .type = FOURCC('i', 's', 'p', 'e'),
+ .width = mWidth,
+ .height = mHeight,
+ }));
+ mOwner->addItem_l({
+ .itemType = "grid",
+ .isPrimary = (mIsPrimary != 0),
+ .isHidden = false,
+ .rows = (uint32_t)mGridRows,
+ .cols = (uint32_t)mGridCols,
+ .width = (uint32_t)mWidth,
+ .height = (uint32_t)mHeight,
+ .properties = mProperties,
+ .dimgRefs = mDimgRefs,
+ });
+ }
+ }
+}
+
void MPEG4Writer::Track::setTimeScale() {
ALOGV("setTimeScale");
// Default time scale
@@ -1855,7 +2066,8 @@
size_t size = 0;
if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC)) {
mMeta->findData(kKeyAVCC, &type, &data, &size);
- } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC)) {
+ } else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC) ||
+ !strcasecmp(mime, MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC)) {
mMeta->findData(kKeyHVCC, &type, &data, &size);
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_MPEG4)
|| !strcasecmp(mime, MEDIA_MIMETYPE_AUDIO_AAC)) {
@@ -1945,14 +2157,16 @@
chunk->mTimeStampUs, chunk->mTrack->getTrackType());
int32_t isFirstSample = true;
+ bool usePrefix = chunk->mTrack->usePrefix();
while (!chunk->mSamples.empty()) {
List<MediaBuffer *>::iterator it = chunk->mSamples.begin();
- off64_t offset = (chunk->mTrack->isAvc() || chunk->mTrack->isHevc())
- ? addMultipleLengthPrefixedSamples_l(*it)
- : addSample_l(*it);
+ size_t bytesWritten;
+ off64_t offset = addSample_l(*it, usePrefix, &bytesWritten);
- if (isFirstSample) {
+ if (chunk->mTrack->isHeic()) {
+ chunk->mTrack->addItemOffsetAndSize(offset, bytesWritten);
+ } else if (isFirstSample) {
chunk->mTrack->addChunkOffset(offset);
isFirstSample = false;
}
@@ -2637,7 +2851,7 @@
(const uint8_t *)buffer->data()
+ buffer->range_offset(),
buffer->range_length());
- } else if (mIsHevc) {
+ } else if (mIsHevc || mIsHeic) {
err = makeHEVCCodecSpecificData(
(const uint8_t *)buffer->data()
+ buffer->range_offset(),
@@ -2662,7 +2876,8 @@
}
// Per-frame metadata sample's size must be smaller than max allowed.
- if (!mIsVideo && !mIsAudio && buffer->range_length() >= kMaxMetadataSize) {
+ if (!mIsVideo && !mIsAudio && !mIsHeic &&
+ buffer->range_length() >= kMaxMetadataSize) {
ALOGW("Buffer size is %zu. Maximum metadata buffer size is %lld for %s track",
buffer->range_length(), (long long)kMaxMetadataSize, trackName);
buffer->release();
@@ -2683,10 +2898,10 @@
buffer->release();
buffer = NULL;
- if (mIsAvc || mIsHevc) StripStartcode(copy);
+ if (usePrefix()) StripStartcode(copy);
size_t sampleSize = copy->range_length();
- if (mIsAvc || mIsHevc) {
+ if (usePrefix()) {
if (mOwner->useNalLengthFour()) {
sampleSize += 4;
} else {
@@ -2948,15 +3163,19 @@
trackProgressStatus(timestampUs);
}
if (!hasMultipleTracks) {
- off64_t offset = (mIsAvc || mIsHevc) ? mOwner->addMultipleLengthPrefixedSamples_l(copy)
- : mOwner->addSample_l(copy);
+ size_t bytesWritten;
+ off64_t offset = mOwner->addSample_l(copy, usePrefix(), &bytesWritten);
- uint32_t count = (mOwner->use32BitFileOffset()
- ? mStcoTableEntries->count()
- : mCo64TableEntries->count());
+ if (mIsHeic) {
+ addItemOffsetAndSize(offset, bytesWritten);
+ } else {
+ uint32_t count = (mOwner->use32BitFileOffset()
+ ? mStcoTableEntries->count()
+ : mCo64TableEntries->count());
- if (count == 0) {
- addChunkOffset(offset);
+ if (count == 0) {
+ addChunkOffset(offset);
+ }
}
copy->release();
copy = NULL;
@@ -2964,7 +3183,10 @@
}
mChunkSamples.push_back(copy);
- if (interleaveDurationUs == 0) {
+ if (mIsHeic) {
+ bufferChunk(0 /*timestampUs*/);
+ ++nChunks;
+ } else if (interleaveDurationUs == 0) {
addOneStscTableEntry(++nChunks, 1);
bufferChunk(timestampUs);
} else {
@@ -2997,42 +3219,49 @@
mOwner->trackProgressStatus(mTrackId, -1, err);
- // Last chunk
- if (!hasMultipleTracks) {
- addOneStscTableEntry(1, mStszTableEntries->count());
- } else if (!mChunkSamples.empty()) {
- addOneStscTableEntry(++nChunks, mChunkSamples.size());
- bufferChunk(timestampUs);
- }
-
- // We don't really know how long the last frame lasts, since
- // there is no frame time after it, just repeat the previous
- // frame's duration.
- if (mStszTableEntries->count() == 1) {
- lastDurationUs = 0; // A single sample's duration
- lastDurationTicks = 0;
- } else {
- ++sampleCount; // Count for the last sample
- }
-
- if (mStszTableEntries->count() <= 2) {
- addOneSttsTableEntry(1, lastDurationTicks);
- if (sampleCount - 1 > 0) {
- addOneSttsTableEntry(sampleCount - 1, lastDurationTicks);
+ if (mIsHeic) {
+ if (!mChunkSamples.empty()) {
+ bufferChunk(0);
+ ++nChunks;
}
} else {
- addOneSttsTableEntry(sampleCount, lastDurationTicks);
- }
-
- // The last ctts box may not have been written yet, and this
- // is to make sure that we write out the last ctts box.
- if (currCttsOffsetTimeTicks == lastCttsOffsetTimeTicks) {
- if (cttsSampleCount > 0) {
- addOneCttsTableEntry(cttsSampleCount, lastCttsOffsetTimeTicks);
+ // Last chunk
+ if (!hasMultipleTracks) {
+ addOneStscTableEntry(1, mStszTableEntries->count());
+ } else if (!mChunkSamples.empty()) {
+ addOneStscTableEntry(++nChunks, mChunkSamples.size());
+ bufferChunk(timestampUs);
}
- }
- mTrackDurationUs += lastDurationUs;
+ // We don't really know how long the last frame lasts, since
+ // there is no frame time after it, just repeat the previous
+ // frame's duration.
+ if (mStszTableEntries->count() == 1) {
+ lastDurationUs = 0; // A single sample's duration
+ lastDurationTicks = 0;
+ } else {
+ ++sampleCount; // Count for the last sample
+ }
+
+ if (mStszTableEntries->count() <= 2) {
+ addOneSttsTableEntry(1, lastDurationTicks);
+ if (sampleCount - 1 > 0) {
+ addOneSttsTableEntry(sampleCount - 1, lastDurationTicks);
+ }
+ } else {
+ addOneSttsTableEntry(sampleCount, lastDurationTicks);
+ }
+
+ // The last ctts box may not have been written yet, and this
+ // is to make sure that we write out the last ctts box.
+ if (currCttsOffsetTimeTicks == lastCttsOffsetTimeTicks) {
+ if (cttsSampleCount > 0) {
+ addOneCttsTableEntry(cttsSampleCount, lastCttsOffsetTimeTicks);
+ }
+ }
+
+ mTrackDurationUs += lastDurationUs;
+ }
mReachedEOS = true;
sendTrackSummary(hasMultipleTracks);
@@ -3054,7 +3283,7 @@
return true;
}
- if (mStszTableEntries->count() == 0) { // no samples written
+ if (!mIsHeic && mStszTableEntries->count() == 0) { // no samples written
ALOGE("The number of recorded samples is 0");
return true;
}
@@ -3200,13 +3429,28 @@
return mEstimatedTrackSizeBytes;
}
+int32_t MPEG4Writer::Track::getMetaSizeIncrease() const {
+ CHECK(mIsHeic);
+ return 20 // 1. 'ispe' property
+ + (8 + mCodecSpecificDataSize) // 2. 'hvcC' property
+ + (20 // 3. extra 'ispe'
+ + (8 + 2 + 2 + mNumTiles * 2) // 4. 'dimg' ref
+ + 12) // 5. ImageGrid in 'idat' (worst case)
+ * (mNumTiles > 1) // - (3~5: applicable only if grid)
+ + (16 // 6. increase to 'iloc'
+ + 21 // 7. increase to 'iinf'
+ + (3 + 2 * 2)) // 8. increase to 'ipma' (worst case)
+ * (mNumTiles + 1); // - (6~8: are per-item)
+}
+
status_t MPEG4Writer::Track::checkCodecSpecificData() const {
const char *mime;
CHECK(mMeta->findCString(kKeyMIMEType, &mime));
if (!strcasecmp(MEDIA_MIMETYPE_AUDIO_AAC, mime) ||
!strcasecmp(MEDIA_MIMETYPE_VIDEO_MPEG4, mime) ||
!strcasecmp(MEDIA_MIMETYPE_VIDEO_AVC, mime) ||
- !strcasecmp(MEDIA_MIMETYPE_VIDEO_HEVC, mime)) {
+ !strcasecmp(MEDIA_MIMETYPE_VIDEO_HEVC, mime) ||
+ !strcasecmp(MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC, mime)) {
if (!mCodecSpecificData ||
mCodecSpecificDataSize <= 0) {
ALOGE("Missing codec specific data");
@@ -3223,7 +3467,10 @@
}
const char *MPEG4Writer::Track::getTrackType() const {
- return mIsAudio ? "Audio" : (mIsVideo ? "Video" : "Metadata");
+ return mIsAudio ? "Audio" :
+ mIsVideo ? "Video" :
+ mIsHeic ? "Image" :
+ "Metadata";
}
void MPEG4Writer::Track::writeTrackHeader(bool use32BitOffset) {
@@ -3793,11 +4040,11 @@
endBox();
}
-void MPEG4Writer::writeHdlr() {
+void MPEG4Writer::writeHdlr(const char *handlerType) {
beginBox("hdlr");
writeInt32(0); // Version, Flags
writeInt32(0); // Predefined
- writeFourcc("mdta");
+ writeFourcc(handlerType);
writeInt32(0); // Reserved[0]
writeInt32(0); // Reserved[1]
writeInt32(0); // Reserved[2]
@@ -3877,19 +4124,283 @@
endBox(); // ilst
}
-void MPEG4Writer::writeMetaBox() {
+void MPEG4Writer::writeMoovLevelMetaBox() {
size_t count = mMetaKeys->countEntries();
if (count == 0) {
return;
}
beginBox("meta");
- writeHdlr();
+ writeHdlr("mdta");
writeKeys();
writeIlst();
endBox();
}
+void MPEG4Writer::writeIlocBox() {
+ beginBox("iloc");
+ // Use version 1 to allow construction method 1 that refers to
+ // data in idat box inside meta box.
+ writeInt32(0x01000000); // Version = 1, Flags = 0
+ writeInt16(0x4400); // offset_size = length_size = 4
+ // base_offset_size = index_size = 0
+
+ // 16-bit item_count
+ size_t itemCount = mItems.size();
+ if (itemCount > 65535) {
+ ALOGW("Dropping excess items: itemCount %zu", itemCount);
+ itemCount = 65535;
+ }
+ writeInt16((uint16_t)itemCount);
+
+ for (size_t i = 0; i < itemCount; i++) {
+ writeInt16(mItems[i].itemId);
+ bool isGrid = mItems[i].isGrid();
+
+ writeInt16(isGrid ? 1 : 0); // construction_method
+ writeInt16(0); // data_reference_index = 0
+ writeInt16(1); // extent_count = 1
+
+ if (isGrid) {
+ // offset into the 'idat' box
+ writeInt32(mNumGrids++ * 8);
+ writeInt32(8);
+ } else {
+ writeInt32(mItems[i].offset);
+ writeInt32(mItems[i].size);
+ }
+ }
+ endBox();
+}
+
+void MPEG4Writer::writeInfeBox(
+ uint16_t itemId, const char *itemType, uint32_t flags) {
+ beginBox("infe");
+ writeInt32(0x02000000 | flags); // Version = 2, Flags = 0
+ writeInt16(itemId);
+ writeInt16(0); //item_protection_index = 0
+ writeFourcc(itemType);
+ writeCString(""); // item_name
+ endBox();
+}
+
+void MPEG4Writer::writeIinfBox() {
+ beginBox("iinf");
+ writeInt32(0); // Version = 0, Flags = 0
+
+ // 16-bit item_count
+ size_t itemCount = mItems.size();
+ if (itemCount > 65535) {
+ ALOGW("Dropping excess items: itemCount %zu", itemCount);
+ itemCount = 65535;
+ }
+
+ writeInt16((uint16_t)itemCount);
+ for (size_t i = 0; i < itemCount; i++) {
+ writeInfeBox(mItems[i].itemId, mItems[i].itemType,
+ mItems[i].isHidden ? 1 : 0);
+ }
+
+ endBox();
+}
+
+void MPEG4Writer::writeIdatBox() {
+ beginBox("idat");
+
+ for (size_t i = 0; i < mItems.size(); i++) {
+ if (mItems[i].isGrid()) {
+ writeInt8(0); // version
+ // flags == 1 means 32-bit width,height
+ int8_t flags = (mItems[i].width > 65535 || mItems[i].height > 65535);
+ writeInt8(flags);
+ writeInt8(mItems[i].rows - 1);
+ writeInt8(mItems[i].cols - 1);
+ if (flags) {
+ writeInt32(mItems[i].width);
+ writeInt32(mItems[i].height);
+ } else {
+ writeInt16((uint16_t)mItems[i].width);
+ writeInt16((uint16_t)mItems[i].height);
+ }
+ }
+ }
+
+ endBox();
+}
+
+void MPEG4Writer::writeIrefBox() {
+ beginBox("iref");
+ writeInt32(0); // Version = 0, Flags = 0
+ {
+ for (size_t i = 0; i < mItems.size(); i++) {
+ if (!mItems[i].isGrid()) {
+ continue;
+ }
+ beginBox("dimg");
+ writeInt16(mItems[i].itemId);
+ size_t refCount = mItems[i].dimgRefs.size();
+ if (refCount > 65535) {
+ ALOGW("too many entries in dimg");
+ refCount = 65535;
+ }
+ writeInt16((uint16_t)refCount);
+ for (size_t refIndex = 0; refIndex < refCount; refIndex++) {
+ writeInt16(mItems[i].dimgRefs[refIndex]);
+ }
+ endBox();
+ }
+ }
+ endBox();
+}
+
+void MPEG4Writer::writePitmBox() {
+ beginBox("pitm");
+ writeInt32(0); // Version = 0, Flags = 0
+ writeInt16(mPrimaryItemId);
+ endBox();
+}
+
+void MPEG4Writer::writeIpcoBox() {
+ beginBox("ipco");
+ size_t numProperties = mProperties.size();
+ if (numProperties > 32767) {
+ ALOGW("Dropping excess properties: numProperties %zu", numProperties);
+ numProperties = 32767;
+ }
+ for (size_t propIndex = 0; propIndex < numProperties; propIndex++) {
+ if (mProperties[propIndex].type == FOURCC('h', 'v', 'c', 'C')) {
+ beginBox("hvcC");
+ sp<ABuffer> hvcc = mProperties[propIndex].hvcc;
+ // Patch avcc's lengthSize field to match the number
+ // of bytes we use to indicate the size of a nal unit.
+ uint8_t *ptr = (uint8_t *)hvcc->data();
+ ptr[21] = (ptr[21] & 0xfc) | (useNalLengthFour() ? 3 : 1);
+ write(hvcc->data(), hvcc->size());
+ endBox();
+ } else if (mProperties[propIndex].type == FOURCC('i', 's', 'p', 'e')) {
+ beginBox("ispe");
+ writeInt32(0); // Version = 0, Flags = 0
+ writeInt32(mProperties[propIndex].width);
+ writeInt32(mProperties[propIndex].height);
+ endBox();
+ } else {
+ ALOGW("Skipping unrecognized property: type 0x%08x",
+ mProperties[propIndex].type);
+ }
+ }
+ endBox();
+}
+
+void MPEG4Writer::writeIpmaBox() {
+ beginBox("ipma");
+ uint32_t flags = (mProperties.size() > 127) ? 1 : 0;
+ writeInt32(flags); // Version = 0
+
+ writeInt32(mAssociationEntryCount);
+ for (size_t itemIndex = 0; itemIndex < mItems.size(); itemIndex++) {
+ const Vector<uint16_t> &properties = mItems[itemIndex].properties;
+ if (properties.empty()) {
+ continue;
+ }
+ writeInt16(mItems[itemIndex].itemId);
+
+ size_t entryCount = properties.size();
+ if (entryCount > 255) {
+ ALOGW("Dropping excess associations: entryCount %zu", entryCount);
+ entryCount = 255;
+ }
+ writeInt8((uint8_t)entryCount);
+ for (size_t propIndex = 0; propIndex < entryCount; propIndex++) {
+ if (flags & 1) {
+ writeInt16((1 << 15) | properties[propIndex]);
+ } else {
+ writeInt8((1 << 7) | properties[propIndex]);
+ }
+ }
+ }
+ endBox();
+}
+
+void MPEG4Writer::writeIprpBox() {
+ beginBox("iprp");
+ writeIpcoBox();
+ writeIpmaBox();
+ endBox();
+}
+
+void MPEG4Writer::writeFileLevelMetaBox() {
+ if (mItems.empty()) {
+ ALOGE("no valid item was found");
+ return;
+ }
+
+ // patch up the mPrimaryItemId and count items with prop associations
+ for (size_t index = 0; index < mItems.size(); index++) {
+ if (mItems[index].isPrimary) {
+ mPrimaryItemId = mItems[index].itemId;
+ }
+
+ if (!mItems[index].properties.empty()) {
+ mAssociationEntryCount++;
+ }
+ }
+
+ if (mPrimaryItemId == 0) {
+ ALOGW("didn't find primary, using first item");
+ mPrimaryItemId = mItems[0].itemId;
+ }
+
+ beginBox("meta");
+ writeInt32(0); // Version = 0, Flags = 0
+ writeHdlr("pict");
+ writeIlocBox();
+ writeIinfBox();
+ writePitmBox();
+ writeIprpBox();
+ if (mNumGrids > 0) {
+ writeIdatBox();
+ writeIrefBox();
+ }
+ endBox();
+}
+
+uint16_t MPEG4Writer::addProperty_l(const ItemProperty &prop) {
+ char typeStr[5];
+ MakeFourCCString(prop.type, typeStr);
+ ALOGV("addProperty_l: %s", typeStr);
+
+ mProperties.push_back(prop);
+
+ // returning 1-based property index
+ return mProperties.size();
+}
+
+uint16_t MPEG4Writer::addItem_l(const ItemInfo &info) {
+ ALOGV("addItem_l: type %s, offset %u, size %u",
+ info.itemType, info.offset, info.size);
+
+ size_t index = mItems.size();
+ mItems.push_back(info);
+
+ // make the item id start at 10000
+ mItems.editItemAt(index).itemId = index + 10000;
+
+#if (LOG_NDEBUG==0)
+ if (!info.properties.empty()) {
+ AString str;
+ for (size_t i = 0; i < info.properties.size(); i++) {
+ if (i > 0) {
+ str.append(", ");
+ }
+ str.append(info.properties[i]);
+ }
+ ALOGV("addItem_l: id %d, properties: %s", mItems[index].itemId, str.c_str());
+ }
+#endif // (LOG_NDEBUG==0)
+
+ return mItems[index].itemId;
+}
+
/*
* Geodata is stored according to ISO-6709 standard.
*/
diff --git a/media/libstagefright/MediaMuxer.cpp b/media/libstagefright/MediaMuxer.cpp
index fb048fe..62daac8 100644
--- a/media/libstagefright/MediaMuxer.cpp
+++ b/media/libstagefright/MediaMuxer.cpp
@@ -23,6 +23,7 @@
#include <media/stagefright/MediaMuxer.h>
+#include <media/mediarecorder.h>
#include <media/MediaSource.h>
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
@@ -38,10 +39,16 @@
namespace android {
+static bool isMp4Format(MediaMuxer::OutputFormat format) {
+ return format == MediaMuxer::OUTPUT_FORMAT_MPEG_4 ||
+ format == MediaMuxer::OUTPUT_FORMAT_THREE_GPP ||
+ format == MediaMuxer::OUTPUT_FORMAT_HEIF;
+}
+
MediaMuxer::MediaMuxer(int fd, OutputFormat format)
: mFormat(format),
mState(UNINITIALIZED) {
- if (format == OUTPUT_FORMAT_MPEG_4 || format == OUTPUT_FORMAT_THREE_GPP) {
+ if (isMp4Format(format)) {
mWriter = new MPEG4Writer(fd);
} else if (format == OUTPUT_FORMAT_WEBM) {
mWriter = new WebmWriter(fd);
@@ -49,6 +56,10 @@
if (mWriter != NULL) {
mFileMeta = new MetaData;
+ if (format == OUTPUT_FORMAT_HEIF) {
+ // Note that the key uses recorder file types.
+ mFileMeta->setInt32(kKeyFileType, output_format::OUTPUT_FORMAT_HEIF);
+ }
mState = INITIALIZED;
}
}
@@ -108,8 +119,8 @@
ALOGE("setLocation() must be called before start().");
return INVALID_OPERATION;
}
- if (mFormat != OUTPUT_FORMAT_MPEG_4 && mFormat != OUTPUT_FORMAT_THREE_GPP) {
- ALOGE("setLocation() is only supported for .mp4 pr .3gp output.");
+ if (!isMp4Format(mFormat)) {
+ ALOGE("setLocation() is only supported for .mp4, .3gp or .heic output.");
return INVALID_OPERATION;
}
diff --git a/media/libstagefright/StagefrightMetadataRetriever.cpp b/media/libstagefright/StagefrightMetadataRetriever.cpp
index dfaa8b6..e2db0f5 100644
--- a/media/libstagefright/StagefrightMetadataRetriever.cpp
+++ b/media/libstagefright/StagefrightMetadataRetriever.cpp
@@ -154,7 +154,8 @@
if (!strncasecmp(mime, "image/", 6)) {
int32_t isPrimary;
- if ((index < 0 && meta->findInt32(kKeyIsPrimaryImage, &isPrimary) && isPrimary)
+ if ((index < 0 && meta->findInt32(
+ kKeyTrackIsDefault, &isPrimary) && isPrimary)
|| (index == imageCount++)) {
break;
}
@@ -490,7 +491,8 @@
}
} else if (!strncasecmp("image/", mime, 6)) {
int32_t isPrimary;
- if (trackMeta->findInt32(kKeyIsPrimaryImage, &isPrimary) && isPrimary) {
+ if (trackMeta->findInt32(
+ kKeyTrackIsDefault, &isPrimary) && isPrimary) {
imagePrimary = imageCount;
CHECK(trackMeta->findInt32(kKeyWidth, &imageWidth));
CHECK(trackMeta->findInt32(kKeyHeight, &imageHeight));
diff --git a/media/libstagefright/Utils.cpp b/media/libstagefright/Utils.cpp
index 6e77f15..65783d7 100644
--- a/media/libstagefright/Utils.cpp
+++ b/media/libstagefright/Utils.cpp
@@ -675,6 +675,10 @@
msg->setInt32("grid-rows", gridRows);
msg->setInt32("grid-cols", gridCols);
}
+ int32_t isPrimary;
+ if (meta->findInt32(kKeyTrackIsDefault, &isPrimary) && isPrimary) {
+ msg->setInt32("is-default", 1);
+ }
}
int32_t colorFormat;
@@ -1308,7 +1312,7 @@
meta->setCString(kKeyMediaLanguage, lang.c_str());
}
- if (mime.startsWith("video/")) {
+ if (mime.startsWith("video/") || mime.startsWith("image/")) {
int32_t width;
int32_t height;
if (msg->findInt32("width", &width) && msg->findInt32("height", &height)) {
@@ -1332,6 +1336,26 @@
meta->setInt32(kKeyDisplayHeight, displayHeight);
}
+ if (mime.startsWith("image/")){
+ int32_t isPrimary;
+ if (msg->findInt32("is-default", &isPrimary) && isPrimary) {
+ meta->setInt32(kKeyTrackIsDefault, 1);
+ }
+ int32_t gridWidth, gridHeight, gridRows, gridCols;
+ if (msg->findInt32("grid-width", &gridWidth)) {
+ meta->setInt32(kKeyGridWidth, gridWidth);
+ }
+ if (msg->findInt32("grid-height", &gridHeight)) {
+ meta->setInt32(kKeyGridHeight, gridHeight);
+ }
+ if (msg->findInt32("grid-rows", &gridRows)) {
+ meta->setInt32(kKeyGridRows, gridRows);
+ }
+ if (msg->findInt32("grid-cols", &gridCols)) {
+ meta->setInt32(kKeyGridCols, gridCols);
+ }
+ }
+
int32_t colorFormat;
if (msg->findInt32("color-format", &colorFormat)) {
meta->setInt32(kKeyColorFormat, colorFormat);
@@ -1448,7 +1472,8 @@
// for transporting the CSD to muxers.
reassembleESDS(csd0, esds.data());
meta->setData(kKeyESDS, kKeyESDS, esds.data(), esds.size());
- } else if (mime == MEDIA_MIMETYPE_VIDEO_HEVC) {
+ } else if (mime == MEDIA_MIMETYPE_VIDEO_HEVC ||
+ mime == MEDIA_MIMETYPE_IMAGE_ANDROID_HEIC) {
std::vector<uint8_t> hvcc(csd0size + 1024);
size_t outsize = reassembleHVCC(csd0, hvcc.data(), hvcc.size(), 4);
meta->setData(kKeyHVCC, kKeyHVCC, hvcc.data(), outsize);
diff --git a/media/libstagefright/include/media/stagefright/MPEG4Writer.h b/media/libstagefright/include/media/stagefright/MPEG4Writer.h
index eba3b32..5bb4193 100644
--- a/media/libstagefright/include/media/stagefright/MPEG4Writer.h
+++ b/media/libstagefright/include/media/stagefright/MPEG4Writer.h
@@ -31,6 +31,7 @@
struct AMessage;
class MediaBuffer;
class MetaData;
+struct ABuffer;
class MPEG4Writer : public MediaWriter {
public:
@@ -132,6 +133,8 @@
status_t startTracks(MetaData *params);
size_t numTracks();
int64_t estimateMoovBoxSize(int32_t bitRate);
+ int64_t estimateFileLevelMetaSize();
+ void writeCachedBoxToFile(const char *type);
struct Chunk {
Track *mTrack; // Owner
@@ -164,6 +167,46 @@
List<ChunkInfo> mChunkInfos; // Chunk infos
Condition mChunkReadyCondition; // Signal that chunks are available
+ // HEIF writing
+ typedef struct _ItemInfo {
+ bool isGrid() const { return !strcmp("grid", itemType); }
+ const char *itemType;
+ uint16_t itemId;
+ bool isPrimary;
+ bool isHidden;
+ union {
+ // image item
+ struct {
+ uint32_t offset;
+ uint32_t size;
+ };
+ // grid item
+ struct {
+ uint32_t rows;
+ uint32_t cols;
+ uint32_t width;
+ uint32_t height;
+ };
+ };
+ Vector<uint16_t> properties;
+ Vector<uint16_t> dimgRefs;
+ } ItemInfo;
+
+ typedef struct _ItemProperty {
+ uint32_t type;
+ int32_t width;
+ int32_t height;
+ sp<ABuffer> hvcc;
+ } ItemProperty;
+
+ bool mHasFileLevelMeta;
+ bool mHasMoovBox;
+ uint32_t mPrimaryItemId;
+ uint32_t mAssociationEntryCount;
+ uint32_t mNumGrids;
+ Vector<ItemInfo> mItems;
+ Vector<ItemProperty> mProperties;
+
// Writer thread handling
status_t startWriterThread();
void stopWriterThread();
@@ -209,9 +252,11 @@
void initInternal(int fd, bool isFirstSession);
// Acquire lock before calling these methods
- off64_t addSample_l(MediaBuffer *buffer);
- off64_t addLengthPrefixedSample_l(MediaBuffer *buffer);
- off64_t addMultipleLengthPrefixedSamples_l(MediaBuffer *buffer);
+ off64_t addSample_l(MediaBuffer *buffer, bool usePrefix, size_t *bytesWritten);
+ void addLengthPrefixedSample_l(MediaBuffer *buffer);
+ void addMultipleLengthPrefixedSamples_l(MediaBuffer *buffer);
+ uint16_t addProperty_l(const ItemProperty &);
+ uint16_t addItem_l(const ItemInfo &);
bool exceedsFileSizeLimit();
bool use32BitFileOffset() const;
@@ -230,10 +275,23 @@
void finishCurrentSession();
void addDeviceMeta();
- void writeHdlr();
+ void writeHdlr(const char *handlerType);
void writeKeys();
void writeIlst();
- void writeMetaBox();
+ void writeMoovLevelMetaBox();
+
+ // HEIF writing
+ void writeIlocBox();
+ void writeInfeBox(uint16_t itemId, const char *type, uint32_t flags);
+ void writeIinfBox();
+ void writeIpcoBox();
+ void writeIpmaBox();
+ void writeIprpBox();
+ void writeIdatBox();
+ void writeIrefBox();
+ void writePitmBox();
+ void writeFileLevelMetaBox();
+
void sendSessionSummary();
void release();
status_t switchFd();
diff --git a/media/libstagefright/include/media/stagefright/MediaMuxer.h b/media/libstagefright/include/media/stagefright/MediaMuxer.h
index 63c3ca5..66f4d72 100644
--- a/media/libstagefright/include/media/stagefright/MediaMuxer.h
+++ b/media/libstagefright/include/media/stagefright/MediaMuxer.h
@@ -48,6 +48,7 @@
OUTPUT_FORMAT_MPEG_4 = 0,
OUTPUT_FORMAT_WEBM = 1,
OUTPUT_FORMAT_THREE_GPP = 2,
+ OUTPUT_FORMAT_HEIF = 3,
OUTPUT_FORMAT_LIST_END // must be last - used to validate format type
};