Merge "Support source device to MSD sink device direct patching"
diff --git a/camera/CameraBase.cpp b/camera/CameraBase.cpp
index aecb70a..0b0f584 100644
--- a/camera/CameraBase.cpp
+++ b/camera/CameraBase.cpp
@@ -29,6 +29,7 @@
#include <binder/IMemory.h>
#include <camera/CameraBase.h>
+#include <camera/CameraUtils.h>
// needed to instantiate
#include <camera/Camera.h>
@@ -124,9 +125,7 @@
{
Mutex::Autolock _l(gLock);
if (gCameraService.get() == 0) {
- char value[PROPERTY_VALUE_MAX];
- property_get("config.disable_cameraservice", value, "0");
- if (strncmp(value, "0", 2) != 0 && strncasecmp(value, "false", 6) != 0) {
+ if (CameraUtils::isCameraServiceDisabled()) {
return gCameraService;
}
diff --git a/camera/CameraUtils.cpp b/camera/CameraUtils.cpp
index 67fc116..f9b1b37 100644
--- a/camera/CameraUtils.cpp
+++ b/camera/CameraUtils.cpp
@@ -23,6 +23,7 @@
#include <system/window.h>
#include <system/graphics.h>
+#include <cutils/properties.h>
#include <utils/Log.h>
namespace android {
@@ -122,4 +123,10 @@
return OK;
}
+bool CameraUtils::isCameraServiceDisabled() {
+ char value[PROPERTY_VALUE_MAX];
+ property_get("config.disable_cameraservice", value, "0");
+ return (strncmp(value, "0", 2) != 0 && strncasecmp(value, "false", 6) != 0);
+}
+
} /* namespace android */
diff --git a/camera/include/camera/CameraUtils.h b/camera/include/camera/CameraUtils.h
index f596f80..a397ccd 100644
--- a/camera/include/camera/CameraUtils.h
+++ b/camera/include/camera/CameraUtils.h
@@ -47,6 +47,11 @@
*/
static bool isNativeHandleMetadata(const sp<IMemory>& imageData);
+ /**
+ * Check if camera service is disabled on this device
+ */
+ static bool isCameraServiceDisabled();
+
private:
CameraUtils();
};
diff --git a/camera/ndk/impl/ACameraManager.cpp b/camera/ndk/impl/ACameraManager.cpp
index 419250c..73cabbf 100644
--- a/camera/ndk/impl/ACameraManager.cpp
+++ b/camera/ndk/impl/ACameraManager.cpp
@@ -24,6 +24,7 @@
#include <utils/Vector.h>
#include <cutils/properties.h>
#include <stdlib.h>
+#include <camera/CameraUtils.h>
#include <camera/VendorTagDescriptor.h>
using namespace android::acam;
@@ -70,12 +71,6 @@
mCameraService.clear();
}
-static bool isCameraServiceDisabled() {
- char value[PROPERTY_VALUE_MAX];
- property_get("config.disable_cameraservice", value, "0");
- return (strncmp(value, "0", 2) != 0 && strncasecmp(value, "false", 6) != 0);
-}
-
sp<hardware::ICameraService> CameraManagerGlobal::getCameraService() {
Mutex::Autolock _l(mLock);
return getCameraServiceLocked();
@@ -83,7 +78,7 @@
sp<hardware::ICameraService> CameraManagerGlobal::getCameraServiceLocked() {
if (mCameraService.get() == nullptr) {
- if (isCameraServiceDisabled()) {
+ if (CameraUtils::isCameraServiceDisabled()) {
return mCameraService;
}
diff --git a/media/codec2/components/hevc/C2SoftHevcDec.cpp b/media/codec2/components/hevc/C2SoftHevcDec.cpp
index 56dd26b..a374dfa 100644
--- a/media/codec2/components/hevc/C2SoftHevcDec.cpp
+++ b/media/codec2/components/hevc/C2SoftHevcDec.cpp
@@ -906,7 +906,6 @@
work->result = C2_CORRUPTED;
return;
}
- continue;
}
if (0 < s_decode_op.u4_pic_wd && 0 < s_decode_op.u4_pic_ht) {
if (mHeaderDecoded == false) {
diff --git a/media/codec2/sfplugin/CCodec.cpp b/media/codec2/sfplugin/CCodec.cpp
index 3b3e7ba..b9ee2e6 100644
--- a/media/codec2/sfplugin/CCodec.cpp
+++ b/media/codec2/sfplugin/CCodec.cpp
@@ -492,7 +492,10 @@
// We used to not report changes to these keys to the client.
const static std::set<std::string> sIgnoredKeys({
KEY_BIT_RATE,
+ KEY_FRAME_RATE,
KEY_MAX_BIT_RATE,
+ KEY_MAX_WIDTH,
+ KEY_MAX_HEIGHT,
"csd-0",
"csd-1",
"csd-2",
@@ -923,19 +926,84 @@
/*
* Handle desired color format.
*/
+ int32_t defaultColorFormat = COLOR_FormatYUV420Flexible;
if ((config->mDomain & (Config::IS_VIDEO | Config::IS_IMAGE))) {
- int32_t format = -1;
+ int32_t format = 0;
+ // Query vendor format for Flexible YUV
+ std::vector<std::unique_ptr<C2Param>> heapParams;
+ C2StoreFlexiblePixelFormatDescriptorsInfo *pixelFormatInfo = nullptr;
+ if (mClient->query(
+ {},
+ {C2StoreFlexiblePixelFormatDescriptorsInfo::PARAM_TYPE},
+ C2_MAY_BLOCK,
+ &heapParams) == C2_OK
+ && heapParams.size() == 1u) {
+ pixelFormatInfo = C2StoreFlexiblePixelFormatDescriptorsInfo::From(
+ heapParams[0].get());
+ } else {
+ pixelFormatInfo = nullptr;
+ }
+ std::optional<uint32_t> flexPixelFormat{};
+ std::optional<uint32_t> flexPlanarPixelFormat{};
+ std::optional<uint32_t> flexSemiPlanarPixelFormat{};
+ if (pixelFormatInfo && *pixelFormatInfo) {
+ for (size_t i = 0; i < pixelFormatInfo->flexCount(); ++i) {
+ const C2FlexiblePixelFormatDescriptorStruct &desc =
+ pixelFormatInfo->m.values[i];
+ if (desc.bitDepth != 8
+ || desc.subsampling != C2Color::YUV_420
+ // TODO(b/180076105): some device report wrong layout
+ // || desc.layout == C2Color::INTERLEAVED_PACKED
+ // || desc.layout == C2Color::INTERLEAVED_ALIGNED
+ || desc.layout == C2Color::UNKNOWN_LAYOUT) {
+ continue;
+ }
+ if (!flexPixelFormat) {
+ flexPixelFormat = desc.pixelFormat;
+ }
+ if (desc.layout == C2Color::PLANAR_PACKED && !flexPlanarPixelFormat) {
+ flexPlanarPixelFormat = desc.pixelFormat;
+ }
+ if (desc.layout == C2Color::SEMIPLANAR_PACKED && !flexSemiPlanarPixelFormat) {
+ flexSemiPlanarPixelFormat = desc.pixelFormat;
+ }
+ }
+ }
if (!msg->findInt32(KEY_COLOR_FORMAT, &format)) {
- /*
- * Also handle default color format (encoders require color format, so this is only
- * needed for decoders.
- */
+ // Also handle default color format (encoders require color format, so this is only
+ // needed for decoders.
if (!(config->mDomain & Config::IS_ENCODER)) {
- format = (surface == nullptr) ? COLOR_FormatYUV420Planar : COLOR_FormatSurface;
+ if (surface == nullptr) {
+ format = flexPixelFormat.value_or(COLOR_FormatYUV420Flexible);
+ } else {
+ format = COLOR_FormatSurface;
+ }
+ defaultColorFormat = format;
+ }
+ } else {
+ if ((config->mDomain & Config::IS_ENCODER) || !surface) {
+ switch (format) {
+ case COLOR_FormatYUV420Flexible:
+ format = flexPixelFormat.value_or(COLOR_FormatYUV420Planar);
+ break;
+ case COLOR_FormatYUV420Planar:
+ case COLOR_FormatYUV420PackedPlanar:
+ format = flexPlanarPixelFormat.value_or(
+ flexPixelFormat.value_or(format));
+ break;
+ case COLOR_FormatYUV420SemiPlanar:
+ case COLOR_FormatYUV420PackedSemiPlanar:
+ format = flexSemiPlanarPixelFormat.value_or(
+ flexPixelFormat.value_or(format));
+ break;
+ default:
+ // No-op
+ break;
+ }
}
}
- if (format >= 0) {
+ if (format != 0) {
msg->setInt32("android._color-format", format);
}
}
@@ -1087,12 +1155,16 @@
// Set desired color format from configuration parameter
int32_t format;
- if (msg->findInt32("android._color-format", &format)) {
- if (config->mDomain & Config::IS_ENCODER) {
- config->mInputFormat->setInt32(KEY_COLOR_FORMAT, format);
- } else {
- config->mOutputFormat->setInt32(KEY_COLOR_FORMAT, format);
+ if (!msg->findInt32(KEY_COLOR_FORMAT, &format)) {
+ format = defaultColorFormat;
+ }
+ if (config->mDomain & Config::IS_ENCODER) {
+ config->mInputFormat->setInt32(KEY_COLOR_FORMAT, format);
+ if (msg->findInt32("android._color-format", &format)) {
+ config->mInputFormat->setInt32("android._color-format", format);
}
+ } else {
+ config->mOutputFormat->setInt32(KEY_COLOR_FORMAT, format);
}
}
@@ -1132,6 +1204,8 @@
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
const std::unique_ptr<Config> &config = *configLocked;
+ config->queryConfiguration(comp);
+
mCallback->onComponentConfigured(config->mInputFormat, config->mOutputFormat);
}
@@ -1637,7 +1711,9 @@
{
Mutexed<std::unique_ptr<Config>>::Locked configLocked(mConfig);
const std::unique_ptr<Config> &config = *configLocked;
+ sp<AMessage> outputFormat = config->mOutputFormat;
config->queryConfiguration(comp);
+ RevertOutputFormatIfNeeded(outputFormat, config->mOutputFormat);
}
(void)mChannel->start(nullptr, nullptr, [&]{
@@ -2051,7 +2127,7 @@
}
if (param->type() == C2PortAllocatorsTuning::input::PARAM_TYPE) {
mInputAllocators.reset(
- C2PortAllocatorsTuning::input::From(params[0].get()));
+ C2PortAllocatorsTuning::input::From(param));
}
}
mInitStatus = OK;
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.cpp b/media/codec2/sfplugin/CCodecBufferChannel.cpp
index ba1d178..4d2700a 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.cpp
+++ b/media/codec2/sfplugin/CCodecBufferChannel.cpp
@@ -30,6 +30,7 @@
#include <android/hardware/cas/native/1.0/IDescrambler.h>
#include <android/hardware/drm/1.0/types.h>
+#include <android-base/properties.h>
#include <android-base/stringprintf.h>
#include <binder/MemoryBase.h>
#include <binder/MemoryDealer.h>
@@ -253,6 +254,16 @@
usesFrameReassembler = true;
input->frameReassembler.process(buffer, &items);
} else {
+ int32_t cvo = 0;
+ if (buffer->meta()->findInt32("cvo", &cvo)) {
+ int32_t rotation = cvo % 360;
+ // change rotation to counter-clock wise.
+ rotation = ((rotation <= 0) ? 0 : 360) - rotation;
+
+ Mutexed<OutputSurface>::Locked output(mOutputSurface);
+ uint64_t frameIndex = work->input.ordinal.frameIndex.peeku();
+ output->rotation[frameIndex] = rotation;
+ }
work->input.buffers.push_back(c2buffer);
if (encryptedBlock) {
work->input.infoBuffers.emplace_back(C2InfoBuffer::CreateLinearBuffer(
@@ -747,6 +758,22 @@
c2Buffer->getInfo(C2StreamRotationInfo::output::PARAM_TYPE));
bool flip = rotation && (rotation->flip & 1);
uint32_t quarters = ((rotation ? rotation->value : 0) / 90) & 3;
+
+ {
+ Mutexed<OutputSurface>::Locked output(mOutputSurface);
+ if (output->surface == nullptr) {
+ ALOGI("[%s] cannot render buffer without surface", mName);
+ return OK;
+ }
+ int64_t frameIndex;
+ buffer->meta()->findInt64("frameIndex", &frameIndex);
+ if (output->rotation.count(frameIndex) != 0) {
+ auto it = output->rotation.find(frameIndex);
+ quarters = (it->second / 90) & 3;
+ output->rotation.erase(it);
+ }
+ }
+
uint32_t transform = 0;
switch (quarters) {
case 0: // no rotation
@@ -790,14 +817,6 @@
hdr10PlusInfo.reset();
}
- {
- Mutexed<OutputSurface>::Locked output(mOutputSurface);
- if (output->surface == nullptr) {
- ALOGI("[%s] cannot render buffer without surface", mName);
- return OK;
- }
- }
-
std::vector<C2ConstGraphicBlock> blocks = c2Buffer->data().graphicBlocks();
if (blocks.size() != 1u) {
ALOGD("[%s] expected 1 graphic block, but got %zu", mName, blocks.size());
@@ -873,7 +892,12 @@
}
return result;
}
- ALOGV("[%s] queue buffer successful", mName);
+
+ if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
+ ALOGD("[%s] queue buffer successful", mName);
+ } else {
+ ALOGV("[%s] queue buffer successful", mName);
+ }
int64_t mediaTimeUs = 0;
(void)buffer->meta()->findInt64("timeUs", &mediaTimeUs);
diff --git a/media/codec2/sfplugin/CCodecBufferChannel.h b/media/codec2/sfplugin/CCodecBufferChannel.h
index b9e8d39..45da003 100644
--- a/media/codec2/sfplugin/CCodecBufferChannel.h
+++ b/media/codec2/sfplugin/CCodecBufferChannel.h
@@ -293,6 +293,7 @@
sp<Surface> surface;
uint32_t generation;
int maxDequeueBuffers;
+ std::map<uint64_t, int> rotation;
};
Mutexed<OutputSurface> mOutputSurface;
diff --git a/media/codec2/sfplugin/CCodecBuffers.cpp b/media/codec2/sfplugin/CCodecBuffers.cpp
index 1b2d3d3..6825dc2 100644
--- a/media/codec2/sfplugin/CCodecBuffers.cpp
+++ b/media/codec2/sfplugin/CCodecBuffers.cpp
@@ -27,6 +27,7 @@
#include <mediadrm/ICrypto.h>
#include "CCodecBuffers.h"
+#include "Codec2Mapper.h"
namespace android {
@@ -315,6 +316,7 @@
// Append information from the front stash entry to outBuffer.
(*outBuffer)->meta()->setInt64("timeUs", entry.timestamp);
(*outBuffer)->meta()->setInt32("flags", entry.flags);
+ (*outBuffer)->meta()->setInt64("frameIndex", entry.ordinal.frameIndex.peekll());
ALOGV("[%s] popFromStashAndRegister: "
"out buffer index = %zu [%p] => %p + %zu (%lld)",
mName, *index, outBuffer->get(),
@@ -1006,18 +1008,32 @@
// track of the flushed work.
}
+static uint32_t extractPixelFormat(const sp<AMessage> &format) {
+ int32_t frameworkColorFormat = 0;
+ if (!format->findInt32("android._color-format", &frameworkColorFormat)) {
+ return PIXEL_FORMAT_UNKNOWN;
+ }
+ uint32_t pixelFormat = PIXEL_FORMAT_UNKNOWN;
+ if (C2Mapper::mapPixelFormatFrameworkToCodec(frameworkColorFormat, &pixelFormat)) {
+ return pixelFormat;
+ }
+ return PIXEL_FORMAT_UNKNOWN;
+}
+
std::unique_ptr<InputBuffers> GraphicInputBuffers::toArrayMode(size_t size) {
std::unique_ptr<InputBuffersArray> array(
new InputBuffersArray(mComponentName.c_str(), "2D-BB-Input[N]"));
array->setPool(mPool);
array->setFormat(mFormat);
+ uint32_t pixelFormat = extractPixelFormat(mFormat);
array->initialize(
mImpl,
size,
- [pool = mPool, format = mFormat, lbp = mLocalBufferPool]() -> sp<Codec2Buffer> {
+ [pool = mPool, format = mFormat, lbp = mLocalBufferPool, pixelFormat]()
+ -> sp<Codec2Buffer> {
C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
return AllocateGraphicBuffer(
- pool, format, HAL_PIXEL_FORMAT_YV12, usage, lbp);
+ pool, format, pixelFormat, usage, lbp);
});
return std::move(array);
}
@@ -1030,7 +1046,7 @@
// TODO: read usage from intf
C2MemoryUsage usage = { C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE };
return AllocateGraphicBuffer(
- mPool, mFormat, HAL_PIXEL_FORMAT_YV12, usage, mLocalBufferPool);
+ mPool, mFormat, extractPixelFormat(mFormat), usage, mLocalBufferPool);
}
// OutputBuffersArray
diff --git a/media/codec2/sfplugin/Codec2Buffer.cpp b/media/codec2/sfplugin/Codec2Buffer.cpp
index 5072323..fc4ee51 100644
--- a/media/codec2/sfplugin/Codec2Buffer.cpp
+++ b/media/codec2/sfplugin/Codec2Buffer.cpp
@@ -212,21 +212,24 @@
* Creates a C2GraphicView <=> MediaImage converter
*
* \param view C2GraphicView object
- * \param colorFormat desired SDK color format for the MediaImage (if this is a flexible format,
- * an attempt is made to simply represent the graphic view as a flexible SDK format
- * without a memcpy)
+ * \param format buffer format
* \param copy whether the converter is used for copy or not
*/
GraphicView2MediaImageConverter(
- const C2GraphicView &view, int32_t colorFormat, bool copy)
+ const C2GraphicView &view, const sp<AMessage> &format, bool copy)
: mInitCheck(NO_INIT),
mView(view),
mWidth(view.width()),
mHeight(view.height()),
- mColorFormat(colorFormat),
mAllocatedDepth(0),
mBackBufferSize(0),
mMediaImage(new ABuffer(sizeof(MediaImage2))) {
+ if (!format->findInt32(KEY_COLOR_FORMAT, &mClientColorFormat)) {
+ mClientColorFormat = COLOR_FormatYUV420Flexible;
+ }
+ if (!format->findInt32("android._color-format", &mComponentColorFormat)) {
+ mComponentColorFormat = COLOR_FormatYUV420Flexible;
+ }
if (view.error() != C2_OK) {
ALOGD("Converter: view.error() = %d", view.error());
mInitCheck = BAD_VALUE;
@@ -247,70 +250,57 @@
uint32_t stride = align(view.crop().width, 2) * divUp(layout.planes[0].allocatedDepth, 8u);
uint32_t vStride = align(view.crop().height, 2);
+ bool tryWrapping = !copy;
+
switch (layout.type) {
- case C2PlanarLayout::TYPE_YUV:
+ case C2PlanarLayout::TYPE_YUV: {
mediaImage->mType = MediaImage2::MEDIA_IMAGE_TYPE_YUV;
if (layout.numPlanes != 3) {
ALOGD("Converter: %d planes for YUV layout", layout.numPlanes);
mInitCheck = BAD_VALUE;
return;
}
- if (layout.planes[0].channel != C2PlaneInfo::CHANNEL_Y
- || layout.planes[1].channel != C2PlaneInfo::CHANNEL_CB
- || layout.planes[2].channel != C2PlaneInfo::CHANNEL_CR
- || layout.planes[0].colSampling != 1
- || layout.planes[0].rowSampling != 1
- || layout.planes[1].colSampling != 2
- || layout.planes[1].rowSampling != 2
- || layout.planes[2].colSampling != 2
- || layout.planes[2].rowSampling != 2) {
- ALOGD("Converter: not YUV420 for YUV layout");
+ C2PlaneInfo yPlane = layout.planes[C2PlanarLayout::PLANE_Y];
+ C2PlaneInfo uPlane = layout.planes[C2PlanarLayout::PLANE_U];
+ C2PlaneInfo vPlane = layout.planes[C2PlanarLayout::PLANE_V];
+ if (yPlane.channel != C2PlaneInfo::CHANNEL_Y
+ || uPlane.channel != C2PlaneInfo::CHANNEL_CB
+ || vPlane.channel != C2PlaneInfo::CHANNEL_CR) {
+ ALOGD("Converter: not YUV layout");
mInitCheck = BAD_VALUE;
return;
}
- switch (mColorFormat) {
- case COLOR_FormatYUV420Flexible:
- if (!copy) {
- // try to map directly. check if the planes are near one another
- const uint8_t *minPtr = mView.data()[0];
- const uint8_t *maxPtr = mView.data()[0];
- int32_t planeSize = 0;
- for (uint32_t i = 0; i < layout.numPlanes; ++i) {
- const C2PlaneInfo &plane = layout.planes[i];
- int64_t planeStride = std::abs(plane.rowInc / plane.colInc);
- ssize_t minOffset = plane.minOffset(
- mWidth / plane.colSampling, mHeight / plane.rowSampling);
- ssize_t maxOffset = plane.maxOffset(
- mWidth / plane.colSampling, mHeight / plane.rowSampling);
- if (minPtr > mView.data()[i] + minOffset) {
- minPtr = mView.data()[i] + minOffset;
- }
- if (maxPtr < mView.data()[i] + maxOffset) {
- maxPtr = mView.data()[i] + maxOffset;
- }
- planeSize += planeStride * divUp(mAllocatedDepth, 8u)
- * align(mHeight, 64) / plane.rowSampling;
- }
-
- if (minPtr == mView.data()[0] && (maxPtr - minPtr + 1) <= planeSize) {
- // FIXME: this is risky as reading/writing data out of bound results
- // in an undefined behavior, but gralloc does assume a
- // contiguous mapping
- for (uint32_t i = 0; i < layout.numPlanes; ++i) {
- const C2PlaneInfo &plane = layout.planes[i];
- mediaImage->mPlane[i].mOffset = mView.data()[i] - minPtr;
- mediaImage->mPlane[i].mColInc = plane.colInc;
- mediaImage->mPlane[i].mRowInc = plane.rowInc;
- mediaImage->mPlane[i].mHorizSubsampling = plane.colSampling;
- mediaImage->mPlane[i].mVertSubsampling = plane.rowSampling;
- }
- mWrapped = new ABuffer(const_cast<uint8_t *>(minPtr),
- maxPtr - minPtr + 1);
- break;
- }
+ bool yuv420888 = yPlane.rowSampling == 1 && yPlane.colSampling == 1
+ && uPlane.rowSampling == 2 && uPlane.colSampling == 2
+ && vPlane.rowSampling == 2 && vPlane.colSampling == 2;
+ if (yuv420888) {
+ for (uint32_t i = 0; i < 3; ++i) {
+ const C2PlaneInfo &plane = layout.planes[i];
+ if (plane.allocatedDepth != 8 || plane.bitDepth != 8) {
+ yuv420888 = false;
+ break;
}
- [[fallthrough]];
-
+ }
+ yuv420888 = yuv420888 && yPlane.colInc == 1 && uPlane.rowInc == vPlane.rowInc;
+ }
+ int32_t copyFormat = mClientColorFormat;
+ if (yuv420888 && mClientColorFormat == COLOR_FormatYUV420Flexible) {
+ if (uPlane.colInc == 2 && vPlane.colInc == 2
+ && yPlane.rowInc == uPlane.rowInc) {
+ copyFormat = COLOR_FormatYUV420PackedSemiPlanar;
+ } else if (uPlane.colInc == 1 && vPlane.colInc == 1
+ && yPlane.rowInc == uPlane.rowInc * 2) {
+ copyFormat = COLOR_FormatYUV420PackedPlanar;
+ }
+ }
+ ALOGV("client_fmt=0x%x y:{colInc=%d rowInc=%d} u:{colInc=%d rowInc=%d} "
+ "v:{colInc=%d rowInc=%d}",
+ mClientColorFormat,
+ yPlane.colInc, yPlane.rowInc,
+ uPlane.colInc, uPlane.rowInc,
+ vPlane.colInc, vPlane.rowInc);
+ switch (copyFormat) {
+ case COLOR_FormatYUV420Flexible:
case COLOR_FormatYUV420Planar:
case COLOR_FormatYUV420PackedPlanar:
mediaImage->mPlane[mediaImage->Y].mOffset = 0;
@@ -330,6 +320,13 @@
mediaImage->mPlane[mediaImage->V].mRowInc = stride / 2;
mediaImage->mPlane[mediaImage->V].mHorizSubsampling = 2;
mediaImage->mPlane[mediaImage->V].mVertSubsampling = 2;
+
+ if (tryWrapping && mClientColorFormat != COLOR_FormatYUV420Flexible) {
+ tryWrapping = yuv420888 && uPlane.colInc == 1 && vPlane.colInc == 1
+ && yPlane.rowInc == uPlane.rowInc * 2
+ && view.data()[0] < view.data()[1]
+ && view.data()[1] < view.data()[2];
+ }
break;
case COLOR_FormatYUV420SemiPlanar:
@@ -351,64 +348,165 @@
mediaImage->mPlane[mediaImage->V].mRowInc = stride;
mediaImage->mPlane[mediaImage->V].mHorizSubsampling = 2;
mediaImage->mPlane[mediaImage->V].mVertSubsampling = 2;
+
+ if (tryWrapping && mClientColorFormat != COLOR_FormatYUV420Flexible) {
+ tryWrapping = yuv420888 && uPlane.colInc == 2 && vPlane.colInc == 2
+ && yPlane.rowInc == uPlane.rowInc
+ && view.data()[0] < view.data()[1]
+ && view.data()[1] < view.data()[2];
+ }
break;
- default:
- ALOGD("Converter: incompactible color format (%d) for YUV layout", mColorFormat);
- mInitCheck = BAD_VALUE;
- return;
+ case COLOR_FormatYUVP010:
+ mediaImage->mPlane[mediaImage->Y].mOffset = 0;
+ mediaImage->mPlane[mediaImage->Y].mColInc = 2;
+ mediaImage->mPlane[mediaImage->Y].mRowInc = stride * 2;
+ mediaImage->mPlane[mediaImage->Y].mHorizSubsampling = 1;
+ mediaImage->mPlane[mediaImage->Y].mVertSubsampling = 1;
+
+ mediaImage->mPlane[mediaImage->U].mOffset = stride * vStride * 2;
+ mediaImage->mPlane[mediaImage->U].mColInc = 4;
+ mediaImage->mPlane[mediaImage->U].mRowInc = stride * 2;
+ mediaImage->mPlane[mediaImage->U].mHorizSubsampling = 2;
+ mediaImage->mPlane[mediaImage->U].mVertSubsampling = 2;
+
+ mediaImage->mPlane[mediaImage->V].mOffset = stride * vStride * 2 + 2;
+ mediaImage->mPlane[mediaImage->V].mColInc = 4;
+ mediaImage->mPlane[mediaImage->V].mRowInc = stride * 2;
+ mediaImage->mPlane[mediaImage->V].mHorizSubsampling = 2;
+ mediaImage->mPlane[mediaImage->V].mVertSubsampling = 2;
+ if (tryWrapping) {
+ tryWrapping = yPlane.allocatedDepth == 16
+ && uPlane.allocatedDepth == 16
+ && vPlane.allocatedDepth == 16
+ && yPlane.bitDepth == 10
+ && uPlane.bitDepth == 10
+ && vPlane.bitDepth == 10
+ && yPlane.rightShift == 6
+ && uPlane.rightShift == 6
+ && vPlane.rightShift == 6
+ && yPlane.rowSampling == 1 && yPlane.colSampling == 1
+ && uPlane.rowSampling == 2 && uPlane.colSampling == 2
+ && vPlane.rowSampling == 2 && vPlane.colSampling == 2
+ && yPlane.colInc == 2
+ && uPlane.colInc == 4
+ && vPlane.colInc == 4
+ && yPlane.rowInc == uPlane.rowInc
+ && yPlane.rowInc == vPlane.rowInc;
+ }
+ break;
+
+ default: {
+ // default to fully planar format --- this will be overridden if wrapping
+ // TODO: keep interleaved format
+ int32_t colInc = divUp(mAllocatedDepth, 8u);
+ int32_t rowInc = stride * colInc / yPlane.colSampling;
+ mediaImage->mPlane[mediaImage->Y].mOffset = 0;
+ mediaImage->mPlane[mediaImage->Y].mColInc = colInc;
+ mediaImage->mPlane[mediaImage->Y].mRowInc = rowInc;
+ mediaImage->mPlane[mediaImage->Y].mHorizSubsampling = yPlane.colSampling;
+ mediaImage->mPlane[mediaImage->Y].mVertSubsampling = yPlane.rowSampling;
+ int32_t offset = rowInc * vStride / yPlane.rowSampling;
+
+ rowInc = stride * colInc / uPlane.colSampling;
+ mediaImage->mPlane[mediaImage->U].mOffset = offset;
+ mediaImage->mPlane[mediaImage->U].mColInc = colInc;
+ mediaImage->mPlane[mediaImage->U].mRowInc = rowInc;
+ mediaImage->mPlane[mediaImage->U].mHorizSubsampling = uPlane.colSampling;
+ mediaImage->mPlane[mediaImage->U].mVertSubsampling = uPlane.rowSampling;
+ offset += rowInc * vStride / uPlane.rowSampling;
+
+ rowInc = stride * colInc / vPlane.colSampling;
+ mediaImage->mPlane[mediaImage->V].mOffset = offset;
+ mediaImage->mPlane[mediaImage->V].mColInc = colInc;
+ mediaImage->mPlane[mediaImage->V].mRowInc = rowInc;
+ mediaImage->mPlane[mediaImage->V].mHorizSubsampling = vPlane.colSampling;
+ mediaImage->mPlane[mediaImage->V].mVertSubsampling = vPlane.rowSampling;
+ break;
+ }
}
break;
+ }
+
case C2PlanarLayout::TYPE_YUVA:
- mediaImage->mType = MediaImage2::MEDIA_IMAGE_TYPE_YUVA;
- // We don't have an SDK YUVA format
- ALOGD("Converter: incompactible color format (%d) for YUVA layout", mColorFormat);
- mInitCheck = BAD_VALUE;
+ ALOGD("Converter: unrecognized color format "
+ "(client %d component %d) for YUVA layout",
+ mClientColorFormat, mComponentColorFormat);
+ mInitCheck = NO_INIT;
return;
case C2PlanarLayout::TYPE_RGB:
- mediaImage->mType = MediaImage2::MEDIA_IMAGE_TYPE_RGB;
- switch (mColorFormat) {
- // TODO media image
- case COLOR_FormatRGBFlexible:
- case COLOR_Format24bitBGR888:
- case COLOR_Format24bitRGB888:
- break;
- default:
- ALOGD("Converter: incompactible color format (%d) for RGB layout", mColorFormat);
- mInitCheck = BAD_VALUE;
- return;
- }
- if (layout.numPlanes != 3) {
- ALOGD("Converter: %d planes for RGB layout", layout.numPlanes);
- mInitCheck = BAD_VALUE;
- return;
- }
- break;
+ ALOGD("Converter: unrecognized color format "
+ "(client %d component %d) for RGB layout",
+ mClientColorFormat, mComponentColorFormat);
+ mInitCheck = NO_INIT;
+ // TODO: support MediaImage layout
+ return;
case C2PlanarLayout::TYPE_RGBA:
- mediaImage->mType = MediaImage2::MEDIA_IMAGE_TYPE_RGBA;
- switch (mColorFormat) {
- // TODO media image
- case COLOR_FormatRGBAFlexible:
- case COLOR_Format32bitABGR8888:
- case COLOR_Format32bitARGB8888:
- case COLOR_Format32bitBGRA8888:
- break;
- default:
- ALOGD("Incompactible color format (%d) for RGBA layout", mColorFormat);
- mInitCheck = BAD_VALUE;
- return;
- }
- if (layout.numPlanes != 4) {
- ALOGD("Converter: %d planes for RGBA layout", layout.numPlanes);
- mInitCheck = BAD_VALUE;
- return;
- }
- break;
+ ALOGD("Converter: unrecognized color format "
+ "(client %d component %d) for RGBA layout",
+ mClientColorFormat, mComponentColorFormat);
+ mInitCheck = NO_INIT;
+ // TODO: support MediaImage layout
+ return;
default:
mediaImage->mType = MediaImage2::MEDIA_IMAGE_TYPE_UNKNOWN;
- ALOGD("Unknown layout");
- mInitCheck = BAD_VALUE;
- return;
+ if (layout.numPlanes == 1) {
+ const C2PlaneInfo &plane = layout.planes[0];
+ if (plane.colInc < 0 || plane.rowInc < 0) {
+ // Copy-only if we have negative colInc/rowInc
+ tryWrapping = false;
+ }
+ mediaImage->mPlane[0].mOffset = 0;
+ mediaImage->mPlane[0].mColInc = std::abs(plane.colInc);
+ mediaImage->mPlane[0].mRowInc = std::abs(plane.rowInc);
+ mediaImage->mPlane[0].mHorizSubsampling = plane.colSampling;
+ mediaImage->mPlane[0].mVertSubsampling = plane.rowSampling;
+ } else {
+ ALOGD("Converter: unrecognized layout: color format (client %d component %d)",
+ mClientColorFormat, mComponentColorFormat);
+ mInitCheck = NO_INIT;
+ return;
+ }
+ break;
+ }
+ if (tryWrapping) {
+ // try to map directly. check if the planes are near one another
+ const uint8_t *minPtr = mView.data()[0];
+ const uint8_t *maxPtr = mView.data()[0];
+ int32_t planeSize = 0;
+ for (uint32_t i = 0; i < layout.numPlanes; ++i) {
+ const C2PlaneInfo &plane = layout.planes[i];
+ int64_t planeStride = std::abs(plane.rowInc / plane.colInc);
+ ssize_t minOffset = plane.minOffset(
+ mWidth / plane.colSampling, mHeight / plane.rowSampling);
+ ssize_t maxOffset = plane.maxOffset(
+ mWidth / plane.colSampling, mHeight / plane.rowSampling);
+ if (minPtr > mView.data()[i] + minOffset) {
+ minPtr = mView.data()[i] + minOffset;
+ }
+ if (maxPtr < mView.data()[i] + maxOffset) {
+ maxPtr = mView.data()[i] + maxOffset;
+ }
+ planeSize += planeStride * divUp(mAllocatedDepth, 8u)
+ * align(mHeight, 64) / plane.rowSampling;
+ }
+
+ if ((maxPtr - minPtr + 1) <= planeSize) {
+ // FIXME: this is risky as reading/writing data out of bound results
+ // in an undefined behavior, but gralloc does assume a
+ // contiguous mapping
+ for (uint32_t i = 0; i < layout.numPlanes; ++i) {
+ const C2PlaneInfo &plane = layout.planes[i];
+ mediaImage->mPlane[i].mOffset = mView.data()[i] - minPtr;
+ mediaImage->mPlane[i].mColInc = plane.colInc;
+ mediaImage->mPlane[i].mRowInc = plane.rowInc;
+ mediaImage->mPlane[i].mHorizSubsampling = plane.colSampling;
+ mediaImage->mPlane[i].mVertSubsampling = plane.rowSampling;
+ }
+ mWrapped = new ABuffer(const_cast<uint8_t *>(minPtr),
+ maxPtr - minPtr + 1);
+ ALOGV("Converter: wrapped (capacity=%zu)", mWrapped->capacity());
+ }
}
mediaImage->mNumPlanes = layout.numPlanes;
mediaImage->mWidth = view.crop().width;
@@ -431,12 +529,12 @@
return;
}
if (plane.allocatedDepth != mAllocatedDepth || plane.bitDepth != bitDepth) {
- ALOGV("different allocatedDepth/bitDepth per plane unsupported");
+ ALOGD("different allocatedDepth/bitDepth per plane unsupported");
mInitCheck = BAD_VALUE;
return;
}
bufferSize += stride * vStride
- / plane.rowSampling / plane.colSampling;
+ / plane.rowSampling / plane.colSampling * divUp(mAllocatedDepth, 8u);
}
mBackBufferSize = bufferSize;
@@ -491,7 +589,8 @@
const C2GraphicView mView;
uint32_t mWidth;
uint32_t mHeight;
- int32_t mColorFormat; ///< SDK color format for MediaImage
+ int32_t mClientColorFormat; ///< SDK color format for MediaImage
+ int32_t mComponentColorFormat; ///< SDK color format from component
sp<ABuffer> mWrapped; ///< wrapped buffer (if we can map C2Buffer to an ABuffer)
uint32_t mAllocatedDepth;
uint32_t mBackBufferSize;
@@ -520,10 +619,7 @@
return nullptr;
}
- int32_t colorFormat = COLOR_FormatYUV420Flexible;
- (void)format->findInt32("color-format", &colorFormat);
-
- GraphicView2MediaImageConverter converter(view, colorFormat, false /* copy */);
+ GraphicView2MediaImageConverter converter(view, format, false /* copy */);
if (converter.initCheck() != OK) {
ALOGD("Converter init failed: %d", converter.initCheck());
return nullptr;
@@ -649,10 +745,7 @@
buffer->data().graphicBlocks()[0].map().get()));
std::unique_ptr<const C2GraphicView> holder;
- int32_t colorFormat = COLOR_FormatYUV420Flexible;
- (void)format->findInt32("color-format", &colorFormat);
-
- GraphicView2MediaImageConverter converter(*view, colorFormat, false /* copy */);
+ GraphicView2MediaImageConverter converter(*view, format, false /* copy */);
if (converter.initCheck() != OK) {
ALOGD("Converter init failed: %d", converter.initCheck());
return nullptr;
@@ -744,12 +837,11 @@
return false;
}
- int32_t colorFormat = COLOR_FormatYUV420Flexible;
- // FIXME: format() is not const, but we cannot change it, so do a const cast here
- const_cast<ConstGraphicBlockBuffer *>(this)->format()->findInt32("color-format", &colorFormat);
-
GraphicView2MediaImageConverter converter(
- buffer->data().graphicBlocks()[0].map().get(), colorFormat, true /* copy */);
+ buffer->data().graphicBlocks()[0].map().get(),
+ // FIXME: format() is not const, but we cannot change it, so do a const cast here
+ const_cast<ConstGraphicBlockBuffer *>(this)->format(),
+ true /* copy */);
if (converter.initCheck() != OK) {
ALOGD("ConstGraphicBlockBuffer::canCopy: converter init failed: %d", converter.initCheck());
return false;
@@ -767,11 +859,9 @@
setRange(0, 0);
return true;
}
- int32_t colorFormat = COLOR_FormatYUV420Flexible;
- format()->findInt32("color-format", &colorFormat);
GraphicView2MediaImageConverter converter(
- buffer->data().graphicBlocks()[0].map().get(), colorFormat, true /* copy */);
+ buffer->data().graphicBlocks()[0].map().get(), format(), true /* copy */);
if (converter.initCheck() != OK) {
ALOGD("ConstGraphicBlockBuffer::copy: converter init failed: %d", converter.initCheck());
return false;
diff --git a/media/codec2/sfplugin/tests/CCodecBuffers_test.cpp b/media/codec2/sfplugin/tests/CCodecBuffers_test.cpp
index ad8f6e5..66b7622 100644
--- a/media/codec2/sfplugin/tests/CCodecBuffers_test.cpp
+++ b/media/codec2/sfplugin/tests/CCodecBuffers_test.cpp
@@ -18,11 +18,12 @@
#include <gtest/gtest.h>
-#include <media/stagefright/foundation/AString.h>
+#include <codec2/hidl/client.h>
#include <media/stagefright/MediaCodecConstants.h>
#include <C2BlockInternal.h>
#include <C2PlatformSupport.h>
+#include <Codec2Mapper.h>
namespace android {
@@ -105,6 +106,318 @@
}
}
+TEST(RawGraphicOutputBuffersTest, FlexYuvColorFormat) {
+ constexpr int32_t kWidth = 320;
+ constexpr int32_t kHeight = 240;
+
+ std::vector<uint32_t> flexPixelFormats({HAL_PIXEL_FORMAT_YCbCr_420_888});
+ std::shared_ptr<Codec2Client> client = Codec2Client::CreateFromService("default");
+ if (client) {
+ // Query vendor format for Flexible YUV
+ std::vector<std::unique_ptr<C2Param>> heapParams;
+ C2StoreFlexiblePixelFormatDescriptorsInfo *pixelFormatInfo = nullptr;
+ if (client->query(
+ {},
+ {C2StoreFlexiblePixelFormatDescriptorsInfo::PARAM_TYPE},
+ C2_MAY_BLOCK,
+ &heapParams) == C2_OK
+ && heapParams.size() == 1u) {
+ pixelFormatInfo = C2StoreFlexiblePixelFormatDescriptorsInfo::From(
+ heapParams[0].get());
+ } else {
+ pixelFormatInfo = nullptr;
+ }
+ if (pixelFormatInfo && *pixelFormatInfo) {
+ for (size_t i = 0; i < pixelFormatInfo->flexCount(); ++i) {
+ const C2FlexiblePixelFormatDescriptorStruct &desc =
+ pixelFormatInfo->m.values[i];
+ if (desc.bitDepth != 8
+ || desc.subsampling != C2Color::YUV_420
+ // TODO(b/180076105): some devices report wrong layouts
+ // || desc.layout == C2Color::INTERLEAVED_PACKED
+ // || desc.layout == C2Color::INTERLEAVED_ALIGNED
+ || desc.layout == C2Color::UNKNOWN_LAYOUT) {
+ continue;
+ }
+ flexPixelFormats.push_back(desc.pixelFormat);
+ }
+ }
+ }
+
+ for (uint32_t pixelFormat : flexPixelFormats) {
+ std::shared_ptr<RawGraphicOutputBuffers> buffers =
+ std::make_shared<RawGraphicOutputBuffers>(
+ AStringPrintf("test pixel format 0x%x", pixelFormat).c_str());
+
+ sp<AMessage> format{new AMessage};
+ format->setInt32(KEY_WIDTH, kWidth);
+ format->setInt32(KEY_HEIGHT, kHeight);
+ format->setInt32(KEY_COLOR_FORMAT, COLOR_FormatYUV420Flexible);
+ int32_t fwkPixelFormat = 0;
+ if (C2Mapper::mapPixelFormatCodecToFramework(pixelFormat, &fwkPixelFormat)) {
+ format->setInt32("android._color-format", fwkPixelFormat);
+ }
+ buffers->setFormat(format);
+
+ std::shared_ptr<C2BlockPool> pool;
+ ASSERT_EQ(OK, GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool));
+
+ std::shared_ptr<C2GraphicBlock> block;
+ ASSERT_EQ(OK, pool->fetchGraphicBlock(
+ kWidth, kHeight, pixelFormat,
+ C2MemoryUsage{C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE}, &block));
+
+ {
+ C2GraphicView view = block->map().get();
+ C2PlanarLayout layout = view.layout();
+
+ // Verify the block is in YUV420 format
+ ASSERT_EQ(C2PlanarLayout::TYPE_YUV, layout.type);
+ ASSERT_EQ(3u, layout.numPlanes);
+ const C2PlaneInfo& yPlane = layout.planes[C2PlanarLayout::PLANE_Y];
+ const C2PlaneInfo& uPlane = layout.planes[C2PlanarLayout::PLANE_U];
+ const C2PlaneInfo& vPlane = layout.planes[C2PlanarLayout::PLANE_V];
+
+ // Y plane
+ ASSERT_EQ(1u, yPlane.colSampling);
+ ASSERT_EQ(1u, yPlane.rowSampling);
+ ASSERT_EQ(8u, yPlane.allocatedDepth);
+ ASSERT_EQ(8u, yPlane.bitDepth);
+ ASSERT_EQ(0u, yPlane.rightShift);
+
+ // U plane
+ ASSERT_EQ(2u, uPlane.colSampling);
+ ASSERT_EQ(2u, uPlane.rowSampling);
+ ASSERT_EQ(8u, uPlane.allocatedDepth);
+ ASSERT_EQ(8u, uPlane.bitDepth);
+ ASSERT_EQ(0u, uPlane.rightShift);
+
+ // V plane
+ ASSERT_EQ(2u, vPlane.colSampling);
+ ASSERT_EQ(2u, vPlane.rowSampling);
+ ASSERT_EQ(8u, vPlane.allocatedDepth);
+ ASSERT_EQ(8u, vPlane.bitDepth);
+ ASSERT_EQ(0u, vPlane.rightShift);
+
+ uint8_t *yRowPtr = view.data()[C2PlanarLayout::PLANE_Y];
+ uint8_t *uRowPtr = view.data()[C2PlanarLayout::PLANE_U];
+ uint8_t *vRowPtr = view.data()[C2PlanarLayout::PLANE_V];
+ for (int32_t row = 0; row < kHeight; ++row) {
+ uint8_t *yPtr = yRowPtr;
+ uint8_t *uPtr = uRowPtr;
+ uint8_t *vPtr = vRowPtr;
+ for (int32_t col = 0; col < kWidth; ++col) {
+ *yPtr = ((row + col) & 0xFF);
+ yPtr += yPlane.colInc;
+
+ if (row < kHeight / 2 && col < kWidth / 2) {
+ *uPtr = ((row + col + 1) & 0xFF);
+ *vPtr = ((row + col + 2) & 0xFF);
+ uPtr += uPlane.colInc;
+ vPtr += vPlane.colInc;
+ }
+ }
+ yRowPtr += yPlane.rowInc;
+ if (row < kHeight / 2) {
+ uRowPtr += uPlane.rowInc;
+ vRowPtr += vPlane.rowInc;
+ }
+ }
+ }
+
+ std::shared_ptr<C2Buffer> c2Buffer = C2Buffer::CreateGraphicBuffer(block->share(
+ block->crop(), C2Fence{}));
+ size_t index;
+ sp<MediaCodecBuffer> clientBuffer;
+ ASSERT_EQ(OK, buffers->registerBuffer(c2Buffer, &index, &clientBuffer));
+ ASSERT_NE(nullptr, clientBuffer);
+ sp<ABuffer> imageData;
+ ASSERT_TRUE(clientBuffer->format()->findBuffer("image-data", &imageData));
+ MediaImage2 *img = (MediaImage2 *)imageData->data();
+ ASSERT_EQ(MediaImage2::MEDIA_IMAGE_TYPE_YUV, img->mType);
+ ASSERT_EQ(3u, img->mNumPlanes);
+ ASSERT_EQ(kWidth, img->mWidth);
+ ASSERT_EQ(kHeight, img->mHeight);
+ ASSERT_EQ(8u, img->mBitDepth);
+ ASSERT_EQ(8u, img->mBitDepthAllocated);
+ const MediaImage2::PlaneInfo &yPlane = img->mPlane[MediaImage2::Y];
+ const MediaImage2::PlaneInfo &uPlane = img->mPlane[MediaImage2::U];
+ const MediaImage2::PlaneInfo &vPlane = img->mPlane[MediaImage2::V];
+ ASSERT_EQ(1u, yPlane.mHorizSubsampling);
+ ASSERT_EQ(1u, yPlane.mVertSubsampling);
+ ASSERT_EQ(2u, uPlane.mHorizSubsampling);
+ ASSERT_EQ(2u, uPlane.mVertSubsampling);
+ ASSERT_EQ(2u, vPlane.mHorizSubsampling);
+ ASSERT_EQ(2u, vPlane.mVertSubsampling);
+
+ uint8_t *yRowPtr = clientBuffer->data() + yPlane.mOffset;
+ uint8_t *uRowPtr = clientBuffer->data() + uPlane.mOffset;
+ uint8_t *vRowPtr = clientBuffer->data() + vPlane.mOffset;
+ for (int32_t row = 0; row < kHeight; ++row) {
+ uint8_t *yPtr = yRowPtr;
+ uint8_t *uPtr = uRowPtr;
+ uint8_t *vPtr = vRowPtr;
+ for (int32_t col = 0; col < kWidth; ++col) {
+ ASSERT_EQ((row + col) & 0xFF, *yPtr);
+ yPtr += yPlane.mColInc;
+ if (row < kHeight / 2 && col < kWidth / 2) {
+ ASSERT_EQ((row + col + 1) & 0xFF, *uPtr);
+ ASSERT_EQ((row + col + 2) & 0xFF, *vPtr);
+ uPtr += uPlane.mColInc;
+ vPtr += vPlane.mColInc;
+ }
+ }
+ yRowPtr += yPlane.mRowInc;
+ if (row < kHeight / 2) {
+ uRowPtr += uPlane.mRowInc;
+ vRowPtr += vPlane.mRowInc;
+ }
+ }
+ }
+}
+
+TEST(RawGraphicOutputBuffersTest, P010ColorFormat) {
+ constexpr int32_t kWidth = 320;
+ constexpr int32_t kHeight = 240;
+
+ std::shared_ptr<RawGraphicOutputBuffers> buffers =
+ std::make_shared<RawGraphicOutputBuffers>("test P010");
+
+ sp<AMessage> format{new AMessage};
+ format->setInt32(KEY_WIDTH, kWidth);
+ format->setInt32(KEY_HEIGHT, kHeight);
+ format->setInt32(KEY_COLOR_FORMAT, COLOR_FormatYUVP010);
+ int32_t fwkPixelFormat = 0;
+ if (C2Mapper::mapPixelFormatCodecToFramework(HAL_PIXEL_FORMAT_YCBCR_P010, &fwkPixelFormat)) {
+ format->setInt32("android._color-format", fwkPixelFormat);
+ }
+ buffers->setFormat(format);
+
+ std::shared_ptr<C2BlockPool> pool;
+ ASSERT_EQ(OK, GetCodec2BlockPool(C2BlockPool::BASIC_GRAPHIC, nullptr, &pool));
+
+ std::shared_ptr<C2GraphicBlock> block;
+ c2_status_t err = pool->fetchGraphicBlock(
+ kWidth, kHeight, HAL_PIXEL_FORMAT_YCBCR_P010,
+ C2MemoryUsage{C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE}, &block);
+ if (err != C2_OK) {
+ GTEST_SKIP();
+ }
+
+ {
+ C2GraphicView view = block->map().get();
+ C2PlanarLayout layout = view.layout();
+
+ // Verify the block is in YUV420 format
+ ASSERT_EQ(C2PlanarLayout::TYPE_YUV, layout.type);
+ ASSERT_EQ(3u, layout.numPlanes);
+ const C2PlaneInfo& yPlane = layout.planes[C2PlanarLayout::PLANE_Y];
+ const C2PlaneInfo& uPlane = layout.planes[C2PlanarLayout::PLANE_U];
+ const C2PlaneInfo& vPlane = layout.planes[C2PlanarLayout::PLANE_V];
+
+ // Y plane
+ ASSERT_EQ(1u, yPlane.colSampling);
+ ASSERT_EQ(1u, yPlane.rowSampling);
+ ASSERT_EQ(16u, yPlane.allocatedDepth);
+ ASSERT_EQ(10u, yPlane.bitDepth);
+ ASSERT_EQ(6u, yPlane.rightShift);
+
+ // U plane
+ ASSERT_EQ(2u, uPlane.colSampling);
+ ASSERT_EQ(2u, uPlane.rowSampling);
+ ASSERT_EQ(16u, uPlane.allocatedDepth);
+ ASSERT_EQ(10u, uPlane.bitDepth);
+ ASSERT_EQ(6u, uPlane.rightShift);
+
+ // V plane
+ ASSERT_EQ(2u, vPlane.colSampling);
+ ASSERT_EQ(2u, vPlane.rowSampling);
+ ASSERT_EQ(16u, vPlane.allocatedDepth);
+ ASSERT_EQ(10u, vPlane.bitDepth);
+ ASSERT_EQ(6u, vPlane.rightShift);
+
+ uint8_t *yRowPtr = view.data()[C2PlanarLayout::PLANE_Y];
+ uint8_t *uRowPtr = view.data()[C2PlanarLayout::PLANE_U];
+ uint8_t *vRowPtr = view.data()[C2PlanarLayout::PLANE_V];
+ for (int32_t row = 0; row < kHeight; ++row) {
+ uint8_t *yPtr = yRowPtr;
+ uint8_t *uPtr = uRowPtr;
+ uint8_t *vPtr = vRowPtr;
+ for (int32_t col = 0; col < kWidth; ++col) {
+ yPtr[0] = ((row + col) & 0x3) << 6;
+ yPtr[1] = ((row + col) & 0x3FC) >> 2;
+ yPtr += yPlane.colInc;
+
+ if (row < kHeight / 2 && col < kWidth / 2) {
+ uPtr[0] = ((row + col + 1) & 0x3) << 6;
+ uPtr[1] = ((row + col + 1) & 0x3FC) >> 2;
+ vPtr[0] = ((row + col + 2) & 0x3) << 6;
+ vPtr[1] = ((row + col + 2) & 0x3FC) >> 2;
+ uPtr += uPlane.colInc;
+ vPtr += vPlane.colInc;
+ }
+ }
+ yRowPtr += yPlane.rowInc;
+ if (row < kHeight / 2) {
+ uRowPtr += uPlane.rowInc;
+ vRowPtr += vPlane.rowInc;
+ }
+ }
+ }
+
+ std::shared_ptr<C2Buffer> c2Buffer = C2Buffer::CreateGraphicBuffer(block->share(
+ block->crop(), C2Fence{}));
+ size_t index;
+ sp<MediaCodecBuffer> clientBuffer;
+ ASSERT_EQ(OK, buffers->registerBuffer(c2Buffer, &index, &clientBuffer));
+ ASSERT_NE(nullptr, clientBuffer);
+ sp<ABuffer> imageData;
+ ASSERT_TRUE(clientBuffer->format()->findBuffer("image-data", &imageData));
+ MediaImage2 *img = (MediaImage2 *)imageData->data();
+ ASSERT_EQ(MediaImage2::MEDIA_IMAGE_TYPE_YUV, img->mType);
+ ASSERT_EQ(3u, img->mNumPlanes);
+ ASSERT_EQ(kWidth, img->mWidth);
+ ASSERT_EQ(kHeight, img->mHeight);
+ ASSERT_EQ(10u, img->mBitDepth);
+ ASSERT_EQ(16u, img->mBitDepthAllocated);
+ const MediaImage2::PlaneInfo &yPlane = img->mPlane[MediaImage2::Y];
+ const MediaImage2::PlaneInfo &uPlane = img->mPlane[MediaImage2::U];
+ const MediaImage2::PlaneInfo &vPlane = img->mPlane[MediaImage2::V];
+ ASSERT_EQ(1u, yPlane.mHorizSubsampling);
+ ASSERT_EQ(1u, yPlane.mVertSubsampling);
+ ASSERT_EQ(2u, uPlane.mHorizSubsampling);
+ ASSERT_EQ(2u, uPlane.mVertSubsampling);
+ ASSERT_EQ(2u, vPlane.mHorizSubsampling);
+ ASSERT_EQ(2u, vPlane.mVertSubsampling);
+
+ uint8_t *yRowPtr = clientBuffer->data() + yPlane.mOffset;
+ uint8_t *uRowPtr = clientBuffer->data() + uPlane.mOffset;
+ uint8_t *vRowPtr = clientBuffer->data() + vPlane.mOffset;
+ for (int32_t row = 0; row < kHeight; ++row) {
+ uint8_t *yPtr = yRowPtr;
+ uint8_t *uPtr = uRowPtr;
+ uint8_t *vPtr = vRowPtr;
+ for (int32_t col = 0; col < kWidth; ++col) {
+ ASSERT_EQ(((row + col) & 0x3) << 6, yPtr[0]);
+ ASSERT_EQ(((row + col) & 0x3FC) >> 2, yPtr[1]);
+ yPtr += yPlane.mColInc;
+ if (row < kHeight / 2 && col < kWidth / 2) {
+ ASSERT_EQ(((row + col + 1) & 0x3) << 6, uPtr[0]);
+ ASSERT_EQ(((row + col + 1) & 0x3FC) >> 2, uPtr[1]);
+ ASSERT_EQ(((row + col + 2) & 0x3) << 6, vPtr[0]);
+ ASSERT_EQ(((row + col + 2) & 0x3FC) >> 2, vPtr[1]);
+ uPtr += uPlane.mColInc;
+ vPtr += vPlane.mColInc;
+ }
+ }
+ yRowPtr += yPlane.mRowInc;
+ if (row < kHeight / 2) {
+ uRowPtr += uPlane.mRowInc;
+ vRowPtr += vPlane.mRowInc;
+ }
+ }
+}
+
class TestGraphicAllocation : public C2GraphicAllocation {
public:
TestGraphicAllocation(
@@ -407,7 +720,6 @@
}
}
}
-
size_t yPlaneSize = stride * kHeight;
size_t uvPlaneSize = stride * kHeight / 4;
size_t capacity = yPlaneSize + uvPlaneSize * 2;
diff --git a/media/codec2/sfplugin/utils/Codec2Mapper.cpp b/media/codec2/sfplugin/utils/Codec2Mapper.cpp
index 903db6c..1390642 100644
--- a/media/codec2/sfplugin/utils/Codec2Mapper.cpp
+++ b/media/codec2/sfplugin/utils/Codec2Mapper.cpp
@@ -958,17 +958,16 @@
*c2Value = HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED;
return true;
case COLOR_FormatYUV420Flexible:
- *c2Value = HAL_PIXEL_FORMAT_YCBCR_420_888;
- return true;
case COLOR_FormatYUV420Planar:
case COLOR_FormatYUV420SemiPlanar:
case COLOR_FormatYUV420PackedPlanar:
case COLOR_FormatYUV420PackedSemiPlanar:
- *c2Value = HAL_PIXEL_FORMAT_YV12;
+ *c2Value = HAL_PIXEL_FORMAT_YCBCR_420_888;
return true;
default:
- // TODO: support some sort of passthrough
- return false;
+ // Passthrough
+ *c2Value = uint32_t(frameworkValue);
+ return true;
}
}
@@ -979,11 +978,16 @@
case HAL_PIXEL_FORMAT_IMPLEMENTATION_DEFINED:
*frameworkValue = COLOR_FormatSurface;
return true;
- case HAL_PIXEL_FORMAT_YV12:
+ case HAL_PIXEL_FORMAT_YCBCR_422_SP:
+ case HAL_PIXEL_FORMAT_YCRCB_420_SP:
+ case HAL_PIXEL_FORMAT_YCBCR_422_I:
case HAL_PIXEL_FORMAT_YCBCR_420_888:
+ case HAL_PIXEL_FORMAT_YV12:
*frameworkValue = COLOR_FormatYUV420Flexible;
return true;
default:
- return false;
+ // Passthrough
+ *frameworkValue = int32_t(c2Value);
+ return true;
}
}
diff --git a/media/codec2/vndk/C2AllocatorGralloc.cpp b/media/codec2/vndk/C2AllocatorGralloc.cpp
index 4d7e619..8e59df1 100644
--- a/media/codec2/vndk/C2AllocatorGralloc.cpp
+++ b/media/codec2/vndk/C2AllocatorGralloc.cpp
@@ -25,6 +25,7 @@
#include <hardware/gralloc.h>
#include <ui/GraphicBufferAllocator.h>
#include <ui/GraphicBufferMapper.h>
+#include <ui/Rect.h>
#include <C2AllocatorGralloc.h>
#include <C2Buffer.h>
@@ -253,7 +254,7 @@
virtual ~C2AllocationGralloc() override;
virtual c2_status_t map(
- C2Rect rect, C2MemoryUsage usage, C2Fence *fence,
+ C2Rect c2Rect, C2MemoryUsage usage, C2Fence *fence,
C2PlanarLayout *layout /* nonnull */, uint8_t **addr /* nonnull */) override;
virtual c2_status_t unmap(
uint8_t **addr /* nonnull */, C2Rect rect, C2Fence *fence /* nullable */) override;
@@ -336,8 +337,12 @@
}
c2_status_t C2AllocationGralloc::map(
- C2Rect rect, C2MemoryUsage usage, C2Fence *fence,
+ C2Rect c2Rect, C2MemoryUsage usage, C2Fence *fence,
C2PlanarLayout *layout /* nonnull */, uint8_t **addr /* nonnull */) {
+ const Rect rect{(int32_t)c2Rect.left, (int32_t)c2Rect.top,
+ (int32_t)(c2Rect.left + c2Rect.width) /* right */,
+ (int32_t)(c2Rect.top + c2Rect.height) /* bottom */};
+
uint64_t grallocUsage = static_cast<C2AndroidMemoryUsage>(usage).asGrallocUsage();
ALOGV("mapping buffer with usage %#llx => %#llx",
(long long)usage.expected, (long long)grallocUsage);
@@ -386,10 +391,7 @@
void *pointer = nullptr;
// TODO: fence
status_t err = GraphicBufferMapper::get().lock(
- const_cast<native_handle_t *>(mBuffer), grallocUsage,
- { (int32_t)rect.left, (int32_t)rect.top,
- (int32_t)rect.width, (int32_t)rect.height },
- &pointer);
+ const_cast<native_handle_t *>(mBuffer), grallocUsage, rect, &pointer);
if (err) {
ALOGE("failed transaction: lock(RGBA_1010102)");
return C2_CORRUPTED;
@@ -464,10 +466,7 @@
void *pointer = nullptr;
// TODO: fence
status_t err = GraphicBufferMapper::get().lock(
- const_cast<native_handle_t*>(mBuffer), grallocUsage,
- { (int32_t)rect.left, (int32_t)rect.top,
- (int32_t)rect.width, (int32_t)rect.height },
- &pointer);
+ const_cast<native_handle_t*>(mBuffer), grallocUsage, rect, &pointer);
if (err) {
ALOGE("failed transaction: lock(RGBA_8888)");
return C2_CORRUPTED;
@@ -524,10 +523,7 @@
void *pointer = nullptr;
// TODO: fence
status_t err = GraphicBufferMapper::get().lock(
- const_cast<native_handle_t*>(mBuffer), grallocUsage,
- { (int32_t)rect.left, (int32_t)rect.top,
- (int32_t)rect.width, (int32_t)rect.height },
- &pointer);
+ const_cast<native_handle_t*>(mBuffer), grallocUsage, rect, &pointer);
if (err) {
ALOGE("failed transaction: lock(BLOB)");
return C2_CORRUPTED;
@@ -536,18 +532,19 @@
break;
}
+ case static_cast<uint32_t>(PixelFormat4::YCBCR_422_SP):
+ // fall-through
+ case static_cast<uint32_t>(PixelFormat4::YCRCB_420_SP):
+ // fall-through
+ case static_cast<uint32_t>(PixelFormat4::YCBCR_422_I):
+ // fall-through
case static_cast<uint32_t>(PixelFormat4::YCBCR_420_888):
// fall-through
- case static_cast<uint32_t>(PixelFormat4::YV12):
- // fall-through
- default: {
+ case static_cast<uint32_t>(PixelFormat4::YV12): {
android_ycbcr ycbcrLayout;
status_t err = GraphicBufferMapper::get().lockYCbCr(
- const_cast<native_handle_t*>(mBuffer), grallocUsage,
- { (int32_t)rect.left, (int32_t)rect.top,
- (int32_t)rect.width, (int32_t)rect.height },
- &ycbcrLayout);
+ const_cast<native_handle_t*>(mBuffer), grallocUsage, rect, &ycbcrLayout);
if (err) {
ALOGE("failed transaction: lockYCbCr");
return C2_CORRUPTED;
@@ -611,6 +608,158 @@
}
break;
}
+
+ case static_cast<uint32_t>(PixelFormat4::YCBCR_P010): {
+ void *pointer = nullptr;
+ status_t err = GraphicBufferMapper::get().lock(
+ const_cast<native_handle_t *>(mBuffer), grallocUsage, rect, &pointer);
+ if (err) {
+ ALOGE("failed transaction: lock(YCBCR_P010)");
+ return C2_CORRUPTED;
+ }
+ addr[C2PlanarLayout::PLANE_Y] = (uint8_t *)pointer;
+ addr[C2PlanarLayout::PLANE_U] = (uint8_t *)pointer + mStride * 2 * rect.height();
+ addr[C2PlanarLayout::PLANE_V] = addr[C2PlanarLayout::PLANE_U] + 2;
+ layout->type = C2PlanarLayout::TYPE_YUV;
+ layout->numPlanes = 3;
+ layout->rootPlanes = 2;
+ layout->planes[C2PlanarLayout::PLANE_Y] = {
+ C2PlaneInfo::CHANNEL_Y, // channel
+ 2, // colInc
+ static_cast<int32_t>(2 * mStride), // rowInc
+ 1, // mColSampling
+ 1, // mRowSampling
+ 16, // allocatedDepth
+ 10, // bitDepth
+ 6, // rightShift
+ C2PlaneInfo::LITTLE_END, // endianness
+ C2PlanarLayout::PLANE_Y, // rootIx
+ 0, // offset
+ };
+ layout->planes[C2PlanarLayout::PLANE_U] = {
+ C2PlaneInfo::CHANNEL_CB, // channel
+ 4, // colInc
+ static_cast<int32_t>(2 * mStride), // rowInc
+ 2, // mColSampling
+ 2, // mRowSampling
+ 16, // allocatedDepth
+ 10, // bitDepth
+ 6, // rightShift
+ C2PlaneInfo::LITTLE_END, // endianness
+ C2PlanarLayout::PLANE_U, // rootIx
+ 0, // offset
+ };
+ layout->planes[C2PlanarLayout::PLANE_V] = {
+ C2PlaneInfo::CHANNEL_CR, // channel
+ 4, // colInc
+ static_cast<int32_t>(2 * mStride), // rowInc
+ 2, // mColSampling
+ 2, // mRowSampling
+ 16, // allocatedDepth
+ 10, // bitDepth
+ 6, // rightShift
+ C2PlaneInfo::LITTLE_END, // endianness
+ C2PlanarLayout::PLANE_U, // rootIx
+ 2, // offset
+ };
+ break;
+ }
+
+ default: {
+ // We don't know what it is, but let's try to lock it.
+ android_ycbcr ycbcrLayout;
+
+ status_t err = GraphicBufferMapper::get().lockYCbCr(
+ const_cast<native_handle_t*>(mBuffer), grallocUsage, rect, &ycbcrLayout);
+ if (err == OK) {
+ addr[C2PlanarLayout::PLANE_Y] = (uint8_t *)ycbcrLayout.y;
+ addr[C2PlanarLayout::PLANE_U] = (uint8_t *)ycbcrLayout.cb;
+ addr[C2PlanarLayout::PLANE_V] = (uint8_t *)ycbcrLayout.cr;
+ layout->type = C2PlanarLayout::TYPE_YUV;
+ layout->numPlanes = 3;
+ layout->rootPlanes = 3;
+ layout->planes[C2PlanarLayout::PLANE_Y] = {
+ C2PlaneInfo::CHANNEL_Y, // channel
+ 1, // colInc
+ (int32_t)ycbcrLayout.ystride, // rowInc
+ 1, // mColSampling
+ 1, // mRowSampling
+ 8, // allocatedDepth
+ 8, // bitDepth
+ 0, // rightShift
+ C2PlaneInfo::NATIVE, // endianness
+ C2PlanarLayout::PLANE_Y, // rootIx
+ 0, // offset
+ };
+ layout->planes[C2PlanarLayout::PLANE_U] = {
+ C2PlaneInfo::CHANNEL_CB, // channel
+ (int32_t)ycbcrLayout.chroma_step, // colInc
+ (int32_t)ycbcrLayout.cstride, // rowInc
+ 2, // mColSampling
+ 2, // mRowSampling
+ 8, // allocatedDepth
+ 8, // bitDepth
+ 0, // rightShift
+ C2PlaneInfo::NATIVE, // endianness
+ C2PlanarLayout::PLANE_U, // rootIx
+ 0, // offset
+ };
+ layout->planes[C2PlanarLayout::PLANE_V] = {
+ C2PlaneInfo::CHANNEL_CR, // channel
+ (int32_t)ycbcrLayout.chroma_step, // colInc
+ (int32_t)ycbcrLayout.cstride, // rowInc
+ 2, // mColSampling
+ 2, // mRowSampling
+ 8, // allocatedDepth
+ 8, // bitDepth
+ 0, // rightShift
+ C2PlaneInfo::NATIVE, // endianness
+ C2PlanarLayout::PLANE_V, // rootIx
+ 0, // offset
+ };
+ // handle interleaved formats
+ intptr_t uvOffset = addr[C2PlanarLayout::PLANE_V] - addr[C2PlanarLayout::PLANE_U];
+ if (uvOffset > 0 && uvOffset < (intptr_t)ycbcrLayout.chroma_step) {
+ layout->rootPlanes = 2;
+ layout->planes[C2PlanarLayout::PLANE_V].rootIx = C2PlanarLayout::PLANE_U;
+ layout->planes[C2PlanarLayout::PLANE_V].offset = uvOffset;
+ } else if (uvOffset < 0 && uvOffset > -(intptr_t)ycbcrLayout.chroma_step) {
+ layout->rootPlanes = 2;
+ layout->planes[C2PlanarLayout::PLANE_U].rootIx = C2PlanarLayout::PLANE_V;
+ layout->planes[C2PlanarLayout::PLANE_U].offset = -uvOffset;
+ }
+ break;
+ }
+
+ // We really don't know what this is; lock the buffer and pass it through ---
+ // the client may know how to interpret it.
+ void *pointer = nullptr;
+ err = GraphicBufferMapper::get().lock(
+ const_cast<native_handle_t *>(mBuffer), grallocUsage, rect, &pointer);
+ if (err) {
+ ALOGE("failed transaction: lock(??? %x)", mFormat);
+ return C2_CORRUPTED;
+ }
+ addr[0] = (uint8_t *)pointer;
+ layout->type = C2PlanarLayout::TYPE_UNKNOWN;
+ layout->numPlanes = 1;
+ layout->rootPlanes = 1;
+ layout->planes[0] = {
+ // TODO: CHANNEL_UNKNOWN?
+ C2PlaneInfo::channel_t(0xFF), // channel
+ 1, // colInc
+ int32_t(mStride), // rowInc
+ 1, // mColSampling
+ 1, // mRowSampling
+ 8, // allocatedDepth
+ 8, // bitDepth
+ 0, // rightShift
+ C2PlaneInfo::NATIVE, // endianness
+ 0, // rootIx
+ 0, // offset
+ };
+ break;
+ }
}
mLocked = true;
diff --git a/media/libeffects/lvm/benchmarks/Android.bp b/media/libeffects/lvm/benchmarks/Android.bp
index 60a9772..8a25b85 100644
--- a/media/libeffects/lvm/benchmarks/Android.bp
+++ b/media/libeffects/lvm/benchmarks/Android.bp
@@ -28,6 +28,7 @@
cc_benchmark {
name: "reverb_benchmark",
vendor: true,
+ host_supported: true,
include_dirs: [
"frameworks/av/media/libeffects/lvm/wrapper/Reverb",
],
diff --git a/media/libeffects/lvm/benchmarks/lvm_benchmark.cpp b/media/libeffects/lvm/benchmarks/lvm_benchmark.cpp
index e2e4a85..bdb66d8 100644
--- a/media/libeffects/lvm/benchmarks/lvm_benchmark.cpp
+++ b/media/libeffects/lvm/benchmarks/lvm_benchmark.cpp
@@ -53,8 +53,6 @@
constexpr size_t kNumChMasks = std::size(kChMasks);
constexpr int kSampleRate = 44100;
-// TODO(b/131240940) Remove once effects are updated to produce mono output
-constexpr size_t kMinOutputChannelCount = 2;
/*******************************************************************
* A test result running on Pixel 3 for comparison.
@@ -64,6 +62,10 @@
* -----------------------------------------------------
* Benchmark Time CPU Iterations
* -----------------------------------------------------
+ * BM_LVM/1/0 52123 ns 51971 ns 13437
+ * BM_LVM/1/1 75397 ns 75175 ns 9382
+ * BM_LVM/1/2 40253 ns 40140 ns 17418
+ * BM_LVM/1/3 19918 ns 19860 ns 35230
* BM_LVM/2/0 62455 ns 62283 ns 11214
* BM_LVM/2/1 110086 ns 109751 ns 6350
* BM_LVM/2/2 44017 ns 43890 ns 15982
@@ -203,7 +205,7 @@
// Run the test
for (auto _ : state) {
- std::vector<float> output(kFrameCount * std::max(channelCount, kMinOutputChannelCount));
+ std::vector<float> output(kFrameCount * channelCount);
benchmark::DoNotOptimize(input.data());
benchmark::DoNotOptimize(output.data());
@@ -224,8 +226,7 @@
}
static void LVMArgs(benchmark::internal::Benchmark* b) {
- // TODO(b/131240940) Test single channel once effects are updated to process mono data
- for (int i = 2; i <= kNumChMasks; i++) {
+ for (int i = FCC_1; i <= kNumChMasks; i++) {
for (int j = 0; j < kNumEffectUuids; ++j) {
b->Args({i, j});
}
diff --git a/media/libeffects/lvm/lib/Android.bp b/media/libeffects/lvm/lib/Android.bp
index ed7ef7f..a044295 100644
--- a/media/libeffects/lvm/lib/Android.bp
+++ b/media/libeffects/lvm/lib/Android.bp
@@ -140,6 +140,7 @@
},
vendor: true,
+ host_supported: true,
srcs: [
"Reverb/src/LVREV_ApplyNewSettings.cpp",
"Reverb/src/LVREV_ClearAudioBuffers.cpp",
diff --git a/media/libeffects/lvm/lib/Bass/src/LVDBE_Control.cpp b/media/libeffects/lvm/lib/Bass/src/LVDBE_Control.cpp
index d860ad0..3fc9e95 100644
--- a/media/libeffects/lvm/lib/Bass/src/LVDBE_Control.cpp
+++ b/media/libeffects/lvm/lib/Bass/src/LVDBE_Control.cpp
@@ -277,8 +277,8 @@
/*
* Create biquad instance
*/
- pInstance->pHPFBiquad.reset(new android::audio_utils::BiquadFilter<LVM_FLOAT>(
- (FCC_1 == pParams->NrChannels) ? FCC_2 : pParams->NrChannels));
+ pInstance->pHPFBiquad.reset(
+ new android::audio_utils::BiquadFilter<LVM_FLOAT>(pParams->NrChannels));
/*
* Update the filters
diff --git a/media/libeffects/lvm/lib/Bass/src/LVDBE_Init.cpp b/media/libeffects/lvm/lib/Bass/src/LVDBE_Init.cpp
index 979644c..761c6ce 100644
--- a/media/libeffects/lvm/lib/Bass/src/LVDBE_Init.cpp
+++ b/media/libeffects/lvm/lib/Bass/src/LVDBE_Init.cpp
@@ -57,10 +57,7 @@
* Create the instance handle if not already initialised
*/
if (*phInstance == LVM_NULL) {
- *phInstance = calloc(1, sizeof(*pInstance));
- }
- if (*phInstance == LVM_NULL) {
- return LVDBE_NULLADDRESS;
+ *phInstance = new LVDBE_Instance_t;
}
pInstance = (LVDBE_Instance_t*)*phInstance;
@@ -185,6 +182,6 @@
free(pInstance->pData);
pInstance->pData = LVM_NULL;
}
- free(pInstance);
+ delete pInstance;
*phInstance = LVM_NULL;
}
diff --git a/media/libeffects/lvm/lib/Bass/src/LVDBE_Process.cpp b/media/libeffects/lvm/lib/Bass/src/LVDBE_Process.cpp
index 8c62e71..0969053 100644
--- a/media/libeffects/lvm/lib/Bass/src/LVDBE_Process.cpp
+++ b/media/libeffects/lvm/lib/Bass/src/LVDBE_Process.cpp
@@ -79,11 +79,7 @@
const LVM_UINT16 NrFrames) // updated to use samples = frames * channels.
{
LVDBE_Instance_t* pInstance = (LVDBE_Instance_t*)hInstance;
-
- /*Extract number of Channels info*/
- // Mono passed in as stereo
- const LVM_INT32 NrChannels =
- pInstance->Params.NrChannels == 1 ? 2 : pInstance->Params.NrChannels;
+ const LVM_INT32 NrChannels = pInstance->Params.NrChannels;
const LVM_INT32 NrSamples = NrChannels * NrFrames;
/* Space to store DBE path computation */
diff --git a/media/libeffects/lvm/lib/Bundle/src/LVM_Buffers.cpp b/media/libeffects/lvm/lib/Bundle/src/LVM_Buffers.cpp
index fbb0fe1..1d913d7 100644
--- a/media/libeffects/lvm/lib/Bundle/src/LVM_Buffers.cpp
+++ b/media/libeffects/lvm/lib/Bundle/src/LVM_Buffers.cpp
@@ -487,10 +487,6 @@
void LVM_BufferUnmanagedOut(LVM_Handle_t hInstance, LVM_UINT16* pNumSamples) {
LVM_Instance_t* pInstance = (LVM_Instance_t*)hInstance;
LVM_INT16 NumChannels = pInstance->NrChannels;
- if (NumChannels == 1) {
- /* Mono input is processed as stereo by LVM module */
- NumChannels = 2;
- }
#undef NrFrames
#define NrFrames (*pNumSamples) // alias for clarity
diff --git a/media/libeffects/lvm/lib/Bundle/src/LVM_Init.cpp b/media/libeffects/lvm/lib/Bundle/src/LVM_Init.cpp
index c1b375e..b092970 100644
--- a/media/libeffects/lvm/lib/Bundle/src/LVM_Init.cpp
+++ b/media/libeffects/lvm/lib/Bundle/src/LVM_Init.cpp
@@ -93,10 +93,7 @@
/*
* Create the instance handle
*/
- *phInstance = (LVM_Handle_t)calloc(1, sizeof(*pInstance));
- if (*phInstance == LVM_NULL) {
- return LVM_NULLADDRESS;
- }
+ *phInstance = new LVM_Instance_t;
pInstance = (LVM_Instance_t*)*phInstance;
pInstance->InstParams = *pInstParams;
@@ -543,7 +540,7 @@
pInstance->pPSAInput = LVM_NULL;
}
- free(*phInstance);
+ delete pInstance;
return;
}
diff --git a/media/libeffects/lvm/lib/Bundle/src/LVM_Process.cpp b/media/libeffects/lvm/lib/Bundle/src/LVM_Process.cpp
index 82c0e68..20058a1 100644
--- a/media/libeffects/lvm/lib/Bundle/src/LVM_Process.cpp
+++ b/media/libeffects/lvm/lib/Bundle/src/LVM_Process.cpp
@@ -111,19 +111,6 @@
}
/*
- * Convert from Mono if necessary
- */
- if (pInstance->Params.SourceFormat == LVM_MONO) {
- MonoTo2I_Float(pInData, /* Source */
- pOutData, /* Destination */
- (LVM_INT16)NumSamples); /* Number of input samples */
- pInput = pOutData;
- pToProcess = pOutData;
- NrChannels = 2;
- ChMask = AUDIO_CHANNEL_OUT_STEREO;
- }
-
- /*
* Process the data with managed buffers
*/
while (SampleCount != 0) {
diff --git a/media/libeffects/lvm/lib/Common/lib/LVM_Types.h b/media/libeffects/lvm/lib/Common/lib/LVM_Types.h
index b95d076..5cdcf35 100644
--- a/media/libeffects/lvm/lib/Common/lib/LVM_Types.h
+++ b/media/libeffects/lvm/lib/Common/lib/LVM_Types.h
@@ -121,6 +121,19 @@
LVM_FS_DUMMY = LVM_MAXENUM
} LVM_Fs_en;
+static inline LVM_Fs_en lvmFsForSampleRate(int sampleRate) {
+ static const std::map<int, LVM_Fs_en> kLVMFsMap = {
+ {8000, LVM_FS_8000}, {11025, LVM_FS_11025}, {12000, LVM_FS_12000},
+ {16000, LVM_FS_16000}, {22050, LVM_FS_22050}, {24000, LVM_FS_24000},
+ {32000, LVM_FS_32000}, {44100, LVM_FS_44100}, {48000, LVM_FS_48000},
+ {88200, LVM_FS_88200}, {96000, LVM_FS_96000}, {176400, LVM_FS_176400},
+ {192000, LVM_FS_192000}};
+ if (kLVMFsMap.find(sampleRate) != kLVMFsMap.end()) {
+ return kLVMFsMap.at(sampleRate);
+ }
+ return LVM_FS_INVALID;
+}
+
/* Memory Types */
typedef enum {
LVM_PERSISTENT_SLOW_DATA = LVM_MEMREGION_PERSISTENT_SLOW_DATA,
diff --git a/media/libeffects/lvm/lib/Common/lib/VectorArithmetic.h b/media/libeffects/lvm/lib/Common/lib/VectorArithmetic.h
index 281d941..18de85b 100644
--- a/media/libeffects/lvm/lib/Common/lib/VectorArithmetic.h
+++ b/media/libeffects/lvm/lib/Common/lib/VectorArithmetic.h
@@ -29,17 +29,18 @@
void Copy_Float(const LVM_FLOAT* src, LVM_FLOAT* dst, LVM_INT16 n);
void Copy_Float_Mc_Stereo(const LVM_FLOAT* src, LVM_FLOAT* dst, LVM_INT16 NrFrames,
LVM_INT32 NrChannels);
-void Copy_Float_Stereo_Mc(const LVM_FLOAT* src, LVM_FLOAT* StereoOut, LVM_FLOAT* dst,
+void Copy_Float_Stereo_Mc(const LVM_FLOAT* src, const LVM_FLOAT* StereoOut, LVM_FLOAT* dst,
LVM_INT16 NrFrames, LVM_INT32 NrChannels);
void Mult3s_Float(const LVM_FLOAT* src, const LVM_FLOAT val, LVM_FLOAT* dst, LVM_INT16 n);
-void DelayMix_Float(const LVM_FLOAT* src, /* Source 1, to be delayed */
- LVM_FLOAT* delay, /* Delay buffer */
- LVM_INT16 size, /* Delay size */
- LVM_FLOAT* dst, /* Source/destination */
- LVM_INT16* pOffset, /* Delay offset */
- LVM_INT16 n); /* Number of stereo samples */
+void DelayMix_Float(const LVM_FLOAT* src, /* Source 1, to be delayed */
+ LVM_FLOAT* delay, /* Delay buffer */
+ LVM_INT16 size, /* Delay size */
+ LVM_FLOAT* dst, /* Source/destination */
+ LVM_INT16* pOffset, /* Delay offset */
+ LVM_INT16 n, /* Number of stereo samples */
+ LVM_INT32 NrChannels); /* Number of channels */
void Add2_Sat_Float(const LVM_FLOAT* src, LVM_FLOAT* dst, LVM_INT16 n);
void Mac3s_Sat_Float(const LVM_FLOAT* src, const LVM_FLOAT val, LVM_FLOAT* dst, LVM_INT16 n);
diff --git a/media/libeffects/lvm/lib/Common/src/Copy_16.cpp b/media/libeffects/lvm/lib/Common/src/Copy_16.cpp
index 7046a94..1fe7470 100644
--- a/media/libeffects/lvm/lib/Common/src/Copy_16.cpp
+++ b/media/libeffects/lvm/lib/Common/src/Copy_16.cpp
@@ -51,25 +51,32 @@
}
// Merge a multichannel source with stereo contained in StereoOut, to dst.
-void Copy_Float_Stereo_Mc(const LVM_FLOAT* src, LVM_FLOAT* StereoOut, LVM_FLOAT* dst,
+void Copy_Float_Stereo_Mc(const LVM_FLOAT* src, const LVM_FLOAT* StereoOut, LVM_FLOAT* dst,
LVM_INT16 NrFrames, /* Number of frames*/
LVM_INT32 NrChannels) {
LVM_INT16 ii, jj;
- // pack dst with stereo information of StereoOut
- // together with the upper channels of src.
- StereoOut += 2 * (NrFrames - 1);
- dst += NrChannels * (NrFrames - 1);
- src += NrChannels * (NrFrames - 1);
- for (ii = NrFrames; ii != 0; ii--) {
- dst[1] = StereoOut[1];
- dst[0] = StereoOut[0]; // copy 1 before 0 is required for NrChannels == 3.
- for (jj = 2; jj < NrChannels; jj++) {
- dst[jj] = src[jj];
+ if (NrChannels >= FCC_2) {
+ // pack dst with stereo information of StereoOut
+ // together with the upper channels of src.
+ StereoOut += 2 * (NrFrames - 1);
+ dst += NrChannels * (NrFrames - 1);
+ src += NrChannels * (NrFrames - 1);
+
+ for (ii = NrFrames; ii != 0; ii--) {
+ dst[1] = StereoOut[1];
+ dst[0] = StereoOut[0]; // copy 1 before 0 is required for NrChannels == 3.
+ for (jj = FCC_2; jj < NrChannels; jj++) {
+ dst[jj] = src[jj];
+ }
+ dst -= NrChannels;
+ src -= NrChannels;
+ StereoOut -= 2;
}
- dst -= NrChannels;
- src -= NrChannels;
- StereoOut -= 2;
+ } else {
+ Copy_Float((const LVM_FLOAT*)StereoOut, /* Source */
+ (LVM_FLOAT*)dst, /* Destination */
+ (LVM_INT16)NrFrames); /* Number of frames */
}
}
/**********************************************************************************/
diff --git a/media/libeffects/lvm/lib/Common/src/DelayMix_16x16.cpp b/media/libeffects/lvm/lib/Common/src/DelayMix_16x16.cpp
index d2537eb..a346636 100644
--- a/media/libeffects/lvm/lib/Common/src/DelayMix_16x16.cpp
+++ b/media/libeffects/lvm/lib/Common/src/DelayMix_16x16.cpp
@@ -26,34 +26,50 @@
LVM_INT16 size, /* Delay size */
LVM_FLOAT* dst, /* Source/destination */
LVM_INT16* pOffset, /* Delay offset */
- LVM_INT16 n) /* Number of stereo samples */
+ LVM_INT16 n, /* Number of samples */
+ LVM_INT32 NrChannels) /* Number of channels */
{
LVM_INT16 i;
LVM_INT16 Offset = *pOffset;
LVM_FLOAT temp;
for (i = 0; i < n; i++) {
- /* Left channel */
- temp = (LVM_FLOAT)((LVM_FLOAT)(*dst + (LVM_FLOAT)delay[Offset]) / 2.0f);
- *dst = temp;
- dst++;
+ if (NrChannels == FCC_1) {
+ temp = (LVM_FLOAT)(*dst + (LVM_FLOAT)delay[Offset]) / 2.0f;
+ *dst = temp;
+ dst++;
- delay[Offset] = *src;
- Offset++;
- src++;
+ delay[Offset] = *src;
+ Offset++;
+ src++;
- /* Right channel */
- temp = (LVM_FLOAT)((LVM_FLOAT)(*dst - (LVM_FLOAT)delay[Offset]) / 2.0f);
- *dst = temp;
- dst++;
+ /* Make the reverb delay buffer a circular buffer */
+ if (Offset >= size) {
+ Offset = 0;
+ }
+ } else {
+ /* Left channel */
+ temp = (LVM_FLOAT)(*dst + (LVM_FLOAT)delay[Offset]) / 2.0f;
+ *dst = temp;
+ dst++;
- delay[Offset] = *src;
- Offset++;
- src++;
+ delay[Offset] = *src;
+ Offset++;
+ src++;
- /* Make the reverb delay buffer a circular buffer */
- if (Offset >= size) {
- Offset = 0;
+ /* Right channel */
+ temp = (LVM_FLOAT)(*dst - (LVM_FLOAT)delay[Offset]) / 2.0f;
+ *dst = temp;
+ dst++;
+
+ delay[Offset] = *src;
+ Offset++;
+ src++;
+
+ /* Make the reverb delay buffer a circular buffer */
+ if (Offset >= size) {
+ Offset = 0;
+ }
}
}
diff --git a/media/libeffects/lvm/lib/Common/src/LVC_MixSoft_1St_2i_D16C31_SAT.cpp b/media/libeffects/lvm/lib/Common/src/LVC_MixSoft_1St_2i_D16C31_SAT.cpp
index 58bc06e..b0aa172 100644
--- a/media/libeffects/lvm/lib/Common/src/LVC_MixSoft_1St_2i_D16C31_SAT.cpp
+++ b/media/libeffects/lvm/lib/Common/src/LVC_MixSoft_1St_2i_D16C31_SAT.cpp
@@ -56,10 +56,11 @@
Mix_Private_FLOAT_st* pInstance[NrChannels];
if (audio_channel_mask_get_representation(ChMask) == AUDIO_CHANNEL_REPRESENTATION_INDEX) {
- for (int i = 0; i < 2; i++) {
+ int loopLimit = (NrChannels == FCC_1) ? NrChannels : FCC_2;
+ for (int i = 0; i < loopLimit; i++) {
pInstance[i] = pMixPrivInst[i];
}
- for (int i = 2; i < NrChannels; i++) {
+ for (int i = loopLimit; i < NrChannels; i++) {
pInstance[i] = pMixPrivInst[2];
}
} else {
diff --git a/media/libeffects/lvm/lib/Eq/src/LVEQNB_Control.cpp b/media/libeffects/lvm/lib/Eq/src/LVEQNB_Control.cpp
index 3ab6afb..7e5caed 100644
--- a/media/libeffects/lvm/lib/Eq/src/LVEQNB_Control.cpp
+++ b/media/libeffects/lvm/lib/Eq/src/LVEQNB_Control.cpp
@@ -311,9 +311,8 @@
/*
* Create biquad instance
*/
- pInstance->eqBiquad.resize(
- pParams->NBands, android::audio_utils::BiquadFilter<LVM_FLOAT>(
- (FCC_1 == pParams->NrChannels) ? FCC_2 : pParams->NrChannels));
+ pInstance->eqBiquad.resize(pParams->NBands,
+ android::audio_utils::BiquadFilter<LVM_FLOAT>(pParams->NrChannels));
LVEQNB_ClearFilterHistory(pInstance);
if (bChange || modeChange) {
diff --git a/media/libeffects/lvm/lib/Eq/src/LVEQNB_Init.cpp b/media/libeffects/lvm/lib/Eq/src/LVEQNB_Init.cpp
index 833ee5d..37e6d4d 100644
--- a/media/libeffects/lvm/lib/Eq/src/LVEQNB_Init.cpp
+++ b/media/libeffects/lvm/lib/Eq/src/LVEQNB_Init.cpp
@@ -52,10 +52,7 @@
LVEQNB_Capabilities_t* pCapabilities, void* pScratch) {
LVEQNB_Instance_t* pInstance;
- *phInstance = calloc(1, sizeof(*pInstance));
- if (phInstance == LVM_NULL) {
- return LVEQNB_NULLADDRESS;
- }
+ *phInstance = new LVEQNB_Instance_t;
pInstance = (LVEQNB_Instance_t*)*phInstance;
pInstance->Capabilities = *pCapabilities;
@@ -146,6 +143,6 @@
free(pInstance->pBiquadType);
pInstance->pBiquadType = LVM_NULL;
}
- free(pInstance);
+ delete pInstance;
*phInstance = LVM_NULL;
}
diff --git a/media/libeffects/lvm/lib/Eq/src/LVEQNB_Process.cpp b/media/libeffects/lvm/lib/Eq/src/LVEQNB_Process.cpp
index 8992803..b177dd4 100644
--- a/media/libeffects/lvm/lib/Eq/src/LVEQNB_Process.cpp
+++ b/media/libeffects/lvm/lib/Eq/src/LVEQNB_Process.cpp
@@ -62,10 +62,7 @@
LVEQNB_Handle_t hInstance, const LVM_FLOAT* pInData, LVM_FLOAT* pOutData,
const LVM_UINT16 NrFrames) { // updated to use samples = frames * channels.
LVEQNB_Instance_t* pInstance = (LVEQNB_Instance_t*)hInstance;
-
- // Mono passed in as stereo
- const LVM_INT32 NrChannels =
- pInstance->Params.NrChannels == 1 ? 2 : pInstance->Params.NrChannels;
+ const LVM_INT32 NrChannels = pInstance->Params.NrChannels;
const LVM_INT32 NrSamples = NrChannels * NrFrames;
/* Check for NULL pointers */
@@ -104,7 +101,6 @@
* Check if band is non-zero dB gain
*/
if (pInstance->pBandDefinitions[i].Gain != 0) {
-
/*
* Select single or double precision as required
*/
diff --git a/media/libeffects/lvm/lib/Reverb/lib/LVREV.h b/media/libeffects/lvm/lib/Reverb/lib/LVREV.h
index 484787a..489bc6f 100644
--- a/media/libeffects/lvm/lib/Reverb/lib/LVREV.h
+++ b/media/libeffects/lvm/lib/Reverb/lib/LVREV.h
@@ -191,6 +191,23 @@
/****************************************************************************************/
/* */
+/* FUNCTION: LVREV_FreeInstance */
+/* */
+/* DESCRIPTION: */
+/* This function is used to free the internal allocations of the module. */
+/* */
+/* PARAMETERS: */
+/* hInstance Instance handle */
+/* */
+/* RETURNS: */
+/* LVREV_SUCCESS free instance succeeded */
+/* LVREV_NULLADDRESS Instance is NULL */
+/* */
+/****************************************************************************************/
+LVREV_ReturnStatus_en LVREV_FreeInstance(LVREV_Handle_t hInstance);
+
+/****************************************************************************************/
+/* */
/* FUNCTION: LVXX_GetControlParameters */
/* */
/* DESCRIPTION: */
diff --git a/media/libeffects/lvm/lib/Reverb/src/LVREV_GetInstanceHandle.cpp b/media/libeffects/lvm/lib/Reverb/src/LVREV_GetInstanceHandle.cpp
index 9a797bd..bf71634 100644
--- a/media/libeffects/lvm/lib/Reverb/src/LVREV_GetInstanceHandle.cpp
+++ b/media/libeffects/lvm/lib/Reverb/src/LVREV_GetInstanceHandle.cpp
@@ -114,7 +114,7 @@
* Set the instance handle if not already initialised
*/
if (*phInstance == LVM_NULL) {
- *phInstance = InstAlloc_AddMember(&SlowData, sizeof(LVREV_Instance_st));
+ *phInstance = new LVREV_Instance_st;
}
pLVREV_Private = (LVREV_Instance_st*)*phInstance;
pLVREV_Private->MemoryTable = *pMemoryTable;
@@ -269,4 +269,27 @@
return LVREV_SUCCESS;
}
+/****************************************************************************************/
+/* */
+/* FUNCTION: LVREV_FreeInstance */
+/* */
+/* DESCRIPTION: */
+/* This function is used to free the internal allocations of the module. */
+/* */
+/* PARAMETERS: */
+/* hInstance Instance handle */
+/* */
+/* RETURNS: */
+/* LVREV_SUCCESS free instance succeeded */
+/* LVREV_NULLADDRESS Instance is NULL */
+/* */
+/****************************************************************************************/
+LVREV_ReturnStatus_en LVREV_FreeInstance(LVREV_Handle_t hInstance) {
+ if (hInstance == LVM_NULL) {
+ return LVREV_NULLADDRESS;
+ }
+
+ delete (LVREV_Instance_st*)hInstance;
+ return LVREV_SUCCESS;
+}
/* End of file */
diff --git a/media/libeffects/lvm/lib/SpectrumAnalyzer/src/LVPSA_Init.cpp b/media/libeffects/lvm/lib/SpectrumAnalyzer/src/LVPSA_Init.cpp
index 9874dcc..5ca8543 100644
--- a/media/libeffects/lvm/lib/SpectrumAnalyzer/src/LVPSA_Init.cpp
+++ b/media/libeffects/lvm/lib/SpectrumAnalyzer/src/LVPSA_Init.cpp
@@ -49,10 +49,7 @@
LVM_UINT32 BufferLength = 0;
/* Set the instance handle if not already initialised */
- *phInstance = calloc(1, sizeof(*pLVPSA_Inst));
- if (*phInstance == LVM_NULL) {
- return LVPSA_ERROR_NULLADDRESS;
- }
+ *phInstance = new LVPSA_InstancePr_t;
pLVPSA_Inst = (LVPSA_InstancePr_t*)*phInstance;
pLVPSA_Inst->pScratch = pScratch;
@@ -191,6 +188,6 @@
free(pLVPSA_Inst->pQPD_Taps);
pLVPSA_Inst->pQPD_Taps = LVM_NULL;
}
- free(pLVPSA_Inst);
+ delete pLVPSA_Inst;
*phInstance = LVM_NULL;
}
diff --git a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_BypassMix.cpp b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_BypassMix.cpp
index efca27d..f805fca 100644
--- a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_BypassMix.cpp
+++ b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_BypassMix.cpp
@@ -200,6 +200,8 @@
LVM_UINT16 NumSamples) {
LVCS_Instance_t* pInstance = (LVCS_Instance_t*)hInstance;
LVCS_BypassMix_t* pConfig = (LVCS_BypassMix_t*)&pInstance->BypassMix;
+ LVM_UINT16 destNumSamples =
+ (pInstance->Params.NrChannels == FCC_1) ? NumSamples : FCC_2 * NumSamples;
/*
* Check if the bypass mixer is enabled
@@ -209,12 +211,12 @@
* Apply the bypass mix
*/
LVC_MixSoft_2St_D16C31_SAT(&pConfig->Mixer_Instance, pProcessed, (LVM_FLOAT*)pUnprocessed,
- pOutData, (LVM_INT16)(2 * NumSamples));
+ pOutData, (LVM_INT16)destNumSamples);
/*
* Apply output gain correction shift
*/
Shift_Sat_Float((LVM_INT16)pConfig->Output_Shift, (LVM_FLOAT*)pOutData,
- (LVM_FLOAT*)pOutData, (LVM_INT16)(2 * NumSamples)); /* Left and right*/
+ (LVM_FLOAT*)pOutData, (LVM_INT16)destNumSamples);
}
return (LVCS_SUCCESS);
diff --git a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Control.cpp b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Control.cpp
index 8f88986..89f2f3b 100644
--- a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Control.cpp
+++ b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Control.cpp
@@ -180,7 +180,9 @@
if (pInstance->bInOperatingModeTransition != LVM_TRUE) {
pInstance->bTimerDone = LVM_FALSE;
pInstance->TimerParams.TimeInMs =
- (LVM_INT16)(((pInstance->Reverberation.DelaySize << 2) /
+ (LVM_INT16)(((pInstance->Params.NrChannels == FCC_1
+ ? pInstance->Reverberation.DelaySize << 3
+ : pInstance->Reverberation.DelaySize << 2) /
pInstance->TimerParams.SamplingRate) +
1);
LVM_Timer_Init(&pInstance->TimerInstance, &pInstance->TimerParams);
diff --git a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Equaliser.cpp b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Equaliser.cpp
index c8ad94e..1746786 100644
--- a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Equaliser.cpp
+++ b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Equaliser.cpp
@@ -74,7 +74,8 @@
pEqualiserCoefTable[Offset].A0, pEqualiserCoefTable[Offset].A1,
pEqualiserCoefTable[Offset].A2, -(pEqualiserCoefTable[Offset].B1),
-(pEqualiserCoefTable[Offset].B2)};
- pInstance->pEqBiquad.reset(new android::audio_utils::BiquadFilter<LVM_FLOAT>(FCC_2, coefs));
+ pInstance->pEqBiquad.reset(new android::audio_utils::BiquadFilter<LVM_FLOAT>(
+ (pParams->NrChannels == FCC_1) ? FCC_1 : FCC_2, coefs));
}
return (LVCS_SUCCESS);
diff --git a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Init.cpp b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Init.cpp
index ba3202f..d60b360 100644
--- a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Init.cpp
+++ b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Init.cpp
@@ -55,10 +55,7 @@
* Create the instance handle if not already initialised
*/
if (*phInstance == LVM_NULL) {
- *phInstance = calloc(1, sizeof(*pInstance));
- }
- if (*phInstance == LVM_NULL) {
- return LVCS_NULLADDRESS;
+ *phInstance = new LVCS_Instance_t;
}
pInstance = (LVCS_Instance_t*)*phInstance;
@@ -123,7 +120,7 @@
if (pInstance == LVM_NULL) {
return;
}
- free(pInstance);
+ delete pInstance;
*phInstance = LVM_NULL;
return;
}
diff --git a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Process.cpp b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Process.cpp
index d18f2c3..6af0f75 100644
--- a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Process.cpp
+++ b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_Process.cpp
@@ -75,16 +75,6 @@
LVM_INT32 channels = pInstance->Params.NrChannels;
#define NrFrames NumSamples // alias for clarity
- /*In case of mono processing, stereo input is created from mono
- *and stored in pInData before applying any of the effects.
- *However we do not update the value pInstance->Params.NrChannels
- *at this point.
- *So to treat the pInData as stereo we are setting channels to 2
- */
- if (channels == 1) {
- channels = 2;
- }
-
pScratch = (LVM_FLOAT*)pInstance->pScratch;
/*
@@ -97,11 +87,16 @@
*/
pInput = pScratch + (2 * NrFrames);
pStIn = pScratch + ((LVCS_SCRATCHBUFFERS - 2) * NrFrames);
- /* The first two channel data is extracted from the input data and
- * copied into pInput buffer
- */
- Copy_Float_Mc_Stereo((LVM_FLOAT*)pInData, (LVM_FLOAT*)pInput, NrFrames, channels);
- Copy_Float((LVM_FLOAT*)pInput, (LVM_FLOAT*)pStIn, (LVM_INT16)(2 * NrFrames));
+ if (channels == FCC_1) {
+ Copy_Float((LVM_FLOAT*)pInData, (LVM_FLOAT*)pInput, (LVM_INT16)NrFrames);
+ Copy_Float((LVM_FLOAT*)pInput, (LVM_FLOAT*)pStIn, (LVM_INT16)NrFrames);
+ } else {
+ /* The first two channel data is extracted from the input data and
+ * copied into pInput buffer
+ */
+ Copy_Float_Mc_Stereo((LVM_FLOAT*)pInData, (LVM_FLOAT*)pInput, NrFrames, channels);
+ Copy_Float((LVM_FLOAT*)pInput, (LVM_FLOAT*)pStIn, (LVM_INT16)(FCC_2 * NrFrames));
+ }
/*
* Call the stereo enhancer
*/
@@ -172,10 +167,10 @@
LVCS_ReturnStatus_en err;
/*Extract number of Channels info*/
LVM_INT32 channels = pInstance->Params.NrChannels;
+ LVM_UINT16 destNumSamples = (channels == FCC_1) ? NumSamples : FCC_2 * NumSamples;
+ LVM_INT32 compGainInterval =
+ (channels == FCC_1) ? LVCS_COMPGAINFRAME : FCC_2 * LVCS_COMPGAINFRAME;
#define NrFrames NumSamples // alias for clarity
- if (channels == 1) {
- channels = 2;
- }
/*
* Check the number of samples is not too large
*/
@@ -227,7 +222,7 @@
if (NumSamples < LVCS_COMPGAINFRAME) {
NonLinComp_Float(Gain, /* Compressor gain setting */
- pStereoOut, pStereoOut, (LVM_INT32)(2 * NrFrames));
+ pStereoOut, pStereoOut, (LVM_INT32)destNumSamples);
} else {
LVM_FLOAT GainStep;
LVM_FLOAT FinalGain;
@@ -266,12 +261,15 @@
if (SampleToProcess > LVCS_COMPGAINFRAME) {
NonLinComp_Float(Gain, /* Compressor gain setting */
- pOutPtr, pOutPtr, (LVM_INT32)(2 * LVCS_COMPGAINFRAME));
- pOutPtr += (2 * LVCS_COMPGAINFRAME);
+ pOutPtr, pOutPtr, compGainInterval);
+ pOutPtr += compGainInterval;
SampleToProcess = (LVM_INT16)(SampleToProcess - LVCS_COMPGAINFRAME);
} else {
NonLinComp_Float(Gain, /* Compressor gain setting */
- pOutPtr, pOutPtr, (LVM_INT32)(2 * SampleToProcess));
+ pOutPtr, pOutPtr,
+ (channels == FCC_1)
+ ? (LVM_INT32)(SampleToProcess)
+ : (LVM_INT32)(FCC_2 * SampleToProcess));
SampleToProcess = 0;
}
}
@@ -297,7 +295,7 @@
LVM_Timer(&pInstance->TimerInstance, (LVM_INT16)NumSamples);
}
}
- Copy_Float_Stereo_Mc(pInData, pStereoOut, pOutData, NrFrames, channels);
+ Copy_Float_Stereo_Mc(pInData, (const LVM_FLOAT*)pStereoOut, pOutData, NrFrames, channels);
} else {
if (pInData != pOutData) {
/*
diff --git a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_ReverbGenerator.cpp b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_ReverbGenerator.cpp
index 15acda9..12b1dc3 100644
--- a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_ReverbGenerator.cpp
+++ b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_ReverbGenerator.cpp
@@ -65,7 +65,6 @@
LVCS_ReverbGenerator_t* pConfig = (LVCS_ReverbGenerator_t*)&pInstance->Reverberation;
const BiquadA012B12CoefsSP_t* pReverbCoefTable;
-
/*
* Initialise the delay and filters if:
* - the sample rate has changed
@@ -79,7 +78,8 @@
*/
Delay = (LVM_UINT16)LVCS_StereoDelayCS[(LVM_UINT16)pParams->SampleRate];
- pConfig->DelaySize = (LVM_INT16)(2 * Delay);
+ pConfig->DelaySize =
+ (pParams->NrChannels == FCC_1) ? (LVM_INT16)Delay : (LVM_INT16)(FCC_2 * Delay);
pConfig->DelayOffset = 0;
LoadConst_Float(0, /* Value */
(LVM_FLOAT*)&pConfig->StereoSamples[0], /* Destination */
@@ -95,8 +95,8 @@
pReverbCoefTable[Offset].A0, pReverbCoefTable[Offset].A1,
pReverbCoefTable[Offset].A2, -(pReverbCoefTable[Offset].B1),
-(pReverbCoefTable[Offset].B2)};
- pInstance->pRevBiquad.reset(
- new android::audio_utils::BiquadFilter<LVM_FLOAT>(FCC_2, coefs));
+ pInstance->pRevBiquad.reset(new android::audio_utils::BiquadFilter<LVM_FLOAT>(
+ (pParams->NrChannels == FCC_1) ? FCC_1 : FCC_2, coefs));
/*
* Setup the mixer
@@ -155,6 +155,9 @@
LVCS_Instance_t* pInstance = (LVCS_Instance_t*)hInstance;
LVCS_ReverbGenerator_t* pConfig = (LVCS_ReverbGenerator_t*)&pInstance->Reverberation;
LVM_FLOAT* pScratch;
+ LVM_INT32 NumChannels = pInstance->Params.NrChannels;
+ LVM_UINT16 destNumSamples =
+ (pInstance->Params.NrChannels == FCC_1) ? NumSamples : FCC_2 * NumSamples;
pScratch = (LVM_FLOAT*)pInstance->pScratch;
@@ -165,9 +168,9 @@
/*
* Reverb not required so just copy the data
*/
- Copy_Float((LVM_FLOAT*)pInData, /* Source */
- (LVM_FLOAT*)pOutData, /* Destination */
- (LVM_INT16)(2 * NumSamples)); /* Left and right */
+ Copy_Float((LVM_FLOAT*)pInData, /* Source */
+ (LVM_FLOAT*)pOutData, /* Destination */
+ (LVM_INT16)destNumSamples); /* Number of frames */
}
/*
@@ -188,9 +191,9 @@
/*
* Copy the input data to the scratch memory
*/
- Copy_Float((LVM_FLOAT*)pInData, /* Source */
- (LVM_FLOAT*)pScratch, /* Destination */
- (LVM_INT16)(2 * NumSamples)); /* Left and right */
+ Copy_Float((LVM_FLOAT*)pInData, /* Source */
+ (LVM_FLOAT*)pScratch, /* Destination */
+ (LVM_INT16)destNumSamples); /* Number of frames */
/*
* Filter the data
@@ -198,13 +201,13 @@
pInstance->pRevBiquad->process(pScratch, pScratch, NumSamples);
Mult3s_Float((LVM_FLOAT*)pScratch, pConfig->ReverbLevel, (LVM_FLOAT*)pScratch,
- (LVM_INT16)(2 * NumSamples));
+ (LVM_INT16)destNumSamples); /* Number of frames */
/*
* Apply the delay mix
*/
DelayMix_Float((LVM_FLOAT*)pScratch, &pConfig->StereoSamples[0], pConfig->DelaySize,
- pOutData, &pConfig->DelayOffset, (LVM_INT16)NumSamples);
+ pOutData, &pConfig->DelayOffset, (LVM_INT16)NumSamples, NumChannels);
}
return (LVCS_SUCCESS);
diff --git a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_StereoEnhancer.cpp b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_StereoEnhancer.cpp
index 00bb26c..e3ff604 100644
--- a/media/libeffects/lvm/lib/StereoWidening/src/LVCS_StereoEnhancer.cpp
+++ b/media/libeffects/lvm/lib/StereoWidening/src/LVCS_StereoEnhancer.cpp
@@ -55,7 +55,6 @@
LVCS_Instance_t* pInstance = (LVCS_Instance_t*)hInstance;
const BiquadA012B12CoefsSP_t* pSESideCoefs;
-
/*
* If the sample rate or speaker type has changed update the filters
*/
@@ -129,6 +128,8 @@
LVCS_StereoEnhancer_t* pConfig = (LVCS_StereoEnhancer_t*)&pInstance->StereoEnhancer;
LVM_FLOAT* pScratch;
pScratch = (LVM_FLOAT*)pInstance->pScratch;
+ LVM_INT32 NumChannels = pInstance->Params.NrChannels;
+ LVM_UINT16 destNumSamples = (NumChannels == FCC_1) ? NumSamples : FCC_2 * NumSamples;
/*
* Check if the Stereo Enhancer is enabled
*/
@@ -136,7 +137,12 @@
/*
* Convert from stereo to middle and side
*/
- From2iToMS_Float(pInData, pScratch, pScratch + NumSamples, (LVM_INT16)NumSamples);
+ if (NumChannels == 1) {
+ // Copy same input to scratch as Middle data
+ Copy_Float((LVM_FLOAT*)pInData, (LVM_FLOAT*)pScratch, (LVM_INT16)NumSamples);
+ } else {
+ From2iToMS_Float(pInData, pScratch, pScratch + NumSamples, (LVM_INT16)NumSamples);
+ }
/*
* Apply filter to the middle signal
@@ -159,18 +165,23 @@
NumSamples);
}
- /*
- * Convert from middle and side to stereo
- */
- MSTo2i_Sat_Float(pScratch, pScratch + NumSamples, pOutData, (LVM_INT16)NumSamples);
+ if (NumChannels == 1) {
+ // Copy processed Middle data from scratch to pOutData
+ Copy_Float((LVM_FLOAT*)pScratch, (LVM_FLOAT*)pOutData, (LVM_INT16)NumSamples);
+ } else {
+ /*
+ * Convert from middle and side to stereo
+ */
+ MSTo2i_Sat_Float(pScratch, pScratch + NumSamples, pOutData, (LVM_INT16)NumSamples);
+ }
} else {
/*
* The stereo enhancer is disabled so just copy the data
*/
- Copy_Float((LVM_FLOAT*)pInData, /* Source */
- (LVM_FLOAT*)pOutData, /* Destination */
- (LVM_INT16)(2 * NumSamples)); /* Left and right */
+ Copy_Float((LVM_FLOAT*)pInData, /* Source */
+ (LVM_FLOAT*)pOutData, /* Destination */
+ (LVM_INT16)destNumSamples); /* Number of frames */
}
return (LVCS_SUCCESS);
diff --git a/media/libeffects/lvm/tests/Android.bp b/media/libeffects/lvm/tests/Android.bp
index 8627c13..639af4d 100644
--- a/media/libeffects/lvm/tests/Android.bp
+++ b/media/libeffects/lvm/tests/Android.bp
@@ -10,6 +10,26 @@
}
cc_test {
+ name: "EffectBundleTest",
+ vendor: true,
+ gtest: true,
+ host_supported: true,
+ test_suites: ["device-tests"],
+ srcs: ["EffectBundleTest.cpp"],
+ static_libs: [
+ "libaudioutils",
+ "libbundlewrapper",
+ "libmusicbundle",
+ ],
+ shared_libs: [
+ "liblog",
+ ],
+ header_libs: [
+ "libhardware_headers",
+ ],
+}
+
+cc_test {
name: "lvmtest",
host_supported: false,
proprietary: true,
@@ -54,8 +74,9 @@
cc_test {
name: "reverb_test",
- host_supported: false,
+ host_supported: true,
proprietary: true,
+ gtest: false,
include_dirs: [
"frameworks/av/media/libeffects/lvm/wrapper/Reverb",
diff --git a/media/libeffects/lvm/tests/EffectBundleTest.cpp b/media/libeffects/lvm/tests/EffectBundleTest.cpp
new file mode 100644
index 0000000..aae09de
--- /dev/null
+++ b/media/libeffects/lvm/tests/EffectBundleTest.cpp
@@ -0,0 +1,335 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <array>
+#include <audio_utils/channels.h>
+#include <audio_utils/primitives.h>
+#include <climits>
+#include <cstdlib>
+#include <gtest/gtest.h>
+#include <hardware/audio_effect.h>
+#include <log/log.h>
+#include <random>
+#include <system/audio.h>
+#include <vector>
+
+extern audio_effect_library_t AUDIO_EFFECT_LIBRARY_INFO_SYM;
+
+// Corresponds to SNR for 1 bit difference between two int16_t signals
+constexpr float kSNRThreshold = 90.308998;
+
+// Update isBassBoost, if the order of effects is updated
+constexpr effect_uuid_t kEffectUuids[] = {
+ // NXP SW BassBoost
+ {0x8631f300, 0x72e2, 0x11df, 0xb57e, {0x00, 0x02, 0xa5, 0xd5, 0xc5, 0x1b}},
+ // NXP SW Virtualizer
+ {0x1d4033c0, 0x8557, 0x11df, 0x9f2d, {0x00, 0x02, 0xa5, 0xd5, 0xc5, 0x1b}},
+ // NXP SW Equalizer
+ {0xce772f20, 0x847d, 0x11df, 0xbb17, {0x00, 0x02, 0xa5, 0xd5, 0xc5, 0x1b}},
+ // NXP SW Volume
+ {0x119341a0, 0x8469, 0x11df, 0x81f9, {0x00, 0x02, 0xa5, 0xd5, 0xc5, 0x1b}},
+};
+
+static bool isBassBoost(const effect_uuid_t* uuid) {
+ // Update this, if the order of effects in kEffectUuids is updated
+ return uuid == &kEffectUuids[0];
+}
+
+constexpr size_t kNumEffectUuids = std::size(kEffectUuids);
+
+constexpr audio_channel_mask_t kChMasks[] = {
+ AUDIO_CHANNEL_OUT_MONO, AUDIO_CHANNEL_OUT_STEREO,
+ AUDIO_CHANNEL_OUT_2POINT1, AUDIO_CHANNEL_OUT_2POINT0POINT2,
+ AUDIO_CHANNEL_OUT_QUAD, AUDIO_CHANNEL_OUT_QUAD_BACK,
+ AUDIO_CHANNEL_OUT_QUAD_SIDE, AUDIO_CHANNEL_OUT_SURROUND,
+ AUDIO_CHANNEL_INDEX_MASK_4, AUDIO_CHANNEL_OUT_2POINT1POINT2,
+ AUDIO_CHANNEL_OUT_3POINT0POINT2, AUDIO_CHANNEL_OUT_PENTA,
+ AUDIO_CHANNEL_INDEX_MASK_5, AUDIO_CHANNEL_OUT_3POINT1POINT2,
+ AUDIO_CHANNEL_OUT_5POINT1, AUDIO_CHANNEL_OUT_5POINT1_BACK,
+ AUDIO_CHANNEL_OUT_5POINT1_SIDE, AUDIO_CHANNEL_INDEX_MASK_6,
+ AUDIO_CHANNEL_OUT_6POINT1, AUDIO_CHANNEL_INDEX_MASK_7,
+ AUDIO_CHANNEL_OUT_5POINT1POINT2, AUDIO_CHANNEL_OUT_7POINT1,
+ AUDIO_CHANNEL_INDEX_MASK_8, AUDIO_CHANNEL_INDEX_MASK_9,
+ AUDIO_CHANNEL_INDEX_MASK_10, AUDIO_CHANNEL_INDEX_MASK_11,
+ AUDIO_CHANNEL_INDEX_MASK_12, AUDIO_CHANNEL_INDEX_MASK_13,
+ AUDIO_CHANNEL_INDEX_MASK_14, AUDIO_CHANNEL_INDEX_MASK_15,
+ AUDIO_CHANNEL_INDEX_MASK_16, AUDIO_CHANNEL_INDEX_MASK_17,
+ AUDIO_CHANNEL_INDEX_MASK_18, AUDIO_CHANNEL_INDEX_MASK_19,
+ AUDIO_CHANNEL_INDEX_MASK_20, AUDIO_CHANNEL_INDEX_MASK_21,
+ AUDIO_CHANNEL_INDEX_MASK_22, AUDIO_CHANNEL_INDEX_MASK_23,
+ AUDIO_CHANNEL_INDEX_MASK_24,
+};
+
+constexpr size_t kNumChMasks = std::size(kChMasks);
+
+constexpr size_t kSampleRates[] = {8000, 11025, 12000, 16000, 22050, 24000, 32000,
+ 44100, 48000, 88200, 96000, 176400, 192000};
+
+constexpr size_t kNumSampleRates = std::size(kSampleRates);
+
+constexpr size_t kFrameCounts[] = {4, 2048};
+
+constexpr size_t kNumFrameCounts = std::size(kFrameCounts);
+
+constexpr size_t kLoopCounts[] = {1, 4};
+
+constexpr size_t kNumLoopCounts = std::size(kLoopCounts);
+
+class EffectBundleHelper {
+ public:
+ EffectBundleHelper(const effect_uuid_t* uuid, size_t chMask, size_t sampleRate,
+ size_t frameCount, size_t loopCount)
+ : mUuid(uuid),
+ mChMask(chMask),
+ mChannelCount(audio_channel_count_from_out_mask(mChMask)),
+ mSampleRate(sampleRate),
+ mFrameCount(frameCount),
+ mLoopCount(loopCount) {}
+ void createEffect();
+ void releaseEffect();
+ void configEffect();
+ void process(float* input, float* output);
+
+ private:
+ const effect_uuid_t* mUuid;
+ const size_t mChMask;
+ const size_t mChannelCount;
+ const size_t mSampleRate;
+ const size_t mFrameCount;
+ const size_t mLoopCount;
+ effect_handle_t mEffectHandle{};
+};
+
+void EffectBundleHelper::createEffect() {
+ int status = AUDIO_EFFECT_LIBRARY_INFO_SYM.create_effect(mUuid, 1, 1, &mEffectHandle);
+ ASSERT_EQ(status, 0) << "create_effect returned an error " << status << "\n";
+}
+
+void EffectBundleHelper::releaseEffect() {
+ int status = AUDIO_EFFECT_LIBRARY_INFO_SYM.release_effect(mEffectHandle);
+ ASSERT_EQ(status, 0) << "release_effect returned an error " << status << "\n";
+}
+
+void EffectBundleHelper::configEffect() {
+ effect_config_t config{};
+ config.inputCfg.samplingRate = config.outputCfg.samplingRate = mSampleRate;
+ config.inputCfg.channels = config.outputCfg.channels = mChMask;
+ config.inputCfg.format = config.outputCfg.format = AUDIO_FORMAT_PCM_FLOAT;
+
+ int reply = 0;
+ uint32_t replySize = sizeof(reply);
+ int status = (*mEffectHandle)
+ ->command(mEffectHandle, EFFECT_CMD_SET_CONFIG, sizeof(effect_config_t),
+ &config, &replySize, &reply);
+ ASSERT_EQ(status, 0) << "command returned an error " << status << "\n";
+ ASSERT_EQ(reply, 0) << "command reply non zero " << reply << "\n";
+
+ status = (*mEffectHandle)
+ ->command(mEffectHandle, EFFECT_CMD_ENABLE, 0, nullptr, &replySize, &reply);
+ ASSERT_EQ(status, 0) << "command enable returned an error " << status << "\n";
+ ASSERT_EQ(reply, 0) << "command reply non zero " << reply << "\n";
+}
+
+void EffectBundleHelper::process(float* input, float* output) {
+ audio_buffer_t inBuffer = {.frameCount = mFrameCount, .f32 = input};
+ audio_buffer_t outBuffer = {.frameCount = mFrameCount, .f32 = output};
+ for (size_t i = 0; i < mLoopCount; i++) {
+ int status = (*mEffectHandle)->process(mEffectHandle, &inBuffer, &outBuffer);
+ ASSERT_EQ(status, 0) << "process returned an error " << status << "\n";
+
+ inBuffer.f32 += mFrameCount * mChannelCount;
+ outBuffer.f32 += mFrameCount * mChannelCount;
+ }
+}
+
+typedef std::tuple<int, int, int, int, int> SingleEffectTestParam;
+class SingleEffectTest : public ::testing::TestWithParam<SingleEffectTestParam> {
+ public:
+ SingleEffectTest()
+ : mChMask(kChMasks[std::get<0>(GetParam())]),
+ mChannelCount(audio_channel_count_from_out_mask(mChMask)),
+ mSampleRate(kSampleRates[std::get<1>(GetParam())]),
+ mFrameCount(kFrameCounts[std::get<2>(GetParam())]),
+ mLoopCount(kLoopCounts[std::get<3>(GetParam())]),
+ mTotalFrameCount(mFrameCount * mLoopCount),
+ mUuid(&kEffectUuids[std::get<4>(GetParam())]) {}
+
+ const size_t mChMask;
+ const size_t mChannelCount;
+ const size_t mSampleRate;
+ const size_t mFrameCount;
+ const size_t mLoopCount;
+ const size_t mTotalFrameCount;
+ const effect_uuid_t* mUuid;
+};
+
+// Tests applying a single effect
+TEST_P(SingleEffectTest, SimpleProcess) {
+ SCOPED_TRACE(testing::Message()
+ << "chMask: " << mChMask << " sampleRate: " << mSampleRate
+ << " frameCount: " << mFrameCount << " loopCount: " << mLoopCount);
+
+ EffectBundleHelper effect(mUuid, mChMask, mSampleRate, mFrameCount, mLoopCount);
+
+ ASSERT_NO_FATAL_FAILURE(effect.createEffect());
+ ASSERT_NO_FATAL_FAILURE(effect.configEffect());
+
+ // Initialize input buffer with deterministic pseudo-random values
+ std::vector<float> input(mTotalFrameCount * mChannelCount);
+ std::vector<float> output(mTotalFrameCount * mChannelCount);
+ std::minstd_rand gen(mChMask);
+ std::uniform_real_distribution<> dis(-1.0f, 1.0f);
+ for (auto& in : input) {
+ in = dis(gen);
+ }
+ ASSERT_NO_FATAL_FAILURE(effect.process(input.data(), output.data()));
+ ASSERT_NO_FATAL_FAILURE(effect.releaseEffect());
+}
+
+INSTANTIATE_TEST_SUITE_P(EffectBundleTestAll, SingleEffectTest,
+ ::testing::Combine(::testing::Range(0, (int)kNumChMasks),
+ ::testing::Range(0, (int)kNumSampleRates),
+ ::testing::Range(0, (int)kNumFrameCounts),
+ ::testing::Range(0, (int)kNumLoopCounts),
+ ::testing::Range(0, (int)kNumEffectUuids)));
+
+typedef std::tuple<int, int, int, int> SingleEffectComparisonTestParam;
+class SingleEffectComparisonTest
+ : public ::testing::TestWithParam<SingleEffectComparisonTestParam> {
+ public:
+ SingleEffectComparisonTest()
+ : mSampleRate(kSampleRates[std::get<0>(GetParam())]),
+ mFrameCount(kFrameCounts[std::get<1>(GetParam())]),
+ mLoopCount(kLoopCounts[std::get<2>(GetParam())]),
+ mTotalFrameCount(mFrameCount * mLoopCount),
+ mUuid(&kEffectUuids[std::get<3>(GetParam())]) {}
+
+ const size_t mSampleRate;
+ const size_t mFrameCount;
+ const size_t mLoopCount;
+ const size_t mTotalFrameCount;
+ const effect_uuid_t* mUuid;
+};
+
+template <typename T>
+float computeSnr(const T* ref, const T* tst, size_t count) {
+ double signal{};
+ double noise{};
+
+ for (size_t i = 0; i < count; ++i) {
+ const double value(ref[i]);
+ const double diff(tst[i] - value);
+ signal += value * value;
+ noise += diff * diff;
+ }
+ // Initialized to a value greater than kSNRThreshold to handle
+ // cases where ref and tst match exactly
+ float snr = kSNRThreshold + 1.0f;
+ if (signal > 0.0f && noise > 0.0f) {
+ snr = 10.f * log(signal / noise);
+ }
+ return snr;
+}
+
+// Compares first two channels in multi-channel output to stereo output when same effect is applied
+TEST_P(SingleEffectComparisonTest, SimpleProcess) {
+ SCOPED_TRACE(testing::Message() << " sampleRate: " << mSampleRate << " frameCount: "
+ << mFrameCount << " loopCount: " << mLoopCount);
+
+ // Initialize mono input buffer with deterministic pseudo-random values
+ std::vector<float> monoInput(mTotalFrameCount);
+
+ std::minstd_rand gen(mSampleRate);
+ std::uniform_real_distribution<> dis(-1.0f, 1.0f);
+ for (auto& in : monoInput) {
+ in = dis(gen);
+ }
+
+ // Generate stereo by repeating mono channel data
+ std::vector<float> stereoInput(mTotalFrameCount * FCC_2);
+ adjust_channels(monoInput.data(), FCC_1, stereoInput.data(), FCC_2, sizeof(float),
+ mTotalFrameCount * sizeof(float) * FCC_1);
+
+ // Apply effect on stereo channels
+ EffectBundleHelper stereoEffect(mUuid, AUDIO_CHANNEL_OUT_STEREO, mSampleRate, mFrameCount,
+ mLoopCount);
+
+ ASSERT_NO_FATAL_FAILURE(stereoEffect.createEffect());
+ ASSERT_NO_FATAL_FAILURE(stereoEffect.configEffect());
+
+ std::vector<float> stereoOutput(mTotalFrameCount * FCC_2);
+ ASSERT_NO_FATAL_FAILURE(stereoEffect.process(stereoInput.data(), stereoOutput.data()));
+ ASSERT_NO_FATAL_FAILURE(stereoEffect.releaseEffect());
+
+ // Convert stereo float data to stereo int16_t to be used as reference
+ std::vector<int16_t> stereoRefI16(mTotalFrameCount * FCC_2);
+ memcpy_to_i16_from_float(stereoRefI16.data(), stereoOutput.data(), mTotalFrameCount * FCC_2);
+
+ for (size_t chMask : kChMasks) {
+ size_t channelCount = audio_channel_count_from_out_mask(chMask);
+ EffectBundleHelper testEffect(mUuid, chMask, mSampleRate, mFrameCount, mLoopCount);
+
+ ASSERT_NO_FATAL_FAILURE(testEffect.createEffect());
+ ASSERT_NO_FATAL_FAILURE(testEffect.configEffect());
+
+ std::vector<float> testInput(mTotalFrameCount * channelCount);
+
+ // Repeat mono channel data to all the channels
+ // adjust_channels() zero fills channels > 2, hence can't be used here
+ for (size_t i = 0; i < mTotalFrameCount; ++i) {
+ auto* fp = &testInput[i * channelCount];
+ std::fill(fp, fp + channelCount, monoInput[i]);
+ }
+
+ std::vector<float> testOutput(mTotalFrameCount * channelCount);
+ ASSERT_NO_FATAL_FAILURE(testEffect.process(testInput.data(), testOutput.data()));
+ ASSERT_NO_FATAL_FAILURE(testEffect.releaseEffect());
+
+ // Extract first two channels
+ std::vector<float> stereoTestOutput(mTotalFrameCount * FCC_2);
+ adjust_channels(testOutput.data(), channelCount, stereoTestOutput.data(), FCC_2,
+ sizeof(float), mTotalFrameCount * sizeof(float) * channelCount);
+
+ // Convert the test data to int16_t
+ std::vector<int16_t> stereoTestI16(mTotalFrameCount * FCC_2);
+ memcpy_to_i16_from_float(stereoTestI16.data(), stereoTestOutput.data(),
+ mTotalFrameCount * FCC_2);
+
+ if (isBassBoost(mUuid)) {
+ // SNR must be above the threshold
+ float snr = computeSnr<int16_t>(stereoRefI16.data(), stereoTestI16.data(),
+ mTotalFrameCount * FCC_2);
+ ASSERT_GT(snr, kSNRThreshold) << "SNR " << snr << "is lower than " << kSNRThreshold;
+ } else {
+ ASSERT_EQ(0,
+ memcmp(stereoRefI16.data(), stereoTestI16.data(), mTotalFrameCount * FCC_2))
+ << "First two channels do not match with stereo output \n";
+ }
+ }
+}
+
+INSTANTIATE_TEST_SUITE_P(EffectBundleTestAll, SingleEffectComparisonTest,
+ ::testing::Combine(::testing::Range(0, (int)kNumSampleRates),
+ ::testing::Range(0, (int)kNumFrameCounts),
+ ::testing::Range(0, (int)kNumLoopCounts),
+ ::testing::Range(0, (int)kNumEffectUuids)));
+
+int main(int argc, char** argv) {
+ ::testing::InitGoogleTest(&argc, argv);
+ int status = RUN_ALL_TESTS();
+ ALOGV("Test result = %d\n", status);
+ return status;
+}
diff --git a/media/libeffects/lvm/tests/build_and_run_all_unit_tests.sh b/media/libeffects/lvm/tests/build_and_run_all_unit_tests.sh
index 7b0ff5e..df7ca5a 100755
--- a/media/libeffects/lvm/tests/build_and_run_all_unit_tests.sh
+++ b/media/libeffects/lvm/tests/build_and_run_all_unit_tests.sh
@@ -102,6 +102,11 @@
((++error_count))
fi
+ # Do not compare cases where -vcBal is in flags and chMask is 0 (due to
+ # stereo computation)
+ if [[ $flags == *"-vcBal:"* ]] && [[ $chMask -eq 0 ]]; then
+ continue
+ fi
# two channel files should be identical to higher channel
# computation (first 2 channels).
diff --git a/media/libeffects/lvm/tests/lvmtest.cpp b/media/libeffects/lvm/tests/lvmtest.cpp
index f107b18..e484a1a 100644
--- a/media/libeffects/lvm/tests/lvmtest.cpp
+++ b/media/libeffects/lvm/tests/lvmtest.cpp
@@ -415,52 +415,11 @@
} else {
return -EINVAL;
}
-
- LVM_Fs_en sampleRate;
- switch (plvmConfigParams->samplingFreq) {
- case 8000:
- sampleRate = LVM_FS_8000;
- break;
- case 11025:
- sampleRate = LVM_FS_11025;
- break;
- case 12000:
- sampleRate = LVM_FS_12000;
- break;
- case 16000:
- sampleRate = LVM_FS_16000;
- break;
- case 22050:
- sampleRate = LVM_FS_22050;
- break;
- case 24000:
- sampleRate = LVM_FS_24000;
- break;
- case 32000:
- sampleRate = LVM_FS_32000;
- break;
- case 44100:
- sampleRate = LVM_FS_44100;
- break;
- case 48000:
- sampleRate = LVM_FS_48000;
- break;
- case 88200:
- sampleRate = LVM_FS_88200;
- break;
- case 96000:
- sampleRate = LVM_FS_96000;
- break;
- case 176400:
- sampleRate = LVM_FS_176400;
- break;
- case 192000:
- sampleRate = LVM_FS_192000;
- break;
- default:
- return -EINVAL;
+ params->SampleRate = lvmFsForSampleRate(plvmConfigParams->samplingFreq);
+ if (params->SampleRate == LVM_FS_INVALID) {
+ ALOGE("lvmControl invalid sampling rate %d", plvmConfigParams->samplingFreq);
+ return -EINVAL;
}
- params->SampleRate = sampleRate;
/* Concert Sound parameters */
params->VirtualizerOperatingMode = plvmConfigParams->csEnable;
@@ -530,19 +489,11 @@
const int ioChannelCount = plvmConfigParams->fChannels;
const int ioFrameSize = ioChannelCount * sizeof(short); // file load size
const int maxChannelCount = std::max(channelCount, ioChannelCount);
- /*
- * Mono input will be converted to 2 channels internally in the process call
- * by copying the same data into the second channel.
- * Hence when channelCount is 1, output buffer should be allocated for
- * 2 channels. The memAllocChCount takes care of allocation of sufficient
- * memory for the output buffer.
- */
- const int memAllocChCount = (channelCount == 1 ? 2 : channelCount);
std::vector<short> in(frameLength * maxChannelCount);
std::vector<short> out(frameLength * maxChannelCount);
std::vector<float> floatIn(frameLength * channelCount);
- std::vector<float> floatOut(frameLength * memAllocChCount);
+ std::vector<float> floatOut(frameLength * channelCount);
int frameCounter = 0;
while (fread(in.data(), ioFrameSize, frameLength, finp) == (size_t)frameLength) {
diff --git a/media/libeffects/lvm/wrapper/Android.bp b/media/libeffects/lvm/wrapper/Android.bp
index 09c4aef..e169e3c 100644
--- a/media/libeffects/lvm/wrapper/Android.bp
+++ b/media/libeffects/lvm/wrapper/Android.bp
@@ -67,6 +67,7 @@
},
vendor: true,
+ host_supported: true,
srcs: ["Reverb/EffectReverb.cpp"],
cppflags: [
@@ -83,7 +84,6 @@
shared_libs: [
"libaudioutils",
"libcutils",
- "libdl",
"liblog",
],
diff --git a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp
index 46dbf7e..9ccccb4 100644
--- a/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp
+++ b/media/libeffects/lvm/wrapper/Bundle/EffectBundle.cpp
@@ -957,51 +957,12 @@
pContext->config = *pConfig;
const LVM_INT16 NrChannels = audio_channel_count_from_out_mask(pConfig->inputCfg.channels);
- switch (pConfig->inputCfg.samplingRate) {
- case 8000:
- SampleRate = LVM_FS_8000;
- pContext->pBundledContext->SamplesPerSecond = 8000 * NrChannels;
- break;
- case 16000:
- SampleRate = LVM_FS_16000;
- pContext->pBundledContext->SamplesPerSecond = 16000 * NrChannels;
- break;
- case 22050:
- SampleRate = LVM_FS_22050;
- pContext->pBundledContext->SamplesPerSecond = 22050 * NrChannels;
- break;
- case 32000:
- SampleRate = LVM_FS_32000;
- pContext->pBundledContext->SamplesPerSecond = 32000 * NrChannels;
- break;
- case 44100:
- SampleRate = LVM_FS_44100;
- pContext->pBundledContext->SamplesPerSecond = 44100 * NrChannels;
- break;
- case 48000:
- SampleRate = LVM_FS_48000;
- pContext->pBundledContext->SamplesPerSecond = 48000 * NrChannels;
- break;
- case 88200:
- SampleRate = LVM_FS_88200;
- pContext->pBundledContext->SamplesPerSecond = 88200 * NrChannels;
- break;
- case 96000:
- SampleRate = LVM_FS_96000;
- pContext->pBundledContext->SamplesPerSecond = 96000 * NrChannels;
- break;
- case 176400:
- SampleRate = LVM_FS_176400;
- pContext->pBundledContext->SamplesPerSecond = 176400 * NrChannels;
- break;
- case 192000:
- SampleRate = LVM_FS_192000;
- pContext->pBundledContext->SamplesPerSecond = 192000 * NrChannels;
- break;
- default:
- ALOGV("\tEffect_setConfig invalid sampling rate %d", pConfig->inputCfg.samplingRate);
- return -EINVAL;
+ SampleRate = lvmFsForSampleRate(pConfig->inputCfg.samplingRate);
+ if (SampleRate == LVM_FS_INVALID) {
+ ALOGV("Effect_setConfig invalid sampling rate %d", pConfig->inputCfg.samplingRate);
+ return -EINVAL;
}
+ pContext->pBundledContext->SamplesPerSecond = pConfig->inputCfg.samplingRate * NrChannels;
if (pContext->pBundledContext->SampleRate != SampleRate ||
pContext->pBundledContext->ChMask != pConfig->inputCfg.channels) {
diff --git a/media/libeffects/lvm/wrapper/Reverb/EffectReverb.cpp b/media/libeffects/lvm/wrapper/Reverb/EffectReverb.cpp
index 9ea70ce..4489e81 100644
--- a/media/libeffects/lvm/wrapper/Reverb/EffectReverb.cpp
+++ b/media/libeffects/lvm/wrapper/Reverb/EffectReverb.cpp
@@ -190,8 +190,8 @@
/* Effect Library Interface Implementation */
-extern "C" int EffectCreate(const effect_uuid_t* uuid, int32_t sessionId __unused,
- int32_t ioId __unused, effect_handle_t* pHandle) {
+extern "C" int EffectCreate(const effect_uuid_t* uuid, int32_t /* sessionId __unused */,
+ int32_t /* ioId __unused */, effect_handle_t* pHandle) {
int ret;
int i;
int length = sizeof(gDescriptors) / sizeof(const effect_descriptor_t*);
@@ -517,6 +517,9 @@
}
}
}
+
+ LvmStatus = LVREV_FreeInstance(pContext->hInstance);
+ LVM_ERROR_CHECK(LvmStatus, "LVREV_FreeInstance", "Reverb_free")
} /* end Reverb_free */
//----------------------------------------------------------------------------
@@ -553,40 +556,10 @@
// ALOGV("\tReverb_setConfig calling memcpy");
pContext->config = *pConfig;
- switch (pConfig->inputCfg.samplingRate) {
- case 8000:
- SampleRate = LVM_FS_8000;
- break;
- case 16000:
- SampleRate = LVM_FS_16000;
- break;
- case 22050:
- SampleRate = LVM_FS_22050;
- break;
- case 32000:
- SampleRate = LVM_FS_32000;
- break;
- case 44100:
- SampleRate = LVM_FS_44100;
- break;
- case 48000:
- SampleRate = LVM_FS_48000;
- break;
- case 88200:
- SampleRate = LVM_FS_88200;
- break;
- case 96000:
- SampleRate = LVM_FS_96000;
- break;
- case 176400:
- SampleRate = LVM_FS_176400;
- break;
- case 192000:
- SampleRate = LVM_FS_192000;
- break;
- default:
- ALOGV("\rReverb_setConfig invalid sampling rate %d", pConfig->inputCfg.samplingRate);
- return -EINVAL;
+ SampleRate = lvmFsForSampleRate(pConfig->inputCfg.samplingRate);
+ if (SampleRate == LVM_FS_INVALID) {
+ ALOGE("Reverb_setConfig invalid sampling rate %d", pConfig->inputCfg.samplingRate);
+ return -EINVAL;
}
if (pContext->SampleRate != SampleRate) {
diff --git a/media/libmedia/IMediaPlayer.cpp b/media/libmedia/IMediaPlayer.cpp
index 134e6fe..8a4b17c 100644
--- a/media/libmedia/IMediaPlayer.cpp
+++ b/media/libmedia/IMediaPlayer.cpp
@@ -697,7 +697,7 @@
}
case SET_DATA_SOURCE_RTP: {
CHECK_INTERFACE(IMediaPlayer, data, reply);
- const String8& rtpParams = data.readString8();
+ String8 rtpParams = data.readString8();
reply->writeInt32(setDataSource(rtpParams));
return NO_ERROR;
}
diff --git a/media/libmedia/IMediaRecorder.cpp b/media/libmedia/IMediaRecorder.cpp
index ac86f72..154988d 100644
--- a/media/libmedia/IMediaRecorder.cpp
+++ b/media/libmedia/IMediaRecorder.cpp
@@ -66,6 +66,7 @@
ENABLE_AUDIO_DEVICE_CALLBACK,
GET_ACTIVE_MICROPHONES,
GET_PORT_ID,
+ GET_RTP_DATA_USAGE,
SET_PREFERRED_MICROPHONE_DIRECTION,
SET_PREFERRED_MICROPHONE_FIELD_DIMENSION,
SET_PRIVACY_SENSITIVE,
@@ -476,6 +477,23 @@
*portId = (audio_port_handle_t)reply.readInt32();
return NO_ERROR;
}
+
+ status_t getRtpDataUsage(uint64_t *bytes)
+ {
+ ALOGV("getRtpDataUsage");
+ if (bytes == nullptr) {
+ return BAD_VALUE;
+ }
+ Parcel data, reply;
+ data.writeInterfaceToken(IMediaRecorder::getInterfaceDescriptor());
+ status_t status = remote()->transact(GET_RTP_DATA_USAGE, data, &reply);
+ if (status != OK
+ || (status = (status_t)reply.readInt32()) != NO_ERROR) {
+ *bytes = 0;
+ return status;
+ }
+ return reply.readUint64(bytes);
+ }
};
IMPLEMENT_META_INTERFACE(MediaRecorder, "android.media.IMediaRecorder");
@@ -759,6 +777,17 @@
}
return NO_ERROR;
}
+ case GET_RTP_DATA_USAGE: {
+ ALOGV("GET_RTP_DATA_USAGE");
+ CHECK_INTERFACE(IMediaRecorder, data, reply);
+ uint64_t bytes;
+ status_t status = getRtpDataUsage(&bytes);
+ reply->writeInt32(status);
+ if (status == NO_ERROR) {
+ reply->writeUint64(bytes);
+ }
+ return NO_ERROR;
+ }
case SET_PREFERRED_MICROPHONE_DIRECTION: {
ALOGV("SET_PREFERRED_MICROPHONE_DIRECTION");
CHECK_INTERFACE(IMediaRecorder, data, reply);
diff --git a/media/libmedia/include/media/IMediaRecorder.h b/media/libmedia/include/media/IMediaRecorder.h
index 651bd5e..6e69782 100644
--- a/media/libmedia/include/media/IMediaRecorder.h
+++ b/media/libmedia/include/media/IMediaRecorder.h
@@ -78,6 +78,7 @@
virtual status_t setPreferredMicrophoneDirection(audio_microphone_direction_t direction) = 0;
virtual status_t setPreferredMicrophoneFieldDimension(float zoom) = 0;
virtual status_t getPortId(audio_port_handle_t *portId) = 0;
+ virtual status_t getRtpDataUsage(uint64_t *bytes) = 0;
};
// ----------------------------------------------------------------------------
diff --git a/media/libmedia/include/media/MediaRecorderBase.h b/media/libmedia/include/media/MediaRecorderBase.h
index 8493f64..d9a7efb 100644
--- a/media/libmedia/include/media/MediaRecorderBase.h
+++ b/media/libmedia/include/media/MediaRecorderBase.h
@@ -77,6 +77,7 @@
virtual status_t setPreferredMicrophoneDirection(audio_microphone_direction_t direction) = 0;
virtual status_t setPreferredMicrophoneFieldDimension(float zoom) = 0;
virtual status_t getPortId(audio_port_handle_t *portId) const = 0;
+ virtual status_t getRtpDataUsage(uint64_t *bytes) = 0;
diff --git a/media/libmedia/include/media/mediaplayer.h b/media/libmedia/include/media/mediaplayer.h
index 9c5f61e..71c0bc5 100644
--- a/media/libmedia/include/media/mediaplayer.h
+++ b/media/libmedia/include/media/mediaplayer.h
@@ -62,6 +62,7 @@
MEDIA_META_DATA = 202,
MEDIA_DRM_INFO = 210,
MEDIA_TIME_DISCONTINUITY = 211,
+ MEDIA_IMS_RX_NOTICE = 300,
MEDIA_AUDIO_ROUTING_CHANGED = 10000,
};
@@ -179,7 +180,10 @@
KEY_PARAMETER_PLAYBACK_RATE_PERMILLE = 1300, // set only
// Set a Parcel containing the value of a parcelled Java AudioAttribute instance
- KEY_PARAMETER_AUDIO_ATTRIBUTES = 1400 // set only
+ KEY_PARAMETER_AUDIO_ATTRIBUTES = 1400, // set only
+
+ // Set a Parcel containing the values of RTP attribute
+ KEY_PARAMETER_RTP_ATTRIBUTES = 2000 // set only
};
// Keep INVOKE_ID_* in sync with MediaPlayer.java.
diff --git a/media/libmedia/include/media/mediarecorder.h b/media/libmedia/include/media/mediarecorder.h
index 6e2d94d..84c92f6 100644
--- a/media/libmedia/include/media/mediarecorder.h
+++ b/media/libmedia/include/media/mediarecorder.h
@@ -270,6 +270,7 @@
status_t setPreferredMicrophoneFieldDimension(float zoom);
status_t getPortId(audio_port_handle_t *portId) const;
+ status_t getRtpDataUsage(uint64_t *bytes);
private:
void doCleanUp();
@@ -291,6 +292,8 @@
bool mIsOutputFileSet;
Mutex mLock;
Mutex mNotifyLock;
+
+ output_format mOutputFormat;
};
}; // namespace android
diff --git a/media/libmedia/mediaplayer.cpp b/media/libmedia/mediaplayer.cpp
index b0db9d5..1554920 100644
--- a/media/libmedia/mediaplayer.cpp
+++ b/media/libmedia/mediaplayer.cpp
@@ -959,6 +959,9 @@
case MEDIA_META_DATA:
ALOGV("Received timed metadata message");
break;
+ case MEDIA_IMS_RX_NOTICE:
+ ALOGV("Received IMS Rx notice message");
+ break;
default:
ALOGV("unrecognized message: (%d, %d, %d)", msg, ext1, ext2);
break;
diff --git a/media/libmedia/mediarecorder.cpp b/media/libmedia/mediarecorder.cpp
index 70655d5..e3cd9d8 100644
--- a/media/libmedia/mediarecorder.cpp
+++ b/media/libmedia/mediarecorder.cpp
@@ -244,6 +244,7 @@
mCurrentState = MEDIA_RECORDER_ERROR;
return ret;
}
+ mOutputFormat = (output_format)of;
mCurrentState = MEDIA_RECORDER_DATASOURCE_CONFIGURED;
return ret;
}
@@ -479,6 +480,13 @@
(MEDIA_RECORDER_PREPARED |
MEDIA_RECORDER_RECORDING |
MEDIA_RECORDER_ERROR));
+
+ // For RTP video, parameter should be set dynamically.
+ if (isInvalidState) {
+ if (mCurrentState == MEDIA_RECORDER_RECORDING &&
+ mOutputFormat == OUTPUT_FORMAT_RTP_AVP)
+ isInvalidState = false;
+ }
if (isInvalidState) {
ALOGE("setParameters is called in an invalid state: %d", mCurrentState);
return INVALID_OPERATION;
@@ -737,6 +745,7 @@
mIsAudioEncoderSet = false;
mIsVideoEncoderSet = false;
mIsOutputFileSet = false;
+ mOutputFormat = OUTPUT_FORMAT_DEFAULT;
}
// Release should be OK in any state
@@ -904,4 +913,14 @@
return mMediaRecorder->getPortId(portId);
}
+status_t MediaRecorder::getRtpDataUsage(uint64_t *bytes)
+{
+ ALOGV("getRtpDataUsage");
+
+ if (mMediaRecorder == NULL) {
+ ALOGE("media recorder is not initialized yet");
+ return INVALID_OPERATION;
+ }
+ return mMediaRecorder->getRtpDataUsage(bytes);
+}
} // namespace android
diff --git a/media/libmediaplayerservice/Android.bp b/media/libmediaplayerservice/Android.bp
index e4c1685..9ab117a 100644
--- a/media/libmediaplayerservice/Android.bp
+++ b/media/libmediaplayerservice/Android.bp
@@ -35,6 +35,7 @@
"android.hardware.media.c2@1.0",
"android.hardware.media.omx@1.0",
"libbase",
+ "libandroid_net",
"libaudioclient",
"libbinder",
"libcamera_client",
diff --git a/media/libmediaplayerservice/MediaRecorderClient.cpp b/media/libmediaplayerservice/MediaRecorderClient.cpp
index 9b1974b..9f16a22 100644
--- a/media/libmediaplayerservice/MediaRecorderClient.cpp
+++ b/media/libmediaplayerservice/MediaRecorderClient.cpp
@@ -28,6 +28,7 @@
#include <binder/IServiceManager.h>
#include <binder/MemoryHeapBase.h>
#include <binder/MemoryBase.h>
+#include <camera/CameraUtils.h>
#include <codec2/hidl/client.h>
#include <cutils/atomic.h>
#include <cutils/properties.h> // for property_get
@@ -423,30 +424,35 @@
sp<IServiceManager> sm = defaultServiceManager();
- // WORKAROUND: We don't know if camera exists here and getService might block for 5 seconds.
- // Use checkService for camera if we don't know it exists.
- static std::atomic<bool> sCameraChecked(false); // once true never becomes false.
- static std::atomic<bool> sCameraVerified(false); // once true never becomes false.
- sp<IBinder> binder = (sCameraVerified || !sCameraChecked)
- ? sm->getService(String16("media.camera")) : sm->checkService(String16("media.camera"));
- // If the device does not have a camera, do not create a death listener for it.
- if (binder != NULL) {
- sCameraVerified = true;
- mDeathNotifiers.emplace_back(
- binder, [l = wp<IMediaRecorderClient>(listener)](){
- sp<IMediaRecorderClient> listener = l.promote();
- if (listener) {
- ALOGV("media.camera service died. "
- "Sending death notification.");
- listener->notify(
- MEDIA_ERROR, MEDIA_ERROR_SERVER_DIED,
- MediaPlayerService::CAMERA_PROCESS_DEATH);
- } else {
- ALOGW("media.camera service died without a death handler.");
- }
- });
+ static const bool sCameraDisabled = CameraUtils::isCameraServiceDisabled();
+
+ if (!sCameraDisabled) {
+ // WORKAROUND: We don't know if camera exists here and getService might block for 5 seconds.
+ // Use checkService for camera if we don't know it exists.
+ static std::atomic<bool> sCameraChecked(false); // once true never becomes false.
+ static std::atomic<bool> sCameraVerified(false); // once true never becomes false.
+
+ sp<IBinder> binder = (sCameraVerified || !sCameraChecked)
+ ? sm->getService(String16("media.camera")) : sm->checkService(String16("media.camera"));
+ // If the device does not have a camera, do not create a death listener for it.
+ if (binder != NULL) {
+ sCameraVerified = true;
+ mDeathNotifiers.emplace_back(
+ binder, [l = wp<IMediaRecorderClient>(listener)](){
+ sp<IMediaRecorderClient> listener = l.promote();
+ if (listener) {
+ ALOGV("media.camera service died. "
+ "Sending death notification.");
+ listener->notify(
+ MEDIA_ERROR, MEDIA_ERROR_SERVER_DIED,
+ MediaPlayerService::CAMERA_PROCESS_DEATH);
+ } else {
+ ALOGW("media.camera service died without a death handler.");
+ }
+ });
+ }
+ sCameraChecked = true;
}
- sCameraChecked = true;
{
using ::android::hidl::base::V1_0::IBase;
@@ -585,4 +591,13 @@
}
return NO_INIT;
}
+
+status_t MediaRecorderClient::getRtpDataUsage(uint64_t *bytes) {
+ ALOGV("getRtpDataUsage");
+ Mutex::Autolock lock(mLock);
+ if (mRecorder != NULL) {
+ return mRecorder->getRtpDataUsage(bytes);
+ }
+ return NO_INIT;
+}
}; // namespace android
diff --git a/media/libmediaplayerservice/MediaRecorderClient.h b/media/libmediaplayerservice/MediaRecorderClient.h
index 12257e5..e041855 100644
--- a/media/libmediaplayerservice/MediaRecorderClient.h
+++ b/media/libmediaplayerservice/MediaRecorderClient.h
@@ -86,6 +86,7 @@
virtual status_t setPreferredMicrophoneDirection(audio_microphone_direction_t direction);
virtual status_t setPreferredMicrophoneFieldDimension(float zoom);
status_t getPortId(audio_port_handle_t *portId) override;
+ virtual status_t getRtpDataUsage(uint64_t *bytes);
private:
friend class MediaPlayerService; // for accessing private constructor
diff --git a/media/libmediaplayerservice/StagefrightRecorder.cpp b/media/libmediaplayerservice/StagefrightRecorder.cpp
index 93cf5bd..ecbdf61 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.cpp
+++ b/media/libmediaplayerservice/StagefrightRecorder.cpp
@@ -17,6 +17,9 @@
//#define LOG_NDEBUG 0
#define LOG_TAG "StagefrightRecorder"
#include <inttypes.h>
+// TODO/workaround: including base logging now as it conflicts with ADebug.h
+// and it must be included first.
+#include <android-base/logging.h>
#include <utils/Log.h>
#include "WebmWriter.h"
@@ -44,6 +47,7 @@
#include <media/stagefright/CameraSourceTimeLapse.h>
#include <media/stagefright/MPEG2TSWriter.h>
#include <media/stagefright/MPEG4Writer.h>
+#include <media/stagefright/MediaCodecConstants.h>
#include <media/stagefright/MediaDefs.h>
#include <media/stagefright/MetaData.h>
#include <media/stagefright/MediaCodecSource.h>
@@ -117,6 +121,11 @@
mAudioSource((audio_source_t)AUDIO_SOURCE_CNT), // initialize with invalid value
mPrivacySensitive(PRIVACY_SENSITIVE_DEFAULT),
mVideoSource(VIDEO_SOURCE_LIST_END),
+ mRTPCVOExtMap(-1),
+ mRTPCVODegrees(0),
+ mRTPSockDscp(0),
+ mRTPSockNetwork(0),
+ mLastSeqNo(0),
mStarted(false),
mSelectedDeviceId(AUDIO_PORT_HANDLE_NONE),
mDeviceCallbackEnabled(false),
@@ -570,6 +579,32 @@
// range that a specific encoder supports. The mismatch between the
// the target and requested bit rate will NOT be treated as an error.
mVideoBitRate = bitRate;
+
+ // A new bitrate(TMMBR) should be applied on runtime as well if OutputFormat is RTP_AVP
+ if (mOutputFormat == OUTPUT_FORMAT_RTP_AVP) {
+ // Regular I frames may overload the network so we reduce the bitrate to allow
+ // margins for the I frame overruns.
+ // Still send requested bitrate (TMMBR) in the reply (TMMBN).
+ const float coefficient = 0.8f;
+ mVideoBitRate = (bitRate * coefficient) / 1000 * 1000;
+ }
+ if (mOutputFormat == OUTPUT_FORMAT_RTP_AVP && mStarted && mPauseStartTimeUs == 0) {
+ mVideoEncoderSource->setEncodingBitrate(mVideoBitRate);
+ ARTPWriter* rtpWriter = static_cast<ARTPWriter*>(mWriter.get());
+ rtpWriter->setTMMBNInfo(mOpponentID, bitRate);
+ }
+
+ return OK;
+}
+
+status_t StagefrightRecorder::setParamVideoBitRateMode(int32_t bitRateMode) {
+ ALOGV("setParamVideoBitRateMode: %d", bitRateMode);
+ // TODO: clarify what bitrate mode of -1 is as these start from 0
+ if (bitRateMode < -1) {
+ ALOGE("Unsupported video bitrate mode: %d", bitRateMode);
+ return BAD_VALUE;
+ }
+ mVideoBitRateMode = bitRateMode;
return OK;
}
@@ -807,6 +842,77 @@
return OK;
}
+status_t StagefrightRecorder::setParamSelfID(int32_t selfID) {
+ ALOGV("setParamSelfID: %x", selfID);
+
+ mSelfID = selfID;
+ return OK;
+}
+
+status_t StagefrightRecorder::setParamVideoOpponentID(int32_t opponentID) {
+ mOpponentID = opponentID;
+ return OK;
+}
+
+status_t StagefrightRecorder::setParamPayloadType(int32_t payloadType) {
+ ALOGV("setParamPayloadType: %d", payloadType);
+
+ mPayloadType = payloadType;
+
+ if (mStarted && mOutputFormat == OUTPUT_FORMAT_RTP_AVP) {
+ mWriter->updatePayloadType(mPayloadType);
+ }
+
+ return OK;
+}
+
+status_t StagefrightRecorder::setRTPCVOExtMap(int32_t extmap) {
+ ALOGV("setRtpCvoExtMap: %d", extmap);
+
+ mRTPCVOExtMap = extmap;
+ return OK;
+}
+
+status_t StagefrightRecorder::setRTPCVODegrees(int32_t cvoDegrees) {
+ Mutex::Autolock autolock(mLock);
+ ALOGV("setRtpCvoDegrees: %d", cvoDegrees);
+
+ mRTPCVODegrees = cvoDegrees;
+
+ if (mStarted && mOutputFormat == OUTPUT_FORMAT_RTP_AVP) {
+ mWriter->updateCVODegrees(mRTPCVODegrees);
+ }
+
+ return OK;
+}
+
+status_t StagefrightRecorder::setParamRtpDscp(int32_t dscp) {
+ ALOGV("setParamRtpDscp: %d", dscp);
+
+ mRTPSockDscp = dscp;
+ return OK;
+}
+
+status_t StagefrightRecorder::setSocketNetwork(int64_t networkHandle) {
+ ALOGV("setSocketNetwork: %llu", (unsigned long long) networkHandle);
+
+ mRTPSockNetwork = networkHandle;
+ if (mStarted && mOutputFormat == OUTPUT_FORMAT_RTP_AVP) {
+ mWriter->updateSocketNetwork(mRTPSockNetwork);
+ }
+ return OK;
+}
+
+status_t StagefrightRecorder::requestIDRFrame() {
+ status_t ret = BAD_VALUE;
+ if (mVideoEncoderSource != NULL) {
+ ret = mVideoEncoderSource->requestIDRFrame();
+ } else {
+ ALOGV("requestIDRFrame: Encoder not ready");
+ }
+ return ret;
+}
+
status_t StagefrightRecorder::setParameter(
const String8 &key, const String8 &value) {
ALOGV("setParameter: key (%s) => value (%s)", key.string(), value.string());
@@ -875,6 +981,11 @@
if (safe_strtoi32(value.string(), &video_bitrate)) {
return setParamVideoEncodingBitRate(video_bitrate);
}
+ } else if (key == "video-param-bitrate-mode") {
+ int32_t video_bitrate_mode;
+ if (safe_strtoi32(value.string(), &video_bitrate_mode)) {
+ return setParamVideoBitRateMode(video_bitrate_mode);
+ }
} else if (key == "video-param-rotation-angle-degrees") {
int32_t degrees;
if (safe_strtoi32(value.string(), °rees)) {
@@ -929,6 +1040,47 @@
if (safe_strtoi32(value.string(), &remotePort)) {
return setParamRtpRemotePort(remotePort);
}
+ } else if (key == "rtp-param-self-id") {
+ int32_t selfID;
+ int64_t temp;
+ if (safe_strtoi64(value.string(), &temp)) {
+ selfID = static_cast<int32_t>(temp);
+ return setParamSelfID(selfID);
+ }
+ } else if (key == "rtp-param-opponent-id") {
+ int32_t opnId;
+ int64_t temp;
+ if (safe_strtoi64(value.string(), &temp)) {
+ opnId = static_cast<int32_t>(temp);
+ return setParamVideoOpponentID(opnId);
+ }
+ } else if (key == "rtp-param-payload-type") {
+ int32_t payloadType;
+ if (safe_strtoi32(value.string(), &payloadType)) {
+ return setParamPayloadType(payloadType);
+ }
+ } else if (key == "rtp-param-ext-cvo-extmap") {
+ int32_t extmap;
+ if (safe_strtoi32(value.string(), &extmap)) {
+ return setRTPCVOExtMap(extmap);
+ }
+ } else if (key == "rtp-param-ext-cvo-degrees") {
+ int32_t degrees;
+ if (safe_strtoi32(value.string(), °rees)) {
+ return setRTPCVODegrees(degrees);
+ }
+ } else if (key == "video-param-request-i-frame") {
+ return requestIDRFrame();
+ } else if (key == "rtp-param-set-socket-dscp") {
+ int32_t dscp;
+ if (safe_strtoi32(value.string(), &dscp)) {
+ return setParamRtpDscp(dscp);
+ }
+ } else if (key == "rtp-param-set-socket-network") {
+ int64_t networkHandle;
+ if (safe_strtoi64(value.string(), &networkHandle)) {
+ return setSocketNetwork(networkHandle);
+ }
} else {
ALOGE("setParameter: failed to find key %s", key.string());
}
@@ -1095,6 +1247,17 @@
sp<MetaData> meta = new MetaData;
int64_t startTimeUs = systemTime() / 1000;
meta->setInt64(kKeyTime, startTimeUs);
+ meta->setInt32(kKeySelfID, mSelfID);
+ meta->setInt32(kKeyPayloadType, mPayloadType);
+ meta->setInt64(kKeySocketNetwork, mRTPSockNetwork);
+ if (mRTPCVOExtMap > 0) {
+ meta->setInt32(kKeyRtpExtMap, mRTPCVOExtMap);
+ meta->setInt32(kKeyRtpCvoDegrees, mRTPCVODegrees);
+ }
+ if (mRTPSockDscp > 0) {
+ meta->setInt32(kKeyRtpDscp, mRTPSockDscp);
+ }
+
status = mWriter->start(meta.get());
break;
}
@@ -1375,7 +1538,7 @@
mVideoEncoderSource = source;
}
- mWriter = new ARTPWriter(mOutputFd, mLocalIp, mLocalPort, mRemoteIp, mRemotePort);
+ mWriter = new ARTPWriter(mOutputFd, mLocalIp, mLocalPort, mRemoteIp, mRemotePort, mLastSeqNo);
mWriter->addSource(source);
mWriter->setListener(mListener);
@@ -1829,7 +1992,13 @@
}
}
+ if (mOutputFormat == OUTPUT_FORMAT_RTP_AVP) {
+ // This indicates that a raw image provided to encoder needs to be rotated.
+ format->setInt32("rotation-degrees", mRotationDegrees);
+ }
+
format->setInt32("bitrate", mVideoBitRate);
+ format->setInt32("bitrate-mode", mVideoBitRateMode);
format->setInt32("frame-rate", mFrameRate);
format->setInt32("i-frame-interval", mIFramesIntervalSec);
@@ -2175,6 +2344,7 @@
if (mWriter != NULL) {
err = mWriter->stop();
+ mLastSeqNo = mWriter->getSequenceNum();
mWriter.clear();
}
@@ -2251,6 +2421,8 @@
mVideoHeight = 144;
mFrameRate = -1;
mVideoBitRate = 192000;
+ // Following MediaCodec's default
+ mVideoBitRateMode = BITRATE_MODE_VBR;
mSampleRate = 8000;
mAudioChannels = 1;
mAudioBitRate = 12200;
@@ -2396,6 +2568,14 @@
return NO_INIT;
}
+status_t StagefrightRecorder::getRtpDataUsage(uint64_t *bytes) {
+ if (mWriter != 0) {
+ *bytes = mWriter->getAccumulativeBytes();
+ return OK;
+ }
+ return NO_INIT;
+}
+
status_t StagefrightRecorder::dump(
int fd, const Vector<String16>& args) const {
ALOGV("dump");
diff --git a/media/libmediaplayerservice/StagefrightRecorder.h b/media/libmediaplayerservice/StagefrightRecorder.h
index 5f02e00..4bba869 100644
--- a/media/libmediaplayerservice/StagefrightRecorder.h
+++ b/media/libmediaplayerservice/StagefrightRecorder.h
@@ -82,6 +82,7 @@
virtual status_t setPreferredMicrophoneDirection(audio_microphone_direction_t direction);
virtual status_t setPreferredMicrophoneFieldDimension(float zoom);
status_t getPortId(audio_port_handle_t *portId) const override;
+ virtual status_t getRtpDataUsage(uint64_t *bytes);
private:
@@ -119,6 +120,7 @@
int32_t mVideoWidth, mVideoHeight;
int32_t mFrameRate;
int32_t mVideoBitRate;
+ int32_t mVideoBitRateMode;
int32_t mAudioBitRate;
int32_t mAudioChannels;
int32_t mSampleRate;
@@ -142,6 +144,14 @@
String8 mRemoteIp;
int32_t mLocalPort;
int32_t mRemotePort;
+ int32_t mSelfID;
+ int32_t mOpponentID;
+ int32_t mPayloadType;
+ int32_t mRTPCVOExtMap;
+ int32_t mRTPCVODegrees;
+ int32_t mRTPSockDscp;
+ int64_t mRTPSockNetwork;
+ uint32_t mLastSeqNo;
int64_t mDurationRecordedUs;
int64_t mStartedRecordingUs;
@@ -209,6 +219,7 @@
status_t setParamCaptureFpsEnable(int32_t timeLapseEnable);
status_t setParamCaptureFps(double fps);
status_t setParamVideoEncodingBitRate(int32_t bitRate);
+ status_t setParamVideoBitRateMode(int32_t bitRateMode);
status_t setParamVideoIFramesInterval(int32_t seconds);
status_t setParamVideoEncoderProfile(int32_t profile);
status_t setParamVideoEncoderLevel(int32_t level);
@@ -227,6 +238,14 @@
status_t setParamRtpLocalPort(int32_t localPort);
status_t setParamRtpRemoteIp(const String8 &remoteIp);
status_t setParamRtpRemotePort(int32_t remotePort);
+ status_t setParamSelfID(int32_t selfID);
+ status_t setParamVideoOpponentID(int32_t opponentID);
+ status_t setParamPayloadType(int32_t payloadType);
+ status_t setRTPCVOExtMap(int32_t extmap);
+ status_t setRTPCVODegrees(int32_t cvoDegrees);
+ status_t setParamRtpDscp(int32_t dscp);
+ status_t setSocketNetwork(int64_t networkHandle);
+ status_t requestIDRFrame();
void clipVideoBitRate();
void clipVideoFrameRate();
void clipVideoFrameWidth();
diff --git a/media/libmediaplayerservice/nuplayer/Android.bp b/media/libmediaplayerservice/nuplayer/Android.bp
index 5a1272a..6d338db 100644
--- a/media/libmediaplayerservice/nuplayer/Android.bp
+++ b/media/libmediaplayerservice/nuplayer/Android.bp
@@ -50,6 +50,7 @@
"frameworks/av/media/libstagefright/mpeg2ts",
"frameworks/av/media/libstagefright/rtsp",
"frameworks/av/media/libstagefright/timedtext",
+ "frameworks/native/include/android",
],
cflags: [
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
index 8b585b1..389249e 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.cpp
@@ -378,6 +378,7 @@
msg->setObject("source", source);
msg->post();
+ mDataSourceType = DATA_SOURCE_TYPE_RTP;
}
void NuPlayer::prepareAsync() {
@@ -1701,6 +1702,12 @@
updateRebufferingTimer(false /* stopping */, false /* exiting */);
}
+void NuPlayer::setTargetBitrate(int bitrate) {
+ if (mSource != NULL) {
+ mSource->setTargetBitrate(bitrate);
+ }
+}
+
void NuPlayer::onPause() {
updatePlaybackTimer(true /* stopping */, "onPause");
@@ -1927,6 +1934,11 @@
format->setInt32("priority", 0 /* realtime */);
+ if (mDataSourceType == DATA_SOURCE_TYPE_RTP) {
+ ALOGV("instantiateDecoder: set decoder error free on stream corrupt.");
+ format->setInt32("corrupt-free", true);
+ }
+
if (!audio) {
AString mime;
CHECK(format->findString("mime", &mime));
@@ -2727,6 +2739,14 @@
break;
}
+ case Source::kWhatIMSRxNotice:
+ {
+ sp<AMessage> IMSRxNotice;
+ CHECK(msg->findMessage("message", &IMSRxNotice));
+ sendIMSRxNotice(IMSRxNotice);
+ break;
+ }
+
default:
TRESPASS();
}
@@ -2829,11 +2849,75 @@
}
}
+void NuPlayer::sendIMSRxNotice(const sp<AMessage> &msg) {
+ int32_t payloadType;
+
+ CHECK(msg->findInt32("payload-type", &payloadType));
+
+ Parcel in;
+ in.writeInt32(payloadType);
+
+ switch (payloadType) {
+ case ARTPSource::RTCP_TSFB: // RTCP TSFB
+ case ARTPSource::RTCP_PSFB: // RTCP PSFB
+ case ARTPSource::RTP_AUTODOWN:
+ {
+ int32_t feedbackType, id;
+ CHECK(msg->findInt32("feedback-type", &feedbackType));
+ CHECK(msg->findInt32("sender", &id));
+ in.writeInt32(feedbackType);
+ in.writeInt32(id);
+ if (payloadType == ARTPSource::RTCP_TSFB) {
+ int32_t bitrate;
+ CHECK(msg->findInt32("bit-rate", &bitrate));
+ in.writeInt32(bitrate);
+ }
+ break;
+ }
+ case ARTPSource::RTP_QUALITY:
+ case ARTPSource::RTP_QUALITY_EMC:
+ {
+ int32_t feedbackType, bitrate;
+ int32_t highestSeqNum, baseSeqNum, prevExpected;
+ int32_t numBufRecv, prevNumBufRecv;
+ CHECK(msg->findInt32("feedback-type", &feedbackType));
+ CHECK(msg->findInt32("bit-rate", &bitrate));
+ CHECK(msg->findInt32("highest-seq-num", &highestSeqNum));
+ CHECK(msg->findInt32("base-seq-num", &baseSeqNum));
+ CHECK(msg->findInt32("prev-expected", &prevExpected));
+ CHECK(msg->findInt32("num-buf-recv", &numBufRecv));
+ CHECK(msg->findInt32("prev-num-buf-recv", &prevNumBufRecv));
+ in.writeInt32(feedbackType);
+ in.writeInt32(bitrate);
+ in.writeInt32(highestSeqNum);
+ in.writeInt32(baseSeqNum);
+ in.writeInt32(prevExpected);
+ in.writeInt32(numBufRecv);
+ in.writeInt32(prevNumBufRecv);
+ break;
+ }
+ case ARTPSource::RTP_CVO:
+ {
+ int32_t cvo;
+ CHECK(msg->findInt32("cvo", &cvo));
+ in.writeInt32(cvo);
+ break;
+ }
+ default:
+ break;
+ }
+
+ notifyListener(MEDIA_IMS_RX_NOTICE, 0, 0, &in);
+}
+
const char *NuPlayer::getDataSourceType() {
switch (mDataSourceType) {
case DATA_SOURCE_TYPE_HTTP_LIVE:
return "HTTPLive";
+ case DATA_SOURCE_TYPE_RTP:
+ return "RTP";
+
case DATA_SOURCE_TYPE_RTSP:
return "RTSP";
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayer.h b/media/libmediaplayerservice/nuplayer/NuPlayer.h
index f316096..adb7075 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayer.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayer.h
@@ -102,6 +102,8 @@
void updateInternalTimers();
+ void setTargetBitrate(int bitrate /* bps */);
+
protected:
virtual ~NuPlayer();
@@ -260,6 +262,7 @@
typedef enum {
DATA_SOURCE_TYPE_NONE,
DATA_SOURCE_TYPE_HTTP_LIVE,
+ DATA_SOURCE_TYPE_RTP,
DATA_SOURCE_TYPE_RTSP,
DATA_SOURCE_TYPE_GENERIC_URL,
DATA_SOURCE_TYPE_GENERIC_FD,
@@ -337,6 +340,7 @@
void sendSubtitleData(const sp<ABuffer> &buffer, int32_t baseIndex);
void sendTimedMetaData(const sp<ABuffer> &buffer);
void sendTimedTextData(const sp<ABuffer> &buffer);
+ void sendIMSRxNotice(const sp<AMessage> &msg);
void writeTrackInfo(Parcel* reply, const sp<AMessage>& format) const;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
index f734439..5b60bbf 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDecoder.cpp
@@ -746,9 +746,15 @@
mOutputBuffers.editItemAt(index) = buffer;
+ int64_t frameIndex;
+ bool frameIndexFound = buffer->meta()->findInt64("frameIndex", &frameIndex);
+
buffer->setRange(offset, size);
buffer->meta()->clear();
buffer->meta()->setInt64("timeUs", timeUs);
+ if (frameIndexFound) {
+ buffer->meta()->setInt64("frameIndex", frameIndex);
+ }
bool eos = flags & MediaCodec::BUFFER_FLAG_EOS;
// we do not expect CODECCONFIG or SYNCFRAME for decoder
@@ -1050,7 +1056,7 @@
uint32_t flags = 0;
CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
- int32_t eos, csd;
+ int32_t eos, csd, cvo;
// we do not expect SYNCFRAME for decoder
if (buffer->meta()->findInt32("eos", &eos) && eos) {
flags |= MediaCodec::BUFFER_FLAG_EOS;
@@ -1058,6 +1064,24 @@
flags |= MediaCodec::BUFFER_FLAG_CODECCONFIG;
}
+ if (buffer->meta()->findInt32("cvo", (int32_t*)&cvo)) {
+ ALOGV("[%s] cvo(%d) found at %lld us", mComponentName.c_str(), cvo, (long long)timeUs);
+ switch (cvo) {
+ case 0:
+ codecBuffer->meta()->setInt32("cvo", MediaCodec::CVO_DEGREE_0);
+ break;
+ case 1:
+ codecBuffer->meta()->setInt32("cvo", MediaCodec::CVO_DEGREE_90);
+ break;
+ case 2:
+ codecBuffer->meta()->setInt32("cvo", MediaCodec::CVO_DEGREE_180);
+ break;
+ case 3:
+ codecBuffer->meta()->setInt32("cvo", MediaCodec::CVO_DEGREE_270);
+ break;
+ }
+ }
+
// Modular DRM
MediaBufferBase *mediaBuf = NULL;
NuPlayerDrm::CryptoInfo *cryptInfo = NULL;
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
index 2d82944..2a50fc2 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerDriver.cpp
@@ -817,7 +817,11 @@
}
status_t NuPlayerDriver::setParameter(
- int /* key */, const Parcel & /* request */) {
+ int key, const Parcel &request ) {
+ if (key == KEY_PARAMETER_RTP_ATTRIBUTES) {
+ mPlayer->setTargetBitrate(request.readInt32());
+ return OK;
+ }
return INVALID_OPERATION;
}
diff --git a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h
index f137c52..bf6b539 100644
--- a/media/libmediaplayerservice/nuplayer/NuPlayerSource.h
+++ b/media/libmediaplayerservice/nuplayer/NuPlayerSource.h
@@ -58,6 +58,7 @@
kWhatInstantiateSecureDecoders,
// Modular DRM
kWhatDrmInfo,
+ kWhatIMSRxNotice,
};
// The provides message is used to notify the player about various
@@ -131,6 +132,8 @@
virtual void setOffloadAudio(bool /* offload */) {}
+ virtual void setTargetBitrate(int32_t) {}
+
// Modular DRM
virtual status_t prepareDrm(
const uint8_t /*uuid*/[16], const Vector<uint8_t> &/*drmSessionId*/,
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.cpp b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
index de1f8a1..b43df38 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.cpp
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.cpp
@@ -42,10 +42,21 @@
mFinalResult(OK),
mBuffering(false),
mInPreparationPhase(true),
- mRTPConn(new ARTPConnection),
+ mRTPConn(new ARTPConnection(ARTPConnection::kViLTEConnection)),
mEOSTimeoutAudio(0),
- mEOSTimeoutVideo(0) {
- ALOGD("RTPSource initialized with rtpParams=%s", rtpParams.string());
+ mEOSTimeoutVideo(0),
+ mFirstAccessUnit(true),
+ mAllTracksHaveTime(false),
+ mNTPAnchorUs(-1),
+ mMediaAnchorUs(-1),
+ mLastMediaTimeUs(-1),
+ mNumAccessUnitsReceived(0),
+ mLastCVOUpdated(-1),
+ mReceivedFirstRTCPPacket(false),
+ mReceivedFirstRTPPacket(false),
+ mPausing(false),
+ mPauseGeneration(0) {
+ ALOGD("RTPSource initialized with rtpParams=%s", rtpParams.string());
}
NuPlayer::RTPSource::~RTPSource() {
@@ -79,6 +90,9 @@
mLooper->registerHandler(mRTPConn);
}
+ CHECK_EQ(mState, (int)DISCONNECTED);
+ mState = CONNECTING;
+
setParameters(mRTPParams);
TrackInfo *info = NULL;
@@ -92,7 +106,7 @@
AString sdp;
ASessionDescription::SDPStringFactory(sdp, info->mLocalIp,
info->mIsAudio, info->mLocalPort, info->mPayloadType, info->mAS, info->mCodecName,
- NULL, info->mWidth, info->mHeight);
+ NULL, info->mWidth, info->mHeight, info->mCVOExtMap);
ALOGD("RTPSource SDP =>\n%s", sdp.c_str());
sp<ASessionDescription> desc = new ASessionDescription;
@@ -101,7 +115,7 @@
int sockRtp, sockRtcp;
ARTPConnection::MakeRTPSocketPair(&sockRtp, &sockRtcp, info->mLocalIp, info->mRemoteIp,
- info->mLocalPort, info->mRemotePort);
+ info->mLocalPort, info->mRemotePort, info->mSocketNetwork);
sp<AMessage> notify = new AMessage('accu', this);
@@ -109,6 +123,9 @@
notify->setSize("trackIndex", i);
// index(i) should be started from 1. 0 is reserved for [root]
mRTPConn->addStream(sockRtp, sockRtcp, desc, i + 1, notify, false);
+ mRTPConn->setSelfID(info->mSelfID);
+ mRTPConn->setJbTime(
+ (info->mJbTimeMs <= 3000 && info->mJbTimeMs >= 40) ? info->mJbTimeMs : 300);
info->mRTPSocket = sockRtp;
info->mRTCPSocket = sockRtcp;
@@ -129,16 +146,18 @@
if (info->mIsAudio) {
mAudioTrack = source;
+ info->mTimeScale = 16000;
} else {
mVideoTrack = source;
+ info->mTimeScale = 90000;
}
info->mSource = source;
+ info->mRTPTime = 0;
+ info->mNormalPlaytimeUs = 0;
+ info->mNPTMappingValid = false;
}
- CHECK_EQ(mState, (int)DISCONNECTED);
- mState = CONNECTING;
-
if (mInPreparationPhase) {
mInPreparationPhase = false;
notifyPrepared();
@@ -271,7 +290,28 @@
setEOSTimeout(audio, 0);
- return source->dequeueAccessUnit(accessUnit);
+ finalResult = source->dequeueAccessUnit(accessUnit);
+ if (finalResult != OK) {
+ return finalResult;
+ }
+
+ int32_t cvo;
+ if ((*accessUnit) != NULL && (*accessUnit)->meta()->findInt32("cvo", &cvo) &&
+ cvo != mLastCVOUpdated) {
+ sp<AMessage> msg = new AMessage();
+ msg->setInt32("payload-type", ARTPSource::RTP_CVO);
+ msg->setInt32("cvo", cvo);
+
+ sp<AMessage> notify = dupNotify();
+ notify->setInt32("what", kWhatIMSRxNotice);
+ notify->setMessage("message", msg);
+ notify->post();
+
+ ALOGV("notify cvo updated (%d)->(%d) to upper layer", mLastCVOUpdated, cvo);
+ mLastCVOUpdated = cvo;
+ }
+
+ return finalResult;
}
sp<AnotherPacketSource> NuPlayer::RTPSource::getSource(bool audio) {
@@ -315,13 +355,18 @@
void NuPlayer::RTPSource::schedulePollBuffering() {
sp<AMessage> msg = new AMessage(kWhatPollBuffering, this);
- msg->post(1000000ll); // 1 second intervals
+ msg->post(kBufferingPollIntervalUs); // 1 second intervals
}
void NuPlayer::RTPSource::onPollBuffering() {
schedulePollBuffering();
}
+bool NuPlayer::RTPSource::isRealTime() const {
+ ALOGD("RTPSource::isRealTime=%d", true);
+ return true;
+}
+
void NuPlayer::RTPSource::onMessageReceived(const sp<AMessage> &msg) {
ALOGV("onMessageReceived =%d", msg->what());
@@ -356,6 +401,22 @@
break;
}
+ int32_t IMSRxNotice;
+ if (msg->findInt32("rtcp-event", &IMSRxNotice)) {
+ int32_t payloadType, feedbackType;
+ CHECK(msg->findInt32("payload-type", &payloadType));
+ CHECK(msg->findInt32("feedback-type", &feedbackType));
+
+ sp<AMessage> notify = dupNotify();
+ notify->setInt32("what", kWhatIMSRxNotice);
+ notify->setMessage("message", msg);
+ notify->post();
+
+ ALOGV("IMSRxNotice \t\t payload : %d feedback : %d",
+ payloadType, feedbackType);
+ break;
+ }
+
size_t trackIndex;
CHECK(msg->findSize("trackIndex", &trackIndex));
@@ -371,6 +432,8 @@
break;
}
+ // Implicitly assert on valid trackIndex here, which we ensure by
+ // never removing tracks.
TrackInfo *info = &mTracks.editItemAt(trackIndex);
sp<AnotherPacketSource> source = info->mSource;
@@ -386,7 +449,6 @@
source->queueAccessUnit(accessUnit);
break;
}
- */
int64_t nptUs =
((double)rtpTime - (double)info->mRTPTime)
@@ -394,7 +456,8 @@
* 1000000ll
+ info->mNormalPlaytimeUs;
- accessUnit->meta()->setInt64("timeUs", nptUs);
+ */
+ accessUnit->meta()->setInt64("timeUs", ALooper::GetNowUs());
source->queueAccessUnit(accessUnit);
}
@@ -447,10 +510,16 @@
}
}
+void NuPlayer::RTPSource::setTargetBitrate(int32_t bitrate) {
+ mRTPConn->setTargetBitrate(bitrate);
+}
+
void NuPlayer::RTPSource::onTimeUpdate(int32_t trackIndex, uint32_t rtpTime, uint64_t ntpTime) {
ALOGV("onTimeUpdate track %d, rtpTime = 0x%08x, ntpTime = %#016llx",
trackIndex, rtpTime, (long long)ntpTime);
+ // convert ntpTime in Q32 seconds to microseconds. Note: this will not lose precision
+ // because ntpTimeUs is at most 52 bits (double holds 53 bits)
int64_t ntpTimeUs = (int64_t)(ntpTime * 1E6 / (1ll << 32));
TrackInfo *track = &mTracks.editItemAt(trackIndex);
@@ -611,6 +680,7 @@
newTrackInfo.mIsAudio = isAudioKey;
mTracks.push(newTrackInfo);
info = &mTracks.editTop();
+ info->mJbTimeMs = 300;
}
if (key == "rtp-param-mime-type") {
@@ -618,10 +688,10 @@
const char *mime = value.string();
const char *delimiter = strchr(mime, '/');
- info->mCodecName = (delimiter + 1);
+ info->mCodecName = delimiter ? (delimiter + 1) : "<none>";
ALOGV("rtp-param-mime-type: mMimeType (%s) => mCodecName (%s)",
- info->mMimeType.string(), info->mCodecName.string());
+ info->mMimeType.string(), info->mCodecName.string());
} else if (key == "video-param-decoder-profile") {
info->mCodecProfile = atoi(value);
} else if (key == "video-param-decoder-level") {
@@ -646,6 +716,15 @@
} else if (key == "rtp-param-rtp-timeout") {
} else if (key == "rtp-param-rtcp-timeout") {
} else if (key == "rtp-param-time-scale") {
+ } else if (key == "rtp-param-self-id") {
+ info->mSelfID = atoi(value);
+ } else if (key == "rtp-param-ext-cvo-extmap") {
+ info->mCVOExtMap = atoi(value);
+ } else if (key == "rtp-param-set-socket-network") {
+ int64_t networkHandle = atoll(value);
+ setSocketNetwork(networkHandle);
+ } else if (key == "rtp-param-jitter-buffer-time") {
+ info->mJbTimeMs = atoi(value);
}
return OK;
@@ -686,6 +765,20 @@
return OK;
}
+void NuPlayer::RTPSource::setSocketNetwork(int64_t networkHandle) {
+ ALOGV("setSocketNetwork: %llu", (unsigned long long)networkHandle);
+
+ TrackInfo *info = NULL;
+ for (size_t i = 0; i < mTracks.size(); ++i) {
+ info = &mTracks.editItemAt(i);
+
+ if (info == NULL)
+ break;
+
+ info->mSocketNetwork = networkHandle;
+ }
+}
+
// Trim both leading and trailing whitespace from the given string.
//static
void NuPlayer::RTPSource::TrimString(String8 *s) {
diff --git a/media/libmediaplayerservice/nuplayer/RTPSource.h b/media/libmediaplayerservice/nuplayer/RTPSource.h
index 6c618ec..3b4f9e9 100644
--- a/media/libmediaplayerservice/nuplayer/RTPSource.h
+++ b/media/libmediaplayerservice/nuplayer/RTPSource.h
@@ -33,6 +33,7 @@
#include "AnotherPacketSource.h"
#include "APacketSource.h"
#include "ARTPConnection.h"
+#include "ARTPSource.h"
#include "ASessionDescription.h"
#include "NuPlayerSource.h"
@@ -70,8 +71,12 @@
int64_t seekTimeUs,
MediaPlayerSeekMode mode = MediaPlayerSeekMode::SEEK_PREVIOUS_SYNC) override;
+ virtual bool isRealTime() const;
+
void onMessageReceived(const sp<AMessage> &msg);
+ virtual void setTargetBitrate(int32_t bitrate) override;
+
protected:
virtual ~RTPSource();
@@ -87,6 +92,8 @@
kWhatSetBufferingSettings = 'sBuS',
};
+ const int64_t kBufferingPollIntervalUs = 1000000ll;
+
enum State {
DISCONNECTED,
CONNECTING,
@@ -109,9 +116,17 @@
String8 mRemoteIp;
int32_t mLocalPort;
int32_t mRemotePort;
+ int64_t mSocketNetwork;
int32_t mTimeScale;
int32_t mAS;
+ /* RTP jitter buffer time in milliseconds */
+ uint32_t mJbTimeMs;
+ /* Unique ID indicates itself */
+ uint32_t mSelfID;
+ /* extmap:<value> for CVO will be set to here */
+ int32_t mCVOExtMap;
+
/* a copy of TrackInfo in RTSPSource */
sp<AnotherPacketSource> mSource;
uint32_t mRTPTime;
@@ -163,6 +178,7 @@
int64_t mMediaAnchorUs;
int64_t mLastMediaTimeUs;
int64_t mNumAccessUnitsReceived;
+ int32_t mLastCVOUpdated;
bool mReceivedFirstRTCPPacket;
bool mReceivedFirstRTPPacket;
bool mPausing;
@@ -190,6 +206,7 @@
status_t setParameters(const String8 ¶ms);
status_t setParameter(const String8 &key, const String8 &value);
+ void setSocketNetwork(int64_t networkHandle);
static void TrimString(String8 *s);
DISALLOW_EVIL_CONSTRUCTORS(RTPSource);
diff --git a/media/libstagefright/ACodec.cpp b/media/libstagefright/ACodec.cpp
index edbd99e..71a4ad8 100644
--- a/media/libstagefright/ACodec.cpp
+++ b/media/libstagefright/ACodec.cpp
@@ -564,6 +564,7 @@
mShutdownInProgress(false),
mExplicitShutdown(false),
mIsLegacyVP9Decoder(false),
+ mIsStreamCorruptFree(false),
mIsLowLatency(false),
mEncoderDelay(0),
mEncoderPadding(0),
@@ -2339,6 +2340,12 @@
mChannelMaskPresent = false;
}
+ int32_t isCorruptFree = 0;
+ if (msg->findInt32("corrupt-free", &isCorruptFree)) {
+ mIsStreamCorruptFree = isCorruptFree == 1 ? true : false;
+ ALOGV("corrupt-free=[%d]", mIsStreamCorruptFree);
+ }
+
int32_t maxInputSize;
if (msg->findInt32("max-input-size", &maxInputSize)) {
err = setMinBufferSize(kPortIndexInput, (size_t)maxInputSize);
@@ -4164,6 +4171,29 @@
ALOGI("setupVideoEncoder succeeded");
}
+ // Video should be encoded as stand straight because RTP protocol
+ // can provide rotation information only if CVO is supported.
+ // This needs to be added to support non-CVO case for video streaming scenario.
+ int32_t rotation = 0;
+ if (msg->findInt32("rotation-degrees", &rotation)) {
+ OMX_CONFIG_ROTATIONTYPE config;
+ InitOMXParams(&config);
+ config.nPortIndex = kPortIndexOutput;
+ status_t err = mOMXNode->getConfig(
+ (OMX_INDEXTYPE)OMX_IndexConfigCommonRotate, &config, sizeof(config));
+ if (err != OK) {
+ ALOGW("Failed to getConfig of OMX_IndexConfigCommonRotate(err %d)", err);
+ }
+ config.nRotation = rotation;
+ err = mOMXNode->setConfig(
+ (OMX_INDEXTYPE)OMX_IndexConfigCommonRotate, &config, sizeof(config));
+
+ ALOGD("Applying encoder-rotation=[%d] to video encoder.", config.nRotation);
+ if (err != OK) {
+ ALOGW("Failed to setConfig of OMX_IndexConfigCommonRotate(err %d)", err);
+ }
+ }
+
return err;
}
@@ -6057,6 +6087,12 @@
return false;
}
+ if (mCodec->mIsStreamCorruptFree && data1 == (OMX_U32)OMX_ErrorStreamCorrupt) {
+ ALOGV("[%s] handle OMX_ErrorStreamCorrupt as a normal operation",
+ mCodec->mComponentName.c_str());
+ return true;
+ }
+
ALOGE("[%s] ERROR(0x%08x)", mCodec->mComponentName.c_str(), data1);
// verify OMX component sends back an error we expect.
@@ -6164,6 +6200,13 @@
return;
}
+ int32_t cvo;
+ if (mCodec->mNativeWindow != NULL && buffer != NULL &&
+ buffer->meta()->findInt32("cvo", &cvo)) {
+ ALOGV("cvo(%d) found in buffer #%u", cvo, bufferID);
+ setNativeWindowRotation(mCodec->mNativeWindow.get(), cvo);
+ }
+
info->mStatus = BufferInfo::OWNED_BY_US;
info->mData = buffer;
@@ -6817,6 +6860,7 @@
ALOGV("onAllocateComponent");
CHECK(mCodec->mOMXNode == NULL);
+ mCodec->mFatalError = false;
sp<AMessage> notify = new AMessage(kWhatOMXMessageList, mCodec);
notify->setInt32("generation", mCodec->mNodeGeneration + 1);
diff --git a/media/libstagefright/HevcUtils.cpp b/media/libstagefright/HevcUtils.cpp
index 0e4eae2..e4794dd 100644
--- a/media/libstagefright/HevcUtils.cpp
+++ b/media/libstagefright/HevcUtils.cpp
@@ -377,6 +377,54 @@
return reader.overRead() ? ERROR_MALFORMED : OK;
}
+void HevcParameterSets::FindHEVCDimensions(const sp<ABuffer> &SpsBuffer, int32_t *width, int32_t *height)
+{
+ ALOGD("FindHEVCDimensions");
+ // See Rec. ITU-T H.265 v3 (04/2015) Chapter 7.3.2.2 for reference
+ ABitReader reader(SpsBuffer->data() + 1, SpsBuffer->size() - 1);
+ // Skip sps_video_parameter_set_id
+ reader.skipBits(4);
+ uint8_t maxSubLayersMinus1 = reader.getBitsWithFallback(3, 0);
+ // Skip sps_temporal_id_nesting_flag;
+ reader.skipBits(1);
+ // Skip general profile
+ reader.skipBits(96);
+ if (maxSubLayersMinus1 > 0) {
+ bool subLayerProfilePresentFlag[8];
+ bool subLayerLevelPresentFlag[8];
+ for (int i = 0; i < maxSubLayersMinus1; ++i) {
+ subLayerProfilePresentFlag[i] = reader.getBitsWithFallback(1, 0);
+ subLayerLevelPresentFlag[i] = reader.getBitsWithFallback(1, 0);
+ }
+ // Skip reserved
+ reader.skipBits(2 * (8 - maxSubLayersMinus1));
+ for (int i = 0; i < maxSubLayersMinus1; ++i) {
+ if (subLayerProfilePresentFlag[i]) {
+ // Skip profile
+ reader.skipBits(88);
+ }
+ if (subLayerLevelPresentFlag[i]) {
+ // Skip sub_layer_level_idc[i]
+ reader.skipBits(8);
+ }
+ }
+ }
+ // Skip sps_seq_parameter_set_id
+ skipUE(&reader);
+ uint8_t chromaFormatIdc = parseUEWithFallback(&reader, 0);
+ if (chromaFormatIdc == 3) {
+ // Skip separate_colour_plane_flag
+ reader.skipBits(1);
+ }
+ skipUE(&reader);
+ skipUE(&reader);
+
+ // pic_width_in_luma_samples
+ *width = parseUEWithFallback(&reader, 0);
+ // pic_height_in_luma_samples
+ *height = parseUEWithFallback(&reader, 0);
+}
+
status_t HevcParameterSets::parsePps(
const uint8_t* data UNUSED_PARAM, size_t size UNUSED_PARAM) {
return OK;
diff --git a/media/libstagefright/MediaCodec.cpp b/media/libstagefright/MediaCodec.cpp
index 0ed0de1..e228a9d 100644
--- a/media/libstagefright/MediaCodec.cpp
+++ b/media/libstagefright/MediaCodec.cpp
@@ -321,11 +321,24 @@
BufferQueue::createBufferQueue(&mProducer, &mConsumer);
mSurface = new Surface(mProducer, false /* controlledByApp */);
struct ConsumerListener : public BnConsumerListener {
- void onFrameAvailable(const BufferItem&) override {}
+ ConsumerListener(const sp<IGraphicBufferConsumer> &consumer) {
+ mConsumer = consumer;
+ }
+ void onFrameAvailable(const BufferItem&) override {
+ BufferItem buffer;
+ // consume buffer
+ sp<IGraphicBufferConsumer> consumer = mConsumer.promote();
+ if (consumer != nullptr && consumer->acquireBuffer(&buffer, 0) == NO_ERROR) {
+ consumer->releaseBuffer(buffer.mSlot, buffer.mFrameNumber,
+ EGL_NO_DISPLAY, EGL_NO_SYNC_KHR, buffer.mFence);
+ }
+ }
+
+ wp<IGraphicBufferConsumer> mConsumer;
void onBuffersReleased() override {}
void onSidebandStreamChanged() override {}
};
- sp<ConsumerListener> listener{new ConsumerListener};
+ sp<ConsumerListener> listener{new ConsumerListener(mConsumer)};
mConsumer->consumerConnect(listener, false);
mConsumer->setConsumerName(String8{"MediaCodec.release"});
mConsumer->setConsumerUsageBits(usage);
@@ -1311,6 +1324,8 @@
// save msg for reset
mConfigureMsg = msg;
+ sp<AMessage> callback = mCallback;
+
status_t err;
std::vector<MediaResourceParcel> resources;
resources.push_back(MediaResource::CodecResource(mFlags & kFlagIsSecure, mIsVideo));
@@ -1335,7 +1350,18 @@
// the configure failure is due to wrong state.
ALOGE("configure failed with err 0x%08x, resetting...", err);
- reset();
+ status_t err2 = reset();
+ if (err2 != OK) {
+ ALOGE("retrying configure: failed to reset codec (%08x)", err2);
+ break;
+ }
+ if (callback != nullptr) {
+ err2 = setCallback(callback);
+ if (err2 != OK) {
+ ALOGE("retrying configure: failed to set callback (%08x)", err2);
+ break;
+ }
+ }
}
if (!isResourceError(err)) {
break;
@@ -1444,6 +1470,8 @@
status_t MediaCodec::start() {
sp<AMessage> msg = new AMessage(kWhatStart, this);
+ sp<AMessage> callback;
+
status_t err;
std::vector<MediaResourceParcel> resources;
resources.push_back(MediaResource::CodecResource(mFlags & kFlagIsSecure, mIsVideo));
@@ -1468,6 +1496,20 @@
ALOGE("retrying start: failed to configure codec");
break;
}
+ if (callback != nullptr) {
+ err = setCallback(callback);
+ if (err != OK) {
+ ALOGE("retrying start: failed to set callback");
+ break;
+ }
+ ALOGD("succeed to set callback for reclaim");
+ }
+ }
+
+ // Keep callback message after the first iteration if necessary.
+ if (i == 0 && mCallback != nullptr && mFlags & kFlagIsAsync) {
+ callback = mCallback;
+ ALOGD("keep callback message for reclaim");
}
sp<AMessage> response;
diff --git a/media/libstagefright/MediaCodecSource.cpp b/media/libstagefright/MediaCodecSource.cpp
index 1395c27..bc656a2 100644
--- a/media/libstagefright/MediaCodecSource.cpp
+++ b/media/libstagefright/MediaCodecSource.cpp
@@ -435,6 +435,30 @@
buffer->release();
}
+status_t MediaCodecSource::setEncodingBitrate(int32_t bitRate) {
+ ALOGV("setEncodingBitrate (%d)", bitRate);
+
+ if (mEncoder == NULL) {
+ ALOGW("setEncodingBitrate (%d) : mEncoder is null", bitRate);
+ return BAD_VALUE;
+ }
+
+ sp<AMessage> params = new AMessage;
+ params->setInt32("video-bitrate", bitRate);
+
+ return mEncoder->setParameters(params);
+}
+
+status_t MediaCodecSource::requestIDRFrame() {
+ if (mEncoder == NULL) {
+ ALOGW("requestIDRFrame : mEncoder is null");
+ return BAD_VALUE;
+ } else {
+ mEncoder->requestIDRFrame();
+ return OK;
+ }
+}
+
MediaCodecSource::MediaCodecSource(
const sp<ALooper> &looper,
const sp<AMessage> &outputFormat,
diff --git a/media/libstagefright/SurfaceUtils.cpp b/media/libstagefright/SurfaceUtils.cpp
index c284ef7..1f569ef 100644
--- a/media/libstagefright/SurfaceUtils.cpp
+++ b/media/libstagefright/SurfaceUtils.cpp
@@ -175,6 +175,22 @@
}
}
+status_t setNativeWindowRotation(
+ ANativeWindow *nativeWindow /* nonnull */, int rotation) {
+
+ int transform = 0;
+ if ((rotation % 90) == 0) {
+ switch ((rotation / 90) & 3) {
+ case 1: transform = HAL_TRANSFORM_ROT_90; break;
+ case 2: transform = HAL_TRANSFORM_ROT_180; break;
+ case 3: transform = HAL_TRANSFORM_ROT_270; break;
+ default: transform = 0; break;
+ }
+ }
+
+ return native_window_set_buffers_transform(nativeWindow, transform);
+}
+
status_t pushBlankBuffersToNativeWindow(ANativeWindow *nativeWindow /* nonnull */) {
status_t err = NO_ERROR;
ANativeWindowBuffer* anb = NULL;
diff --git a/media/libstagefright/include/HevcUtils.h b/media/libstagefright/include/HevcUtils.h
index 0f59631..335de2f 100644
--- a/media/libstagefright/include/HevcUtils.h
+++ b/media/libstagefright/include/HevcUtils.h
@@ -90,6 +90,8 @@
// Note that this method does not write the start code.
bool write(size_t index, uint8_t* dest, size_t size);
status_t makeHvcc(uint8_t *hvcc, size_t *hvccSize, size_t nalSizeLength);
+ void FindHEVCDimensions(
+ const sp<ABuffer> &SpsBuffer, int32_t *width, int32_t *height);
Info getInfo() const { return mInfo; }
diff --git a/media/libstagefright/include/media/stagefright/ACodec.h b/media/libstagefright/include/media/stagefright/ACodec.h
index 8ef9278..4c97b4d 100644
--- a/media/libstagefright/include/media/stagefright/ACodec.h
+++ b/media/libstagefright/include/media/stagefright/ACodec.h
@@ -273,6 +273,7 @@
bool mShutdownInProgress;
bool mExplicitShutdown;
bool mIsLegacyVP9Decoder;
+ bool mIsStreamCorruptFree;
bool mIsLowLatency;
// If "mKeepComponentAllocated" we only transition back to Loaded state
diff --git a/media/libstagefright/include/media/stagefright/MediaCodec.h b/media/libstagefright/include/media/stagefright/MediaCodec.h
index 5ff4e12..a28d479 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodec.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodec.h
@@ -81,6 +81,13 @@
BUFFER_FLAG_MUXER_DATA = 16,
};
+ enum CVODegree {
+ CVO_DEGREE_0 = 0,
+ CVO_DEGREE_90 = 90,
+ CVO_DEGREE_180 = 180,
+ CVO_DEGREE_270 = 270,
+ };
+
enum {
CB_INPUT_AVAILABLE = 1,
CB_OUTPUT_AVAILABLE = 2,
diff --git a/media/libstagefright/include/media/stagefright/MediaCodecConstants.h b/media/libstagefright/include/media/stagefright/MediaCodecConstants.h
index 178d334..6cf1ba0 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodecConstants.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodecConstants.h
@@ -619,6 +619,7 @@
constexpr int32_t COLOR_FormatYUV422SemiPlanar = 24;
constexpr int32_t COLOR_FormatYUV444Flexible = 0x7F444888;
constexpr int32_t COLOR_FormatYUV444Interleaved = 29;
+constexpr int32_t COLOR_FormatYUVP010 = 54;
constexpr int32_t COLOR_QCOM_FormatYUV420SemiPlanar = 0x7fa30c00;
constexpr int32_t COLOR_TI_FormatYUV420PackedSemiPlanar = 0x7f000100;
diff --git a/media/libstagefright/include/media/stagefright/MediaCodecSource.h b/media/libstagefright/include/media/stagefright/MediaCodecSource.h
index 2f98af1..0f7b535 100644
--- a/media/libstagefright/include/media/stagefright/MediaCodecSource.h
+++ b/media/libstagefright/include/media/stagefright/MediaCodecSource.h
@@ -64,12 +64,15 @@
// MediaBufferObserver
virtual void signalBufferReturned(MediaBufferBase *buffer);
+ virtual status_t setEncodingBitrate(int32_t bitRate /* bps */);
// for AHandlerReflector
void onMessageReceived(const sp<AMessage> &msg);
+ status_t requestIDRFrame();
+
protected:
virtual ~MediaCodecSource();
diff --git a/media/libstagefright/include/media/stagefright/MediaWriter.h b/media/libstagefright/include/media/stagefright/MediaWriter.h
index 1f4fbcb..9f20185 100644
--- a/media/libstagefright/include/media/stagefright/MediaWriter.h
+++ b/media/libstagefright/include/media/stagefright/MediaWriter.h
@@ -54,6 +54,11 @@
virtual void setStartTimeOffsetMs(int /*ms*/) {}
virtual int32_t getStartTimeOffsetMs() const { return 0; }
virtual status_t setNextFd(int /*fd*/) { return INVALID_OPERATION; }
+ virtual void updateCVODegrees(int32_t /*cvoDegrees*/) {}
+ virtual void updatePayloadType(int32_t /*payloadType*/) {}
+ virtual void updateSocketNetwork(int64_t /*socketNetwork*/) {}
+ virtual uint32_t getSequenceNum() { return 0; }
+ virtual uint64_t getAccumulativeBytes() { return 0; }
protected:
virtual ~MediaWriter() {}
diff --git a/media/libstagefright/include/media/stagefright/MetaDataBase.h b/media/libstagefright/include/media/stagefright/MetaDataBase.h
index d455093..6f21a80 100644
--- a/media/libstagefright/include/media/stagefright/MetaDataBase.h
+++ b/media/libstagefright/include/media/stagefright/MetaDataBase.h
@@ -247,8 +247,15 @@
// Treat empty track as malformed for MediaRecorder.
kKeyEmptyTrackMalFormed = 'nemt', // bool (int32_t)
- kKeySps = 'sSps', // int32_t, indicates that a buffer is sps.
- kKeyPps = 'sPps', // int32_t, indicates that a buffer is pps.
+ kKeyVps = 'sVps', // int32_t, indicates that a buffer has vps.
+ kKeySps = 'sSps', // int32_t, indicates that a buffer has sps.
+ kKeyPps = 'sPps', // int32_t, indicates that a buffer has pps.
+ kKeySelfID = 'sfid', // int32_t, source ID to identify itself on RTP protocol.
+ kKeyPayloadType = 'pTyp', // int32_t, SDP negotiated payload type.
+ kKeyRtpExtMap = 'extm', // int32_t, rtp extension ID for cvo on RTP protocol.
+ kKeyRtpCvoDegrees = 'cvod', // int32_t, rtp cvo degrees as per 3GPP 26.114.
+ kKeyRtpDscp = 'dscp', // int32_t, DSCP(Differentiated services codepoint) of RFC 2474.
+ kKeySocketNetwork = 'sNet', // int64_t, socket will be bound to network handle.
};
enum {
diff --git a/media/libstagefright/include/media/stagefright/SurfaceUtils.h b/media/libstagefright/include/media/stagefright/SurfaceUtils.h
index ae55c65..35b3fa2 100644
--- a/media/libstagefright/include/media/stagefright/SurfaceUtils.h
+++ b/media/libstagefright/include/media/stagefright/SurfaceUtils.h
@@ -38,6 +38,8 @@
int width, int height, int format, int rotation, int usage, bool reconnect);
void setNativeWindowHdrMetadata(
ANativeWindow *nativeWindow /* nonnull */, HDRStaticInfo *info /* nonnull */);
+status_t setNativeWindowRotation(
+ ANativeWindow *nativeWindow /* nonnull */, int rotation);
status_t pushBlankBuffersToNativeWindow(ANativeWindow *nativeWindow /* nonnull */);
status_t nativeWindowConnect(ANativeWindow *surface, const char *reason);
status_t nativeWindowDisconnect(ANativeWindow *surface, const char *reason);
diff --git a/media/libstagefright/rtsp/AAVCAssembler.cpp b/media/libstagefright/rtsp/AAVCAssembler.cpp
index 13d74e4..2f93d5d 100644
--- a/media/libstagefright/rtsp/AAVCAssembler.cpp
+++ b/media/libstagefright/rtsp/AAVCAssembler.cpp
@@ -25,8 +25,11 @@
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
+#include <media/stagefright/foundation/avc_utils.h>
#include <media/stagefright/foundation/hexdump.h>
+#include <android-base/properties.h>
+
#include <stdint.h>
namespace android {
@@ -37,85 +40,143 @@
mAccessUnitRTPTime(0),
mNextExpectedSeqNoValid(false),
mNextExpectedSeqNo(0),
- mAccessUnitDamaged(false) {
+ mAccessUnitDamaged(false),
+ mFirstIFrameProvided(false),
+ mLastIFrameProvidedAtMs(0),
+ mWidth(0),
+ mHeight(0) {
}
AAVCAssembler::~AAVCAssembler() {
}
+int32_t AAVCAssembler::addNack(
+ const sp<ARTPSource> &source) {
+ List<sp<ABuffer>> *queue = source->queue();
+ int32_t nackCount = 0;
+
+ List<sp<ABuffer> >::iterator it = queue->begin();
+
+ if (it == queue->end()) {
+ return nackCount /* 0 */;
+ }
+
+ uint16_t queueHeadSeqNum = (*it)->int32Data();
+
+ // move to the packet after which RTCP:NACK was sent.
+ for (; it != queue->end(); ++it) {
+ int32_t seqNum = (*it)->int32Data();
+ if (seqNum >= source->mHighestNackNumber) {
+ break;
+ }
+ }
+
+ int32_t nackStartAt = -1;
+
+ while (it != queue->end()) {
+ int32_t seqBeforeLast = (*it)->int32Data();
+ // increase iterator.
+ if ((++it) == queue->end()) {
+ break;
+ }
+ int32_t seqLast = (*it)->int32Data();
+
+ if ((seqLast - seqBeforeLast) < 0) {
+ ALOGD("addNack: found end of seqNum from(%d) to(%d)", seqBeforeLast, seqLast);
+ source->mHighestNackNumber = 0;
+ }
+
+ // missed packet found
+ if (seqLast > (seqBeforeLast + 1) &&
+ // we didn't send RTCP:NACK for this packet yet.
+ (seqLast - 1) > source->mHighestNackNumber) {
+ source->mHighestNackNumber = seqLast - 1;
+ nackStartAt = seqBeforeLast + 1;
+ break;
+ }
+
+ }
+
+ if (nackStartAt != -1) {
+ nackCount = source->mHighestNackNumber - nackStartAt + 1;
+ ALOGD("addNack: nackCount=%d, nackFrom=%d, nackTo=%d", nackCount,
+ nackStartAt, source->mHighestNackNumber);
+
+ uint16_t mask = (uint16_t)(0xffff) >> (16 - nackCount + 1);
+ source->setSeqNumToNACK(nackStartAt, mask, queueHeadSeqNum);
+ }
+
+ return nackCount;
+}
+
ARTPAssembler::AssemblyStatus AAVCAssembler::addNALUnit(
const sp<ARTPSource> &source) {
List<sp<ABuffer> > *queue = source->queue();
+ const uint32_t firstRTPTime = source->mFirstRtpTime;
if (queue->empty()) {
return NOT_ENOUGH_DATA;
}
sp<ABuffer> buffer = *queue->begin();
- int32_t rtpTime;
- CHECK(buffer->meta()->findInt32("rtp-time", &rtpTime));
+ buffer->meta()->setObject("source", source);
+
+ int64_t rtpTime = findRTPTime(firstRTPTime, buffer);
+
int64_t startTime = source->mFirstSysTime / 1000;
int64_t nowTime = ALooper::GetNowUs() / 1000;
int64_t playedTime = nowTime - startTime;
- int32_t playedTimeRtp = source->mFirstRtpTime +
- (((uint32_t)playedTime) * (source->mClockRate / 1000));
- const int32_t jitterTime = source->mClockRate / 5; // 200ms
- int32_t expiredTimeInJb = rtpTime + jitterTime;
+
+ int64_t playedTimeRtp = source->mFirstRtpTime + playedTime * (int64_t)source->mClockRate / 1000;
+ const int64_t jitterTime = source->mJbTimeMs * (int64_t)source->mClockRate / 1000;
+
+ int64_t expiredTimeInJb = rtpTime + jitterTime;
bool isExpired = expiredTimeInJb <= (playedTimeRtp);
- bool isTooLate = expiredTimeInJb < (playedTimeRtp - jitterTime);
- ALOGV("start=%lld, now=%lld, played=%lld", (long long)startTime,
- (long long)nowTime, (long long)playedTime);
- ALOGV("rtp-time(JB)=%d, played-rtp-time(JB)=%d, expired-rtp-time(JB)=%d isExpired=%d",
- rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+ bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);
+ bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));
+
+ if (mShowQueue && mShowQueueCnt < 20) {
+ showCurrentQueue(queue);
+ printNowTimeUs(startTime, nowTime, playedTime);
+ printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+ mShowQueueCnt++;
+ }
+
+ AAVCAssembler::addNack(source);
if (!isExpired) {
ALOGV("buffering in jitter buffer.");
return NOT_ENOUGH_DATA;
}
- if (isTooLate) {
- ALOGV("buffer arrived too lately..");
- ALOGW("start=%lld, now=%lld, played=%lld", (long long)startTime,
- (long long)nowTime, (long long)playedTime);
- ALOGW("rtp-time(JB)=%d, plyed-rtp-time(JB)=%d, exp-rtp-time(JB)=%d diff=%lld isExpired=%d",
- rtpTime, playedTimeRtp, expiredTimeInJb,
- ((long long)playedTimeRtp) - expiredTimeInJb, isExpired);
- ALOGW("expected Seq. NO =%d", buffer->int32Data());
+ if (isTooLate200) {
+ ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");
+ }
- List<sp<ABuffer> >::iterator it = queue->begin();
- while (it != queue->end()) {
- CHECK((*it)->meta()->findInt32("rtp-time", &rtpTime));
- if (rtpTime + jitterTime >= playedTimeRtp) {
- mNextExpectedSeqNo = (*it)->int32Data();
- break;
- }
- it++;
- }
- source->noticeAbandonBuffer();
+ if (isTooLate300) {
+ ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",
+ (long long)(playedTimeRtp - expiredTimeInJb), buffer->int32Data());
+ printNowTimeUs(startTime, nowTime, playedTime);
+ printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+
+ mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTime);
}
if (mNextExpectedSeqNoValid) {
int32_t size = queue->size();
- int32_t cnt = 0;
- List<sp<ABuffer> >::iterator it = queue->begin();
- while (it != queue->end()) {
- if ((uint32_t)(*it)->int32Data() >= mNextExpectedSeqNo) {
- break;
- }
+ int32_t cntRemove = deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
- it = queue->erase(it);
- cnt++;
- }
-
- if (cnt > 0) {
- source->noticeAbandonBuffer(cnt);
- ALOGW("delete %d of %d buffers", cnt, size);
+ if (cntRemove > 0) {
+ source->noticeAbandonBuffer(cntRemove);
+ ALOGW("delete %d of %d buffers", cntRemove, size);
}
if (queue->empty()) {
return NOT_ENOUGH_DATA;
}
}
+ buffer = *queue->begin();
+
if (!mNextExpectedSeqNoValid) {
mNextExpectedSeqNoValid = true;
mNextExpectedSeqNo = (uint32_t)buffer->int32Data();
@@ -171,15 +232,72 @@
}
}
+void AAVCAssembler::checkSpsUpdated(const sp<ABuffer> &buffer) {
+ const uint8_t *data = buffer->data();
+ unsigned nalType = data[0] & 0x1f;
+ if (nalType == 0x7) {
+ int32_t width = 0, height = 0;
+ FindAVCDimensions(buffer, &width, &height);
+ if (width != mWidth || height != mHeight) {
+ mFirstIFrameProvided = false;
+ mWidth = width;
+ mHeight = height;
+ ALOGD("found a new resolution (%u x %u)", mWidth, mHeight);
+ }
+ }
+}
+
+void AAVCAssembler::checkIFrameProvided(const sp<ABuffer> &buffer) {
+ if (buffer->size() == 0) {
+ return;
+ }
+ const uint8_t *data = buffer->data();
+ unsigned nalType = data[0] & 0x1f;
+ if (nalType == 0x5) {
+ mLastIFrameProvidedAtMs = ALooper::GetNowUs() / 1000;
+ if (!mFirstIFrameProvided) {
+ mFirstIFrameProvided = true;
+
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ ALOGD("got First I-frame to be decoded. rtpTime=%d, size=%zu", rtpTime, buffer->size());
+ }
+ }
+}
+
+bool AAVCAssembler::dropFramesUntilIframe(const sp<ABuffer> &buffer) {
+ const uint8_t *data = buffer->data();
+ unsigned nalType = data[0] & 0x1f;
+ if (!mFirstIFrameProvided && nalType < 0x5) {
+ return true;
+ }
+
+ return false;
+}
+
void AAVCAssembler::addSingleNALUnit(const sp<ABuffer> &buffer) {
ALOGV("addSingleNALUnit of size %zu", buffer->size());
#if !LOG_NDEBUG
hexdump(buffer->data(), buffer->size());
#endif
+ checkSpsUpdated(buffer);
+ checkIFrameProvided(buffer);
+
uint32_t rtpTime;
CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ if (dropFramesUntilIframe(buffer)) {
+ sp<ARTPSource> source = nullptr;
+ buffer->meta()->findObject("source", (sp<android::RefBase>*)&source);
+ if (source != nullptr) {
+ ALOGD("Issued FIR to get the I-frame");
+ source->onIssueFIRByAssembler();
+ }
+ ALOGV("Dropping P-frame till I-frame provided. rtpTime %u", rtpTime);
+ return;
+ }
+
if (!mNALUnits.empty() && rtpTime != mAccessUnitRTPTime) {
submitAccessUnit();
}
@@ -264,6 +382,11 @@
size_t totalCount = 1;
bool complete = false;
+ uint32_t rtpTimeStartAt;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTimeStartAt));
+ uint32_t startSeqNo = buffer->int32Data();
+ bool pFrame = nalType == 0x1;
+
if (data[1] & 0x40) {
// Huh? End bit also set on the first buffer.
@@ -272,6 +395,8 @@
complete = true;
} else {
List<sp<ABuffer> >::iterator it = ++queue->begin();
+ int32_t connected = 1;
+ bool snapped = false;
while (it != queue->end()) {
ALOGV("sequence length %zu", totalCount);
@@ -281,26 +406,32 @@
size_t size = buffer->size();
if ((uint32_t)buffer->int32Data() != expectedSeqNo) {
- ALOGV("sequence not complete, expected seqNo %d, got %d",
- expectedSeqNo, (uint32_t)buffer->int32Data());
+ ALOGD("sequence not complete, expected seqNo %u, got %u, nalType %u",
+ expectedSeqNo, (unsigned)buffer->int32Data(), nalType);
+ snapped = true;
- return WRONG_SEQUENCE_NUMBER;
+ if (!pFrame) {
+ return WRONG_SEQUENCE_NUMBER;
+ }
}
+ if (!snapped) {
+ connected++;
+ }
+
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
if (size < 2
|| data[0] != indicator
|| (data[1] & 0x1f) != nalType
- || (data[1] & 0x80)) {
+ || (data[1] & 0x80)
+ || rtpTime != rtpTimeStartAt) {
ALOGV("Ignoring malformed FU buffer.");
// Delete the whole start of the FU.
- it = queue->begin();
- for (size_t i = 0; i <= totalCount; ++i) {
- it = queue->erase(it);
- }
-
mNextExpectedSeqNo = expectedSeqNo + 1;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
return MALFORMED_PACKET;
}
@@ -308,9 +439,17 @@
totalSize += size - 2;
++totalCount;
- expectedSeqNo = expectedSeqNo + 1;
+ expectedSeqNo = (uint32_t)buffer->int32Data() + 1;
if (data[1] & 0x40) {
+ if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo,
+ connected, totalCount, 0.5f)) {
+ mNextExpectedSeqNo = expectedSeqNo;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
+
+ return MALFORMED_PACKET;
+ }
+
// This is the last fragment.
complete = true;
break;
@@ -338,6 +477,8 @@
unit->data()[0] = (nri << 5) | nalType;
size_t offset = 1;
+ int32_t cvo = -1;
+ sp<ARTPSource> source = nullptr;
List<sp<ABuffer> >::iterator it = queue->begin();
for (size_t i = 0; i < totalCount; ++i) {
const sp<ABuffer> &buffer = *it;
@@ -348,6 +489,9 @@
#endif
memcpy(unit->data() + offset, buffer->data() + 2, buffer->size() - 2);
+
+ buffer->meta()->findObject("source", (sp<android::RefBase>*)&source);
+ buffer->meta()->findInt32("cvo", &cvo);
offset += buffer->size() - 2;
it = queue->erase(it);
@@ -355,6 +499,13 @@
unit->setRange(0, totalSize);
+ if (cvo >= 0) {
+ unit->meta()->setInt32("cvo", cvo);
+ }
+ if (source != nullptr) {
+ unit->meta()->setObject("source", source);
+ }
+
addSingleNALUnit(unit);
ALOGV("successfully assembled a NAL unit from fragments.");
@@ -365,7 +516,11 @@
void AAVCAssembler::submitAccessUnit() {
CHECK(!mNALUnits.empty());
- ALOGV("Access unit complete (%zu nal units)", mNALUnits.size());
+ if(android::base::GetBoolProperty("debug.stagefright.fps", false)) {
+ ALOGD("Access unit complete (%zu nal units)", mNALUnits.size());
+ } else {
+ ALOGV("Access unit complete (%zu nal units)", mNALUnits.size());
+ }
size_t totalSize = 0;
for (List<sp<ABuffer> >::iterator it = mNALUnits.begin();
@@ -375,6 +530,7 @@
sp<ABuffer> accessUnit = new ABuffer(totalSize);
size_t offset = 0;
+ int32_t cvo = -1;
for (List<sp<ABuffer> >::iterator it = mNALUnits.begin();
it != mNALUnits.end(); ++it) {
memcpy(accessUnit->data() + offset, "\x00\x00\x00\x01", 4);
@@ -383,6 +539,8 @@
sp<ABuffer> nal = *it;
memcpy(accessUnit->data() + offset, nal->data(), nal->size());
offset += nal->size();
+
+ nal->meta()->findInt32("cvo", &cvo);
}
CopyTimes(accessUnit, *mNALUnits.begin());
@@ -391,6 +549,9 @@
printf(mAccessUnitDamaged ? "X" : ".");
fflush(stdout);
#endif
+ if (cvo >= 0) {
+ accessUnit->meta()->setInt32("cvo", cvo);
+ }
if (mAccessUnitDamaged) {
accessUnit->meta()->setInt32("damaged", true);
@@ -404,22 +565,89 @@
msg->post();
}
+inline int64_t AAVCAssembler::findRTPTime(
+ const uint32_t& firstRTPTime, const sp<ABuffer>& buffer) {
+ /* If you want to +, -, * rtpTime, recommend to declare rtpTime as int64_t.
+ Because rtpTime can be near UINT32_MAX. Beware the overflow. */
+ int64_t rtpTime = 0;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ // If the first overs 2^31 and rtp unders 2^31, the rtp value is overflowed one.
+ int64_t overflowMask = (firstRTPTime & 0x80000000 & ~rtpTime) << 1;
+ return rtpTime | overflowMask;
+}
+
+int32_t AAVCAssembler::pickProperSeq(const Queue *queue,
+ uint32_t first, int64_t play, int64_t jit) {
+ sp<ABuffer> buffer = *(queue->begin());
+ int32_t nextSeqNo = buffer->int32Data();
+
+ Queue::const_iterator it = queue->begin();
+ while (it != queue->end()) {
+ int64_t rtpTime = findRTPTime(first, *it);
+ // if pkt in time exists, that should be the next pivot
+ if (rtpTime + jit >= play) {
+ nextSeqNo = (*it)->int32Data();
+ break;
+ }
+ it++;
+ }
+ return nextSeqNo;
+}
+
+bool AAVCAssembler::recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
+ size_t avail, float goodRatio) {
+ float total = end - start;
+ float valid = connected;
+ float exist = avail;
+ bool isRecycle = (valid / total) >= goodRatio;
+
+ ALOGV("checking p-frame losses.. recvBufs %f valid %f diff %f recycle? %d",
+ exist, valid, total, isRecycle);
+
+ return isRecycle;
+}
+
+int32_t AAVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) {
+ int32_t initSize = queue->size();
+ Queue::iterator it = queue->begin();
+ while (it != queue->end()) {
+ if ((uint32_t)(*it)->int32Data() >= seq) {
+ break;
+ }
+ it++;
+ }
+ queue->erase(queue->begin(), it);
+ return initSize - queue->size();
+}
+
+inline void AAVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {
+ ALOGD("start=%lld, now=%lld, played=%lld",
+ (long long)start, (long long)now, (long long)play);
+}
+
+inline void AAVCAssembler::printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp) {
+ ALOGD("rtp-time(JB)=%lld, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%lld expired=%d",
+ (long long)rtp, (long long)play, (long long)exp, isExp);
+}
+
ARTPAssembler::AssemblyStatus AAVCAssembler::assembleMore(
const sp<ARTPSource> &source) {
AssemblyStatus status = addNALUnit(source);
if (status == MALFORMED_PACKET) {
- mAccessUnitDamaged = true;
+ uint64_t msecsSinceLastIFrame = (ALooper::GetNowUs() / 1000) - mLastIFrameProvidedAtMs;
+ if (msecsSinceLastIFrame > 1000) {
+ ALOGV("request FIR to get a new I-Frame, time since "
+ "last I-Frame %llu ms", (unsigned long long)msecsSinceLastIFrame);
+ source->onIssueFIRByAssembler();
+ }
}
return status;
}
void AAVCAssembler::packetLost() {
CHECK(mNextExpectedSeqNoValid);
- ALOGV("packetLost (expected %d)", mNextExpectedSeqNo);
-
+ ALOGD("packetLost (expected %u)", mNextExpectedSeqNo);
++mNextExpectedSeqNo;
-
- mAccessUnitDamaged = true;
}
void AAVCAssembler::onByeReceived() {
diff --git a/media/libstagefright/rtsp/AAVCAssembler.h b/media/libstagefright/rtsp/AAVCAssembler.h
index e19480c..9d71e2f 100644
--- a/media/libstagefright/rtsp/AAVCAssembler.h
+++ b/media/libstagefright/rtsp/AAVCAssembler.h
@@ -31,6 +31,7 @@
struct AAVCAssembler : public ARTPAssembler {
explicit AAVCAssembler(const sp<AMessage> ¬ify);
+ typedef List<sp<ABuffer> > Queue;
protected:
virtual ~AAVCAssembler();
@@ -45,8 +46,16 @@
bool mNextExpectedSeqNoValid;
uint32_t mNextExpectedSeqNo;
bool mAccessUnitDamaged;
+ bool mFirstIFrameProvided;
+ uint64_t mLastIFrameProvidedAtMs;
+ int32_t mWidth;
+ int32_t mHeight;
List<sp<ABuffer> > mNALUnits;
+ int32_t addNack(const sp<ARTPSource> &source);
+ void checkSpsUpdated(const sp<ABuffer> &buffer);
+ void checkIFrameProvided(const sp<ABuffer> &buffer);
+ bool dropFramesUntilIframe(const sp<ABuffer> &buffer);
AssemblyStatus addNALUnit(const sp<ARTPSource> &source);
void addSingleNALUnit(const sp<ABuffer> &buffer);
AssemblyStatus addFragmentedNALUnit(List<sp<ABuffer> > *queue);
@@ -54,6 +63,14 @@
void submitAccessUnit();
+ inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);
+ int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);
+ bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
+ size_t avail, float goodRatio);
+ int32_t deleteUnitUnderSeq(Queue *q, uint32_t seq);
+ void printNowTimeUs(int64_t start, int64_t now, int64_t play);
+ void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);
+
DISALLOW_EVIL_CONSTRUCTORS(AAVCAssembler);
};
diff --git a/media/libstagefright/rtsp/AHEVCAssembler.cpp b/media/libstagefright/rtsp/AHEVCAssembler.cpp
index c316471..553ea08 100644
--- a/media/libstagefright/rtsp/AHEVCAssembler.cpp
+++ b/media/libstagefright/rtsp/AHEVCAssembler.cpp
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-#define LOG_NDEBUG 0
+//#define LOG_NDEBUG 0
#define LOG_TAG "AHEVCAssembler"
#include <utils/Log.h>
@@ -25,6 +25,7 @@
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
+#include <include/HevcUtils.h>
#include <media/stagefright/foundation/hexdump.h>
#include <stdint.h>
@@ -46,7 +47,11 @@
mAccessUnitRTPTime(0),
mNextExpectedSeqNoValid(false),
mNextExpectedSeqNo(0),
- mAccessUnitDamaged(false) {
+ mAccessUnitDamaged(false),
+ mFirstIFrameProvided(false),
+ mLastIFrameProvidedAtMs(0),
+ mWidth(0),
+ mHeight(0) {
ALOGV("Constructor");
}
@@ -54,42 +59,124 @@
AHEVCAssembler::~AHEVCAssembler() {
}
+int32_t AHEVCAssembler::addNack(
+ const sp<ARTPSource> &source) {
+ List<sp<ABuffer>> *queue = source->queue();
+ int32_t nackCount = 0;
+
+ List<sp<ABuffer> >::iterator it = queue->begin();
+
+ if (it == queue->end()) {
+ return nackCount /* 0 */;
+ }
+
+ uint16_t queueHeadSeqNum = (*it)->int32Data();
+
+ // move to the packet after which RTCP:NACK was sent.
+ for (; it != queue->end(); ++it) {
+ int32_t seqNum = (*it)->int32Data();
+ if (seqNum >= source->mHighestNackNumber) {
+ break;
+ }
+ }
+
+ int32_t nackStartAt = -1;
+
+ while (it != queue->end()) {
+ int32_t seqBeforeLast = (*it)->int32Data();
+ // increase iterator.
+ if ((++it) == queue->end()) {
+ break;
+ }
+
+ int32_t seqLast = (*it)->int32Data();
+
+ if ((seqLast - seqBeforeLast) < 0) {
+ ALOGD("addNack: found end of seqNum from(%d) to(%d)", seqBeforeLast, seqLast);
+ source->mHighestNackNumber = 0;
+ }
+
+ // missed packet found
+ if (seqLast > (seqBeforeLast + 1) &&
+ // we didn't send RTCP:NACK for this packet yet.
+ (seqLast - 1) > source->mHighestNackNumber) {
+ source->mHighestNackNumber = seqLast -1;
+ nackStartAt = seqBeforeLast + 1;
+ break;
+ }
+
+ }
+
+ if (nackStartAt != -1) {
+ nackCount = source->mHighestNackNumber - nackStartAt + 1;
+ ALOGD("addNack: nackCount=%d, nackFrom=%d, nackTo=%d", nackCount,
+ nackStartAt, source->mHighestNackNumber);
+
+ uint16_t mask = (uint16_t)(0xffff) >> (16 - nackCount + 1);
+ source->setSeqNumToNACK(nackStartAt, mask, queueHeadSeqNum);
+ }
+
+ return nackCount;
+}
+
ARTPAssembler::AssemblyStatus AHEVCAssembler::addNALUnit(
const sp<ARTPSource> &source) {
List<sp<ABuffer> > *queue = source->queue();
+ const uint32_t firstRTPTime = source->mFirstRtpTime;
if (queue->empty()) {
return NOT_ENOUGH_DATA;
}
sp<ABuffer> buffer = *queue->begin();
- int32_t rtpTime;
- CHECK(buffer->meta()->findInt32("rtp-time", &rtpTime));
+ buffer->meta()->setObject("source", source);
+ int64_t rtpTime = findRTPTime(firstRTPTime, buffer);
+
int64_t startTime = source->mFirstSysTime / 1000;
int64_t nowTime = ALooper::GetNowUs() / 1000;
int64_t playedTime = nowTime - startTime;
- int32_t playedTimeRtp = source->mFirstRtpTime +
- (((uint32_t)playedTime) * (source->mClockRate / 1000));
- int32_t expiredTimeInJb = rtpTime + (source->mClockRate / 5);
+ int64_t playedTimeRtp = source->mFirstRtpTime + playedTime * (int64_t)source->mClockRate / 1000;
+ const int64_t jitterTime = source->mJbTimeMs * (int64_t)source->mClockRate / 1000;
+
+ int64_t expiredTimeInJb = rtpTime + jitterTime;
bool isExpired = expiredTimeInJb <= (playedTimeRtp);
- ALOGV("start=%lld, now=%lld, played=%lld", (long long)startTime,
- (long long)nowTime, (long long)playedTime);
- ALOGV("rtp-time(JB)=%d, played-rtp-time(JB)=%d, expired-rtp-time(JB)=%d isExpired=%d",
- rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+ bool isTooLate200 = expiredTimeInJb < (playedTimeRtp - jitterTime);
+ bool isTooLate300 = expiredTimeInJb < (playedTimeRtp - (jitterTime * 3 / 2));
+
+ if (mShowQueueCnt < 20) {
+ showCurrentQueue(queue);
+ printNowTimeUs(startTime, nowTime, playedTime);
+ printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+ mShowQueueCnt++;
+ }
+
+ AHEVCAssembler::addNack(source);
if (!isExpired) {
ALOGV("buffering in jitter buffer.");
return NOT_ENOUGH_DATA;
}
- if (mNextExpectedSeqNoValid) {
- List<sp<ABuffer> >::iterator it = queue->begin();
- while (it != queue->end()) {
- if ((uint32_t)(*it)->int32Data() >= mNextExpectedSeqNo) {
- break;
- }
+ if (isTooLate200) {
+ ALOGW("=== WARNING === buffer arrived 200ms late. === WARNING === ");
+ }
- it = queue->erase(it);
+ if (isTooLate300) {
+ ALOGW("buffer arrived after 300ms ... \t Diff in Jb=%lld \t Seq# %d",
+ (long long)(playedTimeRtp - expiredTimeInJb), buffer->int32Data());
+ printNowTimeUs(startTime, nowTime, playedTime);
+ printRTPTime(rtpTime, playedTimeRtp, expiredTimeInJb, isExpired);
+
+ mNextExpectedSeqNo = pickProperSeq(queue, firstRTPTime, playedTimeRtp, jitterTime);
+ }
+
+ if (mNextExpectedSeqNoValid) {
+ int32_t size = queue->size();
+ int32_t cntRemove = deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
+
+ if (cntRemove > 0) {
+ source->noticeAbandonBuffer(cntRemove);
+ ALOGW("delete %d of %d buffers", cntRemove, size);
}
if (queue->empty()) {
@@ -97,6 +184,8 @@
}
}
+ buffer = *queue->begin();
+
if (!mNextExpectedSeqNoValid) {
mNextExpectedSeqNoValid = true;
mNextExpectedSeqNo = (uint32_t)buffer->int32Data();
@@ -152,15 +241,74 @@
}
}
+void AHEVCAssembler::checkSpsUpdated(const sp<ABuffer> &buffer) {
+ if (buffer->size() == 0) {
+ return;
+ }
+ const uint8_t *data = buffer->data();
+ HevcParameterSets paramSets;
+ unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;
+ if (nalType == H265_NALU_SPS) {
+ int32_t width = 0, height = 0;
+ paramSets.FindHEVCDimensions(buffer, &width, &height);
+ ALOGV("existing resolution (%u x %u)", mWidth, mHeight);
+ if (width != mWidth || height != mHeight) {
+ mFirstIFrameProvided = false;
+ mWidth = width;
+ mHeight = height;
+ ALOGD("found a new resolution (%u x %u)", mWidth, mHeight);
+ }
+ }
+}
+
+void AHEVCAssembler::checkIFrameProvided(const sp<ABuffer> &buffer) {
+ if (buffer->size() == 0) {
+ return;
+ }
+ const uint8_t *data = buffer->data();
+ unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;
+ if (nalType > 0x0F && nalType < 0x18) {
+ mLastIFrameProvidedAtMs = ALooper::GetNowUs() / 1000;
+ if (!mFirstIFrameProvided) {
+ mFirstIFrameProvided = true;
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ ALOGD("got First I-frame to be decoded. rtpTime=%d, size=%zu", rtpTime, buffer->size());
+ }
+ }
+}
+
+bool AHEVCAssembler::dropFramesUntilIframe(const sp<ABuffer> &buffer) {
+ if (buffer->size() == 0) {
+ return false;
+ }
+ const uint8_t *data = buffer->data();
+ unsigned nalType = (data[0] >> 1) & H265_NALU_MASK;
+ return !mFirstIFrameProvided && nalType < 0x10;
+}
+
void AHEVCAssembler::addSingleNALUnit(const sp<ABuffer> &buffer) {
ALOGV("addSingleNALUnit of size %zu", buffer->size());
#if !LOG_NDEBUG
hexdump(buffer->data(), buffer->size());
#endif
+ checkSpsUpdated(buffer);
+ checkIFrameProvided(buffer);
uint32_t rtpTime;
CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ if (dropFramesUntilIframe(buffer)) {
+ sp<ARTPSource> source = nullptr;
+ buffer->meta()->findObject("source", (sp<android::RefBase>*)&source);
+ if (source != nullptr) {
+ ALOGD("Issued FIR to get the I-frame");
+ source->onIssueFIRByAssembler();
+ }
+ ALOGD("drop P-frames till an I-frame provided. rtpTime %u", rtpTime);
+ return;
+ }
+
if (!mNALUnits.empty() && rtpTime != mAccessUnitRTPTime) {
submitAccessUnit();
}
@@ -225,7 +373,7 @@
CHECK((indicator & H265_NALU_MASK) == H265_NALU_FU);
- if (size < 2) {
+ if (size < 3) {
ALOGV("Ignoring malformed FU buffer (size = %zu)", size);
queue->erase(queue->begin());
@@ -258,6 +406,11 @@
size_t totalCount = 1;
bool complete = false;
+ uint32_t rtpTimeStartAt;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTimeStartAt));
+ uint32_t startSeqNo = buffer->int32Data();
+ bool pFrame = (nalType < 0x10);
+
if (data[2] & 0x40) {
// Huh? End bit also set on the first buffer.
@@ -266,6 +419,8 @@
complete = true;
} else {
List<sp<ABuffer> >::iterator it = ++queue->begin();
+ int32_t connected = 1;
+ bool snapped = false;
while (it != queue->end()) {
ALOGV("sequence length %zu", totalCount);
@@ -275,26 +430,32 @@
size_t size = buffer->size();
if ((uint32_t)buffer->int32Data() != expectedSeqNo) {
- ALOGV("sequence not complete, expected seqNo %d, got %d",
- expectedSeqNo, (uint32_t)buffer->int32Data());
+ ALOGV("sequence not complete, expected seqNo %u, got %u, nalType %u",
+ expectedSeqNo, (uint32_t)buffer->int32Data(), nalType);
+ snapped = true;
- return WRONG_SEQUENCE_NUMBER;
+ if (!pFrame) {
+ return WRONG_SEQUENCE_NUMBER;
+ }
}
+ if (!snapped) {
+ connected++;
+ }
+
+ uint32_t rtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
if (size < 3
|| ((data[0] >> 1) & H265_NALU_MASK) != indicator
|| (data[2] & H265_NALU_MASK) != nalType
- || (data[2] & 0x80)) {
+ || (data[2] & 0x80)
+ || rtpTime != rtpTimeStartAt) {
ALOGV("Ignoring malformed FU buffer.");
// Delete the whole start of the FU.
- it = queue->begin();
- for (size_t i = 0; i <= totalCount; ++i) {
- it = queue->erase(it);
- }
-
mNextExpectedSeqNo = expectedSeqNo + 1;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
return MALFORMED_PACKET;
}
@@ -302,9 +463,16 @@
totalSize += size - 3;
++totalCount;
- expectedSeqNo = expectedSeqNo + 1;
+ expectedSeqNo = (uint32_t)buffer->int32Data() + 1;
if (data[2] & 0x40) {
+ if (pFrame && !recycleUnit(startSeqNo, expectedSeqNo,
+ connected, totalCount, 0.5f)) {
+ mNextExpectedSeqNo = expectedSeqNo;
+ deleteUnitUnderSeq(queue, mNextExpectedSeqNo);
+
+ return MALFORMED_PACKET;
+ }
// This is the last fragment.
complete = true;
break;
@@ -333,6 +501,7 @@
unit->data()[1] = tid;
size_t offset = 2;
+ int32_t cvo = -1;
List<sp<ABuffer> >::iterator it = queue->begin();
for (size_t i = 0; i < totalCount; ++i) {
const sp<ABuffer> &buffer = *it;
@@ -343,6 +512,7 @@
#endif
memcpy(unit->data() + offset, buffer->data() + 3, buffer->size() - 3);
+ buffer->meta()->findInt32("cvo", &cvo);
offset += buffer->size() - 3;
it = queue->erase(it);
@@ -350,6 +520,10 @@
unit->setRange(0, totalSize);
+ if (cvo >= 0) {
+ unit->meta()->setInt32("cvo", cvo);
+ }
+
addSingleNALUnit(unit);
ALOGV("successfully assembled a NAL unit from fragments.");
@@ -370,6 +544,7 @@
sp<ABuffer> accessUnit = new ABuffer(totalSize);
size_t offset = 0;
+ int32_t cvo = -1;
for (List<sp<ABuffer> >::iterator it = mNALUnits.begin();
it != mNALUnits.end(); ++it) {
memcpy(accessUnit->data() + offset, "\x00\x00\x00\x01", 4);
@@ -378,6 +553,7 @@
sp<ABuffer> nal = *it;
memcpy(accessUnit->data() + offset, nal->data(), nal->size());
offset += nal->size();
+ nal->meta()->findInt32("cvo", &cvo);
}
CopyTimes(accessUnit, *mNALUnits.begin());
@@ -386,6 +562,9 @@
printf(mAccessUnitDamaged ? "X" : ".");
fflush(stdout);
#endif
+ if (cvo >= 0) {
+ accessUnit->meta()->setInt32("cvo", cvo);
+ }
if (mAccessUnitDamaged) {
accessUnit->meta()->setInt32("damaged", true);
@@ -399,22 +578,90 @@
msg->post();
}
+inline int64_t AHEVCAssembler::findRTPTime(
+ const uint32_t& firstRTPTime, const sp<ABuffer>& buffer) {
+ /* If you want to +, -, * rtpTime, recommend to declare rtpTime as int64_t.
+ Because rtpTime can be near UINT32_MAX. Beware the overflow. */
+ int64_t rtpTime = 0;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&rtpTime));
+ // If the first overs 2^31 and rtp unders 2^31, the rtp value is overflowed one.
+ int64_t overflowMask = (firstRTPTime & 0x80000000 & ~rtpTime) << 1;
+ return rtpTime | overflowMask;
+}
+
+int32_t AHEVCAssembler::pickProperSeq(const Queue *queue,
+ uint32_t first, int64_t play, int64_t jit) {
+ sp<ABuffer> buffer = *(queue->begin());
+ int32_t nextSeqNo = buffer->int32Data();
+
+ Queue::const_iterator it = queue->begin();
+ while (it != queue->end()) {
+ int64_t rtpTime = findRTPTime(first, *it);
+ // if pkt in time exists, that should be the next pivot
+ if (rtpTime + jit >= play) {
+ nextSeqNo = (*it)->int32Data();
+ break;
+ }
+ it++;
+ }
+ return nextSeqNo;
+}
+
+bool AHEVCAssembler::recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
+ size_t avail, float goodRatio) {
+ float total = end - start;
+ float valid = connected;
+ float exist = avail;
+ bool isRecycle = (valid / total) >= goodRatio;
+
+ ALOGV("checking p-frame losses.. recvBufs %f valid %f diff %f recycle? %d",
+ exist, valid, total, isRecycle);
+
+ return isRecycle;
+}
+
+int32_t AHEVCAssembler::deleteUnitUnderSeq(Queue *queue, uint32_t seq) {
+ int32_t initSize = queue->size();
+ Queue::iterator it = queue->begin();
+ while (it != queue->end()) {
+ if ((uint32_t)(*it)->int32Data() >= seq) {
+ break;
+ }
+ it++;
+ }
+ queue->erase(queue->begin(), it);
+ return initSize - queue->size();
+}
+
+inline void AHEVCAssembler::printNowTimeUs(int64_t start, int64_t now, int64_t play) {
+ ALOGD("start=%lld, now=%lld, played=%lld",
+ (long long)start, (long long)now, (long long)play);
+}
+
+inline void AHEVCAssembler::printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp) {
+ ALOGD("rtp-time(JB)=%lld, played-rtp-time(JB)=%lld, expired-rtp-time(JB)=%lld expired=%d",
+ (long long)rtp, (long long)play, (long long)exp, isExp);
+}
+
ARTPAssembler::AssemblyStatus AHEVCAssembler::assembleMore(
const sp<ARTPSource> &source) {
AssemblyStatus status = addNALUnit(source);
if (status == MALFORMED_PACKET) {
- mAccessUnitDamaged = true;
+ uint64_t msecsSinceLastIFrame = (ALooper::GetNowUs() / 1000) - mLastIFrameProvidedAtMs;
+ if (msecsSinceLastIFrame > 1000) {
+ ALOGV("request FIR to get a new I-Frame, time after "
+ "last I-Frame in %llu ms", (unsigned long long)msecsSinceLastIFrame);
+ source->onIssueFIRByAssembler();
+ }
}
return status;
}
void AHEVCAssembler::packetLost() {
CHECK(mNextExpectedSeqNoValid);
- ALOGV("packetLost (expected %d)", mNextExpectedSeqNo);
+ ALOGD("packetLost (expected %u)", mNextExpectedSeqNo);
++mNextExpectedSeqNo;
-
- mAccessUnitDamaged = true;
}
void AHEVCAssembler::onByeReceived() {
diff --git a/media/libstagefright/rtsp/AHEVCAssembler.h b/media/libstagefright/rtsp/AHEVCAssembler.h
index cc20622..bf1cded 100644
--- a/media/libstagefright/rtsp/AHEVCAssembler.h
+++ b/media/libstagefright/rtsp/AHEVCAssembler.h
@@ -31,6 +31,8 @@
struct AHEVCAssembler : public ARTPAssembler {
AHEVCAssembler(const sp<AMessage> ¬ify);
+ typedef List<sp<ABuffer> > Queue;
+
protected:
virtual ~AHEVCAssembler();
@@ -45,8 +47,16 @@
bool mNextExpectedSeqNoValid;
uint32_t mNextExpectedSeqNo;
bool mAccessUnitDamaged;
+ bool mFirstIFrameProvided;
+ uint64_t mLastIFrameProvidedAtMs;
+ int32_t mWidth;
+ int32_t mHeight;
List<sp<ABuffer> > mNALUnits;
+ int32_t addNack(const sp<ARTPSource> &source);
+ void checkSpsUpdated(const sp<ABuffer> &buffer);
+ void checkIFrameProvided(const sp<ABuffer> &buffer);
+ bool dropFramesUntilIframe(const sp<ABuffer> &buffer);
AssemblyStatus addNALUnit(const sp<ARTPSource> &source);
void addSingleNALUnit(const sp<ABuffer> &buffer);
AssemblyStatus addFragmentedNALUnit(List<sp<ABuffer> > *queue);
@@ -54,6 +64,14 @@
void submitAccessUnit();
+ inline int64_t findRTPTime(const uint32_t& firstRTPTime, const sp<ABuffer>& buffer);
+ int32_t pickProperSeq(const Queue *q, uint32_t first, int64_t play, int64_t jit);
+ bool recycleUnit(uint32_t start, uint32_t end, uint32_t connected,
+ size_t avail, float goodRatio);
+ int32_t deleteUnitUnderSeq(Queue *queue, uint32_t seq);
+ void printNowTimeUs(int64_t start, int64_t now, int64_t play);
+ void printRTPTime(int64_t rtp, int64_t play, int64_t exp, bool isExp);
+
DISALLOW_EVIL_CONSTRUCTORS(AHEVCAssembler);
};
diff --git a/media/libstagefright/rtsp/ARTPAssembler.cpp b/media/libstagefright/rtsp/ARTPAssembler.cpp
index befc226..52aa3a0 100644
--- a/media/libstagefright/rtsp/ARTPAssembler.cpp
+++ b/media/libstagefright/rtsp/ARTPAssembler.cpp
@@ -14,6 +14,7 @@
* limitations under the License.
*/
+#define LOG_TAG "ARTPAssembler"
#include "ARTPAssembler.h"
#include <media/stagefright/foundation/ABuffer.h>
@@ -21,12 +22,16 @@
#include <media/stagefright/foundation/ALooper.h>
#include <media/stagefright/foundation/AMessage.h>
+#include <android-base/properties.h>
+
#include <stdint.h>
namespace android {
ARTPAssembler::ARTPAssembler()
- : mFirstFailureTimeUs(-1) {
+ : mShowQueueCnt(0),
+ mFirstFailureTimeUs(-1) {
+ mShowQueue = android::base::GetBoolProperty("debug.stagefright.rtp", false);
}
void ARTPAssembler::onPacketReceived(const sp<ARTPSource> &source) {
@@ -141,4 +146,15 @@
return accessUnit;
}
+void ARTPAssembler::showCurrentQueue(List<sp<ABuffer> > *queue) {
+ AString temp("Queue elem size : ");
+ List<sp<ABuffer> >::iterator it = queue->begin();
+ while (it != queue->end()) {
+ temp.append((*it)->size());
+ temp.append(" \t");
+ it++;
+ }
+ ALOGD("%s",temp.c_str());
+};
+
} // namespace android
diff --git a/media/libstagefright/rtsp/ARTPAssembler.h b/media/libstagefright/rtsp/ARTPAssembler.h
index 4082d4c..191f08e 100644
--- a/media/libstagefright/rtsp/ARTPAssembler.h
+++ b/media/libstagefright/rtsp/ARTPAssembler.h
@@ -56,6 +56,11 @@
static sp<ABuffer> MakeCompoundFromPackets(
const List<sp<ABuffer> > &frames);
+ void showCurrentQueue(List<sp<ABuffer> > *queue);
+
+ bool mShowQueue;
+ int32_t mShowQueueCnt;
+
private:
int64_t mFirstFailureTimeUs;
diff --git a/media/libstagefright/rtsp/ARTPConnection.cpp b/media/libstagefright/rtsp/ARTPConnection.cpp
index 12cce00..61c06d1 100644
--- a/media/libstagefright/rtsp/ARTPConnection.cpp
+++ b/media/libstagefright/rtsp/ARTPConnection.cpp
@@ -30,6 +30,8 @@
#include <media/stagefright/foundation/AString.h>
#include <media/stagefright/foundation/hexdump.h>
+#include <android/multinetwork.h>
+
#include <arpa/inet.h>
#include <sys/socket.h>
@@ -67,12 +69,18 @@
struct sockaddr_in6 mRemoteRTCPAddr6;
bool mIsInjected;
+
+ // RTCP Extension for CVO
+ int mCVOExtMap; // will be set to 0 if cvo is not negotiated in sdp
};
ARTPConnection::ARTPConnection(uint32_t flags)
: mFlags(flags),
mPollEventPending(false),
- mLastReceiverReportTimeUs(-1) {
+ mLastReceiverReportTimeUs(-1),
+ mLastBitrateReportTimeUs(-1),
+ mTargetBitrate(-1),
+ mJbTimeMs(300) {
}
ARTPConnection::~ARTPConnection() {
@@ -157,7 +165,7 @@
// static
void ARTPConnection::MakeRTPSocketPair(
int *rtpSocket, int *rtcpSocket, const char *localIp, const char *remoteIp,
- unsigned localPort, unsigned remotePort) {
+ unsigned localPort, unsigned remotePort, int64_t socketNetwork) {
bool isIPv6 = false;
if (strchr(localIp, ':') != NULL)
isIPv6 = true;
@@ -170,6 +178,22 @@
*rtcpSocket = socket(isIPv6 ? AF_INET6 : AF_INET, SOCK_DGRAM, 0);
CHECK_GE(*rtcpSocket, 0);
+ if (socketNetwork != 0) {
+ ALOGD("trying to bind rtp socket(%d) to network(%llu).",
+ *rtpSocket, (unsigned long long)socketNetwork);
+
+ int result = android_setsocknetwork((net_handle_t)socketNetwork, *rtpSocket);
+ if (result != 0) {
+ ALOGW("failed(%d) to bind rtp socket(%d) to network(%llu)",
+ result, *rtpSocket, (unsigned long long)socketNetwork);
+ }
+ result = android_setsocknetwork((net_handle_t)socketNetwork, *rtcpSocket);
+ if (result != 0) {
+ ALOGW("failed(%d) to bind rtcp socket(%d) to network(%llu)",
+ result, *rtcpSocket, (unsigned long long)socketNetwork);
+ }
+ }
+
bumpSocketBufferSize(*rtcpSocket);
struct sockaddr *addr;
@@ -191,8 +215,8 @@
}
int sockopt = 1;
- setsockopt(*rtpSocket, SOL_SOCKET, SO_REUSEPORT, (int *)&sockopt, sizeof(sockopt));
- setsockopt(*rtcpSocket, SOL_SOCKET, SO_REUSEPORT, (int *)&sockopt, sizeof(sockopt));
+ setsockopt(*rtpSocket, SOL_SOCKET, SO_REUSEADDR, (int *)&sockopt, sizeof(sockopt));
+ setsockopt(*rtcpSocket, SOL_SOCKET, SO_REUSEADDR, (int *)&sockopt, sizeof(sockopt));
int sizeSockSt = isIPv6 ? sizeof(addr6) : sizeof(addr4);
@@ -310,6 +334,18 @@
memset(&info->mRemoteRTCPAddr, 0, sizeof(info->mRemoteRTCPAddr));
memset(&info->mRemoteRTCPAddr6, 0, sizeof(info->mRemoteRTCPAddr6));
+ sp<ASessionDescription> sessionDesc = info->mSessionDesc;
+ info->mCVOExtMap = 0;
+ for (size_t i = 1; i < sessionDesc->countTracks(); ++i) {
+ int32_t cvoExtMap;
+ if (sessionDesc->getCvoExtMap(i, &cvoExtMap)) {
+ info->mCVOExtMap = cvoExtMap;
+ ALOGI("urn:3gpp:video-orientation(cvo) found as extmap:%d", info->mCVOExtMap);
+ } else {
+ ALOGI("urn:3gpp:video-orientation(cvo) not found :%d", info->mCVOExtMap);
+ }
+ }
+
if (!injected) {
postPollEvent();
}
@@ -400,17 +436,59 @@
if (err == -ECONNRESET) {
// socket failure, this stream is dead, Jim.
+ sp<AMessage> notify = it->mNotifyMsg->dup();
+ notify->setInt32("rtcp-event", 1);
+ notify->setInt32("payload-type", 400);
+ notify->setInt32("feedback-type", 1);
+ notify->setInt32("sender", it->mSources.valueAt(0)->getSelfID());
+ notify->post();
ALOGW("failed to receive RTP/RTCP datagram.");
it = mStreams.erase(it);
continue;
}
+ // add NACK and FIR that needs to be sent immediately.
+ sp<ABuffer> buffer = new ABuffer(kMaxUDPSize);
+ for (size_t i = 0; i < it->mSources.size(); ++i) {
+ buffer->setRange(0, 0);
+ int cnt = it->mSources.valueAt(i)->addNACK(buffer);
+ if (cnt > 0) {
+ ALOGV("Send NACK for lost %d Packets", cnt);
+ send(&*it, buffer);
+ }
+
+ buffer->setRange(0, 0);
+ it->mSources.valueAt(i)->addFIR(buffer);
+ if (buffer->size() > 0) {
+ ALOGD("Send FIR immediately for lost Packets");
+ send(&*it, buffer);
+ }
+
+ buffer->setRange(0, 0);
+ it->mSources.valueAt(i)->addTMMBR(buffer, mTargetBitrate);
+ mTargetBitrate = -1;
+ if (buffer->size() > 0) {
+ ALOGV("Sending TMMBR...");
+ ssize_t n = send(&*it, buffer);
+
+ if (n != (ssize_t)buffer->size()) {
+ ALOGW("failed to send RTCP TMMBR (%s).",
+ n >= 0 ? "connection gone" : strerror(errno));
+
+ it = mStreams.erase(it);
+ continue;
+ }
+ }
+ }
+
++it;
}
}
int64_t nowUs = ALooper::GetNowUs();
+ checkRxBitrate(nowUs);
+
if (mLastReceiverReportTimeUs <= 0
|| mLastReceiverReportTimeUs + 5000000LL <= nowUs) {
sp<ABuffer> buffer = new ABuffer(kMaxUDPSize);
@@ -445,33 +523,16 @@
if (buffer->size() > 0) {
ALOGV("Sending RR...");
- struct sockaddr* pRemoteRTCPAddr;
- int sizeSockSt;
- if (s->isIPv6) {
- pRemoteRTCPAddr = (struct sockaddr *)&s->mRemoteRTCPAddr6;
- sizeSockSt = sizeof(struct sockaddr_in6);
- } else {
- pRemoteRTCPAddr = (struct sockaddr *)&s->mRemoteRTCPAddr;
- sizeSockSt = sizeof(struct sockaddr_in);
- }
+ ssize_t n = send(s, buffer);
- ssize_t n;
- do {
- n = sendto(
- s->mRTCPSocket, buffer->data(), buffer->size(), 0,
- pRemoteRTCPAddr, sizeSockSt);
- } while (n < 0 && errno == EINTR);
-
- if (n <= 0) {
+ if (n != (ssize_t)buffer->size()) {
ALOGW("failed to send RTCP receiver report (%s).",
- n == 0 ? "connection gone" : strerror(errno));
+ n >= 0 ? "connection gone" : strerror(errno));
it = mStreams.erase(it);
continue;
}
- CHECK_EQ(n, (ssize_t)buffer->size());
-
mLastReceiverReportTimeUs = nowUs;
}
@@ -504,6 +565,10 @@
(!receiveRTP && s->mNumRTCPPacketsReceived == 0)
? sizeSockSt : 0;
+ if (mFlags & kViLTEConnection) {
+ remoteAddrLen = 0;
+ }
+
ssize_t nbytes;
do {
nbytes = recvfrom(
@@ -513,6 +578,7 @@
0,
remoteAddrLen > 0 ? pRemoteRTCPAddr : NULL,
remoteAddrLen > 0 ? &remoteAddrLen : NULL);
+ mCumulativeBytes += nbytes;
} while (nbytes < 0 && errno == EINTR);
if (nbytes <= 0) {
@@ -533,6 +599,36 @@
return err;
}
+ssize_t ARTPConnection::send(const StreamInfo *info, const sp<ABuffer> buffer) {
+ struct sockaddr* pRemoteRTCPAddr;
+ int sizeSockSt;
+
+ /* It seems this isIPv6 variable is useless.
+ * We should remove it to prevent confusion */
+ if (info->isIPv6) {
+ pRemoteRTCPAddr = (struct sockaddr *)&info->mRemoteRTCPAddr6;
+ sizeSockSt = sizeof(struct sockaddr_in6);
+ } else {
+ pRemoteRTCPAddr = (struct sockaddr *)&info->mRemoteRTCPAddr;
+ sizeSockSt = sizeof(struct sockaddr_in);
+ }
+
+ if (mFlags & kViLTEConnection) {
+ ALOGV("ViLTE RTCP");
+ pRemoteRTCPAddr = NULL;
+ sizeSockSt = 0;
+ }
+
+ ssize_t n;
+ do {
+ n = sendto(
+ info->mRTCPSocket, buffer->data(), buffer->size(), 0,
+ pRemoteRTCPAddr, sizeSockSt);
+ } while (n < 0 && errno == EINTR);
+
+ return n;
+}
+
status_t ARTPConnection::parseRTP(StreamInfo *s, const sp<ABuffer> &buffer) {
if (s->mNumRTPPacketsReceived++ == 0) {
sp<AMessage> notify = s->mNotifyMsg->dup();
@@ -554,6 +650,11 @@
return -1;
}
+ if ((data[1] & 0x7f) == 20 /* decimal */) {
+ // Unassigned payload type
+ return -1;
+ }
+
if (data[0] & 0x20) {
// Padding present.
@@ -577,6 +678,7 @@
return -1;
}
+ int32_t cvoDegrees = -1;
if (data[0] & 0x10) {
// Header eXtension present.
@@ -590,13 +692,14 @@
const uint8_t *extensionData = &data[payloadOffset];
size_t extensionLength =
- 4 * (extensionData[2] << 8 | extensionData[3]);
+ (4 * (extensionData[2] << 8 | extensionData[3])) + 4;
- if (size < payloadOffset + 4 + extensionLength) {
+ if (size < payloadOffset + extensionLength) {
return -1;
}
- payloadOffset += 4 + extensionLength;
+ parseRTPExt(s, (const uint8_t *)extensionData, extensionLength, &cvoDegrees);
+ payloadOffset += extensionLength;
}
uint32_t srcId = u32at(&data[8]);
@@ -610,6 +713,9 @@
meta->setInt32("rtp-time", rtpTime);
meta->setInt32("PT", data[1] & 0x7f);
meta->setInt32("M", data[1] >> 7);
+ if (cvoDegrees >= 0) {
+ meta->setInt32("cvo", cvoDegrees);
+ }
buffer->setInt32Data(u16at(&data[2]));
buffer->setRange(payloadOffset, size - payloadOffset);
@@ -619,11 +725,65 @@
return OK;
}
+status_t ARTPConnection::parseRTPExt(StreamInfo *s,
+ const uint8_t *extHeader, size_t extLen, int32_t *cvoDegrees) {
+ if (extLen < 4)
+ return -1;
+
+ uint16_t header = (extHeader[0] << 8) | (extHeader[1]);
+ bool isOnebyteHeader = false;
+
+ if (header == 0xBEDE) {
+ isOnebyteHeader = true;
+ } else if (header == 0x1000) {
+ ALOGW("parseRTPExt: two-byte header is not implemented yet");
+ return -1;
+ } else {
+ ALOGW("parseRTPExt: can not recognize header");
+ return -1;
+ }
+
+ const uint8_t *extPayload = extHeader + 4;
+ extLen -= 4;
+ size_t offset = 0; //start from first payload of rtp extension.
+ // one-byte header parser
+ while (isOnebyteHeader && offset < extLen) {
+ uint8_t extmapId = extPayload[offset] >> 4;
+ uint8_t length = (extPayload[offset] & 0xF) + 1;
+ offset++;
+
+ // padding case
+ if (extmapId == 0)
+ continue;
+
+ uint8_t data[16]; // maximum length value
+ for (uint8_t j = 0; offset + j <= extLen && j < length; j++) {
+ data[j] = extPayload[offset + j];
+ }
+
+ offset += length;
+
+ if (extmapId == s->mCVOExtMap) {
+ *cvoDegrees = (int32_t)data[0];
+ return OK;
+ }
+ }
+
+ return BAD_VALUE;
+}
+
status_t ARTPConnection::parseRTCP(StreamInfo *s, const sp<ABuffer> &buffer) {
if (s->mNumRTCPPacketsReceived++ == 0) {
sp<AMessage> notify = s->mNotifyMsg->dup();
notify->setInt32("first-rtcp", true);
notify->post();
+
+ ALOGI("send first-rtcp event to upper layer as ImsRxNotice");
+ sp<AMessage> imsNotify = s->mNotifyMsg->dup();
+ imsNotify->setInt32("rtcp-event", 1);
+ imsNotify->setInt32("payload-type", 101);
+ imsNotify->setInt32("feedback-type", 0);
+ imsNotify->post();
}
const uint8_t *data = buffer->data();
@@ -674,8 +834,12 @@
break;
case 205: // TSFB (transport layer specific feedback)
+ parseTSFB(s, data, headerLength);
+ break;
case 206: // PSFB (payload specific feedback)
// hexdump(data, headerLength);
+ parsePSFB(s, data, headerLength);
+ ALOGI("RTCP packet type %u of size %zu", (unsigned)data[1], headerLength);
break;
case 203:
@@ -712,6 +876,12 @@
sp<ARTPSource> source = findSource(s, id);
+ // Report a final stastics to be used for rtp data usage.
+ int64_t nowUs = ALooper::GetNowUs();
+ int32_t timeDiff = (nowUs - mLastBitrateReportTimeUs) / 1000000ll;
+ int32_t bitrate = mCumulativeBytes * 8 / timeDiff;
+ source->notifyPktInfo(bitrate, true /* isRegular */);
+
source->byeReceived();
return OK;
@@ -744,6 +914,144 @@
return 0;
}
+status_t ARTPConnection::parseTSFB(
+ StreamInfo *s, const uint8_t *data, size_t size) {
+ if (size < 12) {
+ // broken packet
+ return -1;
+ }
+
+ uint8_t msgType = data[0] & 0x1f;
+ uint32_t id = u32at(&data[4]);
+
+ const uint8_t *ptr = &data[12];
+ size -= 12;
+
+ using namespace std;
+ size_t FCISize;
+ switch(msgType) {
+ case 1: // Generic NACK
+ {
+ FCISize = 4;
+ while (size >= FCISize) {
+ uint16_t PID = u16at(&ptr[0]); // lost packet RTP number
+ uint16_t BLP = u16at(&ptr[2]); // Bitmask of following Lost Packets
+
+ size -= FCISize;
+ ptr += FCISize;
+
+ AString list_of_losts;
+ list_of_losts.append(PID);
+ for (int i=0 ; i<16 ; i++) {
+ bool is_lost = BLP & (0x1 << i);
+ if (is_lost) {
+ list_of_losts.append(", ");
+ list_of_losts.append(PID + i);
+ }
+ }
+ ALOGI("Opponent losts packet of RTP %s", list_of_losts.c_str());
+ }
+ break;
+ }
+ case 3: // TMMBR
+ case 4: // TMMBN
+ {
+ FCISize = 8;
+ while (size >= FCISize) {
+ uint32_t MxTBR = u32at(&ptr[4]);
+ uint32_t MxTBRExp = MxTBR >> 26;
+ uint32_t MxTBRMantissa = (MxTBR >> 9) & 0x01FFFF;
+ uint32_t overhead = MxTBR & 0x01FF;
+
+ size -= FCISize;
+ ptr += FCISize;
+
+ uint32_t bitRate = (1 << MxTBRExp) * MxTBRMantissa;
+
+ if (msgType == 3)
+ ALOGI("Op -> UE Req Tx bitrate : %d X 2^%d = %d",
+ MxTBRMantissa, MxTBRExp, bitRate);
+ else if (msgType == 4)
+ ALOGI("OP -> UE Noti Rx bitrate : %d X 2^%d = %d",
+ MxTBRMantissa, MxTBRExp, bitRate);
+
+ sp<AMessage> notify = s->mNotifyMsg->dup();
+ notify->setInt32("rtcp-event", 1);
+ notify->setInt32("payload-type", 205);
+ notify->setInt32("feedback-type", msgType);
+ notify->setInt32("sender", id);
+ notify->setInt32("bit-rate", bitRate);
+ notify->post();
+ ALOGI("overhead : %d", overhead);
+ }
+ break;
+ }
+ default:
+ {
+ ALOGI("Not supported TSFB type %d", msgType);
+ break;
+ }
+ }
+
+ return 0;
+}
+
+status_t ARTPConnection::parsePSFB(
+ StreamInfo *s, const uint8_t *data, size_t size) {
+ if (size < 12) {
+ // broken packet
+ return -1;
+ }
+
+ uint8_t msgType = data[0] & 0x1f;
+ uint32_t id = u32at(&data[4]);
+
+ const uint8_t *ptr = &data[12];
+ size -= 12;
+
+ using namespace std;
+ switch(msgType) {
+ case 1: // Picture Loss Indication (PLI)
+ {
+ if (size > 0) {
+ // PLI does not need parameters
+ break;
+ };
+ sp<AMessage> notify = s->mNotifyMsg->dup();
+ notify->setInt32("rtcp-event", 1);
+ notify->setInt32("payload-type", 206);
+ notify->setInt32("feedback-type", msgType);
+ notify->setInt32("sender", id);
+ notify->post();
+ ALOGI("PLI detected.");
+ break;
+ }
+ case 4: // Full Intra Request (FIR)
+ {
+ if (size < 4) {
+ break;
+ }
+ uint32_t requestedId = u32at(&ptr[0]);
+ if (requestedId == (uint32_t)mSelfID) {
+ sp<AMessage> notify = s->mNotifyMsg->dup();
+ notify->setInt32("rtcp-event", 1);
+ notify->setInt32("payload-type", 206);
+ notify->setInt32("feedback-type", msgType);
+ notify->setInt32("sender", id);
+ notify->post();
+ ALOGI("FIR detected.");
+ }
+ break;
+ }
+ default:
+ {
+ ALOGI("Not supported PSFB type %d", msgType);
+ break;
+ }
+ }
+
+ return 0;
+}
sp<ARTPSource> ARTPConnection::findSource(StreamInfo *info, uint32_t srcId) {
sp<ARTPSource> source;
ssize_t index = info->mSources.indexOfKey(srcId);
@@ -753,6 +1061,12 @@
source = new ARTPSource(
srcId, info->mSessionDesc, info->mIndex, info->mNotifyMsg);
+ if (mFlags & kViLTEConnection) {
+ source->setPeriodicFIR(false);
+ }
+
+ source->setSelfID(mSelfID);
+ source->setJbTime(mJbTimeMs > 0 ? mJbTimeMs : 300);
info->mSources.add(srcId, source);
} else {
source = info->mSources.valueAt(index);
@@ -768,6 +1082,78 @@
msg->post();
}
+void ARTPConnection::setSelfID(const uint32_t selfID) {
+ mSelfID = selfID;
+}
+
+void ARTPConnection::setJbTime(const uint32_t jbTimeMs) {
+ mJbTimeMs = jbTimeMs;
+}
+
+void ARTPConnection::setTargetBitrate(int32_t targetBitrate) {
+ mTargetBitrate = targetBitrate;
+}
+
+void ARTPConnection::checkRxBitrate(int64_t nowUs) {
+ if (mLastBitrateReportTimeUs <= 0) {
+ mCumulativeBytes = 0;
+ mLastBitrateReportTimeUs = nowUs;
+ }
+ else if (mLastEarlyNotifyTimeUs + 100000ll <= nowUs) {
+ int32_t timeDiff = (nowUs - mLastBitrateReportTimeUs) / 1000000ll;
+ int32_t bitrate = mCumulativeBytes * 8 / timeDiff;
+ mLastEarlyNotifyTimeUs = nowUs;
+
+ List<StreamInfo>::iterator it = mStreams.begin();
+ while (it != mStreams.end()) {
+ StreamInfo *s = &*it;
+ if (s->mIsInjected) {
+ ++it;
+ continue;
+ }
+ for (size_t i = 0; i < s->mSources.size(); ++i) {
+ sp<ARTPSource> source = s->mSources.valueAt(i);
+ if (source->isNeedToEarlyNotify()) {
+ source->notifyPktInfo(bitrate, false /* isRegular */);
+ mLastEarlyNotifyTimeUs = nowUs + (1000000ll * 3600 * 24); // after 1 day
+ }
+ }
+ ++it;
+ }
+ }
+ else if (mLastBitrateReportTimeUs + 1000000ll <= nowUs) {
+ int32_t timeDiff = (nowUs - mLastBitrateReportTimeUs) / 1000000ll;
+ int32_t bitrate = mCumulativeBytes * 8 / timeDiff;
+ ALOGI("Actual Rx bitrate : %d bits/sec", bitrate);
+
+ sp<ABuffer> buffer = new ABuffer(kMaxUDPSize);
+ List<StreamInfo>::iterator it = mStreams.begin();
+ while (it != mStreams.end()) {
+ StreamInfo *s = &*it;
+ if (s->mIsInjected) {
+ ++it;
+ continue;
+ }
+
+ if (s->mNumRTCPPacketsReceived == 0) {
+ // We have never received any RTCP packets on this stream,
+ // we don't even know where to send a report.
+ ++it;
+ continue;
+ }
+
+ buffer->setRange(0, 0);
+ for (size_t i = 0; i < s->mSources.size(); ++i) {
+ sp<ARTPSource> source = s->mSources.valueAt(i);
+ source->notifyPktInfo(bitrate, true /* isRegular */);
+ }
+ ++it;
+ }
+ mCumulativeBytes = 0;
+ mLastBitrateReportTimeUs = nowUs;
+ mLastEarlyNotifyTimeUs = nowUs;
+ }
+}
void ARTPConnection::onInjectPacket(const sp<AMessage> &msg) {
int32_t index;
CHECK(msg->findInt32("index", &index));
@@ -795,4 +1181,3 @@
}
} // namespace android
-
diff --git a/media/libstagefright/rtsp/ARTPConnection.h b/media/libstagefright/rtsp/ARTPConnection.h
index 889ec30..a37ac0e 100644
--- a/media/libstagefright/rtsp/ARTPConnection.h
+++ b/media/libstagefright/rtsp/ARTPConnection.h
@@ -30,6 +30,7 @@
struct ARTPConnection : public AHandler {
enum Flags {
kRegularlyRequestFIR = 2,
+ kViLTEConnection = 4,
};
explicit ARTPConnection(uint32_t flags = 0);
@@ -44,6 +45,10 @@
void injectPacket(int index, const sp<ABuffer> &buffer);
+ void setSelfID(const uint32_t selfID);
+ void setJbTime(const uint32_t jbTimeMs);
+ void setTargetBitrate(int32_t targetBitrate);
+
// Creates a pair of UDP datagram sockets bound to adjacent ports
// (the rtpSocket is bound to an even port, the rtcpSocket to the
// next higher port).
@@ -55,7 +60,7 @@
static void MakeRTPSocketPair(
int *rtpSocket, int *rtcpSocket,
const char *localIp, const char *remoteIp,
- unsigned localPort, unsigned remotePort);
+ unsigned localPort, unsigned remotePort, int64_t socketNetwork = 0);
protected:
virtual ~ARTPConnection();
@@ -78,18 +83,32 @@
bool mPollEventPending;
int64_t mLastReceiverReportTimeUs;
+ int64_t mLastBitrateReportTimeUs;
+ int64_t mLastEarlyNotifyTimeUs;
+
+ int32_t mSelfID;
+ int32_t mTargetBitrate;
+
+ uint32_t mJbTimeMs;
+
+ int32_t mCumulativeBytes;
void onAddStream(const sp<AMessage> &msg);
void onRemoveStream(const sp<AMessage> &msg);
void onPollStreams();
void onInjectPacket(const sp<AMessage> &msg);
void onSendReceiverReports();
+ void checkRxBitrate(int64_t nowUs);
status_t receive(StreamInfo *info, bool receiveRTP);
+ ssize_t send(const StreamInfo *info, const sp<ABuffer> buffer);
status_t parseRTP(StreamInfo *info, const sp<ABuffer> &buffer);
+ status_t parseRTPExt(StreamInfo *s, const uint8_t *extData, size_t extLen, int32_t *cvoDegrees);
status_t parseRTCP(StreamInfo *info, const sp<ABuffer> &buffer);
status_t parseSR(StreamInfo *info, const uint8_t *data, size_t size);
+ status_t parseTSFB(StreamInfo *info, const uint8_t *data, size_t size);
+ status_t parsePSFB(StreamInfo *info, const uint8_t *data, size_t size);
status_t parseBYE(StreamInfo *info, const uint8_t *data, size_t size);
sp<ARTPSource> findSource(StreamInfo *info, uint32_t id);
diff --git a/media/libstagefright/rtsp/ARTPSource.cpp b/media/libstagefright/rtsp/ARTPSource.cpp
index e271ac1..3fdf8e4 100644
--- a/media/libstagefright/rtsp/ARTPSource.cpp
+++ b/media/libstagefright/rtsp/ARTPSource.cpp
@@ -34,9 +34,11 @@
#include <media/stagefright/foundation/ADebug.h>
#include <media/stagefright/foundation/AMessage.h>
+#include <strings.h>
+
namespace android {
-static const uint32_t kSourceID = 0xdeadbeef;
+static uint32_t kSourceID = 0xdeadbeef;
ARTPSource::ARTPSource(
uint32_t id,
@@ -46,15 +48,21 @@
mFirstRtpTime(0),
mFirstSysTime(0),
mClockRate(0),
+ mJbTimeMs(300), // default jitter buffer time is 300ms.
+ mFirstSsrc(0),
+ mHighestNackNumber(0),
mID(id),
mHighestSeqNumber(0),
mPrevExpected(0),
mBaseSeqNumber(0),
mNumBuffersReceived(0),
mPrevNumBuffersReceived(0),
+ mPrevExpectedForRR(0),
+ mPrevNumBuffersReceivedForRR(0),
mLastNTPTime(0),
mLastNTPTimeUpdateUs(0),
mIssueFIRRequests(false),
+ mIssueFIRByAssembler(false),
mLastFIRRequestUs(-1),
mNextFIRSeqNo((rand() * 256.0) / RAND_MAX),
mNotify(notify) {
@@ -120,20 +128,29 @@
bool ARTPSource::queuePacket(const sp<ABuffer> &buffer) {
uint32_t seqNum = (uint32_t)buffer->int32Data();
+ int32_t ssrc = 0;
+ buffer->meta()->findInt32("ssrc", &ssrc);
+
if (mNumBuffersReceived++ == 0 && mFirstSysTime == 0) {
- int32_t firstRtpTime;
- CHECK(buffer->meta()->findInt32("rtp-time", &firstRtpTime));
+ uint32_t firstRtpTime;
+ CHECK(buffer->meta()->findInt32("rtp-time", (int32_t *)&firstRtpTime));
mFirstSysTime = ALooper::GetNowUs();
mHighestSeqNumber = seqNum;
mBaseSeqNumber = seqNum;
mFirstRtpTime = firstRtpTime;
- ALOGV("first-rtp arrived: first-rtp-time=%d, sys-time=%lld, seq-num=%u",
- mFirstRtpTime, (long long)mFirstSysTime, mHighestSeqNumber);
+ mFirstSsrc = ssrc;
+ ALOGD("first-rtp arrived: first-rtp-time=%d, sys-time=%lld, seq-num=%u, ssrc=%d",
+ mFirstRtpTime, (long long)mFirstSysTime, mHighestSeqNumber, mFirstSsrc);
mClockRate = 90000;
mQueue.push_back(buffer);
return true;
}
+ if (mFirstSsrc != ssrc) {
+ ALOGW("Discarding a buffer due to unexpected ssrc");
+ return false;
+ }
+
// Only the lower 16-bit of the sequence numbers are transmitted,
// derive the high-order bits by choosing the candidate closest
// to the highest sequence number (extended to 32 bits) received so far.
@@ -196,20 +213,34 @@
}
void ARTPSource::addFIR(const sp<ABuffer> &buffer) {
- if (!mIssueFIRRequests) {
+ if (!mIssueFIRRequests && !mIssueFIRByAssembler) {
return;
}
+ bool send = false;
int64_t nowUs = ALooper::GetNowUs();
- if (mLastFIRRequestUs >= 0 && mLastFIRRequestUs + 5000000LL > nowUs) {
- // Send FIR requests at most every 5 secs.
+ int64_t usecsSinceLastFIR = nowUs - mLastFIRRequestUs;
+ if (mLastFIRRequestUs < 0) {
+ // A first FIR, just send it.
+ send = true;
+ } else if (mIssueFIRByAssembler && (usecsSinceLastFIR > 1000000)) {
+ // A FIR issued by Assembler.
+ // Send it if last FIR is not sent within a sec.
+ send = true;
+ } else if (mIssueFIRRequests && (usecsSinceLastFIR > 5000000)) {
+ // A FIR issued periodically regardless packet loss.
+ // Send it if last FIR is not sent within 5 secs.
+ send = true;
+ }
+
+ if (!send) {
return;
}
mLastFIRRequestUs = nowUs;
if (buffer->size() + 20 > buffer->capacity()) {
- ALOGW("RTCP buffer too small to accomodate FIR.");
+ ALOGW("RTCP buffer too small to accommodate FIR.");
return;
}
@@ -218,7 +249,7 @@
data[0] = 0x80 | 4;
data[1] = 206; // PSFB
data[2] = 0;
- data[3] = 4;
+ data[3] = 4; // total (4+1) * sizeof(int32_t) = 20 bytes
data[4] = kSourceID >> 24;
data[5] = (kSourceID >> 16) & 0xff;
data[6] = (kSourceID >> 8) & 0xff;
@@ -240,14 +271,16 @@
data[18] = 0x00;
data[19] = 0x00;
- buffer->setRange(buffer->offset(), buffer->size() + 20);
+ buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
+
+ mIssueFIRByAssembler = false;
ALOGV("Added FIR request.");
}
void ARTPSource::addReceiverReport(const sp<ABuffer> &buffer) {
if (buffer->size() + 32 > buffer->capacity()) {
- ALOGW("RTCP buffer too small to accomodate RR.");
+ ALOGW("RTCP buffer too small to accommodate RR.");
return;
}
@@ -255,16 +288,16 @@
// According to appendix A.3 in RFC 3550
uint32_t expected = mHighestSeqNumber - mBaseSeqNumber + 1;
- int64_t intervalExpected = expected - mPrevExpected;
- int64_t intervalReceived = mNumBuffersReceived - mPrevNumBuffersReceived;
+ int64_t intervalExpected = expected - mPrevExpectedForRR;
+ int64_t intervalReceived = mNumBuffersReceived - mPrevNumBuffersReceivedForRR;
int64_t intervalPacketLost = intervalExpected - intervalReceived;
if (intervalExpected > 0 && intervalPacketLost > 0) {
fraction = (intervalPacketLost << 8) / intervalExpected;
}
- mPrevExpected = expected;
- mPrevNumBuffersReceived = mNumBuffersReceived;
+ mPrevExpectedForRR = expected;
+ mPrevNumBuffersReceivedForRR = mNumBuffersReceived;
int32_t cumulativePacketLost = (int32_t)expected - mNumBuffersReceived;
uint8_t *data = buffer->data() + buffer->size();
@@ -272,7 +305,7 @@
data[0] = 0x80 | 1;
data[1] = 201; // RR
data[2] = 0;
- data[3] = 7;
+ data[3] = 7; // total (7+1) * sizeof(int32_t) = 32 bytes
data[4] = kSourceID >> 24;
data[5] = (kSourceID >> 16) & 0xff;
data[6] = (kSourceID >> 8) & 0xff;
@@ -318,12 +351,210 @@
data[30] = (DLSR >> 8) & 0xff;
data[31] = DLSR & 0xff;
- buffer->setRange(buffer->offset(), buffer->size() + 32);
+ buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
+}
+
+void ARTPSource::addTMMBR(const sp<ABuffer> &buffer, int32_t targetBitrate) {
+ if (buffer->size() + 20 > buffer->capacity()) {
+ ALOGW("RTCP buffer too small to accommodate RR.");
+ return;
+ }
+
+ if (targetBitrate <= 0) {
+ return;
+ }
+
+ uint8_t *data = buffer->data() + buffer->size();
+
+ data[0] = 0x80 | 3; // TMMBR
+ data[1] = 205; // TSFB
+ data[2] = 0;
+ data[3] = 4; // total (4+1) * sizeof(int32_t) = 20 bytes
+ data[4] = kSourceID >> 24;
+ data[5] = (kSourceID >> 16) & 0xff;
+ data[6] = (kSourceID >> 8) & 0xff;
+ data[7] = kSourceID & 0xff;
+
+ *(int32_t*)(&data[8]) = 0; // 4 bytes blank
+
+ data[12] = mID >> 24;
+ data[13] = (mID >> 16) & 0xff;
+ data[14] = (mID >> 8) & 0xff;
+ data[15] = mID & 0xff;
+
+ // Find the first bit '1' from left & right side of the value.
+ int32_t leftEnd = 31 - __builtin_clz(targetBitrate);
+ int32_t rightEnd = ffs(targetBitrate) - 1;
+
+ // Mantissa have only 17bit space by RTCP specification.
+ if ((leftEnd - rightEnd) > 16) {
+ rightEnd = leftEnd - 16;
+ }
+ int32_t mantissa = targetBitrate >> rightEnd;
+
+ data[16] = ((rightEnd << 2) & 0xfc) | ((mantissa & 0x18000) >> 15);
+ data[17] = (mantissa & 0x07f80) >> 7;
+ data[18] = (mantissa & 0x0007f) << 1;
+ data[19] = 40; // 40 bytes overhead;
+
+ buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
+
+ ALOGI("UE -> Op Req Rx bitrate : %d ", mantissa << rightEnd);
+}
+
+int ARTPSource::addNACK(const sp<ABuffer> &buffer) {
+ constexpr size_t kMaxFCIs = 10; // max number of FCIs
+ if (buffer->size() + (3 + kMaxFCIs) * sizeof(int32_t) > buffer->capacity()) {
+ ALOGW("RTCP buffer too small to accommodate NACK.");
+ return -1;
+ }
+
+ uint8_t *data = buffer->data() + buffer->size();
+
+ data[0] = 0x80 | 1; // Generic NACK
+ data[1] = 205; // TSFB
+ data[2] = 0;
+ data[3] = 0; // will be decided later
+ data[4] = kSourceID >> 24;
+ data[5] = (kSourceID >> 16) & 0xff;
+ data[6] = (kSourceID >> 8) & 0xff;
+ data[7] = kSourceID & 0xff;
+
+ data[8] = mID >> 24;
+ data[9] = (mID >> 16) & 0xff;
+ data[10] = (mID >> 8) & 0xff;
+ data[11] = mID & 0xff;
+
+ List<int> list;
+ List<int>::iterator it;
+ getSeqNumToNACK(list, kMaxFCIs);
+ size_t cnt = 0;
+
+ int *FCI = (int *)(data + 12);
+ for (it = list.begin(); it != list.end() && cnt < kMaxFCIs; it++) {
+ *(FCI + cnt) = *it;
+ cnt++;
+ }
+
+ data[3] = (3 + cnt) - 1; // total (3 + #ofFCI) * sizeof(int32_t) byte
+
+ buffer->setRange(buffer->offset(), buffer->size() + (data[3] + 1) * sizeof(int32_t));
+
+ return cnt;
+}
+
+int ARTPSource::getSeqNumToNACK(List<int>& list, int size) {
+ AutoMutex _l(mMapLock);
+ int cnt = 0;
+
+ std::map<uint16_t, infoNACK>::iterator it;
+ for(it = mNACKMap.begin(); it != mNACKMap.end() && cnt < size; it++) {
+ infoNACK &info_it = it->second;
+ if (info_it.needToNACK) {
+ info_it.needToNACK = false;
+ // switch LSB to MSB for sending N/W
+ uint32_t FCI;
+ uint8_t *temp = (uint8_t *)&FCI;
+ temp[0] = (info_it.seqNum >> 8) & 0xff;
+ temp[1] = (info_it.seqNum) & 0xff;
+ temp[2] = (info_it.mask >> 8) & 0xff;
+ temp[3] = (info_it.mask) & 0xff;
+
+ list.push_back(FCI);
+ cnt++;
+ }
+ }
+
+ return cnt;
+}
+
+void ARTPSource::setSeqNumToNACK(uint16_t seqNum, uint16_t mask, uint16_t nowJitterHeadSeqNum) {
+ AutoMutex _l(mMapLock);
+ infoNACK info = {seqNum, mask, nowJitterHeadSeqNum, true};
+ std::map<uint16_t, infoNACK>::iterator it;
+
+ it = mNACKMap.find(seqNum);
+ if (it != mNACKMap.end()) {
+ infoNACK &info_it = it->second;
+ // renew if (mask or head seq) is changed
+ if ((info_it.mask != mask) || (info_it.nowJitterHeadSeqNum != nowJitterHeadSeqNum)) {
+ info_it = info;
+ }
+ } else {
+ mNACKMap[seqNum] = info;
+ }
+
+ // delete all NACK far from current Jitter's first sequence number
+ it = mNACKMap.begin();
+ while (it != mNACKMap.end()) {
+ infoNACK &info_it = it->second;
+
+ int diff = nowJitterHeadSeqNum - info_it.nowJitterHeadSeqNum;
+ if (diff > 100) {
+ ALOGV("Delete %d pkt from NACK map ", info_it.seqNum);
+ it = mNACKMap.erase(it);
+ } else {
+ it++;
+ }
+ }
+
+}
+
+uint32_t ARTPSource::getSelfID() {
+ return kSourceID;
+}
+
+void ARTPSource::setSelfID(const uint32_t selfID) {
+ kSourceID = selfID;
+}
+
+void ARTPSource::setJbTime(const uint32_t jbTimeMs) {
+ mJbTimeMs = jbTimeMs;
+}
+
+void ARTPSource::setPeriodicFIR(bool enable) {
+ ALOGD("setPeriodicFIR %d", enable);
+ mIssueFIRRequests = enable;
+}
+
+bool ARTPSource::isNeedToEarlyNotify() {
+ uint32_t expected = mHighestSeqNumber - mBaseSeqNumber + 1;
+ int32_t intervalExpectedInNow = expected - mPrevExpected;
+ int32_t intervalReceivedInNow = mNumBuffersReceived - mPrevNumBuffersReceived;
+
+ if (intervalExpectedInNow - intervalReceivedInNow > 5)
+ return true;
+ return false;
+}
+
+void ARTPSource::notifyPktInfo(int32_t bitrate, bool isRegular) {
+ int32_t payloadType = isRegular ? RTP_QUALITY : RTP_QUALITY_EMC;
+
+ sp<AMessage> notify = mNotify->dup();
+ notify->setInt32("rtcp-event", 1);
+ notify->setInt32("payload-type", payloadType);
+ notify->setInt32("feedback-type", 0);
+ // sending target bitrate up to application to share rtp quality.
+ notify->setInt32("bit-rate", bitrate);
+ notify->setInt32("highest-seq-num", mHighestSeqNumber);
+ notify->setInt32("base-seq-num", mBaseSeqNumber);
+ notify->setInt32("prev-expected", mPrevExpected);
+ notify->setInt32("num-buf-recv", mNumBuffersReceived);
+ notify->setInt32("prev-num-buf-recv", mPrevNumBuffersReceived);
+ notify->post();
+
+ if (isRegular) {
+ uint32_t expected = mHighestSeqNumber - mBaseSeqNumber + 1;
+ mPrevExpected = expected;
+ mPrevNumBuffersReceived = mNumBuffersReceived;
+ }
+}
+
+void ARTPSource::onIssueFIRByAssembler() {
+ mIssueFIRByAssembler = true;
}
void ARTPSource::noticeAbandonBuffer(int cnt) {
mNumBuffersReceived -= cnt;
}
} // namespace android
-
-
diff --git a/media/libstagefright/rtsp/ARTPSource.h b/media/libstagefright/rtsp/ARTPSource.h
index 12de18d..c51fd8a 100644
--- a/media/libstagefright/rtsp/ARTPSource.h
+++ b/media/libstagefright/rtsp/ARTPSource.h
@@ -23,6 +23,9 @@
#include <media/stagefright/foundation/ABase.h>
#include <utils/List.h>
#include <utils/RefBase.h>
+#include <utils/Thread.h>
+
+#include <map>
namespace android {
@@ -37,6 +40,17 @@
const sp<ASessionDescription> &sessionDesc, size_t index,
const sp<AMessage> ¬ify);
+ enum {
+ RTP_FIRST_PACKET = 100,
+ RTCP_FIRST_PACKET = 101,
+ RTP_QUALITY = 102,
+ RTP_QUALITY_EMC = 103,
+ RTCP_TSFB = 205,
+ RTCP_PSFB = 206,
+ RTP_CVO = 300,
+ RTP_AUTODOWN = 400,
+ };
+
void processRTPPacket(const sp<ABuffer> &buffer);
void timeUpdate(uint32_t rtpTime, uint64_t ntpTime);
void byeReceived();
@@ -45,29 +59,59 @@
void addReceiverReport(const sp<ABuffer> &buffer);
void addFIR(const sp<ABuffer> &buffer);
+ void addTMMBR(const sp<ABuffer> &buffer, int32_t targetBitrate);
+ int addNACK(const sp<ABuffer> &buffer);
+ void setSeqNumToNACK(uint16_t seqNum, uint16_t mask, uint16_t nowJitterHeadSeqNum);
+ uint32_t getSelfID();
+ void setSelfID(const uint32_t selfID);
+ void setJbTime(const uint32_t jbTimeMs);
+ void setPeriodicFIR(bool enable);
+ bool isNeedToEarlyNotify();
+ void notifyPktInfo(int32_t bitrate, bool isRegular);
+ // FIR needs to be sent by missing packet or broken video image.
+ void onIssueFIRByAssembler();
void noticeAbandonBuffer(int cnt=1);
int32_t mFirstSeqNumber;
- int32_t mFirstRtpTime;
+ uint32_t mFirstRtpTime;
int64_t mFirstSysTime;
int32_t mClockRate;
+ uint32_t mJbTimeMs;
+ int32_t mFirstSsrc;
+ int32_t mHighestNackNumber;
+
private:
+
uint32_t mID;
uint32_t mHighestSeqNumber;
uint32_t mPrevExpected;
uint32_t mBaseSeqNumber;
int32_t mNumBuffersReceived;
int32_t mPrevNumBuffersReceived;
+ uint32_t mPrevExpectedForRR;
+ int32_t mPrevNumBuffersReceivedForRR;
List<sp<ABuffer> > mQueue;
sp<ARTPAssembler> mAssembler;
+ typedef struct infoNACK {
+ uint16_t seqNum;
+ uint16_t mask;
+ uint16_t nowJitterHeadSeqNum;
+ bool needToNACK;
+ } infoNACK;
+
+ Mutex mMapLock;
+ std::map<uint16_t, infoNACK> mNACKMap;
+ int getSeqNumToNACK(List<int>& list, int size);
+
uint64_t mLastNTPTime;
int64_t mLastNTPTimeUpdateUs;
bool mIssueFIRRequests;
+ bool mIssueFIRByAssembler;
int64_t mLastFIRRequestUs;
uint8_t mNextFIRSeqNo;
diff --git a/media/libstagefright/rtsp/ARTPWriter.cpp b/media/libstagefright/rtsp/ARTPWriter.cpp
index 65c8189..ec70952 100644
--- a/media/libstagefright/rtsp/ARTPWriter.cpp
+++ b/media/libstagefright/rtsp/ARTPWriter.cpp
@@ -20,8 +20,6 @@
#include "ARTPWriter.h"
-#include <fcntl.h>
-
#include <media/stagefright/MediaSource.h>
#include <media/stagefright/foundation/ABuffer.h>
#include <media/stagefright/foundation/ADebug.h>
@@ -32,6 +30,9 @@
#include <media/stagefright/MetaData.h>
#include <utils/ByteOrder.h>
+#include <fcntl.h>
+#include <strings.h>
+
#define PT 97
#define PT_STR "97"
@@ -42,21 +43,30 @@
#define H264_NALU_PFRAME 0x1
#define H265_NALU_MASK 0x3F
-#define H265_NALU_VPS 0x40
-#define H265_NALU_SPS 0x42
-#define H265_NALU_PPS 0x44
+#define H265_NALU_VPS 0x20
+#define H265_NALU_SPS 0x21
+#define H265_NALU_PPS 0x22
+#define IPV4_HEADER_SIZE 20
+#define IPV6_HEADER_SIZE 40
#define UDP_HEADER_SIZE 8
+#define TCPIPV4_HEADER_SIZE (IPV4_HEADER_SIZE + UDP_HEADER_SIZE)
+#define TCPIPV6_HEADER_SIZE (IPV6_HEADER_SIZE + UDP_HEADER_SIZE)
+#define TCPIP_HEADER_SIZE TCPIPV4_HEADER_SIZE
#define RTP_HEADER_SIZE 12
-#define RTP_HEADER_EXT_SIZE 1
+#define RTP_HEADER_EXT_SIZE 8
#define RTP_FU_HEADER_SIZE 2
-#define RTP_PAYLOAD_ROOM_SIZE 140
+#define RTP_PAYLOAD_ROOM_SIZE 100 // ROOM size for IPv6 header, ESP and etc.
namespace android {
// static const size_t kMaxPacketSize = 65507; // maximum payload in UDP over IP
-static const size_t kMaxPacketSize = 1500;
+static const size_t kMaxPacketSize = 1280;
+static char kCNAME[255] = "someone@somewhere";
+
+static const size_t kTrafficRecorderMaxEntries = 128;
+static const size_t kTrafficRecorderMaxTimeSpanMs = 2000;
static int UniformRand(int limit) {
return ((double)rand() * limit) / RAND_MAX;
@@ -66,7 +76,9 @@
: mFlags(0),
mFd(dup(fd)),
mLooper(new ALooper),
- mReflector(new AHandlerReflector<ARTPWriter>(this)) {
+ mReflector(new AHandlerReflector<ARTPWriter>(this)),
+ mTrafficRec(new TrafficRecorder<uint32_t /* Time */, Bytes>(
+ kTrafficRecorderMaxEntries, kTrafficRecorderMaxTimeSpanMs)) {
CHECK_GE(fd, 0);
mIsIPv6 = false;
@@ -112,11 +124,13 @@
}
ARTPWriter::ARTPWriter(int fd, String8& localIp, int localPort, String8& remoteIp,
- int remotePort)
+ int remotePort, uint32_t seqNo)
: mFlags(0),
mFd(dup(fd)),
mLooper(new ALooper),
- mReflector(new AHandlerReflector<ARTPWriter>(this)) {
+ mReflector(new AHandlerReflector<ARTPWriter>(this)),
+ mTrafficRec(new TrafficRecorder<uint32_t /* Time */, Bytes>(
+ kTrafficRecorderMaxEntries, kTrafficRecorderMaxTimeSpanMs)) {
CHECK_GE(fd, 0);
mIsIPv6 = false;
@@ -125,9 +139,13 @@
mLooper->start();
makeSocketPairAndBind(localIp, localPort, remoteIp , remotePort);
+ mVPSBuf = NULL;
mSPSBuf = NULL;
mPPSBuf = NULL;
+ initState();
+ mSeqNo = seqNo; // Must use explicit # of seq for RTP continuity
+
#if LOG_TO_FILES
mRTPFd = open(
"/data/misc/rtpout.bin",
@@ -144,6 +162,21 @@
}
ARTPWriter::~ARTPWriter() {
+ if (mVPSBuf != NULL) {
+ mVPSBuf->release();
+ mVPSBuf = NULL;
+ }
+
+ if (mSPSBuf != NULL) {
+ mSPSBuf->release();
+ mSPSBuf = NULL;
+ }
+
+ if (mPPSBuf != NULL) {
+ mPPSBuf->release();
+ mPPSBuf = NULL;
+ }
+
#if LOG_TO_FILES
close(mRTCPFd);
mRTCPFd = -1;
@@ -160,16 +193,29 @@
close(mFd);
mFd = -1;
+}
- if(mSPSBuf != NULL) {
- mSPSBuf->release();
- mSPSBuf = NULL;
- }
+void ARTPWriter::initState() {
+ if (mSourceID == 0)
+ mSourceID = rand();
+ mPayloadType = 0;
+ if (mSeqNo == 0)
+ mSeqNo = UniformRand(65536);
+ mRTPTimeBase = 0;
+ mNumRTPSent = 0;
+ mNumRTPOctetsSent = 0;
+ mLastRTPTime = 0;
+ mLastNTPTime = 0;
- if(mPPSBuf != NULL) {
- mPPSBuf->release();
- mPPSBuf = NULL;
- }
+ mOpponentID = 0;
+ mBitrate = 192000;
+
+ mNumSRsSent = 0;
+ mRTPCVOExtMap = -1;
+ mRTPCVODegrees = 0;
+ mRTPSockNetwork = 0;
+
+ mMode = INVALID;
}
status_t ARTPWriter::addSource(const sp<MediaSource> &source) {
@@ -182,26 +228,42 @@
return (mFlags & kFlagEOS) != 0;
}
-status_t ARTPWriter::start(MetaData * /* params */) {
+status_t ARTPWriter::start(MetaData * params) {
Mutex::Autolock autoLock(mLock);
if (mFlags & kFlagStarted) {
return INVALID_OPERATION;
}
mFlags &= ~kFlagEOS;
- mSourceID = rand();
- mSeqNo = UniformRand(65536);
- mRTPTimeBase = 0;
- mNumRTPSent = 0;
- mNumRTPOctetsSent = 0;
- mLastRTPTime = 0;
- mLastNTPTime = 0;
- mNumSRsSent = 0;
+ initState();
const char *mime;
CHECK(mSource->getFormat()->findCString(kKeyMIMEType, &mime));
- mMode = INVALID;
+ int32_t selfID = 0;
+ if (params->findInt32(kKeySelfID, &selfID))
+ mSourceID = selfID;
+
+ int32_t payloadType = 0;
+ if (params->findInt32(kKeyPayloadType, &payloadType))
+ mPayloadType = payloadType;
+
+ int32_t rtpExtMap = 0;
+ if (params->findInt32(kKeyRtpExtMap, &rtpExtMap))
+ mRTPCVOExtMap = rtpExtMap;
+
+ int32_t rtpCVODegrees = 0;
+ if (params->findInt32(kKeyRtpCvoDegrees, &rtpCVODegrees))
+ mRTPCVODegrees = rtpCVODegrees;
+
+ int32_t dscp = 0;
+ if (params->findInt32(kKeyRtpDscp, &dscp))
+ updateSocketDscp(dscp);
+
+ int64_t sockNetwork = 0;
+ if (params->findInt64(kKeySocketNetwork, &sockNetwork))
+ updateSocketNetwork(sockNetwork);
+
if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_AVC)) {
mMode = H264;
} else if (!strcasecmp(mime, MEDIA_MIMETYPE_VIDEO_HEVC)) {
@@ -243,12 +305,9 @@
return OK;
}
-// return size of SPS if there is more NAL unit found following to SPS.
-static uint32_t StripStartcode(MediaBufferBase *buffer) {
- uint32_t nalSize = 0;
-
+static void StripStartcode(MediaBufferBase *buffer) {
if (buffer->range_length() < 4) {
- return 0;
+ return;
}
const uint8_t *ptr =
@@ -258,55 +317,129 @@
buffer->set_range(
buffer->range_offset() + 4, buffer->range_length() - 4);
}
-
- ptr = (const uint8_t *)buffer->data() + buffer->range_offset();
-
- if ((*ptr & H264_NALU_MASK) == H264_NALU_SPS) {
- for (uint32_t i = 0; i < buffer->range_length(); i++) {
-
- if (!memcmp(ptr + i, "\x00\x00\x00\x01", 4)) {
- // Now, we found one more NAL unit in the media buffer.
- // Mostly, it will be a PPS.
- nalSize = i;
- ALOGV("SPS found. size=%d", nalSize);
- }
- }
- }
-
- return nalSize;
}
-static void SpsPpsParser(MediaBufferBase *mediaBuffer,
- MediaBufferBase **spsBuffer, MediaBufferBase **ppsBuffer, uint32_t spsSize) {
+static const uint8_t SPCSize = 4; // Start Prefix Code Size
+static const uint8_t startPrefixCode[SPCSize] = {0, 0, 0, 1};
+static const uint8_t spcKMPidx[SPCSize] = {0, 0, 2, 0};
+static void SpsPpsParser(MediaBufferBase *buffer,
+ MediaBufferBase **spsBuffer, MediaBufferBase **ppsBuffer) {
- if (mediaBuffer == NULL || mediaBuffer->range_length() < 4)
- return;
+ while (buffer->range_length() > 0) {
+ const uint8_t *NALPtr = (const uint8_t *)buffer->data() + buffer->range_offset();
- if((*spsBuffer) != NULL) {
- (*spsBuffer)->release();
- (*spsBuffer) = NULL;
+ MediaBufferBase **targetPtr = NULL;
+ if ((*NALPtr & H264_NALU_MASK) == H264_NALU_SPS) {
+ targetPtr = spsBuffer;
+ } else if ((*NALPtr & H264_NALU_MASK) == H264_NALU_PPS) {
+ targetPtr = ppsBuffer;
+ } else {
+ return;
+ }
+ ALOGV("SPS(7) or PPS(8) found. Type %d", *NALPtr & H264_NALU_MASK);
+
+ uint32_t bufferSize = buffer->range_length();
+ MediaBufferBase *&target = *targetPtr;
+ uint32_t i = 0, j = 0;
+ bool isBoundFound = false;
+ for (i = 0; i < bufferSize; i++) {
+ while (j > 0 && NALPtr[i] != startPrefixCode[j]) {
+ j = spcKMPidx[j - 1];
+ }
+ if (NALPtr[i] == startPrefixCode[j]) {
+ j++;
+ if (j == SPCSize) {
+ isBoundFound = true;
+ break;
+ }
+ }
+ }
+
+ uint32_t targetSize;
+ if (target != NULL) {
+ target->release();
+ }
+ // note that targetSize is never 0 as the first byte is never part
+ // of a start prefix
+ if (isBoundFound) {
+ targetSize = i - SPCSize + 1;
+ target = MediaBufferBase::Create(targetSize);
+ memcpy(target->data(),
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ targetSize);
+ buffer->set_range(buffer->range_offset() + targetSize + SPCSize,
+ buffer->range_length() - targetSize - SPCSize);
+ } else {
+ targetSize = bufferSize;
+ target = MediaBufferBase::Create(targetSize);
+ memcpy(target->data(),
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ targetSize);
+ buffer->set_range(buffer->range_offset() + bufferSize, 0);
+ return;
+ }
}
+}
- if((*ppsBuffer) != NULL) {
- (*ppsBuffer)->release();
- (*ppsBuffer) = NULL;
- }
+static void VpsSpsPpsParser(MediaBufferBase *buffer,
+ MediaBufferBase **vpsBuffer, MediaBufferBase **spsBuffer, MediaBufferBase **ppsBuffer) {
- // we got sps/pps but startcode of sps is striped.
- (*spsBuffer) = MediaBufferBase::Create(spsSize);
- int32_t ppsSize = mediaBuffer->range_length() - spsSize - 4/*startcode*/;
- (*ppsBuffer) = MediaBufferBase::Create(ppsSize);
- memcpy((*spsBuffer)->data(),
- (const uint8_t *)mediaBuffer->data() + mediaBuffer->range_offset(),
- spsSize);
+ while (buffer->range_length() > 0) {
+ const uint8_t *NALPtr = (const uint8_t *)buffer->data() + buffer->range_offset();
+ uint8_t nalType = ((*NALPtr) >> 1) & H265_NALU_MASK;
- if (ppsSize > 0) {
- ALOGV("PPS found. size=%d", (int)ppsSize);
- mediaBuffer->set_range(mediaBuffer->range_offset() + spsSize + 4/*startcode*/,
- mediaBuffer->range_length() - spsSize - 4/*startcode*/);
- memcpy((*ppsBuffer)->data(),
- (const uint8_t *)mediaBuffer->data() + mediaBuffer->range_offset(),
- ppsSize);
+ MediaBufferBase **targetPtr = NULL;
+ if (nalType == H265_NALU_VPS) {
+ targetPtr = vpsBuffer;
+ } else if (nalType == H265_NALU_SPS) {
+ targetPtr = spsBuffer;
+ } else if (nalType == H265_NALU_PPS) {
+ targetPtr = ppsBuffer;
+ } else {
+ return;
+ }
+ ALOGV("VPS(32) SPS(33) or PPS(34) found. Type %d", nalType);
+
+ uint32_t bufferSize = buffer->range_length();
+ MediaBufferBase *&target = *targetPtr;
+ uint32_t i = 0, j = 0;
+ bool isBoundFound = false;
+ for (i = 0; i < bufferSize; i++) {
+ while (j > 0 && NALPtr[i] != startPrefixCode[j]) {
+ j = spcKMPidx[j - 1];
+ }
+ if (NALPtr[i] == startPrefixCode[j]) {
+ j++;
+ if (j == SPCSize) {
+ isBoundFound = true;
+ break;
+ }
+ }
+ }
+
+ if (target != NULL) {
+ target->release();
+ }
+ uint32_t targetSize;
+ // note that targetSize is never 0 as the first byte is never part
+ // of a start prefix
+ if (isBoundFound) {
+ targetSize = i - SPCSize + 1;
+ target = MediaBufferBase::Create(j);
+ memcpy(target->data(),
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ j);
+ buffer->set_range(buffer->range_offset() + targetSize + SPCSize,
+ buffer->range_length() - targetSize - SPCSize);
+ } else {
+ targetSize = bufferSize;
+ target = MediaBufferBase::Create(targetSize);
+ memcpy(target->data(),
+ (const uint8_t *)buffer->data() + buffer->range_offset(),
+ targetSize);
+ buffer->set_range(buffer->range_offset() + bufferSize, 0);
+ return;
+ }
}
}
@@ -314,7 +447,9 @@
switch (msg->what()) {
case kWhatStart:
{
- CHECK_EQ(mSource->start(), (status_t)OK);
+ sp<MetaData> meta = new MetaData();
+ meta->setInt64(kKeyTime, 10ll);
+ CHECK_EQ(mSource->start(meta.get()), (status_t)OK);
#if 0
if (mMode == H264) {
@@ -387,6 +522,18 @@
}
}
+void ARTPWriter::setTMMBNInfo(uint32_t opponentID, uint32_t bitrate) {
+ mOpponentID = opponentID;
+ mBitrate = bitrate;
+
+ sp<ABuffer> buffer = new ABuffer(65536);
+ buffer->setRange(0, 0);
+
+ addTMMBN(buffer);
+
+ send(buffer, true /* isRTCP */);
+}
+
void ARTPWriter::onRead(const sp<AMessage> &msg) {
MediaBufferBase *mediaBuf;
status_t err = mSource->read(&mediaBuf);
@@ -403,15 +550,17 @@
ALOGV("read buffer of size %zu", mediaBuf->range_length());
if (mMode == H264) {
- uint32_t spsSize = 0;
- if ((spsSize = StripStartcode(mediaBuf)) > 0) {
- SpsPpsParser(mediaBuf, &mSPSBuf, &mPPSBuf, spsSize);
- } else {
+ StripStartcode(mediaBuf);
+ SpsPpsParser(mediaBuf, &mSPSBuf, &mPPSBuf);
+ if (mediaBuf->range_length() > 0) {
sendAVCData(mediaBuf);
}
} else if (mMode == H265) {
StripStartcode(mediaBuf);
- sendHEVCData(mediaBuf);
+ VpsSpsPpsParser(mediaBuf, &mVPSBuf, &mSPSBuf, &mPPSBuf);
+ if (mediaBuf->range_length() > 0) {
+ sendHEVCData(mediaBuf);
+ }
} else if (mMode == H263) {
sendH263Data(mediaBuf);
} else if (mMode == AMR_NB || mMode == AMR_WB) {
@@ -456,10 +605,22 @@
remAddr = (struct sockaddr *)&mRTPAddr;
}
+ // Unseal code if moderator is needed (prevent overflow of instant bandwidth)
+ // Set limit bits per period through the moderator.
+ // ex) 6KByte/10ms = 48KBit/10ms = 4.8MBit/s instant limit
+ // ModerateInstantTraffic(10, 6 * 1024);
+
ssize_t n = sendto(isRTCP ? mRTCPSocket : mRTPSocket,
buffer->data(), buffer->size(), 0, remAddr, sizeSockSt);
- CHECK_EQ(n, (ssize_t)buffer->size());
+ if (n != (ssize_t)buffer->size()) {
+ ALOGW("packets can not be sent. ret=%d, buf=%d", (int)n, (int)buffer->size());
+ } else {
+ // Record current traffic & Print bits while last 1sec (1000ms)
+ mTrafficRec->writeBytes(buffer->size() +
+ (mIsIPv6 ? TCPIPV6_HEADER_SIZE : TCPIPV4_HEADER_SIZE));
+ mTrafficRec->printAccuBitsForLastPeriod(1000, 1000);
+ }
#if LOG_TO_FILES
int fd = isRTCP ? mRTCPFd : mRTPFd;
@@ -524,7 +685,6 @@
data[offset++] = 1; // CNAME
- static const char *kCNAME = "someone@somewhere";
data[offset++] = strlen(kCNAME);
memcpy(&data[offset], kCNAME, strlen(kCNAME));
@@ -561,9 +721,55 @@
buffer->setRange(buffer->offset(), buffer->size() + offset);
}
+void ARTPWriter::addTMMBN(const sp<ABuffer> &buffer) {
+ if (buffer->size() + 20 > buffer->capacity()) {
+ ALOGW("RTCP buffer too small to accommodate SR.");
+ return;
+ }
+ if (mOpponentID == 0)
+ return;
+
+ uint8_t *data = buffer->data() + buffer->size();
+
+ data[0] = 0x80 | 4; // TMMBN
+ data[1] = 205; // TSFB
+ data[2] = 0;
+ data[3] = 4; // total (4+1) * sizeof(int32_t) = 20 bytes
+ data[4] = mSourceID >> 24;
+ data[5] = (mSourceID >> 16) & 0xff;
+ data[6] = (mSourceID >> 8) & 0xff;
+ data[7] = mSourceID & 0xff;
+
+ *(int32_t*)(&data[8]) = 0; // 4 bytes blank
+
+ data[12] = mOpponentID >> 24;
+ data[13] = (mOpponentID >> 16) & 0xff;
+ data[14] = (mOpponentID >> 8) & 0xff;
+ data[15] = mOpponentID & 0xff;
+
+ // Find the first bit '1' from left & right side of the value.
+ int32_t leftEnd = 31 - __builtin_clz(mBitrate);
+ int32_t rightEnd = ffs(mBitrate) - 1;
+
+ // Mantissa have only 17bit space by RTCP specification.
+ if ((leftEnd - rightEnd) > 16) {
+ rightEnd = leftEnd - 16;
+ }
+ int32_t mantissa = mBitrate >> rightEnd;
+
+ data[16] = ((rightEnd << 2) & 0xfc) | ((mantissa & 0x18000) >> 15);
+ data[17] = (mantissa & 0x07f80) >> 7;
+ data[18] = (mantissa & 0x0007f) << 1;
+ data[19] = 40; // 40 bytes overhead;
+
+ buffer->setRange(buffer->offset(), buffer->size() + 20);
+
+ ALOGI("UE -> Op Noti Tx bitrate : %d ", mantissa << rightEnd);
+}
+
// static
uint64_t ARTPWriter::GetNowNTP() {
- uint64_t nowUs = ALooper::GetNowUs();
+ uint64_t nowUs = systemTime(SYSTEM_TIME_REALTIME) / 1000ll;
nowUs += ((70LL * 365 + 17) * 24) * 60 * 60 * 1000000LL;
@@ -715,11 +921,13 @@
}
void ARTPWriter::sendSPSPPSIfIFrame(MediaBufferBase *mediaBuf, int64_t timeUs) {
+ CHECK(mediaBuf->range_length() > 0);
const uint8_t *mediaData =
(const uint8_t *)mediaBuf->data() + mediaBuf->range_offset();
- if ((mediaData[0] & H264_NALU_MASK) != H264_NALU_IFRAME)
+ if ((mediaData[0] & H264_NALU_MASK) != H264_NALU_IFRAME) {
return;
+ }
if (mSPSBuf != NULL) {
mSPSBuf->meta_data().setInt64(kKeyTime, timeUs);
@@ -734,6 +942,35 @@
}
}
+void ARTPWriter::sendVPSSPSPPSIfIFrame(MediaBufferBase *mediaBuf, int64_t timeUs) {
+ CHECK(mediaBuf->range_length() > 0);
+ const uint8_t *mediaData =
+ (const uint8_t *)mediaBuf->data() + mediaBuf->range_offset();
+
+ int nalType = ((mediaData[0] >> 1) & H265_NALU_MASK);
+ if (!(nalType >= 16 && nalType <= 21) /*H265_NALU_IFRAME*/) {
+ return;
+ }
+
+ if (mVPSBuf != NULL) {
+ mVPSBuf->meta_data().setInt64(kKeyTime, timeUs);
+ mVPSBuf->meta_data().setInt32(kKeyVps, 1);
+ sendHEVCData(mVPSBuf);
+ }
+
+ if (mSPSBuf != NULL) {
+ mSPSBuf->meta_data().setInt64(kKeyTime, timeUs);
+ mSPSBuf->meta_data().setInt32(kKeySps, 1);
+ sendHEVCData(mSPSBuf);
+ }
+
+ if (mPPSBuf != NULL) {
+ mPPSBuf->meta_data().setInt64(kKeyTime, timeUs);
+ mPPSBuf->meta_data().setInt32(kKeyPps, 1);
+ sendHEVCData(mPPSBuf);
+ }
+}
+
void ARTPWriter::sendHEVCData(MediaBufferBase *mediaBuf) {
// 12 bytes RTP header + 2 bytes for the FU-indicator and FU-header.
CHECK_GE(kMaxPacketSize, 12u + 2u);
@@ -741,21 +978,33 @@
int64_t timeUs;
CHECK(mediaBuf->meta_data().findInt64(kKeyTime, &timeUs));
- sendSPSPPSIfIFrame(mediaBuf, timeUs);
+ sendVPSSPSPPSIfIFrame(mediaBuf, timeUs);
uint32_t rtpTime = mRTPTimeBase + (timeUs * 9 / 100ll);
+ CHECK(mediaBuf->range_length() > 0);
const uint8_t *mediaData =
(const uint8_t *)mediaBuf->data() + mediaBuf->range_offset();
+ int32_t isNonVCL = 0;
+ if (mediaBuf->meta_data().findInt32(kKeyVps, &isNonVCL) ||
+ mediaBuf->meta_data().findInt32(kKeySps, &isNonVCL) ||
+ mediaBuf->meta_data().findInt32(kKeyPps, &isNonVCL)) {
+ isNonVCL = 1;
+ }
+
sp<ABuffer> buffer = new ABuffer(kMaxPacketSize);
- if (mediaBuf->range_length() + UDP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_PAYLOAD_ROOM_SIZE
- <= buffer->capacity()) {
+ if (mediaBuf->range_length() + TCPIP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_HEADER_EXT_SIZE
+ + RTP_PAYLOAD_ROOM_SIZE <= buffer->capacity()) {
// The data fits into a single packet
uint8_t *data = buffer->data();
data[0] = 0x80;
- data[1] = (1 << 7) | PT; // M-bit
+ if (isNonVCL) {
+ data[1] = mPayloadType; // Marker bit should not be set in case of Non-VCL
+ } else {
+ data[1] = (1 << 7) | mPayloadType; // M-bit
+ }
data[2] = (mSeqNo >> 8) & 0xff;
data[3] = mSeqNo & 0xff;
data[4] = rtpTime >> 24;
@@ -788,16 +1037,16 @@
while (offset < mediaBuf->range_length()) {
size_t size = mediaBuf->range_length() - offset;
bool lastPacket = true;
- if (size + UDP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_FU_HEADER_SIZE +
- RTP_PAYLOAD_ROOM_SIZE > buffer->capacity()) {
+ if (size + TCPIP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_HEADER_EXT_SIZE +
+ RTP_FU_HEADER_SIZE + RTP_PAYLOAD_ROOM_SIZE > buffer->capacity()) {
lastPacket = false;
- size = buffer->capacity() - UDP_HEADER_SIZE - RTP_HEADER_SIZE -
- RTP_FU_HEADER_SIZE - RTP_PAYLOAD_ROOM_SIZE;
+ size = buffer->capacity() - TCPIP_HEADER_SIZE - RTP_HEADER_SIZE -
+ RTP_HEADER_EXT_SIZE - RTP_FU_HEADER_SIZE - RTP_PAYLOAD_ROOM_SIZE;
}
uint8_t *data = buffer->data();
data[0] = 0x80;
- data[1] = (lastPacket ? (1 << 7) : 0x00) | PT; // M-bit
+ data[1] = (lastPacket ? (1 << 7) : 0x00) | mPayloadType; // M-bit
data[2] = (mSeqNo >> 8) & 0xff;
data[3] = mSeqNo & 0xff;
data[4] = rtpTime >> 24;
@@ -870,6 +1119,7 @@
uint32_t rtpTime = mRTPTimeBase + (timeUs * 9 / 100LL);
+ CHECK(mediaBuf->range_length() > 0);
const uint8_t *mediaData =
(const uint8_t *)mediaBuf->data() + mediaBuf->range_offset();
@@ -880,16 +1130,19 @@
isSpsPps = true;
}
+ mTrafficRec->updateClock(ALooper::GetNowUs() / 1000);
sp<ABuffer> buffer = new ABuffer(kMaxPacketSize);
- if (mediaBuf->range_length() + UDP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_PAYLOAD_ROOM_SIZE
- <= buffer->capacity()) {
+ if (mediaBuf->range_length() + TCPIP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_HEADER_EXT_SIZE
+ + RTP_PAYLOAD_ROOM_SIZE <= buffer->capacity()) {
// The data fits into a single packet
uint8_t *data = buffer->data();
data[0] = 0x80;
+ if (mRTPCVOExtMap > 0)
+ data[0] |= 0x10;
if (isSpsPps)
- data[1] = PT; // Marker bit should not be set in case of sps/pps
+ data[1] = mPayloadType; // Marker bit should not be set in case of sps/pps
else
- data[1] = (1 << 7) | PT;
+ data[1] = (1 << 7) | mPayloadType;
data[2] = (mSeqNo >> 8) & 0xff;
data[3] = mSeqNo & 0xff;
data[4] = rtpTime >> 24;
@@ -901,16 +1154,51 @@
data[10] = (mSourceID >> 8) & 0xff;
data[11] = mSourceID & 0xff;
- memcpy(&data[12],
+ int rtpExtIndex = 0;
+ if (mRTPCVOExtMap > 0) {
+ /*
+ 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | 0xBE | 0xDE | length=3 |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | ID | L=0 | data | ID | L=1 | data...
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ ...data | 0 (pad) | 0 (pad) | ID | L=3 |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+ | data |
+ +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+ In the one-byte header form of extensions, the 16-bit value required
+ by the RTP specification for a header extension, labeled in the RTP
+ specification as "defined by profile", takes the fixed bit pattern
+ 0xBEDE (the first version of this specification was written on the
+ feast day of the Venerable Bede).
+ */
+ data[12] = 0xBE;
+ data[13] = 0xDE;
+ // put a length of RTP Extension.
+ data[14] = 0x00;
+ data[15] = 0x01;
+ // put extmap of RTP assigned for CVO.
+ data[16] = (mRTPCVOExtMap << 4) | 0x0;
+ // put image degrees as per CVO specification.
+ data[17] = mRTPCVODegrees;
+ data[18] = 0x0;
+ data[19] = 0x0;
+ rtpExtIndex = 8;
+ }
+
+ memcpy(&data[12 + rtpExtIndex],
mediaData, mediaBuf->range_length());
- buffer->setRange(0, mediaBuf->range_length() + 12);
+ buffer->setRange(0, mediaBuf->range_length() + (12 + rtpExtIndex));
send(buffer, false /* isRTCP */);
++mSeqNo;
++mNumRTPSent;
- mNumRTPOctetsSent += buffer->size() - 12;
+ mNumRTPOctetsSent += buffer->size() - (12 + rtpExtIndex);
} else {
// FU-A
@@ -921,16 +1209,18 @@
while (offset < mediaBuf->range_length()) {
size_t size = mediaBuf->range_length() - offset;
bool lastPacket = true;
- if (size + UDP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_FU_HEADER_SIZE +
- RTP_PAYLOAD_ROOM_SIZE > buffer->capacity()) {
+ if (size + TCPIP_HEADER_SIZE + RTP_HEADER_SIZE + RTP_HEADER_EXT_SIZE +
+ RTP_FU_HEADER_SIZE + RTP_PAYLOAD_ROOM_SIZE > buffer->capacity()) {
lastPacket = false;
- size = buffer->capacity() - UDP_HEADER_SIZE - RTP_HEADER_SIZE -
- RTP_FU_HEADER_SIZE - RTP_PAYLOAD_ROOM_SIZE;
+ size = buffer->capacity() - TCPIP_HEADER_SIZE - RTP_HEADER_SIZE -
+ RTP_HEADER_EXT_SIZE - RTP_FU_HEADER_SIZE - RTP_PAYLOAD_ROOM_SIZE;
}
uint8_t *data = buffer->data();
data[0] = 0x80;
- data[1] = (lastPacket ? (1 << 7) : 0x00) | PT; // M-bit
+ if (lastPacket && mRTPCVOExtMap > 0)
+ data[0] |= 0x10;
+ data[1] = (lastPacket ? (1 << 7) : 0x00) | mPayloadType; // M-bit
data[2] = (mSeqNo >> 8) & 0xff;
data[3] = mSeqNo & 0xff;
data[4] = rtpTime >> 24;
@@ -942,24 +1232,37 @@
data[10] = (mSourceID >> 8) & 0xff;
data[11] = mSourceID & 0xff;
- data[12] = 28 | (nalType & 0xe0);
+ int rtpExtIndex = 0;
+ if (lastPacket && mRTPCVOExtMap > 0) {
+ data[12] = 0xBE;
+ data[13] = 0xDE;
+ data[14] = 0x00;
+ data[15] = 0x01;
+ data[16] = (mRTPCVOExtMap << 4) | 0x0;
+ data[17] = mRTPCVODegrees;
+ data[18] = 0x0;
+ data[19] = 0x0;
+ rtpExtIndex = 8;
+ }
+
+ data[12 + rtpExtIndex] = 28 | (nalType & 0xe0);
CHECK(!firstPacket || !lastPacket);
- data[13] =
+ data[13 + rtpExtIndex] =
(firstPacket ? 0x80 : 0x00)
| (lastPacket ? 0x40 : 0x00)
| (nalType & 0x1f);
- memcpy(&data[14], &mediaData[offset], size);
+ memcpy(&data[14 + rtpExtIndex], &mediaData[offset], size);
- buffer->setRange(0, 14 + size);
+ buffer->setRange(0, 14 + rtpExtIndex + size);
send(buffer, false /* isRTCP */);
++mSeqNo;
++mNumRTPSent;
- mNumRTPOctetsSent += buffer->size() - 12;
+ mNumRTPOctetsSent += buffer->size() - (12 + rtpExtIndex);
firstPacket = false;
offset += size;
@@ -1001,7 +1304,7 @@
uint8_t *data = buffer->data();
data[0] = 0x80;
- data[1] = (lastPacket ? 0x80 : 0x00) | PT; // M-bit
+ data[1] = (lastPacket ? 0x80 : 0x00) | mPayloadType; // M-bit
data[2] = (mSeqNo >> 8) & 0xff;
data[3] = mSeqNo & 0xff;
data[4] = rtpTime >> 24;
@@ -1032,6 +1335,58 @@
mLastNTPTime = GetNowNTP();
}
+void ARTPWriter::updateCVODegrees(int32_t cvoDegrees) {
+ Mutex::Autolock autoLock(mLock);
+ mRTPCVODegrees = cvoDegrees;
+}
+
+void ARTPWriter::updatePayloadType(int32_t payloadType) {
+ Mutex::Autolock autoLock(mLock);
+ mPayloadType = payloadType;
+}
+
+void ARTPWriter::updateSocketDscp(int32_t dscp) {
+ mRtpLayer3Dscp = dscp << 2;
+
+ /* mRtpLayer3Dscp will be mapped to WMM(Wifi) as per operator's requirement */
+ if (setsockopt(mRTPSocket, IPPROTO_IP, IP_TOS,
+ (int *)&mRtpLayer3Dscp, sizeof(mRtpLayer3Dscp)) < 0) {
+ ALOGE("failed to set dscp on rtpsock. err=%s", strerror(errno));
+ } else {
+ ALOGD("successfully set dscp on rtpsock. opt=%d", mRtpLayer3Dscp);
+ setsockopt(mRTCPSocket, IPPROTO_IP, IP_TOS,
+ (int *)&mRtpLayer3Dscp, sizeof(mRtpLayer3Dscp));
+ ALOGD("successfully set dscp on rtcpsock. opt=%d", mRtpLayer3Dscp);
+ }
+}
+
+void ARTPWriter::updateSocketNetwork(int64_t socketNetwork) {
+ mRTPSockNetwork = (net_handle_t)socketNetwork;
+ ALOGI("trying to bind rtp socket(%d) to network(%llu).",
+ mRTPSocket, (unsigned long long)mRTPSockNetwork);
+
+ int result = android_setsocknetwork(mRTPSockNetwork, mRTPSocket);
+ if (result != 0) {
+ ALOGW("failed(%d) to bind rtp socket(%d) to network(%llu)",
+ result, mRTPSocket, (unsigned long long)mRTPSockNetwork);
+ }
+ result = android_setsocknetwork(mRTPSockNetwork, mRTCPSocket);
+ if (result != 0) {
+ ALOGW("failed(%d) to bind rtcp socket(%d) to network(%llu)",
+ result, mRTCPSocket, (unsigned long long)mRTPSockNetwork);
+ }
+ ALOGI("done. bind rtp socket(%d) to network(%llu)",
+ mRTPSocket, (unsigned long long)mRTPSockNetwork);
+}
+
+uint32_t ARTPWriter::getSequenceNum() {
+ return mSeqNo;
+}
+
+uint64_t ARTPWriter::getAccumulativeBytes() {
+ return mTrafficRec->readBytesForTotal();
+}
+
static size_t getFrameSize(bool isWide, unsigned FT) {
static const size_t kFrameSizeNB[8] = {
95, 103, 118, 134, 148, 159, 204, 244
@@ -1083,7 +1438,7 @@
// The data fits into a single packet
uint8_t *data = buffer->data();
data[0] = 0x80;
- data[1] = PT;
+ data[1] = mPayloadType;
if (mNumRTPSent == 0) {
// Signal start of talk-spurt.
data[1] |= 0x80; // M-bit
@@ -1141,6 +1496,11 @@
void ARTPWriter::makeSocketPairAndBind(String8& localIp, int localPort,
String8& remoteIp, int remotePort) {
+ static char kSomeone[16] = "someone@";
+ int nameLength = strlen(kSomeone);
+ memcpy(kCNAME, kSomeone, nameLength);
+ memcpy(kCNAME + nameLength, localIp.c_str(), localIp.length() + 1);
+
if (localIp.contains(":"))
mIsIPv6 = true;
else
@@ -1152,8 +1512,8 @@
CHECK_GE(mRTCPSocket, 0);
int sockopt = 1;
- setsockopt(mRTPSocket, SOL_SOCKET, SO_REUSEPORT, (int *)&sockopt, sizeof(sockopt));
- setsockopt(mRTCPSocket, SOL_SOCKET, SO_REUSEPORT, (int *)&sockopt, sizeof(sockopt));
+ setsockopt(mRTPSocket, SOL_SOCKET, SO_REUSEADDR, (int *)&sockopt, sizeof(sockopt));
+ setsockopt(mRTCPSocket, SOL_SOCKET, SO_REUSEADDR, (int *)&sockopt, sizeof(sockopt));
if (mIsIPv6) {
memset(&mLocalAddr6, 0, sizeof(mLocalAddr6));
@@ -1195,7 +1555,7 @@
if (bind(mRTPSocket, localAddr, sizeSockSt) == -1) {
ALOGE("failed to bind rtp %s:%d err=%s", localIp.string(), localPort, strerror(errno));
} else {
- ALOGI("succeed to bind rtp %s:%d", localIp.string(), localPort);
+ ALOGD("succeed to bind rtp %s:%d", localIp.string(), localPort);
}
if (mIsIPv6)
@@ -1206,14 +1566,19 @@
if (bind(mRTCPSocket, localAddr, sizeSockSt) == -1) {
ALOGE("failed to bind rtcp %s:%d err=%s", localIp.string(), localPort + 1, strerror(errno));
} else {
- ALOGI("succeed to bind rtcp %s:%d", localIp.string(), localPort + 1);
+ ALOGD("succeed to bind rtcp %s:%d", localIp.string(), localPort + 1);
}
+}
- if (mIsIPv6)
- mLocalAddr6.sin6_port = htons((uint16_t)localPort);
- else
- mLocalAddr.sin_port = htons((uint16_t)localPort);
+// TODO : Develop more advanced moderator based on AS & TMMBR value
+void ARTPWriter::ModerateInstantTraffic(uint32_t samplePeriod, uint32_t limitBytes) {
+ unsigned int bytes = mTrafficRec->readBytesForLastPeriod(samplePeriod);
+ if (bytes > limitBytes) {
+ ALOGI("Nuclear moderator. #seq = %d \t\t %d bits / 10ms",
+ mSeqNo, bytes * 8);
+ usleep(4000);
+ mTrafficRec->updateClock(ALooper::GetNowUs() / 1000);
+ }
}
} // namespace android
-
diff --git a/media/libstagefright/rtsp/ARTPWriter.h b/media/libstagefright/rtsp/ARTPWriter.h
index 46df94b..28d6ec5 100644
--- a/media/libstagefright/rtsp/ARTPWriter.h
+++ b/media/libstagefright/rtsp/ARTPWriter.h
@@ -27,6 +27,9 @@
#include <arpa/inet.h>
#include <sys/socket.h>
+#include <android/multinetwork.h>
+#include "TrafficRecorder.h"
+
#define LOG_TO_FILES 0
namespace android {
@@ -37,15 +40,23 @@
struct ARTPWriter : public MediaWriter {
explicit ARTPWriter(int fd);
explicit ARTPWriter(int fd, String8& localIp, int localPort,
- String8& remoteIp, int remotePort);
+ String8& remoteIp, int remotePort,
+ uint32_t seqNo);
virtual status_t addSource(const sp<MediaSource> &source);
virtual bool reachedEOS();
virtual status_t start(MetaData *params);
virtual status_t stop();
virtual status_t pause();
+ void updateCVODegrees(int32_t cvoDegrees);
+ void updatePayloadType(int32_t payloadType);
+ void updateSocketDscp(int32_t dscp);
+ void updateSocketNetwork(int64_t socketNetwork);
+ uint32_t getSequenceNum();
+ virtual uint64_t getAccumulativeBytes() override;
virtual void onMessageReceived(const sp<AMessage> &msg);
+ virtual void setTMMBNInfo(uint32_t opponentID, uint32_t bitrate);
protected:
virtual ~ARTPWriter();
@@ -86,15 +97,19 @@
struct sockaddr_in6 mLocalAddr6;
struct sockaddr_in6 mRTPAddr6;
struct sockaddr_in6 mRTCPAddr6;
+ int32_t mRtpLayer3Dscp;
+ net_handle_t mRTPSockNetwork;
AString mProfileLevel;
AString mSeqParamSet;
AString mPicParamSet;
+ MediaBufferBase *mVPSBuf;
MediaBufferBase *mSPSBuf;
MediaBufferBase *mPPSBuf;
uint32_t mSourceID;
+ uint32_t mPayloadType;
uint32_t mSeqNo;
uint32_t mRTPTimeBase;
uint32_t mNumRTPSent;
@@ -102,7 +117,14 @@
uint32_t mLastRTPTime;
uint64_t mLastNTPTime;
+ uint32_t mOpponentID;
+ uint32_t mBitrate;
+ typedef uint64_t Bytes;
+ sp<TrafficRecorder<uint32_t /* Time */, Bytes> > mTrafficRec;
+
int32_t mNumSRsSent;
+ int32_t mRTPCVOExtMap;
+ int32_t mRTPCVODegrees;
enum {
INVALID,
@@ -115,16 +137,19 @@
static uint64_t GetNowNTP();
+ void initState();
void onRead(const sp<AMessage> &msg);
void onSendSR(const sp<AMessage> &msg);
void addSR(const sp<ABuffer> &buffer);
void addSDES(const sp<ABuffer> &buffer);
+ void addTMMBN(const sp<ABuffer> &buffer);
void makeH264SPropParamSets(MediaBufferBase *buffer);
void dumpSessionDesc();
void sendBye();
+ void sendVPSSPSPPSIfIFrame(MediaBufferBase *mediaBuf, int64_t timeUs);
void sendSPSPPSIfIFrame(MediaBufferBase *mediaBuf, int64_t timeUs);
void sendHEVCData(MediaBufferBase *mediaBuf);
void sendAVCData(MediaBufferBase *mediaBuf);
@@ -134,6 +159,7 @@
void send(const sp<ABuffer> &buffer, bool isRTCP);
void makeSocketPairAndBind(String8& localIp, int localPort, String8& remoteIp, int remotePort);
+ void ModerateInstantTraffic(uint32_t samplePeriod, uint32_t limitBytes);
DISALLOW_EVIL_CONSTRUCTORS(ARTPWriter);
};
diff --git a/media/libstagefright/rtsp/ASessionDescription.cpp b/media/libstagefright/rtsp/ASessionDescription.cpp
index d8fde76..5b5b4b1 100644
--- a/media/libstagefright/rtsp/ASessionDescription.cpp
+++ b/media/libstagefright/rtsp/ASessionDescription.cpp
@@ -27,6 +27,8 @@
namespace android {
+constexpr unsigned kDefaultAs = 960; // kbps?
+
ASessionDescription::ASessionDescription()
: mIsValid(false) {
}
@@ -103,7 +105,7 @@
key.setTo(line, 0, colonPos);
if (key == "a=fmtp" || key == "a=rtpmap"
- || key == "a=framesize") {
+ || key == "a=framesize" || key == "a=extmap") {
ssize_t spacePos = line.find(" ", colonPos + 1);
if (spacePos < 0) {
return false;
@@ -201,6 +203,33 @@
return true;
}
+bool ASessionDescription::getCvoExtMap(
+ size_t index, int32_t *cvoExtMap) const {
+ CHECK_GE(index, 0u);
+ CHECK_LT(index, mTracks.size());
+
+ AString key, value;
+ *cvoExtMap = 0;
+
+ const Attribs &track = mTracks.itemAt(index);
+ for (size_t i = 0; i < track.size(); i++) {
+ value = track.valueAt(i);
+ if (value.size() > 0 && strcmp(value.c_str(), "urn:3gpp:video-orientation") == 0) {
+ key = track.keyAt(i);
+ break;
+ }
+ }
+
+ if (key.size() > 0) {
+ const char *colonPos = strrchr(key.c_str(), ':');
+ colonPos++;
+ *cvoExtMap = atoi(colonPos);
+ return true;
+ }
+
+ return false;
+}
+
void ASessionDescription::getFormatType(
size_t index, unsigned long *PT,
AString *desc, AString *params) const {
@@ -347,8 +376,9 @@
// static
void ASessionDescription::SDPStringFactory(AString &sdp,
- const char *ip, bool isAudio, unsigned port, unsigned payloadType,
- unsigned as, const char *codec, const char *fmtp, int32_t width, int32_t height)
+ const char *ip, bool isAudio, unsigned port, unsigned payloadType,
+ unsigned as, const char *codec, const char *fmtp,
+ int32_t width, int32_t height, int32_t cvoExtMap)
{
bool isIPv4 = (AString(ip).find("::") == -1) ? true : false;
sdp.clear();
@@ -364,15 +394,16 @@
sdp.append("\r\n");
sdp.append("c= IN IP");
- if(isIPv4)
- sdp.append("4 ");
- else
- sdp.append("6 ");
+ if (isIPv4) {
+ sdp.append("4 ");
+ } else {
+ sdp.append("6 ");
+ }
sdp.append(ip);
sdp.append("\r\n");
sdp.append("b=AS:");
- sdp.append(as > 0 ? as : 960);
+ sdp.append(as > 0 ? as : kDefaultAs);
sdp.append("\r\n");
sdp.append("a=rtpmap:");
@@ -383,7 +414,7 @@
sdp.append(isAudio ? "8000" : "90000");
sdp.append("\r\n");
- if(fmtp != NULL) {
+ if (fmtp != NULL) {
sdp.append("a=fmtp:");
sdp.append(payloadType);
sdp.append(" ");
@@ -391,7 +422,7 @@
sdp.append("\r\n");
}
- if(width > 0 && height > 0) {
+ if (!isAudio && width > 0 && height > 0) {
sdp.append("a=framesize:");
sdp.append(payloadType);
sdp.append(" ");
@@ -401,6 +432,14 @@
sdp.append("\r\n");
}
+ if (cvoExtMap > 0) {
+ sdp.append("a=extmap:");
+ sdp.append(cvoExtMap);
+ sdp.append(" ");
+ sdp.append("urn:3gpp:video-orientation");
+ sdp.append("\r\n");
+ }
+
ALOGV("SDPStringFactory => %s", sdp.c_str());
}
diff --git a/media/libstagefright/rtsp/ASessionDescription.h b/media/libstagefright/rtsp/ASessionDescription.h
index bd92916..91f5442 100644
--- a/media/libstagefright/rtsp/ASessionDescription.h
+++ b/media/libstagefright/rtsp/ASessionDescription.h
@@ -40,6 +40,8 @@
size_t countTracks() const;
void getFormat(size_t index, AString *value) const;
+ bool getCvoExtMap(size_t index, int32_t *cvoExtMap) const;
+
void getFormatType(
size_t index, unsigned long *PT,
AString *desc, AString *params) const;
@@ -65,7 +67,7 @@
static void SDPStringFactory(AString &sdp, const char *ip, bool isAudio, unsigned port,
unsigned payloadType, unsigned as, const char *codec, const char *fmtp = NULL,
- int32_t width = 0, int32_t height = 0);
+ int32_t width = 0, int32_t height = 0, int32_t cvoExtMap = 0);
protected:
virtual ~ASessionDescription();
diff --git a/media/libstagefright/rtsp/Android.bp b/media/libstagefright/rtsp/Android.bp
index d50f774..dcadbaf 100644
--- a/media/libstagefright/rtsp/Android.bp
+++ b/media/libstagefright/rtsp/Android.bp
@@ -40,6 +40,7 @@
],
shared_libs: [
+ "libandroid_net",
"libcrypto",
"libdatasource",
"libmedia",
@@ -48,6 +49,7 @@
include_dirs: [
"frameworks/av/media/libstagefright",
"frameworks/native/include/media/openmax",
+ "frameworks/native/include/android",
],
arch: {
diff --git a/media/libstagefright/rtsp/MyHandler.h b/media/libstagefright/rtsp/MyHandler.h
index 7f025a5..0fdf431 100644
--- a/media/libstagefright/rtsp/MyHandler.h
+++ b/media/libstagefright/rtsp/MyHandler.h
@@ -1032,6 +1032,11 @@
break;
}
+ int32_t rtcpEvent;
+ if (msg->findInt32("rtcp-event", &rtcpEvent)) {
+ break;
+ }
+
++mNumAccessUnitsReceived;
postAccessUnitTimeoutCheck();
diff --git a/media/libstagefright/rtsp/QualManager.cpp b/media/libstagefright/rtsp/QualManager.cpp
new file mode 100644
index 0000000..37aa326
--- /dev/null
+++ b/media/libstagefright/rtsp/QualManager.cpp
@@ -0,0 +1,174 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "QualManager"
+
+#include <algorithm>
+
+#include <sys/prctl.h>
+#include <utils/Log.h>
+
+#include "QualManager.h"
+
+namespace android {
+
+QualManager::Watcher::Watcher(int32_t timeLimit)
+ : Thread(false), mWatching(false), mSwitch(false),
+ mTimeLimit(timeLimit * 1000000LL) // timeLimit ms
+{
+}
+
+bool QualManager::Watcher::isExpired() const
+{
+ return mSwitch;
+}
+
+void QualManager::Watcher::setup() {
+ AutoMutex _l(mMyLock);
+ if (mWatching == false) {
+ mWatching = true;
+ mMyCond.signal();
+ }
+}
+
+void QualManager::Watcher::release() {
+ AutoMutex _l(mMyLock);
+ if (mSwitch) {
+ ALOGW("%s DISARMED", name);
+ mSwitch = false;
+ }
+ if (mWatching == true) {
+ ALOGW("%s DISARMED", name);
+ mWatching = false;
+ mMyCond.signal();
+ }
+}
+
+void QualManager::Watcher::exit() {
+ AutoMutex _l(mMyLock);
+ // The order is important to avoid dead lock.
+ Thread::requestExit();
+ mMyCond.signal();
+}
+
+QualManager::Watcher::~Watcher() {
+ ALOGI("%s thread dead", name);
+}
+
+bool QualManager::Watcher::threadLoop() {
+ AutoMutex _l(mMyLock);
+#if defined(__linux__)
+ prctl(PR_GET_NAME, name, 0, 0, 0);
+#endif
+ while (!exitPending()) {
+ ALOGW("%s Timer init", name);
+ mMyCond.wait(mMyLock); // waits as non-watching state
+ if (exitPending())
+ return false;
+ ALOGW("%s timer BOOM after %d msec", name, (int)(mTimeLimit / 1000000LL));
+ mMyCond.waitRelative(mMyLock, mTimeLimit); // waits as watching satte
+ if (mWatching == true) {
+ mSwitch = true;
+ ALOGW("%s BOOM!!!!", name);
+ }
+ mWatching = false;
+ }
+ return false;
+}
+
+
+QualManager::QualManager()
+ : mMinBitrate(-1), mMaxBitrate(-1),
+ mTargetBitrate(512000), mLastTargetBitrate(-1),
+ mLastSetBitrateTime(0), mIsNewTargetBitrate(false)
+{
+ VFPWatcher = new Watcher(3000); //Very Few Packet Watcher
+ VFPWatcher->run("VeryFewPtk");
+ LBRWatcher = new Watcher(10000); //Low Bit Rate Watcher
+ LBRWatcher->run("LowBitRate");
+}
+
+QualManager::~QualManager() {
+ VFPWatcher->exit();
+ LBRWatcher->exit();
+}
+
+int32_t QualManager::getTargetBitrate() {
+ if (mIsNewTargetBitrate) {
+ mIsNewTargetBitrate = false;
+ mLastTargetBitrate = clampingBitrate(mTargetBitrate);
+ mTargetBitrate = mLastTargetBitrate;
+ return mTargetBitrate;
+ } else {
+ return -1;
+ }
+}
+
+bool QualManager::isNeedToDowngrade() {
+ return LBRWatcher->isExpired();
+}
+
+void QualManager::setTargetBitrate(uint8_t fraction, int64_t nowUs, bool isTooLowPkts) {
+ /* Too Low Packet. Maybe opponent is switching camera.
+ * If this condition goes longer, we should down bitrate.
+ */
+ if (isTooLowPkts) {
+ VFPWatcher->setup();
+ } else {
+ VFPWatcher->release();
+ }
+
+ if ((fraction > (256 * 5 / 100) && !isTooLowPkts) || VFPWatcher->isExpired()) {
+ // loss more than 5% or VFPWatcher BOOMED
+ mTargetBitrate -= mBitrateStep * 3;
+ } else if (fraction <= (256 * 2 /100)) {
+ // loss less than 2%
+ mTargetBitrate += mBitrateStep;
+ }
+
+ if (mTargetBitrate > mMaxBitrate) {
+ mTargetBitrate = mMaxBitrate + mBitrateStep;
+ } else if (mTargetBitrate < mMinBitrate) {
+ LBRWatcher->setup();
+ mTargetBitrate = mMinBitrate - mBitrateStep;
+ }
+
+ if (mLastTargetBitrate != clampingBitrate(mTargetBitrate) ||
+ nowUs - mLastSetBitrateTime > 5000000ll) {
+ mIsNewTargetBitrate = true;
+ mLastSetBitrateTime = nowUs;
+ }
+}
+
+void QualManager::setMinMaxBitrate(int32_t min, int32_t max) {
+ mMinBitrate = min;
+ mMaxBitrate = max;
+ mBitrateStep = (max - min) / 8;
+}
+
+void QualManager::setBitrateData(int32_t bitrate, int64_t /*now*/) {
+ // A bitrate that is considered packetloss also should be good.
+ if (bitrate >= mMinBitrate && mTargetBitrate >= mMinBitrate) {
+ LBRWatcher->release();
+ } else if (bitrate < mMinBitrate){
+ LBRWatcher->setup();
+ }
+}
+
+int32_t QualManager::clampingBitrate(int32_t bitrate) {
+ return std::min(std::max(mMinBitrate, bitrate), mMaxBitrate);
+}
+} // namespace android
diff --git a/media/libstagefright/rtsp/QualManager.h b/media/libstagefright/rtsp/QualManager.h
new file mode 100644
index 0000000..a7dc921
--- /dev/null
+++ b/media/libstagefright/rtsp/QualManager.h
@@ -0,0 +1,75 @@
+/*
+ * Copyright (C) 2018 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef QUAL_MANAGER_H_
+
+#define QUAL_MANAGER_H_
+
+#include <stdint.h>
+#include <utils/Thread.h>
+
+namespace android {
+class QualManager {
+public:
+ QualManager();
+ ~QualManager();
+
+ int32_t getTargetBitrate();
+ bool isNeedToDowngrade();
+
+ void setTargetBitrate(uint8_t fraction, int64_t nowUs, bool isTooLowPkts);
+ void setMinMaxBitrate(int32_t min, int32_t max);
+ void setBitrateData(int32_t bitrate, int64_t now);
+private:
+ class Watcher : public Thread
+ {
+ public:
+ Watcher(int32_t timeLimit);
+
+ void setup();
+ void release();
+ void exit();
+ bool isExpired() const;
+ private:
+ virtual ~Watcher();
+ virtual bool threadLoop();
+
+ char name[32] = {0,};
+
+ Condition mMyCond;
+ Mutex mMyLock;
+
+ bool mWatching;
+ bool mSwitch;
+ const nsecs_t mTimeLimit;
+ };
+ sp<Watcher> VFPWatcher;
+ sp<Watcher> LBRWatcher;
+ int32_t mMinBitrate;
+ int32_t mMaxBitrate;
+ int32_t mBitrateStep;
+
+ int32_t mTargetBitrate;
+ int32_t mLastTargetBitrate;
+ int64_t mLastSetBitrateTime;
+
+ bool mIsNewTargetBitrate;
+
+ int32_t clampingBitrate(int32_t bitrate);
+};
+} //namespace android
+
+#endif // QUAL_MANAGER_H_
diff --git a/media/libstagefright/rtsp/TrafficRecorder.h b/media/libstagefright/rtsp/TrafficRecorder.h
new file mode 100644
index 0000000..8ba8f90
--- /dev/null
+++ b/media/libstagefright/rtsp/TrafficRecorder.h
@@ -0,0 +1,179 @@
+/*
+ * Copyright (C) 2010 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef A_TRAFFIC_RECORDER_H_
+
+#define A_TRAFFIC_RECORDER_H_
+
+#include <android-base/logging.h>
+#include <utils/RefBase.h>
+
+namespace android {
+
+// Circular array to save recent amount of bytes
+template <class Time, class Bytes>
+class TrafficRecorder : public RefBase {
+private:
+ constexpr static size_t kMinNumEntries = 4;
+ constexpr static size_t kMaxNumEntries = 1024;
+
+ size_t mSize;
+ size_t mSizeMask;
+ Time *mTimeArray = NULL;
+ Bytes *mBytesArray = NULL;
+ size_t mHeadIdx;
+ size_t mTailIdx;
+
+ int mLastReadIdx;
+
+ const Time mRecordLimit;
+ Time mClock;
+ Time mLastTimeOfPrint;
+ Bytes mAccuBytes;
+
+public:
+ TrafficRecorder(size_t size, Time accuTimeLimit);
+ virtual ~TrafficRecorder();
+
+ void init();
+ void updateClock(Time now);
+ Bytes readBytesForTotal();
+ Bytes readBytesForLastPeriod(Time period);
+ void writeBytes(Bytes bytes);
+ void printAccuBitsForLastPeriod(Time period, Time unit);
+};
+
+template <class Time, class Bytes>
+TrafficRecorder<Time, Bytes>::TrafficRecorder(size_t size, Time recordLimit)
+ : mRecordLimit(recordLimit) {
+ if (size > kMaxNumEntries) {
+ LOG(VERBOSE) << "Limiting TrafficRecorder size to " << kMaxNumEntries;
+ size = kMaxNumEntries;
+ } else if (size < kMinNumEntries) {
+ LOG(VERBOSE) << "Limiting TrafficRecorder size to " << kMaxNumEntries;
+ size = kMinNumEntries;
+ }
+
+ size_t exp = ((sizeof(size_t) == 8) ?
+ 64 - __builtin_clzl(size - 1) :
+ 32 - __builtin_clz(size - 1));
+ mSize = (1ul << exp); // size = 2^exp
+ mSizeMask = mSize - 1;
+
+ LOG(VERBOSE) << "TrafficRecorder Init size " << mSize;
+ mTimeArray = new Time[mSize];
+ mBytesArray = new Bytes[mSize];
+
+ init();
+}
+
+template <class Time, class Bytes>
+TrafficRecorder<Time, Bytes>::~TrafficRecorder() {
+ delete[] mTimeArray;
+ delete[] mBytesArray;
+}
+
+template <class Time, class Bytes>
+void TrafficRecorder<Time, Bytes>::init() {
+ mHeadIdx = 0;
+ mTailIdx = mSizeMask;
+ for (int i = 0 ; i < mSize ; i++) {
+ mTimeArray[i] = 0;
+ mBytesArray[i] = 0;
+ }
+ mClock = 0;
+ mLastReadIdx = 0;
+ mLastTimeOfPrint = 0;
+ mAccuBytes = 0;
+}
+
+template <class Time, class Bytes>
+void TrafficRecorder<Time, Bytes>::updateClock(Time now) {
+ mClock = now;
+}
+
+template <class Time, class Bytes>
+Bytes TrafficRecorder<Time, Bytes>::readBytesForTotal() {
+ return mAccuBytes;
+}
+
+template <class Time, class Bytes>
+Bytes TrafficRecorder<Time, Bytes>::readBytesForLastPeriod(Time period) {
+ // Not enough data
+ if (period > mClock)
+ return 0;
+
+ Bytes bytes = 0;
+ int i = mHeadIdx;
+ while (i != mTailIdx) {
+ LOG(VERBOSE) << "READ " << i << " time " << mTimeArray[i]
+ << " \t EndOfPeriod " << mClock - period
+ << "\t\t Bytes:" << mBytesArray[i] << "\t\t Accu: " << bytes;
+ if (mTimeArray[i] < mClock - period) {
+ break;
+ }
+ bytes += mBytesArray[i];
+ i = (i - 1) & mSizeMask;
+ }
+ mLastReadIdx = (i + 1) & mSizeMask;
+
+ return bytes;
+}
+
+template <class Time, class Bytes>
+void TrafficRecorder<Time, Bytes>::writeBytes(Bytes bytes) {
+ int writeIdx;
+ if (mClock == mTimeArray[mHeadIdx]) {
+ writeIdx = mHeadIdx;
+ mBytesArray[writeIdx] += bytes;
+ } else {
+ writeIdx = (mHeadIdx + 1) & mSizeMask;
+ mTimeArray[writeIdx] = mClock;
+ mBytesArray[writeIdx] = bytes;
+ }
+
+ LOG(VERBOSE) << "WRITE " << writeIdx << " time " << mClock;
+ if (writeIdx == mTailIdx) {
+ mTailIdx = (mTailIdx + 1) & mSizeMask;
+ }
+
+ mHeadIdx = writeIdx;
+ mAccuBytes += bytes;
+}
+
+template <class Time, class Bytes>
+void TrafficRecorder<Time, Bytes>::printAccuBitsForLastPeriod(Time period, Time unit) {
+ Time timeSinceLastPrint = mClock - mLastTimeOfPrint;
+ if (timeSinceLastPrint < period)
+ return;
+
+ Bytes sum = readBytesForLastPeriod(period);
+ Time readPeriod = mClock - mTimeArray[mLastReadIdx];
+
+ float numOfUnit = (float)(readPeriod) / (unit + FLT_MIN);
+ ALOGD("Actual Tx period %.3f unit \t %.0f bytes (%.0f Kbits)/Unit",
+ numOfUnit, sum / numOfUnit, sum * 8.f / numOfUnit / 1000.f);
+ mLastTimeOfPrint = mClock;
+
+ if (mClock - mTimeArray[mTailIdx] < mRecordLimit) {
+ // Size is not enough to record bytes for mRecordLimit period
+ ALOGW("Traffic recorder size is not enough. mRecordLimit %d", mRecordLimit);
+ }
+}
+
+} // namespace android
+
+#endif // A_TRAFFIC_RECORDER_H_
diff --git a/media/libstagefright/writer_fuzzers/README.md b/media/libstagefright/writer_fuzzers/README.md
index 0d21031..6f95ecc 100644
--- a/media/libstagefright/writer_fuzzers/README.md
+++ b/media/libstagefright/writer_fuzzers/README.md
@@ -29,7 +29,7 @@
| Parameter| Valid Values| Configured Value|
|------------- |-------------| ----- |
-| `mime` | 0. `audio/3gpp` 1. `audio/amr-wb` 2. `audio/vorbis` 3. `audio/opus` 4. `audio/mp4a-latm` 5. `video/avc` 6. `video/hevc` 7. `video/mp4v-es` 8. `video/3gpp` 9. `video/x-vnd.on2.vp8` 10. `video/x-vnd.on2.vp9` | All the bits of 2nd byte of data for first track and 11th byte of data for second track (if present) modulus 10 |
+| `mime` | 0. `audio/3gpp` 1. `audio/amr-wb` 2. `audio/vorbis` 3. `audio/opus` 4. `audio/mp4a-latm` 5. `audio/mpeg` 6. `audio/mpeg-L1` 7. `audio/mpeg-L2` 8. `audio/midi` 9. `audio/qcelp` 10. `audio/g711-alaw` 11. `audio/g711-mlaw` 12. `audio/flac` 13. `audio/aac-adts` 14. `audio/gsm` 15. `audio/ac3` 16. `audio/eac3` 17. `audio/eac3-joc` 18. `audio/ac4` 19. `audio/scrambled` 20. `audio/alac` 21. `audio/x-ms-wma` 22. `audio/x-adpcm-ms` 23. `audio/x-adpcm-dvi-ima` 24. `video/avc` 25. `video/hevc` 26. `video/mp4v-es` 27. `video/3gpp` 28. `video/x-vnd.on2.vp8` 29. `video/x-vnd.on2.vp9` 30. `video/av01` 31. `video/mpeg2` 32. `video/dolby-vision` 33. `video/scrambled` 34. `video/divx` 35. `video/divx3` 36. `video/xvid` 37. `video/x-motion-jpeg` 38. `text/3gpp-tt` 39. `application/x-subrip` 40. `text/vtt` 41. `text/cea-608` 42. `text/cea-708` 43. `application/x-id3v4` | All the bits of 2nd byte of data for first track and 11th byte of data for second track and 20th byte of data for third track(if present) modulus 44 |
| `channel-count` | In the range `0 to INT32_MAX` | All the bits of 3rd byte to 6th bytes of data if first track is audio and 12th to 15th bytes of data if second track is audio |
| `sample-rate` | In the range `1 to INT32_MAX` | All the bits of 7th byte to 10th bytes of data if first track is audio and 16th to 19th bytes of data if second track is audio |
| `height` | In the range `0 to INT32_MAX` | All the bits of 3rd byte to 6th bytes of data if first track is video and 12th to 15th bytes of data if second track is video |
diff --git a/media/libstagefright/writer_fuzzers/WriterFuzzerBase.cpp b/media/libstagefright/writer_fuzzers/WriterFuzzerBase.cpp
index 844db39..ee7af70 100644
--- a/media/libstagefright/writer_fuzzers/WriterFuzzerBase.cpp
+++ b/media/libstagefright/writer_fuzzers/WriterFuzzerBase.cpp
@@ -53,7 +53,7 @@
return mNumCsds[trackIndex];
}
-vector<FrameData> WriterFuzzerBase::BufferSource::getFrameList(int32_t trackIndex) {
+vector<FrameData> &WriterFuzzerBase::BufferSource::getFrameList(int32_t trackIndex) {
return mFrameList[trackIndex];
}
@@ -92,9 +92,8 @@
} else {
break;
}
- mFrameList[trackIndex].insert(
- mFrameList[trackIndex].begin(),
- FrameData{static_cast<int32_t>(bufferSize), flags, pts, framePtr});
+ mFrameList[trackIndex].insert(mFrameList[trackIndex].begin(),
+ FrameData{bufferSize, flags, pts, framePtr});
bytesRemaining -= (frameSize + kMarkerSize + kMarkerSuffixSize);
--mReadIndex;
}
@@ -105,31 +104,36 @@
* Scenario where input data does not contain the custom frame markers.
* Hence feed the entire data as single frame.
*/
- mFrameList[0].emplace_back(
- FrameData{static_cast<int32_t>(mSize - readIndexStart), 0, 0, mData + readIndexStart});
+ mFrameList[0].emplace_back(FrameData{mSize - readIndexStart, 0, 0, mData + readIndexStart});
}
}
bool WriterFuzzerBase::BufferSource::getTrackInfo(int32_t trackIndex) {
- if (mSize <= mReadIndex + 2 * sizeof(int) + sizeof(uint8_t)) {
+ if (mSize <= mReadIndex + sizeof(uint8_t)) {
return false;
}
size_t mimeTypeIdx = mData[mReadIndex] % kSupportedMimeTypes;
char *mime = (char *)supportedMimeTypes[mimeTypeIdx].c_str();
mParams[trackIndex].mime = mime;
- ++mReadIndex;
+ mReadIndex += sizeof(uint8_t);
- if (!strncmp(mime, "audio/", 6)) {
- copy(mData + mReadIndex, mData + mReadIndex + sizeof(int),
- reinterpret_cast<char *>(&mParams[trackIndex].channelCount));
- copy(mData + mReadIndex + sizeof(int), mData + mReadIndex + 2 * sizeof(int),
- reinterpret_cast<char *>(&mParams[trackIndex].sampleRate));
+ if (mSize > mReadIndex + 2 * sizeof(int32_t)) {
+ if (!strncmp(mime, "audio/", 6)) {
+ copy(mData + mReadIndex, mData + mReadIndex + sizeof(int32_t),
+ reinterpret_cast<char *>(&mParams[trackIndex].channelCount));
+ copy(mData + mReadIndex + sizeof(int32_t), mData + mReadIndex + 2 * sizeof(int32_t),
+ reinterpret_cast<char *>(&mParams[trackIndex].sampleRate));
+ } else if (!strncmp(mime, "video/", 6)) {
+ copy(mData + mReadIndex, mData + mReadIndex + sizeof(int32_t),
+ reinterpret_cast<char *>(&mParams[trackIndex].height));
+ copy(mData + mReadIndex + sizeof(int32_t), mData + mReadIndex + 2 * sizeof(int32_t),
+ reinterpret_cast<char *>(&mParams[trackIndex].width));
+ }
+ mReadIndex += 2 * sizeof(int32_t);
} else {
- copy(mData + mReadIndex, mData + mReadIndex + sizeof(int),
- reinterpret_cast<char *>(&mParams[trackIndex].height));
- copy(mData + mReadIndex + sizeof(int), mData + mReadIndex + 2 * sizeof(int),
- reinterpret_cast<char *>(&mParams[trackIndex].width));
+ if (strncmp(mime, "text/", 5) && strncmp(mime, "application/", 12)) {
+ return false;
+ }
}
- mReadIndex += 2 * sizeof(int);
return true;
}
@@ -173,7 +177,7 @@
}
format->setInt32("channel-count", params.channelCount);
format->setInt32("sample-rate", params.sampleRate);
- } else {
+ } else if (!strncmp(params.mime, "video/", 6)) {
format->setInt32("width", params.width);
format->setInt32("height", params.height);
}
@@ -193,11 +197,10 @@
mWriter->start(mFileMeta.get());
}
-void WriterFuzzerBase::sendBuffersToWriter(sp<MediaAdapter> ¤tTrack, int32_t trackIndex) {
- int32_t numCsds = mBufferSource->getNumCsds(trackIndex);
+void WriterFuzzerBase::sendBuffersToWriter(sp<MediaAdapter> ¤tTrack, int32_t trackIndex,
+ int32_t startFrameIndex, int32_t endFrameIndex) {
vector<FrameData> bufferInfo = mBufferSource->getFrameList(trackIndex);
- int32_t range = bufferInfo.size();
- for (int idx = numCsds; idx < range; ++idx) {
+ for (int idx = startFrameIndex; idx < endFrameIndex; ++idx) {
sp<ABuffer> buffer = new ABuffer((void *)bufferInfo[idx].buf, bufferInfo[idx].size);
MediaBuffer *mediaBuffer = new MediaBuffer(buffer);
@@ -209,7 +212,7 @@
// Just set the kKeyDecodingTime as the presentation time for now.
sampleMetaData.setInt64(kKeyDecodingTime, bufferInfo[idx].timeUs);
- if (bufferInfo[idx].flags == 1) {
+ if (bufferInfo[idx].flags == SampleFlag::SYNC_FLAG) {
sampleMetaData.setInt32(kKeyIsSyncFrame, true);
}
@@ -218,6 +221,28 @@
}
}
+void WriterFuzzerBase::sendBuffersInterleave(int32_t numTracks, uint8_t numBuffersInterleave) {
+ int32_t currentFrameIndex[numTracks], remainingNumFrames[numTracks], numTrackFramesDone;
+ for (int32_t idx = 0; idx < numTracks; ++idx) {
+ currentFrameIndex[idx] = mBufferSource->getNumCsds(idx);
+ remainingNumFrames[idx] = mBufferSource->getFrameList(idx).size() - currentFrameIndex[idx];
+ }
+ do {
+ numTrackFramesDone = numTracks;
+ for (int32_t idx = 0; idx < numTracks; ++idx) {
+ if (remainingNumFrames[idx] > 0) {
+ int32_t numFramesInterleave =
+ min(remainingNumFrames[idx], static_cast<int32_t>(numBuffersInterleave));
+ sendBuffersToWriter(mCurrentTrack[idx], idx, currentFrameIndex[idx],
+ currentFrameIndex[idx] + numFramesInterleave);
+ currentFrameIndex[idx] += numFramesInterleave;
+ remainingNumFrames[idx] -= numFramesInterleave;
+ --numTrackFramesDone;
+ }
+ }
+ } while (numTrackFramesDone < numTracks);
+}
+
void WriterFuzzerBase::initFileWriterAndProcessData(const uint8_t *data, size_t size) {
if (!createOutputFile()) {
return;
@@ -225,6 +250,14 @@
if (!createWriter()) {
return;
}
+
+ if (size < 1) {
+ return;
+ }
+ uint8_t numBuffersInterleave = (data[0] == 0 ? 1 : data[0]);
+ ++data;
+ --size;
+
mBufferSource = new BufferSource(data, size);
if (!mBufferSource) {
return;
@@ -246,9 +279,7 @@
addWriterSource(idx);
}
start();
- for (int32_t idx = 0; idx < mNumTracks; ++idx) {
- sendBuffersToWriter(mCurrentTrack[idx], idx);
- }
+ sendBuffersInterleave(mNumTracks, numBuffersInterleave);
for (int32_t idx = 0; idx < mNumTracks; ++idx) {
if (mCurrentTrack[idx]) {
mCurrentTrack[idx]->stop();
diff --git a/media/libstagefright/writer_fuzzers/include/WriterFuzzerBase.h b/media/libstagefright/writer_fuzzers/include/WriterFuzzerBase.h
index da06463..4315322 100644
--- a/media/libstagefright/writer_fuzzers/include/WriterFuzzerBase.h
+++ b/media/libstagefright/writer_fuzzers/include/WriterFuzzerBase.h
@@ -34,7 +34,7 @@
using namespace std;
constexpr uint32_t kMimeSize = 128;
-constexpr uint8_t kMaxTrackCount = 2;
+constexpr uint8_t kMaxTrackCount = 3;
constexpr uint32_t kMaxCSDStrlen = 16;
constexpr uint32_t kCodecConfigFlag = 32;
@@ -49,25 +49,65 @@
};
struct FrameData {
- int32_t size;
+ size_t size;
uint8_t flags;
int64_t timeUs;
const uint8_t* buf;
};
-static string supportedMimeTypes[] = {
- "audio/3gpp", "audio/amr-wb", "audio/vorbis", "audio/opus",
- "audio/mp4a-latm", "video/avc", "video/hevc", "video/mp4v-es",
- "video/3gpp", "video/x-vnd.on2.vp8", "video/x-vnd.on2.vp9",
-};
+static string supportedMimeTypes[] = {"audio/3gpp",
+ "audio/amr-wb",
+ "audio/vorbis",
+ "audio/opus",
+ "audio/mp4a-latm",
+ "audio/mpeg",
+ "audio/mpeg-L1",
+ "audio/mpeg-L2",
+ "audio/midi",
+ "audio/qcelp",
+ "audio/g711-alaw",
+ "audio/g711-mlaw",
+ "audio/flac",
+ "audio/aac-adts",
+ "audio/gsm",
+ "audio/ac3",
+ "audio/eac3",
+ "audio/eac3-joc",
+ "audio/ac4",
+ "audio/scrambled",
+ "audio/alac",
+ "audio/x-ms-wma",
+ "audio/x-adpcm-ms",
+ "audio/x-adpcm-dvi-ima",
+ "video/avc",
+ "video/hevc",
+ "video/mp4v-es",
+ "video/3gpp",
+ "video/x-vnd.on2.vp8",
+ "video/x-vnd.on2.vp9",
+ "video/av01",
+ "video/mpeg2",
+ "video/dolby-vision",
+ "video/scrambled",
+ "video/divx",
+ "video/divx3",
+ "video/xvid",
+ "video/x-motion-jpeg",
+ "text/3gpp-tt",
+ "application/x-subrip",
+ "text/vtt",
+ "text/cea-608",
+ "text/cea-708",
+ "application/x-id3v4"};
-enum {
+enum SampleFlag {
DEFAULT_FLAG = 0,
SYNC_FLAG = 1,
ENCRYPTED_FLAG = 2,
};
-static uint8_t flagTypes[] = {DEFAULT_FLAG, SYNC_FLAG, ENCRYPTED_FLAG};
+static uint8_t flagTypes[] = {SampleFlag::DEFAULT_FLAG, SampleFlag::SYNC_FLAG,
+ SampleFlag::ENCRYPTED_FLAG};
class WriterFuzzerBase {
public:
@@ -105,7 +145,10 @@
void start();
- void sendBuffersToWriter(sp<MediaAdapter>& currentTrack, int32_t trackIndex);
+ void sendBuffersToWriter(sp<MediaAdapter>& currentTrack, int32_t trackIndex,
+ int32_t startFrameIndex, int32_t endFrameIndex);
+
+ void sendBuffersInterleave(int32_t numTracks, uint8_t numBuffersInterleave);
void initFileWriterAndProcessData(const uint8_t* data, size_t size);
@@ -126,7 +169,7 @@
void getFrameInfo();
ConfigFormat getConfigFormat(int32_t trackIndex);
int32_t getNumCsds(int32_t trackIndex);
- vector<FrameData> getFrameList(int32_t trackIndex);
+ vector<FrameData>& getFrameList(int32_t trackIndex);
private:
bool isMarker() { return (memcmp(&mData[mReadIndex], kMarker, kMarkerSize) == 0); }
diff --git a/media/ndk/Android.bp b/media/ndk/Android.bp
index b3b5d54..e9ea386 100644
--- a/media/ndk/Android.bp
+++ b/media/ndk/Android.bp
@@ -114,6 +114,7 @@
static_libs: [
"libgrallocusage",
+ "libnativehelper_lazy",
],
header_libs: [
@@ -142,7 +143,6 @@
"libgui",
"libui",
"libmediandk_utils",
- "libnativehelper",
],
export_header_lib_headers: ["jni_headers"],
diff --git a/services/camera/libcameraservice/libcameraservice_fuzzer/Android.bp b/services/camera/libcameraservice/libcameraservice_fuzzer/Android.bp
new file mode 100644
index 0000000..df4ef95
--- /dev/null
+++ b/services/camera/libcameraservice/libcameraservice_fuzzer/Android.bp
@@ -0,0 +1,55 @@
+/******************************************************************************
+ *
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *****************************************************************************
+ * Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore
+ */
+
+cc_fuzz {
+ name: "camera_service_fuzzer",
+ srcs: [
+ "camera_service_fuzzer.cpp",
+ ],
+ header_libs: [
+ "libmedia_headers",
+ ],
+ shared_libs: [
+ "libbinder",
+ "libbase",
+ "libutils",
+ "libcutils",
+ "libcameraservice",
+ "libcamera_client",
+ "libui",
+ "libgui",
+ "android.hardware.camera.common@1.0",
+ "android.hardware.camera.provider@2.4",
+ "android.hardware.camera.provider@2.5",
+ "android.hardware.camera.provider@2.6",
+ "android.hardware.camera.device@1.0",
+ "android.hardware.camera.device@3.2",
+ "android.hardware.camera.device@3.3",
+ "android.hardware.camera.device@3.4",
+ "android.hardware.camera.device@3.5",
+ "android.hardware.camera.device@3.6",
+ ],
+ fuzz_config: {
+ cc: [
+ "android-media-fuzzing-reports@google.com",
+ ],
+ componentid: 155276,
+ },
+}
diff --git a/services/camera/libcameraservice/libcameraservice_fuzzer/README.md b/services/camera/libcameraservice/libcameraservice_fuzzer/README.md
new file mode 100644
index 0000000..c703845
--- /dev/null
+++ b/services/camera/libcameraservice/libcameraservice_fuzzer/README.md
@@ -0,0 +1,59 @@
+# Fuzzer for libcameraservice
+
+## Plugin Design Considerations
+The fuzzer plugin is designed based on the understanding of the
+library and tries to achieve the following:
+
+##### Maximize code coverage
+The configuration parameters are not hardcoded, but instead selected based on
+incoming data. This ensures more code paths are reached by the fuzzer.
+
+libcameraservice supports the following parameters:
+1. Camera Type (parameter name: `cameraType`)
+2. Camera API Version (parameter name: `cameraAPIVersion`)
+3. Event ID (parameter name: `eventId`)
+4. Camera Sound Kind (parameter name: `soundKind`)
+5. Shell Command (parameter name: `shellCommand`)
+
+| Parameter| Valid Values| Configured Value|
+|------------- |-------------| ----- |
+| `cameraType` | 0. `CAMERA_TYPE_BACKWARD_COMPATIBLE` 1. `CAMERA_TYPE_ALL` | Value obtained from FuzzedDataProvider |
+| `cameraAPIVersion` | 0. `API_VERSION_1` 1. `API_VERSION_2` | Value obtained from FuzzedDataProvider |
+| `eventId` | 0. `EVENT_USER_SWITCHED` 1. `EVENT_NONE` | Value obtained from FuzzedDataProvider |
+| `soundKind` | 0. `SOUND_SHUTTER` 1. `SOUND_RECORDING_START` 2. `SOUND_RECORDING_STOP`| Value obtained from FuzzedDataProvider |
+| `shellCommand` | 0. `set-uid-state` 1. `reset-uid-state` 2. `get-uid-state` 3. `set-rotate-and-crop` 4. `get-rotate-and-crop` 5. `help`| Value obtained from FuzzedDataProvider |
+
+This also ensures that the plugin is always deterministic for any given input.
+
+##### Maximize utilization of input data
+The plugin tolerates any kind of input (empty, huge,
+malformed, etc) and doesn't `exit()` on any input and thereby increasing the
+chance of identifying vulnerabilities.
+
+## Build
+
+This describes steps to build camera_service_fuzzer binary.
+
+### Android
+
+#### Steps to build
+Build the fuzzer
+```
+ $ mm -j$(nproc) camera_service_fuzzer
+```
+
+#### Steps to run
+Create a directory CORPUS_DIR
+```
+ $ adb shell mkdir CORPUS_DIR
+```
+
+To run on device
+```
+ $ adb sync data
+ $ adb shell /data/fuzz/arm64/camera_service_fuzzer/camera_service_fuzzer CORPUS_DIR
+```
+
+## References:
+ * http://llvm.org/docs/LibFuzzer.html
+ * https://github.com/google/oss-fuzz
diff --git a/services/camera/libcameraservice/libcameraservice_fuzzer/camera_service_fuzzer.cpp b/services/camera/libcameraservice/libcameraservice_fuzzer/camera_service_fuzzer.cpp
new file mode 100644
index 0000000..54550a5
--- /dev/null
+++ b/services/camera/libcameraservice/libcameraservice_fuzzer/camera_service_fuzzer.cpp
@@ -0,0 +1,433 @@
+/******************************************************************************
+ *
+ * Copyright (C) 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at:
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ *****************************************************************************
+ * Originally developed and contributed by Ittiam Systems Pvt. Ltd, Bangalore
+ */
+
+#include <CameraService.h>
+#include <android/hardware/ICameraServiceListener.h>
+#include <gui/Surface.h>
+#include <gui/SurfaceComposerClient.h>
+#include <private/android_filesystem_config.h>
+#include "fuzzer/FuzzedDataProvider.h"
+
+using namespace android;
+using namespace hardware;
+using namespace std;
+
+const int32_t kPreviewThreshold = 8;
+const nsecs_t kPreviewTimeout = 5000000000; // .5 [s.]
+const nsecs_t kEventTimeout = 10000000000; // 1 [s.]
+const size_t kMaxNumLines = USHRT_MAX;
+const size_t kMinArgs = 1;
+const size_t kMaxArgs = 5;
+const int32_t kCamType[] = {hardware::ICameraService::CAMERA_TYPE_BACKWARD_COMPATIBLE,
+ hardware::ICameraService::CAMERA_TYPE_ALL};
+const int kCameraApiVersion[] = {android::CameraService::API_VERSION_1,
+ android::CameraService::API_VERSION_2};
+const int kLayerMetadata[] = {
+ 0x00100000 /*GRALLOC_USAGE_RENDERSCRIPT*/, 0x00000003 /*GRALLOC_USAGE_SW_READ_OFTEN*/,
+ 0x00000100 /*GRALLOC_USAGE_HW_TEXTURE*/, 0x00000800 /*GRALLOC_USAGE_HW_COMPOSER*/,
+ 0x00000200 /*GRALLOC_USAGE_HW_RENDER*/, 0x00010000 /*GRALLOC_USAGE_HW_VIDEO_ENCODER*/};
+const int kCameraMsg[] = {0x001 /*CAMERA_MSG_ERROR*/,
+ 0x002 /*CAMERA_MSG_SHUTTER*/,
+ 0x004 /*CAMERA_MSG_FOCUS*/,
+ 0x008 /*CAMERA_MSG_ZOOM*/,
+ 0x010 /*CAMERA_MSG_PREVIEW_FRAME*/,
+ 0x020 /*CAMERA_MSG_VIDEO_FRAME */,
+ 0x040 /*CAMERA_MSG_POSTVIEW_FRAME*/,
+ 0x080 /*CAMERA_MSG_RAW_IMAGE */,
+ 0x100 /*CAMERA_MSG_COMPRESSED_IMAGE*/,
+ 0x200 /*CAMERA_MSG_RAW_IMAGE_NOTIFY*/,
+ 0x400 /*CAMERA_MSG_PREVIEW_METADATA*/,
+ 0x800 /*CAMERA_MSG_FOCUS_MOVE*/};
+const int32_t kEventId[] = {ICameraService::EVENT_USER_SWITCHED, ICameraService::EVENT_NONE};
+const android::CameraService::sound_kind kSoundKind[] = {
+ android::CameraService::SOUND_SHUTTER, android::CameraService::SOUND_RECORDING_START,
+ android::CameraService::SOUND_RECORDING_STOP};
+const String16 kShellCmd[] = {String16("set-uid-state"), String16("reset-uid-state"),
+ String16("get-uid-state"), String16("set-rotate-and-crop"),
+ String16("get-rotate-and-crop"), String16("help")};
+const size_t kNumLayerMetaData = size(kLayerMetadata);
+const size_t kNumCameraMsg = size(kCameraMsg);
+const size_t kNumSoundKind = size(kSoundKind);
+const size_t kNumShellCmd = size(kShellCmd);
+
+class CameraFuzzer : public ::android::hardware::BnCameraClient {
+ public:
+ CameraFuzzer() = default;
+ ~CameraFuzzer() { deInit(); }
+ bool init();
+ void process(const uint8_t *data, size_t size);
+ void deInit();
+
+ private:
+ FuzzedDataProvider *mFuzzedDataProvider = nullptr;
+ sp<CameraService> mCameraService = nullptr;
+ sp<SurfaceComposerClient> mComposerClient = nullptr;
+ int32_t mNumCameras = 0;
+ size_t mPreviewBufferCount = 0;
+ bool mAutoFocusMessage = false;
+ bool mSnapshotNotification = false;
+ mutable Mutex mPreviewLock;
+ mutable Condition mPreviewCondition;
+ mutable Mutex mAutoFocusLock;
+ mutable Condition mAutoFocusCondition;
+ mutable Mutex mSnapshotLock;
+ mutable Condition mSnapshotCondition;
+
+ void getNumCameras();
+ void getCameraInformation(int32_t cameraId);
+ void invokeCameraAPIs();
+ void invokeCameraSound();
+ void invokeDump();
+ void invokeShellCommand();
+ void invokeNotifyCalls();
+
+ // CameraClient interface
+ void notifyCallback(int32_t msgType, int32_t, int32_t) override;
+ void dataCallback(int32_t msgType, const sp<IMemory> &, camera_frame_metadata_t *) override;
+ void dataCallbackTimestamp(nsecs_t, int32_t, const sp<IMemory> &) override{};
+ void recordingFrameHandleCallbackTimestamp(nsecs_t, native_handle_t *) override{};
+ void recordingFrameHandleCallbackTimestampBatch(
+ const std::vector<nsecs_t> &, const std::vector<native_handle_t *> &) override{};
+ status_t waitForPreviewStart();
+ status_t waitForEvent(Mutex &mutex, Condition &condition, bool &flag);
+};
+
+void CameraFuzzer::notifyCallback(int32_t msgType, int32_t, int32_t) {
+ if (CAMERA_MSG_FOCUS == msgType) {
+ Mutex::Autolock l(mAutoFocusLock);
+ mAutoFocusMessage = true;
+ mAutoFocusCondition.broadcast();
+ }
+};
+
+void CameraFuzzer::dataCallback(int32_t msgType, const sp<IMemory> & /*data*/,
+ camera_frame_metadata_t *) {
+ switch (msgType) {
+ case CAMERA_MSG_PREVIEW_FRAME: {
+ Mutex::Autolock l(mPreviewLock);
+ ++mPreviewBufferCount;
+ mPreviewCondition.broadcast();
+ break;
+ }
+ case CAMERA_MSG_COMPRESSED_IMAGE: {
+ Mutex::Autolock l(mSnapshotLock);
+ mSnapshotNotification = true;
+ mSnapshotCondition.broadcast();
+ break;
+ }
+ default:
+ break;
+ }
+};
+
+status_t CameraFuzzer::waitForPreviewStart() {
+ status_t rc = NO_ERROR;
+ Mutex::Autolock l(mPreviewLock);
+ mPreviewBufferCount = 0;
+
+ while (mPreviewBufferCount < kPreviewThreshold) {
+ rc = mPreviewCondition.waitRelative(mPreviewLock, kPreviewTimeout);
+ if (NO_ERROR != rc) {
+ break;
+ }
+ }
+
+ return rc;
+}
+
+status_t CameraFuzzer::waitForEvent(Mutex &mutex, Condition &condition, bool &flag) {
+ status_t rc = NO_ERROR;
+ Mutex::Autolock l(mutex);
+ flag = false;
+
+ while (!flag) {
+ rc = condition.waitRelative(mutex, kEventTimeout);
+ if (NO_ERROR != rc) {
+ break;
+ }
+ }
+
+ return rc;
+}
+
+bool CameraFuzzer::init() {
+ setuid(AID_MEDIA);
+ mCameraService = new CameraService();
+ if (mCameraService) {
+ return true;
+ }
+ return false;
+}
+
+void CameraFuzzer::deInit() {
+ if (mCameraService) {
+ mCameraService = nullptr;
+ }
+ if (mComposerClient) {
+ mComposerClient->dispose();
+ }
+}
+
+void CameraFuzzer::getNumCameras() {
+ bool shouldPassInvalidCamType = mFuzzedDataProvider->ConsumeBool();
+ int32_t camType;
+ if (shouldPassInvalidCamType) {
+ camType = mFuzzedDataProvider->ConsumeIntegral<int32_t>();
+ } else {
+ camType = kCamType[mFuzzedDataProvider->ConsumeBool()];
+ }
+ mCameraService->getNumberOfCameras(camType, &mNumCameras);
+}
+
+void CameraFuzzer::getCameraInformation(int32_t cameraId) {
+ String16 cameraIdStr = String16(String8::format("%d", cameraId));
+ bool isSupported = false;
+ mCameraService->supportsCameraApi(
+ cameraIdStr, kCameraApiVersion[mFuzzedDataProvider->ConsumeBool()], &isSupported);
+ mCameraService->isHiddenPhysicalCamera(cameraIdStr, &isSupported);
+
+ String16 parameters;
+ mCameraService->getLegacyParameters(cameraId, ¶meters);
+
+ std::vector<hardware::camera2::utils::ConcurrentCameraIdCombination> concurrentCameraIds;
+ mCameraService->getConcurrentCameraIds(&concurrentCameraIds);
+
+ hardware::camera2::params::VendorTagDescriptorCache cache;
+ mCameraService->getCameraVendorTagCache(&cache);
+
+ CameraInfo cameraInfo;
+ mCameraService->getCameraInfo(cameraId, &cameraInfo);
+
+ CameraMetadata metadata;
+ mCameraService->getCameraCharacteristics(cameraIdStr, &metadata);
+}
+
+void CameraFuzzer::invokeCameraSound() {
+ mCameraService->increaseSoundRef();
+ mCameraService->decreaseSoundRef();
+ bool shouldPassInvalidPlaySound = mFuzzedDataProvider->ConsumeBool();
+ bool shouldPassInvalidLockSound = mFuzzedDataProvider->ConsumeBool();
+ android::CameraService::sound_kind playSound, lockSound;
+ if (shouldPassInvalidPlaySound) {
+ playSound = static_cast<android::CameraService::sound_kind>(
+ mFuzzedDataProvider->ConsumeIntegral<size_t>());
+ } else {
+ playSound =
+ kSoundKind[mFuzzedDataProvider->ConsumeIntegralInRange<size_t>(0, kNumSoundKind - 1)];
+ }
+
+ if (shouldPassInvalidLockSound) {
+ lockSound = static_cast<android::CameraService::sound_kind>(
+ mFuzzedDataProvider->ConsumeIntegral<size_t>());
+ } else {
+ lockSound =
+ kSoundKind[mFuzzedDataProvider->ConsumeIntegralInRange<size_t>(0, kNumSoundKind - 1)];
+ }
+ mCameraService->playSound(playSound);
+ mCameraService->loadSoundLocked(lockSound);
+}
+
+void CameraFuzzer::invokeDump() {
+ Vector<String16> args;
+ size_t numberOfLines = mFuzzedDataProvider->ConsumeIntegralInRange<size_t>(0, kMaxNumLines);
+ for (size_t lineIdx = 0; lineIdx < numberOfLines; ++lineIdx) {
+ args.add(static_cast<String16>(mFuzzedDataProvider->ConsumeRandomLengthString().c_str()));
+ }
+ const char *fileName = "logDumpFile";
+ int fd = memfd_create(fileName, MFD_ALLOW_SEALING);
+ mCameraService->dump(fd, args);
+ close(fd);
+}
+
+void CameraFuzzer::invokeShellCommand() {
+ int in = mFuzzedDataProvider->ConsumeIntegral<int>();
+ int out = mFuzzedDataProvider->ConsumeIntegral<int>();
+ int err = mFuzzedDataProvider->ConsumeIntegral<int>();
+ Vector<String16> args;
+ size_t numArgs = mFuzzedDataProvider->ConsumeIntegralInRange<size_t>(kMinArgs, kMaxArgs);
+ for (size_t argsIdx = 0; argsIdx < numArgs; ++argsIdx) {
+ bool shouldPassInvalidCommand = mFuzzedDataProvider->ConsumeBool();
+ if (shouldPassInvalidCommand) {
+ args.add(
+ static_cast<String16>(mFuzzedDataProvider->ConsumeRandomLengthString().c_str()));
+ } else {
+ args.add(kShellCmd[mFuzzedDataProvider->ConsumeIntegralInRange<size_t>(
+ 0, kNumShellCmd - 1)]);
+ }
+ }
+ mCameraService->shellCommand(in, out, err, args);
+}
+
+void CameraFuzzer::invokeNotifyCalls() {
+ mCameraService->notifyMonitoredUids();
+ int64_t newState = mFuzzedDataProvider->ConsumeIntegral<int64_t>();
+ mCameraService->notifyDeviceStateChange(newState);
+ std::vector<int32_t> args;
+ size_t numArgs = mFuzzedDataProvider->ConsumeIntegralInRange<size_t>(kMinArgs, kMaxArgs);
+ for (size_t argsIdx = 0; argsIdx < numArgs; ++argsIdx) {
+ args.push_back(mFuzzedDataProvider->ConsumeIntegral<int32_t>());
+ }
+ bool shouldPassInvalidEvent = mFuzzedDataProvider->ConsumeBool();
+ int32_t eventId;
+ if (shouldPassInvalidEvent) {
+ eventId = mFuzzedDataProvider->ConsumeIntegral<int32_t>();
+ } else {
+ eventId = kEventId[mFuzzedDataProvider->ConsumeBool()];
+ }
+ mCameraService->notifySystemEvent(eventId, args);
+}
+
+void CameraFuzzer::invokeCameraAPIs() {
+ for (int32_t cameraId = 0; cameraId < mNumCameras; ++cameraId) {
+ getCameraInformation(cameraId);
+
+ const String16 opPackageName("com.fuzzer.poc");
+ ::android::binder::Status rc;
+ sp<ICamera> cameraDevice;
+
+ rc = mCameraService->connect(this, cameraId, opPackageName, AID_MEDIA, AID_ROOT,
+ &cameraDevice);
+ if (!rc.isOk()) {
+ // camera not connected
+ return;
+ }
+ if (cameraDevice) {
+ sp<Surface> previewSurface;
+ sp<SurfaceControl> surfaceControl;
+ CameraParameters params(cameraDevice->getParameters());
+ String8 focusModes(params.get(CameraParameters::KEY_SUPPORTED_FOCUS_MODES));
+ bool isAFSupported = false;
+ const char *focusMode = nullptr;
+
+ if (focusModes.contains(CameraParameters::FOCUS_MODE_AUTO)) {
+ isAFSupported = true;
+ } else if (focusModes.contains(CameraParameters::FOCUS_MODE_CONTINUOUS_PICTURE)) {
+ isAFSupported = true;
+ focusMode = CameraParameters::FOCUS_MODE_CONTINUOUS_PICTURE;
+ } else if (focusModes.contains(CameraParameters::FOCUS_MODE_CONTINUOUS_VIDEO)) {
+ isAFSupported = true;
+ focusMode = CameraParameters::FOCUS_MODE_CONTINUOUS_VIDEO;
+ } else if (focusModes.contains(CameraParameters::FOCUS_MODE_MACRO)) {
+ isAFSupported = true;
+ focusMode = CameraParameters::FOCUS_MODE_MACRO;
+ }
+ if (nullptr != focusMode) {
+ params.set(CameraParameters::KEY_FOCUS_MODE, focusMode);
+ cameraDevice->setParameters(params.flatten());
+ }
+ int previewWidth, previewHeight;
+ params.getPreviewSize(&previewWidth, &previewHeight);
+
+ mComposerClient = new SurfaceComposerClient;
+ mComposerClient->initCheck();
+
+ bool shouldPassInvalidLayerMetaData = mFuzzedDataProvider->ConsumeBool();
+ int layerMetaData;
+ if (shouldPassInvalidLayerMetaData) {
+ layerMetaData = mFuzzedDataProvider->ConsumeIntegral<int>();
+ } else {
+ layerMetaData = kLayerMetadata[mFuzzedDataProvider->ConsumeIntegralInRange<size_t>(
+ 0, kNumLayerMetaData - 1)];
+ }
+ surfaceControl = mComposerClient->createSurface(
+ String8("Test Surface"), previewWidth, previewHeight,
+ CameraParameters::previewFormatToEnum(params.getPreviewFormat()), layerMetaData);
+
+ if (surfaceControl.get() != nullptr) {
+ SurfaceComposerClient::Transaction{}
+ .setLayer(surfaceControl, 0x7fffffff)
+ .show(surfaceControl)
+ .apply();
+
+ previewSurface = surfaceControl->getSurface();
+ cameraDevice->setPreviewTarget(previewSurface->getIGraphicBufferProducer());
+ }
+ cameraDevice->setPreviewCallbackFlag(CAMERA_FRAME_CALLBACK_FLAG_CAMCORDER);
+
+ Vector<Size> pictureSizes;
+ params.getSupportedPictureSizes(pictureSizes);
+
+ for (size_t i = 0; i < pictureSizes.size(); ++i) {
+ params.setPictureSize(pictureSizes[i].width, pictureSizes[i].height);
+ cameraDevice->setParameters(params.flatten());
+ cameraDevice->startPreview();
+ waitForPreviewStart();
+ cameraDevice->autoFocus();
+ waitForEvent(mAutoFocusLock, mAutoFocusCondition, mAutoFocusMessage);
+ bool shouldPassInvalidCameraMsg = mFuzzedDataProvider->ConsumeBool();
+ int msgType;
+ if (shouldPassInvalidCameraMsg) {
+ msgType = mFuzzedDataProvider->ConsumeIntegral<int>();
+ } else {
+ msgType = kCameraMsg[mFuzzedDataProvider->ConsumeIntegralInRange<size_t>(
+ 0, kNumCameraMsg - 1)];
+ }
+ cameraDevice->takePicture(msgType);
+
+ waitForEvent(mSnapshotLock, mSnapshotCondition, mSnapshotNotification);
+ }
+
+ Vector<Size> videoSizes;
+ params.getSupportedVideoSizes(videoSizes);
+
+ for (size_t i = 0; i < videoSizes.size(); ++i) {
+ params.setVideoSize(videoSizes[i].width, videoSizes[i].height);
+
+ cameraDevice->setParameters(params.flatten());
+ cameraDevice->startPreview();
+ waitForPreviewStart();
+ cameraDevice->setVideoBufferMode(
+ android::hardware::BnCamera::VIDEO_BUFFER_MODE_BUFFER_QUEUE);
+ cameraDevice->setVideoTarget(previewSurface->getIGraphicBufferProducer());
+ cameraDevice->startRecording();
+ cameraDevice->stopRecording();
+ }
+ cameraDevice->stopPreview();
+ cameraDevice->disconnect();
+ }
+ }
+}
+
+void CameraFuzzer::process(const uint8_t *data, size_t size) {
+ mFuzzedDataProvider = new FuzzedDataProvider(data, size);
+ getNumCameras();
+ invokeCameraSound();
+ if (mNumCameras > 0) {
+ invokeCameraAPIs();
+ }
+ invokeDump();
+ invokeShellCommand();
+ invokeNotifyCalls();
+ delete mFuzzedDataProvider;
+}
+
+extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+ if (size < 1) {
+ return 0;
+ }
+ sp<CameraFuzzer> camerafuzzer = new CameraFuzzer();
+ if (!camerafuzzer) {
+ return 0;
+ }
+ if (camerafuzzer->init()) {
+ camerafuzzer->process(data, size);
+ }
+ return 0;
+}
diff --git a/services/mediametrics/Android.bp b/services/mediametrics/Android.bp
index 8ddd3f0..d11720b 100644
--- a/services/mediametrics/Android.bp
+++ b/services/mediametrics/Android.bp
@@ -30,7 +30,7 @@
"modernize-loop-convert",
"modernize-make-shared",
"modernize-make-unique",
- "modernize-pass-by-value",
+ // "modernize-pass-by-value", // found in TimeMachine.h
"modernize-raw-string-literal",
"modernize-redundant-void-arg",
"modernize-replace-auto-ptr",
@@ -38,13 +38,13 @@
"modernize-return-braced-init-list",
"modernize-shrink-to-fit",
"modernize-unary-static-assert",
- "modernize-use-auto", // debatable - auto can obscure type
+ // "modernize-use-auto", // found in MediaMetricsService.h, debatable - auto can obscure type
"modernize-use-bool-literals",
"modernize-use-default-member-init",
"modernize-use-emplace",
"modernize-use-equals-default",
"modernize-use-equals-delete",
- "modernize-use-nodiscard",
+ // "modernize-use-nodiscard", // found in TimeMachine.h
"modernize-use-noexcept",
"modernize-use-nullptr",
"modernize-use-override",
@@ -57,6 +57,10 @@
// Remove some pedantic stylistic requirements.
"-google-readability-casting", // C++ casts not always necessary and may be verbose
"-google-readability-todo", // do not require TODO(info)
+
+ "-bugprone-unhandled-self-assignment", // found in TimeMachine.h
+ "-bugprone-suspicious-string-compare", // found in TimeMachine.h
+ "-cert-oop54-cpp", // found in TransactionLog.h
]
cc_defaults {
@@ -88,8 +92,7 @@
tidy_checks: tidy_errors,
tidy_checks_as_errors: tidy_errors,
tidy_flags: [
- "-format-style='file'",
- "--header-filter='frameworks/av/services/mediametrics/'",
+ "-format-style=file",
],
}