Merge "Camera: Integrate dynamic depth processing"
diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp
index 6052a06..c63feb2 100644
--- a/services/camera/libcameraservice/Android.bp
+++ b/services/camera/libcameraservice/Android.bp
@@ -28,6 +28,7 @@
"common/CameraDeviceBase.cpp",
"common/CameraOfflineSessionBase.cpp",
"common/CameraProviderManager.cpp",
+ "common/DepthPhotoProcessor.cpp",
"common/FrameProcessorBase.cpp",
"api1/CameraClient.cpp",
"api1/Camera2Client.cpp",
@@ -91,10 +92,12 @@
"libmediautils",
"libcamera_client",
"libcamera_metadata",
+ "libdynamic_depth",
"libfmq",
"libgui",
"libhardware",
"libhidlbase",
+ "libimage_io",
"libjpeg",
"libmedia_codeclist",
"libmedia_omx",
@@ -102,6 +105,7 @@
"libsensorprivacy",
"libstagefright",
"libstagefright_foundation",
+ "libxml2",
"libyuv",
"android.frameworks.cameraservice.common@2.0",
"android.frameworks.cameraservice.service@2.0",
@@ -143,40 +147,3 @@
}
-cc_library_shared {
- name: "libdepthphoto",
-
- srcs: [
- "utils/ExifUtils.cpp",
- "common/DepthPhotoProcessor.cpp",
- ],
-
- shared_libs: [
- "libimage_io",
- "libdynamic_depth",
- "libxml2",
- "liblog",
- "libutilscallstack",
- "libutils",
- "libcutils",
- "libjpeg",
- "libmemunreachable",
- "libexif",
- "libcamera_client",
- ],
-
- include_dirs: [
- "external/dynamic_depth/includes",
- "external/dynamic_depth/internal",
- ],
-
- export_include_dirs: ["."],
-
- cflags: [
- "-Wall",
- "-Wextra",
- "-Werror",
- "-Wno-ignored-qualifiers",
- ],
-
-}
diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp
index 0b91016..ac6f8d6 100644
--- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp
+++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp
@@ -20,7 +20,6 @@
#include "api1/client2/JpegProcessor.h"
#include "common/CameraProviderManager.h"
-#include "dlfcn.h"
#include <gui/Surface.h>
#include <utils/Log.h>
#include <utils/Trace.h>
@@ -43,9 +42,7 @@
mBlobBufferAcquired(false),
mProducerListener(new ProducerListener()),
mMaxJpegSize(-1),
- mIsLogicalCamera(false),
- mDepthPhotoLibHandle(nullptr),
- mDepthPhotoProcess(nullptr) {
+ mIsLogicalCamera(false) {
sp<CameraDeviceBase> cameraDevice = device.promote();
if (cameraDevice.get() != nullptr) {
CameraMetadata staticInfo = cameraDevice->info();
@@ -83,19 +80,6 @@
}
getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes);
-
- mDepthPhotoLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
- if (mDepthPhotoLibHandle != nullptr) {
- mDepthPhotoProcess = reinterpret_cast<camera3::process_depth_photo_frame> (
- dlsym(mDepthPhotoLibHandle, camera3::kDepthPhotoProcessFunction));
- if (mDepthPhotoProcess == nullptr) {
- ALOGE("%s: Failed to link to depth photo process function: %s", __FUNCTION__,
- dlerror());
- }
- } else {
- ALOGE("%s: Failed to link to depth photo library: %s", __FUNCTION__, dlerror());
- }
-
}
}
@@ -108,11 +92,6 @@
mDepthSurface.clear();
mDepthConsumer = nullptr;
mDepthSurface = nullptr;
- if (mDepthPhotoLibHandle != nullptr) {
- dlclose(mDepthPhotoLibHandle);
- mDepthPhotoLibHandle = nullptr;
- }
- mDepthPhotoProcess = nullptr;
}
void DepthCompositeStream::compilePendingInputLocked() {
@@ -356,7 +335,7 @@
}
size_t actualJpegSize = 0;
- res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
+ res = processDepthPhotoFrame(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
if (res != 0) {
ALOGE("%s: Depth photo processing failed: %s (%d)", __FUNCTION__, strerror(-res), res);
outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1);
@@ -583,11 +562,6 @@
return NO_ERROR;
}
- if ((mDepthPhotoLibHandle == nullptr) || (mDepthPhotoProcess == nullptr)) {
- ALOGE("%s: Depth photo library is not present!", __FUNCTION__);
- return NO_INIT;
- }
-
if (mOutputSurface.get() == nullptr) {
ALOGE("%s: No valid output surface set!", __FUNCTION__);
return NO_INIT;
diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h
index 28a7826..60c6b1e 100644
--- a/services/camera/libcameraservice/api2/DepthCompositeStream.h
+++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h
@@ -126,8 +126,6 @@
std::vector<std::tuple<size_t, size_t>> mSupportedDepthSizes;
std::vector<float> mIntrinsicCalibration, mLensDistortion;
bool mIsLogicalCamera;
- void* mDepthPhotoLibHandle;
- process_depth_photo_frame mDepthPhotoProcess;
// Keep all incoming Depth buffer timestamps pending further processing.
std::vector<int64_t> mInputDepthBuffers;
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp
index 42698c5..0f74a48 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.cpp
+++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp
@@ -716,31 +716,6 @@
}
}
-bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() {
- static bool libraryPresent = false;
- static bool initialized = false;
- if (initialized) {
- return libraryPresent;
- } else {
- initialized = true;
- }
-
- void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
- if (depthLibHandle == nullptr) {
- return false;
- }
-
- auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction);
- if (processFunc != nullptr) {
- libraryPresent = true;
- } else {
- libraryPresent = false;
- }
- dlclose(depthLibHandle);
-
- return libraryPresent;
-}
-
status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {
uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;
uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;
@@ -788,11 +763,6 @@
return OK;
}
- if(!isDepthPhotoLibraryPresent()) {
- // Depth photo processing library is not present, nothing more to do.
- return OK;
- }
-
std::vector<int32_t> dynamicDepthEntries;
for (const auto& it : supportedDynamicDepthSizes) {
int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h
index 651b8a1..58df0e8 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.h
+++ b/services/camera/libcameraservice/common/CameraProviderManager.h
@@ -545,7 +545,6 @@
void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,
const std::vector<int64_t>& blobDurations,
std::vector<int64_t> *dynamicDepthDurations /*out*/);
- static bool isDepthPhotoLibraryPresent();
static void getSupportedDynamicDepthSizes(
const std::vector<std::tuple<size_t, size_t>>& blobSizes,
const std::vector<std::tuple<size_t, size_t>>& depthSizes,
diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
index 94541d8..c995670 100644
--- a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
+++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
@@ -410,7 +410,7 @@
return DepthMap::FromData(depthParams, items);
}
-extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
+int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) {
if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) ||
(depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) {
diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.h b/services/camera/libcameraservice/common/DepthPhotoProcessor.h
index ba5ca9e..09b6935 100644
--- a/services/camera/libcameraservice/common/DepthPhotoProcessor.h
+++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.h
@@ -64,9 +64,7 @@
mOrientation(DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES) {}
};
-static const char *kDepthPhotoLibrary = "libdepthphoto.so";
-static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame";
-typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/,
+int processDepthPhotoFrame(DepthPhotoInputFrame /*inputFrame*/,
size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/,
size_t* /*depthPhotoActualSize out*/);
diff --git a/services/camera/libcameraservice/tests/DepthProcessorTest.cpp b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp
index 2162514..673c149 100644
--- a/services/camera/libcameraservice/tests/DepthProcessorTest.cpp
+++ b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp
@@ -20,7 +20,6 @@
#include <array>
#include <random>
-#include <dlfcn.h>
#include <gtest/gtest.h>
#include "../common/DepthPhotoProcessor.h"
@@ -36,19 +35,6 @@
static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);
static const size_t kSeed = 1234;
-void linkToDepthPhotoLibrary(void **libHandle /*out*/,
- process_depth_photo_frame *processFrameFunc /*out*/) {
- ASSERT_NE(libHandle, nullptr);
- ASSERT_NE(processFrameFunc, nullptr);
-
- *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
- if (*libHandle != nullptr) {
- *processFrameFunc = reinterpret_cast<camera3::process_depth_photo_frame> (
- dlsym(*libHandle, kDepthPhotoProcessFunction));
- ASSERT_NE(*processFrameFunc, nullptr);
- }
-}
-
void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,
bool switchDimensions, std::vector<uint8_t> *colorJpegBuffer /*out*/) {
ASSERT_NE(colorJpegBuffer, nullptr);
@@ -91,26 +77,9 @@
}
}
-TEST(DepthProcessorTest, LinkToLibray) {
- void *libHandle;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle != nullptr) {
- dlclose(libHandle);
- }
-}
-
TEST(DepthProcessorTest, BadInput) {
- void *libHandle;
int jpegQuality = 95;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle == nullptr) {
- // Depth library no present, nothing more to test.
- return;
- }
-
DepthPhotoInputFrame inputFrame;
// Worst case both depth and confidence maps have the same size as the main color image.
inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
@@ -128,37 +97,27 @@
inputFrame.mMainJpegWidth = kTestBufferWidth;
inputFrame.mMainJpegHeight = kTestBufferHeight;
inputFrame.mJpegQuality = jpegQuality;
- ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+ ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
inputFrame.mMainJpegSize = colorJpegBuffer.size();
- ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+ ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
inputFrame.mDepthMapBuffer = depth16Buffer.data();
inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
inputFrame.mDepthMapHeight = kTestBufferHeight;
- ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr,
+ ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), nullptr,
&actualDepthPhotoSize), 0);
- ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr),
- 0);
-
- dlclose(libHandle);
+ ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+ nullptr), 0);
}
TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
- void *libHandle;
int jpegQuality = 95;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle == nullptr) {
- // Depth library no present, nothing more to test.
- return;
- }
-
std::vector<uint8_t> colorJpegBuffer;
generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
/*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer);
@@ -180,7 +139,7 @@
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
- ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+ ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));
@@ -196,21 +155,11 @@
ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
-
- dlclose(libHandle);
}
TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
- void *libHandle;
int jpegQuality = 95;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle == nullptr) {
- // Depth library no present, nothing more to test.
- return;
- }
-
ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,
ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,
ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };
@@ -242,8 +191,8 @@
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
- ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
- &actualDepthPhotoSize), 0);
+ ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
+ depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) &&
(depthPhotoBuffer.size() >= actualDepthPhotoSize));
@@ -281,21 +230,11 @@
ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
}
}
-
- dlclose(libHandle);
}
TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
- void *libHandle;
int jpegQuality = 95;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle == nullptr) {
- // Depth library no present, nothing more to test.
- return;
- }
-
// In case of physical rotation, the EXIF orientation must always be 0.
auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES;
DepthPhotoOrientation depthOrientations[] = {
@@ -339,8 +278,8 @@
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
- ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
- &actualDepthPhotoSize), 0);
+ ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
+ depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) &&
(depthPhotoBuffer.size() >= actualDepthPhotoSize));
@@ -377,6 +316,4 @@
ASSERT_EQ(confidenceMapWidth, expectedWidth);
ASSERT_EQ(confidenceMapHeight, expectedHeight);
}
-
- dlclose(libHandle);
}