Camera: Integrate dynamic depth processing
Integrate dynamic depth processing as part of
the camera service library.
Dynamic linking is no longer required as legacy
devices with small system partitions are not
supported.
Bug: 132449311
Test:
atest
cts/tests/camera/src/android/hardware/camera2/cts/StillCaptureTest.java#testDynamicDepthCapture
atest
cts/tests/camera/src/android/hardware/camera2/cts/ImageReaderTest.java#testDynamicDepth
atest
cts/tests/camera/src/android/hardware/camera2/cts/ExtendedCameraCharacteristicsTest.java#testDepthOutputCharacteristics
cameraservice_test --gtest_filter=DepthProcessorTest.*
Change-Id: Ie8befc5c0635e3e08c7ad8cac7b056cdf5aa3548
diff --git a/services/camera/libcameraservice/Android.bp b/services/camera/libcameraservice/Android.bp
index 6052a06..c63feb2 100644
--- a/services/camera/libcameraservice/Android.bp
+++ b/services/camera/libcameraservice/Android.bp
@@ -28,6 +28,7 @@
"common/CameraDeviceBase.cpp",
"common/CameraOfflineSessionBase.cpp",
"common/CameraProviderManager.cpp",
+ "common/DepthPhotoProcessor.cpp",
"common/FrameProcessorBase.cpp",
"api1/CameraClient.cpp",
"api1/Camera2Client.cpp",
@@ -91,10 +92,12 @@
"libmediautils",
"libcamera_client",
"libcamera_metadata",
+ "libdynamic_depth",
"libfmq",
"libgui",
"libhardware",
"libhidlbase",
+ "libimage_io",
"libjpeg",
"libmedia_codeclist",
"libmedia_omx",
@@ -102,6 +105,7 @@
"libsensorprivacy",
"libstagefright",
"libstagefright_foundation",
+ "libxml2",
"libyuv",
"android.frameworks.cameraservice.common@2.0",
"android.frameworks.cameraservice.service@2.0",
@@ -143,40 +147,3 @@
}
-cc_library_shared {
- name: "libdepthphoto",
-
- srcs: [
- "utils/ExifUtils.cpp",
- "common/DepthPhotoProcessor.cpp",
- ],
-
- shared_libs: [
- "libimage_io",
- "libdynamic_depth",
- "libxml2",
- "liblog",
- "libutilscallstack",
- "libutils",
- "libcutils",
- "libjpeg",
- "libmemunreachable",
- "libexif",
- "libcamera_client",
- ],
-
- include_dirs: [
- "external/dynamic_depth/includes",
- "external/dynamic_depth/internal",
- ],
-
- export_include_dirs: ["."],
-
- cflags: [
- "-Wall",
- "-Wextra",
- "-Werror",
- "-Wno-ignored-qualifiers",
- ],
-
-}
diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp
index 0b91016..ac6f8d6 100644
--- a/services/camera/libcameraservice/api2/DepthCompositeStream.cpp
+++ b/services/camera/libcameraservice/api2/DepthCompositeStream.cpp
@@ -20,7 +20,6 @@
#include "api1/client2/JpegProcessor.h"
#include "common/CameraProviderManager.h"
-#include "dlfcn.h"
#include <gui/Surface.h>
#include <utils/Log.h>
#include <utils/Trace.h>
@@ -43,9 +42,7 @@
mBlobBufferAcquired(false),
mProducerListener(new ProducerListener()),
mMaxJpegSize(-1),
- mIsLogicalCamera(false),
- mDepthPhotoLibHandle(nullptr),
- mDepthPhotoProcess(nullptr) {
+ mIsLogicalCamera(false) {
sp<CameraDeviceBase> cameraDevice = device.promote();
if (cameraDevice.get() != nullptr) {
CameraMetadata staticInfo = cameraDevice->info();
@@ -83,19 +80,6 @@
}
getSupportedDepthSizes(staticInfo, &mSupportedDepthSizes);
-
- mDepthPhotoLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
- if (mDepthPhotoLibHandle != nullptr) {
- mDepthPhotoProcess = reinterpret_cast<camera3::process_depth_photo_frame> (
- dlsym(mDepthPhotoLibHandle, camera3::kDepthPhotoProcessFunction));
- if (mDepthPhotoProcess == nullptr) {
- ALOGE("%s: Failed to link to depth photo process function: %s", __FUNCTION__,
- dlerror());
- }
- } else {
- ALOGE("%s: Failed to link to depth photo library: %s", __FUNCTION__, dlerror());
- }
-
}
}
@@ -108,11 +92,6 @@
mDepthSurface.clear();
mDepthConsumer = nullptr;
mDepthSurface = nullptr;
- if (mDepthPhotoLibHandle != nullptr) {
- dlclose(mDepthPhotoLibHandle);
- mDepthPhotoLibHandle = nullptr;
- }
- mDepthPhotoProcess = nullptr;
}
void DepthCompositeStream::compilePendingInputLocked() {
@@ -356,7 +335,7 @@
}
size_t actualJpegSize = 0;
- res = mDepthPhotoProcess(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
+ res = processDepthPhotoFrame(depthPhoto, finalJpegBufferSize, dstBuffer, &actualJpegSize);
if (res != 0) {
ALOGE("%s: Depth photo processing failed: %s (%d)", __FUNCTION__, strerror(-res), res);
outputANW->cancelBuffer(mOutputSurface.get(), anb, /*fence*/ -1);
@@ -583,11 +562,6 @@
return NO_ERROR;
}
- if ((mDepthPhotoLibHandle == nullptr) || (mDepthPhotoProcess == nullptr)) {
- ALOGE("%s: Depth photo library is not present!", __FUNCTION__);
- return NO_INIT;
- }
-
if (mOutputSurface.get() == nullptr) {
ALOGE("%s: No valid output surface set!", __FUNCTION__);
return NO_INIT;
diff --git a/services/camera/libcameraservice/api2/DepthCompositeStream.h b/services/camera/libcameraservice/api2/DepthCompositeStream.h
index 28a7826..60c6b1e 100644
--- a/services/camera/libcameraservice/api2/DepthCompositeStream.h
+++ b/services/camera/libcameraservice/api2/DepthCompositeStream.h
@@ -126,8 +126,6 @@
std::vector<std::tuple<size_t, size_t>> mSupportedDepthSizes;
std::vector<float> mIntrinsicCalibration, mLensDistortion;
bool mIsLogicalCamera;
- void* mDepthPhotoLibHandle;
- process_depth_photo_frame mDepthPhotoProcess;
// Keep all incoming Depth buffer timestamps pending further processing.
std::vector<int64_t> mInputDepthBuffers;
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp
index 23f7884..c8c31f3 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.cpp
+++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp
@@ -716,31 +716,6 @@
}
}
-bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() {
- static bool libraryPresent = false;
- static bool initialized = false;
- if (initialized) {
- return libraryPresent;
- } else {
- initialized = true;
- }
-
- void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
- if (depthLibHandle == nullptr) {
- return false;
- }
-
- auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction);
- if (processFunc != nullptr) {
- libraryPresent = true;
- } else {
- libraryPresent = false;
- }
- dlclose(depthLibHandle);
-
- return libraryPresent;
-}
-
status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {
uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;
uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;
@@ -788,11 +763,6 @@
return OK;
}
- if(!isDepthPhotoLibraryPresent()) {
- // Depth photo processing library is not present, nothing more to do.
- return OK;
- }
-
std::vector<int32_t> dynamicDepthEntries;
for (const auto& it : supportedDynamicDepthSizes) {
int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h
index 651b8a1..58df0e8 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.h
+++ b/services/camera/libcameraservice/common/CameraProviderManager.h
@@ -545,7 +545,6 @@
void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,
const std::vector<int64_t>& blobDurations,
std::vector<int64_t> *dynamicDepthDurations /*out*/);
- static bool isDepthPhotoLibraryPresent();
static void getSupportedDynamicDepthSizes(
const std::vector<std::tuple<size_t, size_t>>& blobSizes,
const std::vector<std::tuple<size_t, size_t>>& depthSizes,
diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
index 94541d8..c995670 100644
--- a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
+++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
@@ -410,7 +410,7 @@
return DepthMap::FromData(depthParams, items);
}
-extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
+int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) {
if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) ||
(depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) {
diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.h b/services/camera/libcameraservice/common/DepthPhotoProcessor.h
index ba5ca9e..09b6935 100644
--- a/services/camera/libcameraservice/common/DepthPhotoProcessor.h
+++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.h
@@ -64,9 +64,7 @@
mOrientation(DepthPhotoOrientation::DEPTH_ORIENTATION_0_DEGREES) {}
};
-static const char *kDepthPhotoLibrary = "libdepthphoto.so";
-static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame";
-typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/,
+int processDepthPhotoFrame(DepthPhotoInputFrame /*inputFrame*/,
size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/,
size_t* /*depthPhotoActualSize out*/);
diff --git a/services/camera/libcameraservice/tests/DepthProcessorTest.cpp b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp
index 2162514..673c149 100644
--- a/services/camera/libcameraservice/tests/DepthProcessorTest.cpp
+++ b/services/camera/libcameraservice/tests/DepthProcessorTest.cpp
@@ -20,7 +20,6 @@
#include <array>
#include <random>
-#include <dlfcn.h>
#include <gtest/gtest.h>
#include "../common/DepthPhotoProcessor.h"
@@ -36,19 +35,6 @@
static const size_t kTestBufferDepthSize (kTestBufferWidth * kTestBufferHeight);
static const size_t kSeed = 1234;
-void linkToDepthPhotoLibrary(void **libHandle /*out*/,
- process_depth_photo_frame *processFrameFunc /*out*/) {
- ASSERT_NE(libHandle, nullptr);
- ASSERT_NE(processFrameFunc, nullptr);
-
- *libHandle = dlopen(kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
- if (*libHandle != nullptr) {
- *processFrameFunc = reinterpret_cast<camera3::process_depth_photo_frame> (
- dlsym(*libHandle, kDepthPhotoProcessFunction));
- ASSERT_NE(*processFrameFunc, nullptr);
- }
-}
-
void generateColorJpegBuffer(int jpegQuality, ExifOrientation orientationValue, bool includeExif,
bool switchDimensions, std::vector<uint8_t> *colorJpegBuffer /*out*/) {
ASSERT_NE(colorJpegBuffer, nullptr);
@@ -91,26 +77,9 @@
}
}
-TEST(DepthProcessorTest, LinkToLibray) {
- void *libHandle;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle != nullptr) {
- dlclose(libHandle);
- }
-}
-
TEST(DepthProcessorTest, BadInput) {
- void *libHandle;
int jpegQuality = 95;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle == nullptr) {
- // Depth library no present, nothing more to test.
- return;
- }
-
DepthPhotoInputFrame inputFrame;
// Worst case both depth and confidence maps have the same size as the main color image.
inputFrame.mMaxJpegSize = inputFrame.mMainJpegSize * 3;
@@ -128,37 +97,27 @@
inputFrame.mMainJpegWidth = kTestBufferWidth;
inputFrame.mMainJpegHeight = kTestBufferHeight;
inputFrame.mJpegQuality = jpegQuality;
- ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+ ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
inputFrame.mMainJpegBuffer = reinterpret_cast<const char*> (colorJpegBuffer.data());
inputFrame.mMainJpegSize = colorJpegBuffer.size();
- ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+ ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
inputFrame.mDepthMapBuffer = depth16Buffer.data();
inputFrame.mDepthMapWidth = inputFrame.mDepthMapStride = kTestBufferWidth;
inputFrame.mDepthMapHeight = kTestBufferHeight;
- ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), nullptr,
+ ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), nullptr,
&actualDepthPhotoSize), 0);
- ASSERT_NE(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(), nullptr),
- 0);
-
- dlclose(libHandle);
+ ASSERT_NE(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+ nullptr), 0);
}
TEST(DepthProcessorTest, BasicDepthPhotoValidation) {
- void *libHandle;
int jpegQuality = 95;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle == nullptr) {
- // Depth library no present, nothing more to test.
- return;
- }
-
std::vector<uint8_t> colorJpegBuffer;
generateColorJpegBuffer(jpegQuality, ExifOrientation::ORIENTATION_UNDEFINED,
/*includeExif*/ false, /*switchDimensions*/ false, &colorJpegBuffer);
@@ -180,7 +139,7 @@
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
- ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
+ ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
&actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) && (depthPhotoBuffer.size() >= actualDepthPhotoSize));
@@ -196,21 +155,11 @@
ASSERT_EQ(NV12Compressor::findJpegSize(depthPhotoBuffer.data() + mainJpegSize,
actualDepthPhotoSize - mainJpegSize, &depthMapSize), OK);
ASSERT_TRUE((depthMapSize > 0) && (depthMapSize < (actualDepthPhotoSize - mainJpegSize)));
-
- dlclose(libHandle);
}
TEST(DepthProcessorTest, TestDepthPhotoExifOrientation) {
- void *libHandle;
int jpegQuality = 95;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle == nullptr) {
- // Depth library no present, nothing more to test.
- return;
- }
-
ExifOrientation exifOrientations[] = { ExifOrientation::ORIENTATION_UNDEFINED,
ExifOrientation::ORIENTATION_0_DEGREES, ExifOrientation::ORIENTATION_90_DEGREES,
ExifOrientation::ORIENTATION_180_DEGREES, ExifOrientation::ORIENTATION_270_DEGREES };
@@ -242,8 +191,8 @@
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
- ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
- &actualDepthPhotoSize), 0);
+ ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
+ depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) &&
(depthPhotoBuffer.size() >= actualDepthPhotoSize));
@@ -281,21 +230,11 @@
ASSERT_EQ(confidenceJpegExifOrientation, exifOrientation);
}
}
-
- dlclose(libHandle);
}
TEST(DepthProcessorTest, TestDephtPhotoPhysicalRotation) {
- void *libHandle;
int jpegQuality = 95;
- process_depth_photo_frame processFunc;
- linkToDepthPhotoLibrary(&libHandle, &processFunc);
- if (libHandle == nullptr) {
- // Depth library no present, nothing more to test.
- return;
- }
-
// In case of physical rotation, the EXIF orientation must always be 0.
auto exifOrientation = ExifOrientation::ORIENTATION_0_DEGREES;
DepthPhotoOrientation depthOrientations[] = {
@@ -339,8 +278,8 @@
std::vector<uint8_t> depthPhotoBuffer(inputFrame.mMaxJpegSize);
size_t actualDepthPhotoSize = 0;
- ASSERT_EQ(processFunc(inputFrame, depthPhotoBuffer.size(), depthPhotoBuffer.data(),
- &actualDepthPhotoSize), 0);
+ ASSERT_EQ(processDepthPhotoFrame(inputFrame, depthPhotoBuffer.size(),
+ depthPhotoBuffer.data(), &actualDepthPhotoSize), 0);
ASSERT_TRUE((actualDepthPhotoSize > 0) &&
(depthPhotoBuffer.size() >= actualDepthPhotoSize));
@@ -377,6 +316,4 @@
ASSERT_EQ(confidenceMapWidth, expectedWidth);
ASSERT_EQ(confidenceMapHeight, expectedHeight);
}
-
- dlclose(libHandle);
}