Camera: Link dynamically to Depth photo library
Move all depth photo specific processing in a separate
library and link to it dynamically.
Bug: 109735087
Test: Camera CTS
Change-Id: I00a20b26fc9a1d127ad962a36b5b554dd36f0d41
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.cpp b/services/camera/libcameraservice/common/CameraProviderManager.cpp
index d6d34e9..3059b07 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.cpp
+++ b/services/camera/libcameraservice/common/CameraProviderManager.cpp
@@ -24,6 +24,8 @@
#include <algorithm>
#include <chrono>
+#include "common/DepthPhotoProcessor.h"
+#include <dlfcn.h>
#include <future>
#include <inttypes.h>
#include <hardware/camera_common.h>
@@ -606,6 +608,31 @@
}
}
+bool CameraProviderManager::ProviderInfo::DeviceInfo3::isDepthPhotoLibraryPresent() {
+ static bool libraryPresent = false;
+ static bool initialized = false;
+ if (initialized) {
+ return libraryPresent;
+ } else {
+ initialized = true;
+ }
+
+ void* depthLibHandle = dlopen(camera3::kDepthPhotoLibrary, RTLD_NOW | RTLD_LOCAL);
+ if (depthLibHandle == nullptr) {
+ return false;
+ }
+
+ auto processFunc = dlsym(depthLibHandle, camera3::kDepthPhotoProcessFunction);
+ if (processFunc != nullptr) {
+ libraryPresent = true;
+ } else {
+ libraryPresent = false;
+ }
+ dlclose(depthLibHandle);
+
+ return libraryPresent;
+}
+
status_t CameraProviderManager::ProviderInfo::DeviceInfo3::addDynamicDepthTags() {
uint32_t depthExclTag = ANDROID_DEPTH_DEPTH_IS_EXCLUSIVE;
uint32_t depthSizesTag = ANDROID_DEPTH_AVAILABLE_DEPTH_STREAM_CONFIGURATIONS;
@@ -654,6 +681,11 @@
return OK;
}
+ if(!isDepthPhotoLibraryPresent()) {
+ // Depth photo processing library is not present, nothing more to do.
+ return OK;
+ }
+
std::vector<int32_t> dynamicDepthEntries;
for (const auto& it : supportedDynamicDepthSizes) {
int32_t entry[4] = {HAL_PIXEL_FORMAT_BLOB, static_cast<int32_t> (std::get<0>(it)),
diff --git a/services/camera/libcameraservice/common/CameraProviderManager.h b/services/camera/libcameraservice/common/CameraProviderManager.h
index 80ec130..fbd7d2e 100644
--- a/services/camera/libcameraservice/common/CameraProviderManager.h
+++ b/services/camera/libcameraservice/common/CameraProviderManager.h
@@ -483,6 +483,7 @@
void getSupportedDynamicDepthDurations(const std::vector<int64_t>& depthDurations,
const std::vector<int64_t>& blobDurations,
std::vector<int64_t> *dynamicDepthDurations /*out*/);
+ static bool isDepthPhotoLibraryPresent();
static void getSupportedDynamicDepthSizes(
const std::vector<std::tuple<size_t, size_t>>& blobSizes,
const std::vector<std::tuple<size_t, size_t>>& depthSizes,
diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
new file mode 100644
index 0000000..a945aca
--- /dev/null
+++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.cpp
@@ -0,0 +1,340 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "Camera3-DepthPhotoProcessor"
+#define ATRACE_TAG ATRACE_TAG_CAMERA
+//#define LOG_NDEBUG 0
+//
+
+#include "DepthPhotoProcessor.h"
+
+#include <dynamic_depth/camera.h>
+#include <dynamic_depth/cameras.h>
+#include <dynamic_depth/container.h>
+#include <dynamic_depth/device.h>
+#include <dynamic_depth/dimension.h>
+#include <dynamic_depth/dynamic_depth.h>
+#include <dynamic_depth/point.h>
+#include <dynamic_depth/pose.h>
+#include <dynamic_depth/profile.h>
+#include <dynamic_depth/profiles.h>
+#include <jpeglib.h>
+#include <math.h>
+#include <sstream>
+#include <utils/Errors.h>
+#include <utils/Log.h>
+#include <xmpmeta/xmp_data.h>
+#include <xmpmeta/xmp_writer.h>
+
+using dynamic_depth::Camera;
+using dynamic_depth::Cameras;
+using dynamic_depth::CameraParams;
+using dynamic_depth::Container;
+using dynamic_depth::DepthFormat;
+using dynamic_depth::DepthMap;
+using dynamic_depth::DepthMapParams;
+using dynamic_depth::DepthUnits;
+using dynamic_depth::Device;
+using dynamic_depth::DeviceParams;
+using dynamic_depth::Dimension;
+using dynamic_depth::Image;
+using dynamic_depth::ImagingModel;
+using dynamic_depth::ImagingModelParams;
+using dynamic_depth::Item;
+using dynamic_depth::Pose;
+using dynamic_depth::Profile;
+using dynamic_depth::Profiles;
+
+namespace android {
+namespace camera3 {
+
+status_t encodeGrayscaleJpeg(size_t width, size_t height, uint8_t *in, void *out,
+ const size_t maxOutSize, uint8_t jpegQuality, size_t &actualSize) {
+ status_t ret;
+ // libjpeg is a C library so we use C-style "inheritance" by
+ // putting libjpeg's jpeg_destination_mgr first in our custom
+ // struct. This allows us to cast jpeg_destination_mgr* to
+ // CustomJpegDestMgr* when we get it passed to us in a callback.
+ struct CustomJpegDestMgr : public jpeg_destination_mgr {
+ JOCTET *mBuffer;
+ size_t mBufferSize;
+ size_t mEncodedSize;
+ bool mSuccess;
+ } dmgr;
+
+ jpeg_compress_struct cinfo = {};
+ jpeg_error_mgr jerr;
+
+ // Initialize error handling with standard callbacks, but
+ // then override output_message (to print to ALOG) and
+ // error_exit to set a flag and print a message instead
+ // of killing the whole process.
+ cinfo.err = jpeg_std_error(&jerr);
+
+ cinfo.err->output_message = [](j_common_ptr cinfo) {
+ char buffer[JMSG_LENGTH_MAX];
+
+ /* Create the message */
+ (*cinfo->err->format_message)(cinfo, buffer);
+ ALOGE("libjpeg error: %s", buffer);
+ };
+
+ cinfo.err->error_exit = [](j_common_ptr cinfo) {
+ (*cinfo->err->output_message)(cinfo);
+ if(cinfo->client_data) {
+ auto & dmgr = *static_cast<CustomJpegDestMgr*>(cinfo->client_data);
+ dmgr.mSuccess = false;
+ }
+ };
+
+ // Now that we initialized some callbacks, let's create our compressor
+ jpeg_create_compress(&cinfo);
+ dmgr.mBuffer = static_cast<JOCTET*>(out);
+ dmgr.mBufferSize = maxOutSize;
+ dmgr.mEncodedSize = 0;
+ dmgr.mSuccess = true;
+ cinfo.client_data = static_cast<void*>(&dmgr);
+
+ // These lambdas become C-style function pointers and as per C++11 spec
+ // may not capture anything.
+ dmgr.init_destination = [](j_compress_ptr cinfo) {
+ auto & dmgr = static_cast<CustomJpegDestMgr&>(*cinfo->dest);
+ dmgr.next_output_byte = dmgr.mBuffer;
+ dmgr.free_in_buffer = dmgr.mBufferSize;
+ ALOGV("%s:%d jpeg start: %p [%zu]",
+ __FUNCTION__, __LINE__, dmgr.mBuffer, dmgr.mBufferSize);
+ };
+
+ dmgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) {
+ ALOGV("%s:%d Out of buffer", __FUNCTION__, __LINE__);
+ return 0;
+ };
+
+ dmgr.term_destination = [](j_compress_ptr cinfo) {
+ auto & dmgr = static_cast<CustomJpegDestMgr&>(*cinfo->dest);
+ dmgr.mEncodedSize = dmgr.mBufferSize - dmgr.free_in_buffer;
+ ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__, dmgr.mEncodedSize);
+ };
+ cinfo.dest = reinterpret_cast<struct jpeg_destination_mgr*>(&dmgr);
+ cinfo.image_width = width;
+ cinfo.image_height = height;
+ cinfo.input_components = 1;
+ cinfo.in_color_space = JCS_GRAYSCALE;
+
+ // Initialize defaults and then override what we want
+ jpeg_set_defaults(&cinfo);
+
+ jpeg_set_quality(&cinfo, jpegQuality, 1);
+ jpeg_set_colorspace(&cinfo, JCS_GRAYSCALE);
+ cinfo.raw_data_in = 0;
+ cinfo.dct_method = JDCT_IFAST;
+
+ cinfo.comp_info[0].h_samp_factor = 1;
+ cinfo.comp_info[1].h_samp_factor = 1;
+ cinfo.comp_info[2].h_samp_factor = 1;
+ cinfo.comp_info[0].v_samp_factor = 1;
+ cinfo.comp_info[1].v_samp_factor = 1;
+ cinfo.comp_info[2].v_samp_factor = 1;
+
+ jpeg_start_compress(&cinfo, TRUE);
+
+ for (size_t i = 0; i < cinfo.image_height; i++) {
+ auto currentRow = static_cast<JSAMPROW>(in + i*width);
+ jpeg_write_scanlines(&cinfo, ¤tRow, /*num_lines*/1);
+ }
+
+ jpeg_finish_compress(&cinfo);
+
+ actualSize = dmgr.mEncodedSize;
+ if (dmgr.mSuccess) {
+ ret = NO_ERROR;
+ } else {
+ ret = UNKNOWN_ERROR;
+ }
+
+ return ret;
+}
+
+std::unique_ptr<dynamic_depth::DepthMap> processDepthMapFrame(DepthPhotoInputFrame inputFrame,
+ std::vector<std::unique_ptr<Item>> *items /*out*/) {
+ std::vector<float> points, confidence;
+
+ size_t pointCount = inputFrame.mDepthMapWidth * inputFrame.mDepthMapHeight;
+ points.reserve(pointCount);
+ confidence.reserve(pointCount);
+ float near = UINT16_MAX;
+ float far = .0f;
+ for (size_t i = 0; i < inputFrame.mDepthMapHeight; i++) {
+ for (size_t j = 0; j < inputFrame.mDepthMapWidth; j++) {
+ // Android densely packed depth map. The units for the range are in
+ // millimeters and need to be scaled to meters.
+ // The confidence value is encoded in the 3 most significant bits.
+ // The confidence data needs to be additionally normalized with
+ // values 1.0f, 0.0f representing maximum and minimum confidence
+ // respectively.
+ auto value = inputFrame.mDepthMapBuffer[i*inputFrame.mDepthMapStride + j];
+ auto point = static_cast<float>(value & 0x1FFF) / 1000.f;
+ points.push_back(point);
+
+ auto conf = (value >> 13) & 0x7;
+ float normConfidence = (conf == 0) ? 1.f : (static_cast<float>(conf) - 1) / 7.f;
+ confidence.push_back(normConfidence);
+
+ if (near > point) {
+ near = point;
+ }
+ if (far < point) {
+ far = point;
+ }
+ }
+ }
+
+ if (near == far) {
+ ALOGE("%s: Near and far range values must not match!", __FUNCTION__);
+ return nullptr;
+ }
+
+ std::vector<uint8_t> pointsQuantized, confidenceQuantized;
+ pointsQuantized.reserve(pointCount); confidenceQuantized.reserve(pointCount);
+ auto pointIt = points.begin();
+ auto confidenceIt = confidence.begin();
+ while ((pointIt != points.end()) && (confidenceIt != confidence.end())) {
+ pointsQuantized.push_back(floorf(((far * (*pointIt - near)) /
+ (*pointIt * (far - near))) * 255.0f));
+ confidenceQuantized.push_back(floorf(*confidenceIt * 255.0f));
+ confidenceIt++; pointIt++;
+ }
+
+ DepthMapParams depthParams(DepthFormat::kRangeInverse, near, far, DepthUnits::kMeters,
+ "android/depthmap");
+ depthParams.confidence_uri = "android/confidencemap";
+ depthParams.mime = "image/jpeg";
+ depthParams.depth_image_data.resize(inputFrame.mMaxJpegSize);
+ depthParams.confidence_data.resize(inputFrame.mMaxJpegSize);
+ size_t actualJpegSize;
+ auto ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight,
+ pointsQuantized.data(), depthParams.depth_image_data.data(), inputFrame.mMaxJpegSize,
+ inputFrame.mJpegQuality, actualJpegSize);
+ if (ret != NO_ERROR) {
+ ALOGE("%s: Depth map compression failed!", __FUNCTION__);
+ return nullptr;
+ }
+ depthParams.depth_image_data.resize(actualJpegSize);
+
+ ret = encodeGrayscaleJpeg(inputFrame.mDepthMapWidth, inputFrame.mDepthMapHeight,
+ confidenceQuantized.data(), depthParams.confidence_data.data(), inputFrame.mMaxJpegSize,
+ inputFrame.mJpegQuality, actualJpegSize);
+ if (ret != NO_ERROR) {
+ ALOGE("%s: Confidence map compression failed!", __FUNCTION__);
+ return nullptr;
+ }
+ depthParams.confidence_data.resize(actualJpegSize);
+
+ return DepthMap::FromData(depthParams, items);
+}
+
+extern "C" int processDepthPhotoFrame(DepthPhotoInputFrame inputFrame, size_t depthPhotoBufferSize,
+ void* depthPhotoBuffer /*out*/, size_t* depthPhotoActualSize /*out*/) {
+ if ((inputFrame.mMainJpegBuffer == nullptr) || (inputFrame.mDepthMapBuffer == nullptr) ||
+ (depthPhotoBuffer == nullptr) || (depthPhotoActualSize == nullptr)) {
+ return BAD_VALUE;
+ }
+
+ std::vector<std::unique_ptr<Item>> items;
+ std::vector<std::unique_ptr<Camera>> cameraList;
+ auto image = Image::FromDataForPrimaryImage("android/mainimage", &items);
+ std::unique_ptr<CameraParams> cameraParams(new CameraParams(std::move(image)));
+ if (cameraParams == nullptr) {
+ ALOGE("%s: Failed to initialize camera parameters", __FUNCTION__);
+ return BAD_VALUE;
+ }
+
+ cameraParams->depth_map = processDepthMapFrame(inputFrame, &items);
+ if (cameraParams->depth_map == nullptr) {
+ ALOGE("%s: Depth map processing failed!", __FUNCTION__);
+ return BAD_VALUE;
+ }
+
+ // It is not possible to generate an imaging model without instrinsic calibration.
+ if (inputFrame.mIsInstrinsicCalibrationValid) {
+ // The camera intrinsic calibration layout is as follows:
+ // [focalLengthX, focalLengthY, opticalCenterX, opticalCenterY, skew]
+ const dynamic_depth::Point<double> focalLength(inputFrame.mInstrinsicCalibration[0],
+ inputFrame.mInstrinsicCalibration[1]);
+ const Dimension imageSize(inputFrame.mMainJpegWidth, inputFrame.mMainJpegHeight);
+ ImagingModelParams imagingParams(focalLength, imageSize);
+ imagingParams.principal_point.x = inputFrame.mInstrinsicCalibration[2];
+ imagingParams.principal_point.y = inputFrame.mInstrinsicCalibration[3];
+ imagingParams.skew = inputFrame.mInstrinsicCalibration[4];
+
+ // The camera lens distortion contains the following lens correction coefficients.
+ // [kappa_1, kappa_2, kappa_3 kappa_4, kappa_5]
+ if (inputFrame.mIsLensDistortionValid) {
+ // According to specification the lens distortion coefficients should be ordered
+ // as [1, kappa_4, kappa_1, kappa_5, kappa_2, 0, kappa_3, 0]
+ float distortionData[] = {1.f, inputFrame.mLensDistortion[3],
+ inputFrame.mLensDistortion[0], inputFrame.mLensDistortion[4],
+ inputFrame.mLensDistortion[1], 0.f, inputFrame.mLensDistortion[2], 0.f};
+ auto distortionDataLength = sizeof(distortionData) / sizeof(distortionData[0]);
+ imagingParams.distortion.reserve(distortionDataLength);
+ imagingParams.distortion.insert(imagingParams.distortion.end(), distortionData,
+ distortionData + distortionDataLength);
+ }
+
+ cameraParams->imaging_model = ImagingModel::FromData(imagingParams);
+ }
+
+ if (inputFrame.mIsLogical) {
+ cameraParams->trait = dynamic_depth::CameraTrait::LOGICAL;
+ } else {
+ cameraParams->trait = dynamic_depth::CameraTrait::PHYSICAL;
+ }
+
+ cameraList.emplace_back(Camera::FromData(std::move(cameraParams)));
+
+ auto deviceParams = std::make_unique<DeviceParams> (Cameras::FromCameraArray(&cameraList));
+ deviceParams->container = Container::FromItems(&items);
+ std::vector<std::unique_ptr<Profile>> profileList;
+ profileList.emplace_back(Profile::FromData("DepthPhoto", {0}));
+ deviceParams->profiles = Profiles::FromProfileArray(&profileList);
+ std::unique_ptr<Device> device = Device::FromData(std::move(deviceParams));
+ if (device == nullptr) {
+ ALOGE("%s: Failed to initialize camera device", __FUNCTION__);
+ return BAD_VALUE;
+ }
+
+ std::istringstream inputJpegStream(
+ std::string(inputFrame.mMainJpegBuffer, inputFrame.mMainJpegSize));
+ std::ostringstream outputJpegStream;
+ if (!WriteImageAndMetadataAndContainer(&inputJpegStream, device.get(), &outputJpegStream)) {
+ ALOGE("%s: Failed writing depth output", __FUNCTION__);
+ return BAD_VALUE;
+ }
+
+ *depthPhotoActualSize = static_cast<size_t> (outputJpegStream.tellp());
+ if (*depthPhotoActualSize > depthPhotoBufferSize) {
+ ALOGE("%s: Depth photo output buffer not sufficient, needed %zu actual %zu", __FUNCTION__,
+ *depthPhotoActualSize, depthPhotoBufferSize);
+ return NO_MEMORY;
+ }
+
+ memcpy(depthPhotoBuffer, outputJpegStream.str().c_str(), *depthPhotoActualSize);
+
+ return 0;
+}
+
+}; // namespace camera3
+}; // namespace android
diff --git a/services/camera/libcameraservice/common/DepthPhotoProcessor.h b/services/camera/libcameraservice/common/DepthPhotoProcessor.h
new file mode 100644
index 0000000..19889a1
--- /dev/null
+++ b/services/camera/libcameraservice/common/DepthPhotoProcessor.h
@@ -0,0 +1,67 @@
+/*
+ * Copyright (C) 2019 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_PROCESSOR_H
+#define ANDROID_SERVERS_CAMERA_CAMERA3_DEPTH_PROCESSOR_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+namespace android {
+namespace camera3 {
+
+struct DepthPhotoInputFrame {
+ const char* mMainJpegBuffer;
+ size_t mMainJpegSize;
+ size_t mMainJpegWidth, mMainJpegHeight;
+ uint16_t* mDepthMapBuffer;
+ size_t mDepthMapWidth, mDepthMapHeight, mDepthMapStride;
+ size_t mMaxJpegSize;
+ uint8_t mJpegQuality;
+ uint8_t mIsLogical;
+ float mInstrinsicCalibration[5];
+ uint8_t mIsInstrinsicCalibrationValid;
+ float mLensDistortion[5];
+ uint8_t mIsLensDistortionValid;
+
+ DepthPhotoInputFrame() :
+ mMainJpegBuffer(nullptr),
+ mMainJpegSize(0),
+ mMainJpegWidth(0),
+ mMainJpegHeight(0),
+ mDepthMapBuffer(nullptr),
+ mDepthMapWidth(0),
+ mDepthMapHeight(0),
+ mDepthMapStride(0),
+ mMaxJpegSize(0),
+ mJpegQuality(100),
+ mIsLogical(0),
+ mInstrinsicCalibration{0.f},
+ mIsInstrinsicCalibrationValid(0),
+ mLensDistortion{0.f},
+ mIsLensDistortionValid(0) {}
+};
+
+static const char *kDepthPhotoLibrary = "libdepthphoto.so";
+static const char *kDepthPhotoProcessFunction = "processDepthPhotoFrame";
+typedef int (*process_depth_photo_frame) (DepthPhotoInputFrame /*inputFrame*/,
+ size_t /*depthPhotoBufferSize*/, void* /*depthPhotoBuffer out*/,
+ size_t* /*depthPhotoActualSize out*/);
+
+}; // namespace camera3
+}; // namespace android
+
+#endif