CameraService: Implement SCALER_ROTATE_AND_CROP_AUTO, part 1
When an app sets SCALER_ROTATE_AND_CROP to AUTO, the camera service
needs to select the right ROTATE_AND_CROP mode given the application
UI state at the moment, received from the window manager.
In addition, some of the metadata in the active array coordinate
system needs to be converted to/from the cropped+rotated coordinate
system to ensure roundtripping UI information works as before.
Also ensure that the available rotate and crop metadata field is
always available, with a value of NONE if nothing else.
This commit adds support for doing the coordinate transforms and
overriding AUTO to a concrete value; it does not wire up a connection
to another system service to receive the correct override value, but
does add a command to set the override value for all current camera
clients.
Test: New CTS tests pass, unit tests for RotateAndCropMapper pass
Bug: 134631897
Change-Id: Icc45530e2cfbaf838a1e4d04e4fd2aef8122e8e1
diff --git a/services/camera/libcameraservice/device3/Camera3Device.cpp b/services/camera/libcameraservice/device3/Camera3Device.cpp
index 23e26ce..d38bfee 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.cpp
+++ b/services/camera/libcameraservice/device3/Camera3Device.cpp
@@ -351,6 +351,10 @@
mZoomRatioMappers[mId.c_str()] = ZoomRatioMapper(&mDeviceInfo,
mSupportNativeZoomRatio, usePrecorrectArray);
+ if (RotateAndCropMapper::isNeeded(&mDeviceInfo)) {
+ mRotateAndCropMappers.emplace(mId.c_str(), &mDeviceInfo);
+ }
+
return OK;
}
@@ -881,17 +885,12 @@
// Setup burst Id and request Id
newRequest->mResultExtras.burstId = burstId++;
- if (metadataIt->begin()->metadata.exists(ANDROID_REQUEST_ID)) {
- if (metadataIt->begin()->metadata.find(ANDROID_REQUEST_ID).count == 0) {
- CLOGE("RequestID entry exists; but must not be empty in metadata");
- return BAD_VALUE;
- }
- newRequest->mResultExtras.requestId = metadataIt->begin()->metadata.find(
- ANDROID_REQUEST_ID).data.i32[0];
- } else {
+ auto requestIdEntry = metadataIt->begin()->metadata.find(ANDROID_REQUEST_ID);
+ if (requestIdEntry.count == 0) {
CLOGE("RequestID does not exist in metadata");
return BAD_VALUE;
}
+ newRequest->mResultExtras.requestId = requestIdEntry.data.i32[0];
requestList->push_back(newRequest);
@@ -1049,8 +1048,8 @@
mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber,
mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags,
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
- mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor,
- mInputStream, mOutputStreams, listener, *this, *this, *mInterface
+ mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
+ mTagMonitor, mInputStream, mOutputStreams, listener, *this, *this, *mInterface
};
for (const auto& result : results) {
@@ -1106,8 +1105,8 @@
mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber,
mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags,
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
- mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor,
- mInputStream, mOutputStreams, listener, *this, *this, *mInterface
+ mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
+ mTagMonitor, mInputStream, mOutputStreams, listener, *this, *this, *mInterface
};
for (const auto& result : results) {
@@ -1145,8 +1144,8 @@
mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber,
mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags,
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
- mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor,
- mInputStream, mOutputStreams, listener, *this, *this, *mInterface
+ mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
+ mTagMonitor, mInputStream, mOutputStreams, listener, *this, *this, *mInterface
};
for (const auto& msg : msgs) {
camera3::notify(states, msg);
@@ -2222,7 +2221,7 @@
const PhysicalCameraSettingsList &request, const SurfaceMap &surfaceMap) {
ATRACE_CALL();
- sp<CaptureRequest> newRequest = new CaptureRequest;
+ sp<CaptureRequest> newRequest = new CaptureRequest();
newRequest->mSettingsList = request;
camera_metadata_entry_t inputStreams =
@@ -2293,6 +2292,15 @@
newRequest->mSettingsList.begin()->metadata.erase(ANDROID_REQUEST_OUTPUT_STREAMS);
newRequest->mBatchSize = 1;
+ auto rotateAndCropEntry =
+ newRequest->mSettingsList.begin()->metadata.find(ANDROID_SCALER_ROTATE_AND_CROP);
+ if (rotateAndCropEntry.count > 0 &&
+ rotateAndCropEntry.data.u8[0] == ANDROID_SCALER_ROTATE_AND_CROP_AUTO) {
+ newRequest->mRotateAndCropAuto = true;
+ } else {
+ newRequest->mRotateAndCropAuto = false;
+ }
+
return newRequest;
}
@@ -2730,7 +2738,7 @@
int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
bool hasAppCallback, nsecs_t maxExpectedDuration,
std::set<String8>& physicalCameraIds, bool isStillCapture,
- bool isZslCapture, const std::set<std::string>& cameraIdsWithZoom,
+ bool isZslCapture, bool rotateAndCropAuto, const std::set<std::string>& cameraIdsWithZoom,
const SurfaceMap& outputSurfaces) {
ATRACE_CALL();
std::lock_guard<std::mutex> l(mInFlightLock);
@@ -2738,7 +2746,7 @@
ssize_t res;
res = mInFlightMap.add(frameNumber, InFlightRequest(numBuffers, resultExtras, hasInput,
hasAppCallback, maxExpectedDuration, physicalCameraIds, isStillCapture, isZslCapture,
- cameraIdsWithZoom, outputSurfaces));
+ rotateAndCropAuto, cameraIdsWithZoom, outputSurfaces));
if (res < 0) return res;
if (mInFlightMap.size() == 1) {
@@ -3725,6 +3733,7 @@
mLatestRequestId(NAME_NOT_FOUND),
mCurrentAfTriggerId(0),
mCurrentPreCaptureTriggerId(0),
+ mRotateAndCropOverride(ANDROID_SCALER_ROTATE_AND_CROP_NONE),
mRepeatingLastFrameNumber(
hardware::camera2::ICameraDeviceUser::NO_IN_FLIGHT_REPEATING_FRAMES),
mPrepareVideoStream(false),
@@ -4358,8 +4367,11 @@
bool triggersMixedIn = (triggerCount > 0 || mPrevTriggers > 0);
mPrevTriggers = triggerCount;
+ bool rotateAndCropChanged = overrideAutoRotateAndCrop(captureRequest);
+
// If the request is the same as last, or we had triggers last time
- bool newRequest = (mPrevRequest != captureRequest || triggersMixedIn) &&
+ bool newRequest =
+ (mPrevRequest != captureRequest || triggersMixedIn || rotateAndCropChanged) &&
// Request settings are all the same within one batch, so only treat the first
// request in a batch as new
!(batchedRequest && i > 0);
@@ -4419,6 +4431,21 @@
return INVALID_OPERATION;
}
}
+ if (captureRequest->mRotateAndCropAuto) {
+ for (it = captureRequest->mSettingsList.begin();
+ it != captureRequest->mSettingsList.end(); it++) {
+ auto mapper = parent->mRotateAndCropMappers.find(it->cameraId);
+ if (mapper != parent->mRotateAndCropMappers.end()) {
+ res = mapper->second.updateCaptureRequest(&(it->metadata));
+ if (res != OK) {
+ SET_ERR("RequestThread: Unable to correct capture requests "
+ "for rotate-and-crop for request %d: %s (%d)",
+ halRequest->frame_number, strerror(-res), res);
+ return INVALID_OPERATION;
+ }
+ }
+ }
+ }
}
}
@@ -4617,7 +4644,8 @@
/*hasInput*/halRequest->input_buffer != NULL,
hasCallback,
calculateMaxExpectedDuration(halRequest->settings),
- requestedPhysicalCameras, isStillCapture, isZslCapture, mPrevCameraIdsWithZoom,
+ requestedPhysicalCameras, isStillCapture, isZslCapture,
+ captureRequest->mRotateAndCropAuto, mPrevCameraIdsWithZoom,
(mUseHalBufManager) ? uniqueSurfaceIdMap :
SurfaceMap{});
ALOGVV("%s: registered in flight requestId = %" PRId32 ", frameNumber = %" PRId64
@@ -4763,6 +4791,17 @@
streamsToKeep, offlineSessionInfo, offlineSession, bufferRecords);
}
+status_t Camera3Device::RequestThread::setRotateAndCropAutoBehavior(
+ camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue) {
+ ATRACE_CALL();
+ Mutex::Autolock l(mTriggerMutex);
+ if (rotateAndCropValue == ANDROID_SCALER_ROTATE_AND_CROP_AUTO) {
+ return BAD_VALUE;
+ }
+ mRotateAndCropOverride = rotateAndCropValue;
+ return OK;
+}
+
nsecs_t Camera3Device::getExpectedInFlightDuration() {
ATRACE_CALL();
std::lock_guard<std::mutex> l(mInFlightLock);
@@ -5278,6 +5317,32 @@
return OK;
}
+bool Camera3Device::RequestThread::overrideAutoRotateAndCrop(
+ const sp<CaptureRequest> &request) {
+ ATRACE_CALL();
+
+ if (request->mRotateAndCropAuto) {
+ Mutex::Autolock l(mTriggerMutex);
+ CameraMetadata &metadata = request->mSettingsList.begin()->metadata;
+
+ auto rotateAndCropEntry = metadata.find(ANDROID_SCALER_ROTATE_AND_CROP);
+ if (rotateAndCropEntry.count > 0) {
+ if (rotateAndCropEntry.data.u8[0] == mRotateAndCropOverride) {
+ return false;
+ } else {
+ rotateAndCropEntry.data.u8[0] = mRotateAndCropOverride;
+ return true;
+ }
+ } else {
+ uint8_t rotateAndCrop_u8 = mRotateAndCropOverride;
+ metadata.update(ANDROID_SCALER_ROTATE_AND_CROP,
+ &rotateAndCrop_u8, 1);
+ return true;
+ }
+ }
+ return false;
+}
+
/**
* PreparerThread inner class methods
*/
@@ -5813,7 +5878,7 @@
mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber,
mNextShutterFrameNumber, mNextReprocessShutterFrameNumber,
mNextZslStillShutterFrameNumber, mDeviceInfo, mPhysicalDeviceInfoMap,
- mDistortionMappers, mZoomRatioMappers);
+ mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers);
*session = new Camera3OfflineSession(mId, inputStream, offlineStreamSet,
std::move(bufferRecords), offlineReqs, offlineStates, offlineSession);
@@ -5889,4 +5954,15 @@
}
}
+status_t Camera3Device::setRotateAndCropAutoBehavior(
+ camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue) {
+ ATRACE_CALL();
+ Mutex::Autolock il(mInterfaceLock);
+ Mutex::Autolock l(mLock);
+ if (mRequestThread == nullptr) {
+ return INVALID_OPERATION;
+ }
+ return mRequestThread->setRotateAndCropAutoBehavior(rotateAndCropValue);
+}
+
}; // namespace android
diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h
index 7279baf..0764320 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.h
+++ b/services/camera/libcameraservice/device3/Camera3Device.h
@@ -48,6 +48,7 @@
#include "device3/Camera3BufferManager.h"
#include "device3/DistortionMapper.h"
#include "device3/ZoomRatioMapper.h"
+#include "device3/RotateAndCropMapper.h"
#include "device3/InFlightRequest.h"
#include "device3/Camera3OutputInterface.h"
#include "device3/Camera3OfflineSession.h"
@@ -222,6 +223,15 @@
std::vector<sp<camera3::Camera3StreamInterface>> getAllStreams() override;
/**
+ * Set the current behavior for the ROTATE_AND_CROP control when in AUTO.
+ *
+ * The value must be one of the ROTATE_AND_CROP_* values besides AUTO,
+ * and defaults to NONE.
+ */
+ status_t setRotateAndCropAutoBehavior(
+ camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue);
+
+ /**
* Helper functions to map between framework and HIDL values
*/
static hardware::graphics::common::V1_0::PixelFormat mapToPixelFormat(int frameworkFormat);
@@ -501,6 +511,10 @@
int mBatchSize;
// Whether this request is from a repeating or repeating burst.
bool mRepeating;
+ // Whether this request has ROTATE_AND_CROP_AUTO set, so needs both
+ // overriding of ROTATE_AND_CROP value and adjustment of coordinates
+ // in several other controls in both the request and the result
+ bool mRotateAndCropAuto;
};
typedef List<sp<CaptureRequest> > RequestList;
@@ -824,6 +838,9 @@
/*out*/sp<hardware::camera::device::V3_6::ICameraOfflineSession>* offlineSession,
/*out*/camera3::BufferRecords* bufferRecords);
+ status_t setRotateAndCropAutoBehavior(
+ camera_metadata_enum_android_scaler_rotate_and_crop_t rotateAndCropValue);
+
protected:
virtual bool threadLoop();
@@ -840,7 +857,10 @@
// HAL workaround: Make sure a trigger ID always exists if
// a trigger does
- status_t addDummyTriggerIds(const sp<CaptureRequest> &request);
+ status_t addDummyTriggerIds(const sp<CaptureRequest> &request);
+
+ // Override rotate_and_crop control if needed; returns true if the current value was changed
+ bool overrideAutoRotateAndCrop(const sp<CaptureRequest> &request);
static const nsecs_t kRequestTimeout = 50e6; // 50 ms
@@ -962,6 +982,7 @@
TriggerMap mTriggerReplacedMap;
uint32_t mCurrentAfTriggerId;
uint32_t mCurrentPreCaptureTriggerId;
+ camera_metadata_enum_android_scaler_rotate_and_crop_t mRotateAndCropOverride;
int64_t mRepeatingLastFrameNumber;
@@ -993,8 +1014,8 @@
status_t registerInFlight(uint32_t frameNumber,
int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
bool callback, nsecs_t maxExpectedDuration, std::set<String8>& physicalCameraIds,
- bool isStillCapture, bool isZslCapture, const std::set<std::string>& cameraIdsWithZoom,
- const SurfaceMap& outputSurfaces);
+ bool isStillCapture, bool isZslCapture, bool rotateAndCropAuto,
+ const std::set<std::string>& cameraIdsWithZoom, const SurfaceMap& outputSurfaces);
/**
* Tracking for idle detection
@@ -1113,6 +1134,11 @@
*/
std::unordered_map<std::string, camera3::ZoomRatioMapper> mZoomRatioMappers;
+ /**
+ * RotateAndCrop mapper support
+ */
+ std::unordered_map<std::string, camera3::RotateAndCropMapper> mRotateAndCropMappers;
+
// Debug tracker for metadata tag value changes
// - Enabled with the -m <taglist> option to dumpsys, such as
// dumpsys -m android.control.aeState,android.control.aeMode
diff --git a/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp b/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp
index 0f05632..8150de3 100644
--- a/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OfflineSession.cpp
@@ -71,6 +71,7 @@
mPhysicalDeviceInfoMap(offlineStates.mPhysicalDeviceInfoMap),
mDistortionMappers(offlineStates.mDistortionMappers),
mZoomRatioMappers(offlineStates.mZoomRatioMappers),
+ mRotateAndCropMappers(offlineStates.mRotateAndCropMappers),
mStatus(STATUS_UNINITIALIZED) {
ATRACE_CALL();
ALOGV("%s: Created offline session for camera %s", __FUNCTION__, mId.string());
@@ -252,8 +253,8 @@
mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber,
mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags,
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
- mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor,
- mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords
+ mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
+ mTagMonitor, mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords
};
std::lock_guard<std::mutex> lock(mProcessCaptureResultLock);
@@ -290,8 +291,8 @@
mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber,
mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags,
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
- mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor,
- mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords
+ mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
+ mTagMonitor, mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords
};
std::lock_guard<std::mutex> lock(mProcessCaptureResultLock);
@@ -323,8 +324,8 @@
mNextReprocessResultFrameNumber, mNextZslStillResultFrameNumber,
mUseHalBufManager, mUsePartialResult, mNeedFixupMonochromeTags,
mNumPartialResults, mVendorTagId, mDeviceInfo, mPhysicalDeviceInfoMap,
- mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mTagMonitor,
- mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords
+ mResultMetadataQueue, mDistortionMappers, mZoomRatioMappers, mRotateAndCropMappers,
+ mTagMonitor, mInputStream, mOutputStreams, listener, *this, *this, mBufferRecords
};
for (const auto& msg : msgs) {
camera3::notify(states, msg);
diff --git a/services/camera/libcameraservice/device3/Camera3OfflineSession.h b/services/camera/libcameraservice/device3/Camera3OfflineSession.h
index b6b8a7d..969220f 100644
--- a/services/camera/libcameraservice/device3/Camera3OfflineSession.h
+++ b/services/camera/libcameraservice/device3/Camera3OfflineSession.h
@@ -33,6 +33,7 @@
#include "device3/DistortionMapper.h"
#include "device3/InFlightRequest.h"
#include "device3/Camera3OutputUtils.h"
+#include "device3/RotateAndCropMapper.h"
#include "device3/ZoomRatioMapper.h"
#include "utils/TagMonitor.h"
#include "utils/LatencyHistogram.h"
@@ -62,7 +63,9 @@
const CameraMetadata& deviceInfo,
const std::unordered_map<std::string, CameraMetadata>& physicalDeviceInfoMap,
const std::unordered_map<std::string, camera3::DistortionMapper>& distortionMappers,
- const std::unordered_map<std::string, camera3::ZoomRatioMapper>& zoomRatioMappers) :
+ const std::unordered_map<std::string, camera3::ZoomRatioMapper>& zoomRatioMappers,
+ const std::unordered_map<std::string, camera3::RotateAndCropMapper>&
+ rotateAndCropMappers) :
mTagMonitor(tagMonitor), mVendorTagId(vendorTagId),
mUseHalBufManager(useHalBufManager), mNeedFixupMonochromeTags(needFixupMonochromeTags),
mUsePartialResult(usePartialResult), mNumPartialResults(numPartialResults),
@@ -75,7 +78,8 @@
mDeviceInfo(deviceInfo),
mPhysicalDeviceInfoMap(physicalDeviceInfoMap),
mDistortionMappers(distortionMappers),
- mZoomRatioMappers(zoomRatioMappers) {}
+ mZoomRatioMappers(zoomRatioMappers),
+ mRotateAndCropMappers(rotateAndCropMappers) {}
const TagMonitor& mTagMonitor;
const metadata_vendor_id_t mVendorTagId;
@@ -106,6 +110,8 @@
const std::unordered_map<std::string, camera3::DistortionMapper>& mDistortionMappers;
const std::unordered_map<std::string, camera3::ZoomRatioMapper>& mZoomRatioMappers;
+
+ const std::unordered_map<std::string, camera3::RotateAndCropMapper>& mRotateAndCropMappers;
};
/**
@@ -228,6 +234,8 @@
std::unordered_map<std::string, camera3::ZoomRatioMapper> mZoomRatioMappers;
+ std::unordered_map<std::string, camera3::RotateAndCropMapper> mRotateAndCropMappers;
+
mutable std::mutex mLock;
enum Status {
diff --git a/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp b/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
index eb7b666..39b7db4 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
+++ b/services/camera/libcameraservice/device3/Camera3OutputUtils.cpp
@@ -191,7 +191,7 @@
CaptureResultExtras &resultExtras,
CameraMetadata &collectedPartialResult,
uint32_t frameNumber,
- bool reprocess, bool zslStillCapture,
+ bool reprocess, bool zslStillCapture, bool rotateAndCropAuto,
const std::set<std::string>& cameraIdsWithZoom,
const std::vector<PhysicalCaptureResultInfo>& physicalMetadatas) {
ATRACE_CALL();
@@ -277,10 +277,25 @@
return;
}
+ // Fix up result metadata to account for rotateAndCrop in AUTO mode
+ if (rotateAndCropAuto) {
+ auto mapper = states.rotateAndCropMappers.find(states.cameraId.c_str());
+ if (mapper != states.rotateAndCropMappers.end()) {
+ res = mapper->second.updateCaptureResult(
+ &captureResult.mMetadata);
+ if (res != OK) {
+ SET_ERR("Unable to correct capture result rotate-and-crop for frame %d: %s (%d)",
+ frameNumber, strerror(-res), res);
+ return;
+ }
+ }
+ }
+
for (auto& physicalMetadata : captureResult.mPhysicalMetadatas) {
String8 cameraId8(physicalMetadata.mPhysicalCameraId);
- if (states.distortionMappers.find(cameraId8.c_str()) != states.distortionMappers.end()) {
- res = states.distortionMappers[cameraId8.c_str()].correctCaptureResult(
+ auto mapper = states.distortionMappers.find(cameraId8.c_str());
+ if (mapper != states.distortionMappers.end()) {
+ res = mapper->second.correctCaptureResult(
&physicalMetadata.mPhysicalCameraMetadata);
if (res != OK) {
SET_ERR("Unable to correct physical capture result metadata for frame %d: %s (%d)",
@@ -592,7 +607,8 @@
sendCaptureResult(states, metadata, request.resultExtras,
collectedPartialResult, frameNumber,
hasInputBufferInRequest, request.zslCapture && request.stillCapture,
- request.cameraIdsWithZoom, request.physicalMetadatas);
+ request.rotateAndCropAuto, request.cameraIdsWithZoom,
+ request.physicalMetadatas);
}
}
removeInFlightRequestIfReadyLocked(states, idx);
@@ -886,7 +902,7 @@
r.pendingMetadata, r.resultExtras,
r.collectedPartialResult, msg.frame_number,
r.hasInputBuffer, r.zslCapture && r.stillCapture,
- r.cameraIdsWithZoom, r.physicalMetadatas);
+ r.rotateAndCropAuto, r.cameraIdsWithZoom, r.physicalMetadatas);
}
bool timestampIncreasing = !(r.zslCapture || r.hasInputBuffer);
returnOutputBuffers(
diff --git a/services/camera/libcameraservice/device3/Camera3OutputUtils.h b/services/camera/libcameraservice/device3/Camera3OutputUtils.h
index 47d8095..300df5b 100644
--- a/services/camera/libcameraservice/device3/Camera3OutputUtils.h
+++ b/services/camera/libcameraservice/device3/Camera3OutputUtils.h
@@ -29,6 +29,7 @@
#include "device3/BufferUtils.h"
#include "device3/DistortionMapper.h"
#include "device3/ZoomRatioMapper.h"
+#include "device3/RotateAndCropMapper.h"
#include "device3/InFlightRequest.h"
#include "device3/Camera3Stream.h"
#include "device3/Camera3OutputStreamInterface.h"
@@ -79,6 +80,7 @@
std::unique_ptr<ResultMetadataQueue>& fmq;
std::unordered_map<std::string, camera3::DistortionMapper>& distortionMappers;
std::unordered_map<std::string, camera3::ZoomRatioMapper>& zoomRatioMappers;
+ std::unordered_map<std::string, camera3::RotateAndCropMapper>& rotateAndCropMappers;
TagMonitor& tagMonitor;
sp<Camera3Stream> inputStream;
StreamSet& outputStreams;
diff --git a/services/camera/libcameraservice/device3/CoordinateMapper.cpp b/services/camera/libcameraservice/device3/CoordinateMapper.cpp
index d62f397..a0bbe54 100644
--- a/services/camera/libcameraservice/device3/CoordinateMapper.cpp
+++ b/services/camera/libcameraservice/device3/CoordinateMapper.cpp
@@ -24,6 +24,7 @@
/**
* Metadata keys to correct when adjusting coordinates for distortion correction
+ * or for crop and rotate
*/
// Both capture request and result
@@ -33,7 +34,7 @@
ANDROID_CONTROL_AWB_REGIONS
};
-// Both capture request and result
+// Both capture request and result, not applicable to crop and rotate
constexpr std::array<uint32_t, 1> CoordinateMapper::kRectsToCorrect = {
ANDROID_SCALER_CROP_REGION,
};
diff --git a/services/camera/libcameraservice/device3/InFlightRequest.h b/services/camera/libcameraservice/device3/InFlightRequest.h
index ceeaa71..424043b 100644
--- a/services/camera/libcameraservice/device3/InFlightRequest.h
+++ b/services/camera/libcameraservice/device3/InFlightRequest.h
@@ -91,6 +91,9 @@
// Indicates a ZSL capture request
bool zslCapture;
+ // Indicates that ROTATE_AND_CROP was set to AUTO
+ bool rotateAndCropAuto;
+
// Requested camera ids (both logical and physical) with zoomRatio != 1.0f
std::set<std::string> cameraIdsWithZoom;
@@ -112,13 +115,14 @@
maxExpectedDuration(kDefaultExpectedDuration),
skipResultMetadata(false),
stillCapture(false),
- zslCapture(false) {
+ zslCapture(false),
+ rotateAndCropAuto(false) {
}
InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput,
bool hasAppCallback, nsecs_t maxDuration,
const std::set<String8>& physicalCameraIdSet, bool isStillCapture,
- bool isZslCapture, const std::set<std::string>& idsWithZoom,
+ bool isZslCapture, bool rotateAndCropAuto, const std::set<std::string>& idsWithZoom,
const SurfaceMap& outSurfaces = SurfaceMap{}) :
shutterTimestamp(0),
sensorTimestamp(0),
@@ -133,6 +137,7 @@
physicalCameraIds(physicalCameraIdSet),
stillCapture(isStillCapture),
zslCapture(isZslCapture),
+ rotateAndCropAuto(rotateAndCropAuto),
cameraIdsWithZoom(idsWithZoom),
outputSurfaces(outSurfaces) {
}
diff --git a/services/camera/libcameraservice/device3/RotateAndCropMapper.cpp b/services/camera/libcameraservice/device3/RotateAndCropMapper.cpp
new file mode 100644
index 0000000..3718f54
--- /dev/null
+++ b/services/camera/libcameraservice/device3/RotateAndCropMapper.cpp
@@ -0,0 +1,331 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define LOG_TAG "Camera3-RotCropMapper"
+#define ATRACE_TAG ATRACE_TAG_CAMERA
+//#define LOG_NDEBUG 0
+
+#include <algorithm>
+#include <cmath>
+
+#include "device3/RotateAndCropMapper.h"
+
+namespace android {
+
+namespace camera3 {
+
+bool RotateAndCropMapper::isNeeded(const CameraMetadata* deviceInfo) {
+ auto entry = deviceInfo->find(ANDROID_SCALER_AVAILABLE_ROTATE_AND_CROP_MODES);
+ for (size_t i = 0; i < entry.count; i++) {
+ if (entry.data.u8[i] == ANDROID_SCALER_ROTATE_AND_CROP_AUTO) return true;
+ }
+ return false;
+}
+
+RotateAndCropMapper::RotateAndCropMapper(const CameraMetadata* deviceInfo) {
+ auto entry = deviceInfo->find(ANDROID_SENSOR_INFO_ACTIVE_ARRAY_SIZE);
+ if (entry.count != 4) return;
+
+ mArrayWidth = entry.data.i32[2];
+ mArrayHeight = entry.data.i32[3];
+ mArrayAspect = static_cast<float>(mArrayWidth) / mArrayHeight;
+ mRotateAspect = 1.f/mArrayAspect;
+}
+
+/**
+ * Adjust capture request when rotate and crop AUTO is enabled
+ */
+status_t RotateAndCropMapper::updateCaptureRequest(CameraMetadata *request) {
+ auto entry = request->find(ANDROID_SCALER_ROTATE_AND_CROP);
+ if (entry.count == 0) return OK;
+ uint8_t rotateMode = entry.data.u8[0];
+ if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_NONE) return OK;
+
+ int32_t cx = 0;
+ int32_t cy = 0;
+ int32_t cw = mArrayWidth;
+ int32_t ch = mArrayHeight;
+ entry = request->find(ANDROID_SCALER_CROP_REGION);
+ if (entry.count == 4) {
+ cx = entry.data.i32[0];
+ cy = entry.data.i32[1];
+ cw = entry.data.i32[2];
+ ch = entry.data.i32[3];
+ }
+
+ // User inputs are relative to the rotated-and-cropped view, so convert back
+ // to active array coordinates. To be more specific, the application is
+ // calculating coordinates based on the crop rectangle and the active array,
+ // even though the view the user sees is the cropped-and-rotated one. So we
+ // need to adjust the coordinates so that a point that would be on the
+ // top-left corner of the crop region is mapped to the top-left corner of
+ // the rotated-and-cropped fov within the crop region, and the same for the
+ // bottom-right corner.
+ //
+ // Since the zoom ratio control scales everything uniformly (so an app does
+ // not need to adjust anything if it wants to put a metering region on the
+ // top-left quadrant of the preview FOV, when changing zoomRatio), it does
+ // not need to be factored into this calculation at all.
+ //
+ // ->+x active array aw
+ // |+--------------------------------------------------------------------+
+ // v| |
+ // +y| a 1 cw 2 b |
+ // | +=========*HHHHHHHHHHHHHHH*===========+ |
+ // | I H rw H I |
+ // | I H H I |
+ // | I H H I |
+ //ah | ch I H rh H I crop region |
+ // | I H H I |
+ // | I H H I |
+ // | I H rotate region H I |
+ // | +=========*HHHHHHHHHHHHHHH*===========+ |
+ // | d 4 3 c |
+ // | |
+ // +--------------------------------------------------------------------+
+ //
+ // aw , ah = active array width,height
+ // cw , ch = crop region width,height
+ // rw , rh = rotated-and-cropped region width,height
+ // aw / ah = array aspect = rh / rw = 1 / rotated aspect
+ // Coordinate mappings:
+ // ROTATE_AND_CROP_90: point a -> point 2
+ // point c -> point 4 = +x -> +y, +y -> -x
+ // ROTATE_AND_CROP_180: point a -> point c
+ // point c -> point a = +x -> -x, +y -> -y
+ // ROTATE_AND_CROP_270: point a -> point 4
+ // point c -> point 2 = +x -> -y, +y -> +x
+
+ float cropAspect = static_cast<float>(cw) / ch;
+ float transformMat[4] = {0, 0,
+ 0, 0};
+ float xShift = 0;
+ float yShift = 0;
+
+ if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_180) {
+ transformMat[0] = -1;
+ transformMat[3] = -1;
+ xShift = cw;
+ yShift = ch;
+ } else {
+ float rw = cropAspect > mRotateAspect ?
+ ch * mRotateAspect : // pillarbox, not full width
+ cw; // letterbox or 1:1, full width
+ float rh = cropAspect >= mRotateAspect ?
+ ch : // pillarbox or 1:1, full height
+ cw / mRotateAspect; // letterbox, not full height
+ switch (rotateMode) {
+ case ANDROID_SCALER_ROTATE_AND_CROP_90:
+ transformMat[1] = -rw / ch; // +y -> -x
+ transformMat[2] = rh / cw; // +x -> +y
+ xShift = (cw + rw) / 2; // left edge of crop to right edge of rotated
+ yShift = (ch - rh) / 2; // top edge of crop to top edge of rotated
+ break;
+ case ANDROID_SCALER_ROTATE_AND_CROP_270:
+ transformMat[1] = rw / ch; // +y -> +x
+ transformMat[2] = -rh / cw; // +x -> -y
+ xShift = (cw - rw) / 2; // left edge of crop to left edge of rotated
+ yShift = (ch + rh) / 2; // top edge of crop to bottom edge of rotated
+ break;
+ default:
+ ALOGE("%s: Unexpected rotate mode: %d", __FUNCTION__, rotateMode);
+ return BAD_VALUE;
+ }
+ }
+
+ for (auto regionTag : kMeteringRegionsToCorrect) {
+ entry = request->find(regionTag);
+ for (size_t i = 0; i < entry.count; i += 5) {
+ int32_t weight = entry.data.i32[i + 4];
+ if (weight == 0) {
+ continue;
+ }
+ transformPoints(entry.data.i32 + i, 2, transformMat, xShift, yShift, cx, cy);
+ swapRectToMinFirst(entry.data.i32 + i);
+ }
+ }
+
+ return OK;
+}
+
+/**
+ * Adjust capture result when rotate and crop AUTO is enabled
+ */
+status_t RotateAndCropMapper::updateCaptureResult(CameraMetadata *result) {
+ auto entry = result->find(ANDROID_SCALER_ROTATE_AND_CROP);
+ if (entry.count == 0) return OK;
+ uint8_t rotateMode = entry.data.u8[0];
+ if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_NONE) return OK;
+
+ int32_t cx = 0;
+ int32_t cy = 0;
+ int32_t cw = mArrayWidth;
+ int32_t ch = mArrayHeight;
+ entry = result->find(ANDROID_SCALER_CROP_REGION);
+ if (entry.count == 4) {
+ cx = entry.data.i32[0];
+ cy = entry.data.i32[1];
+ cw = entry.data.i32[2];
+ ch = entry.data.i32[3];
+ }
+
+ // HAL inputs are relative to the full active array, so convert back to
+ // rotated-and-cropped coordinates for apps. To be more specific, the
+ // application is calculating coordinates based on the crop rectangle and
+ // the active array, even though the view the user sees is the
+ // cropped-and-rotated one. So we need to adjust the coordinates so that a
+ // point that would be on the top-left corner of the rotate-and-cropped
+ // region is mapped to the top-left corner of the crop region, and the same
+ // for the bottom-right corner.
+ //
+ // Since the zoom ratio control scales everything uniformly (so an app does
+ // not need to adjust anything if it wants to put a metering region on the
+ // top-left quadrant of the preview FOV, when changing zoomRatio), it does
+ // not need to be factored into this calculation at all.
+ //
+ // Also note that round-tripping between original request and final result
+ // fields can't be perfect, since the intermediate values have to be
+ // integers on a smaller range than the original crop region range. That
+ // means that multiple input values map to a single output value in
+ // adjusting a request, so when adjusting a result, the original answer may
+ // not be obtainable. Given that aspect ratios are rarely > 16/9, the
+ // round-trip values should generally only be off by 1 at most.
+ //
+ // ->+x active array aw
+ // |+--------------------------------------------------------------------+
+ // v| |
+ // +y| a 1 cw 2 b |
+ // | +=========*HHHHHHHHHHHHHHH*===========+ |
+ // | I H rw H I |
+ // | I H H I |
+ // | I H H I |
+ //ah | ch I H rh H I crop region |
+ // | I H H I |
+ // | I H H I |
+ // | I H rotate region H I |
+ // | +=========*HHHHHHHHHHHHHHH*===========+ |
+ // | d 4 3 c |
+ // | |
+ // +--------------------------------------------------------------------+
+ //
+ // aw , ah = active array width,height
+ // cw , ch = crop region width,height
+ // rw , rh = rotated-and-cropped region width,height
+ // aw / ah = array aspect = rh / rw = 1 / rotated aspect
+ // Coordinate mappings:
+ // ROTATE_AND_CROP_90: point 2 -> point a
+ // point 4 -> point c = +x -> -y, +y -> +x
+ // ROTATE_AND_CROP_180: point c -> point a
+ // point a -> point c = +x -> -x, +y -> -y
+ // ROTATE_AND_CROP_270: point 4 -> point a
+ // point 2 -> point c = +x -> +y, +y -> -x
+
+ float cropAspect = static_cast<float>(cw) / ch;
+ float transformMat[4] = {0, 0,
+ 0, 0};
+ float xShift = 0;
+ float yShift = 0;
+ float rx = 0; // top-left corner of rotated region
+ float ry = 0;
+ if (rotateMode == ANDROID_SCALER_ROTATE_AND_CROP_180) {
+ transformMat[0] = -1;
+ transformMat[3] = -1;
+ xShift = cw;
+ yShift = ch;
+ rx = cx;
+ ry = cy;
+ } else {
+ float rw = cropAspect > mRotateAspect ?
+ ch * mRotateAspect : // pillarbox, not full width
+ cw; // letterbox or 1:1, full width
+ float rh = cropAspect >= mRotateAspect ?
+ ch : // pillarbox or 1:1, full height
+ cw / mRotateAspect; // letterbox, not full height
+ rx = cx + (cw - rw) / 2;
+ ry = cy + (ch - rh) / 2;
+ switch (rotateMode) {
+ case ANDROID_SCALER_ROTATE_AND_CROP_90:
+ transformMat[1] = ch / rw; // +y -> +x
+ transformMat[2] = -cw / rh; // +x -> -y
+ xShift = -(cw - rw) / 2; // left edge of rotated to left edge of cropped
+ yShift = ry - cy + ch; // top edge of rotated to bottom edge of cropped
+ break;
+ case ANDROID_SCALER_ROTATE_AND_CROP_270:
+ transformMat[1] = -ch / rw; // +y -> -x
+ transformMat[2] = cw / rh; // +x -> +y
+ xShift = (cw + rw) / 2; // left edge of rotated to left edge of cropped
+ yShift = (ch - rh) / 2; // top edge of rotated to bottom edge of cropped
+ break;
+ default:
+ ALOGE("%s: Unexpected rotate mode: %d", __FUNCTION__, rotateMode);
+ return BAD_VALUE;
+ }
+ }
+
+ for (auto regionTag : kMeteringRegionsToCorrect) {
+ entry = result->find(regionTag);
+ for (size_t i = 0; i < entry.count; i += 5) {
+ int32_t weight = entry.data.i32[i + 4];
+ if (weight == 0) {
+ continue;
+ }
+ transformPoints(entry.data.i32 + i, 2, transformMat, xShift, yShift, rx, ry);
+ swapRectToMinFirst(entry.data.i32 + i);
+ }
+ }
+
+ for (auto pointsTag: kResultPointsToCorrectNoClamp) {
+ entry = result->find(pointsTag);
+ transformPoints(entry.data.i32, entry.count / 2, transformMat, xShift, yShift, rx, ry);
+ if (pointsTag == ANDROID_STATISTICS_FACE_RECTANGLES) {
+ for (size_t i = 0; i < entry.count; i += 4) {
+ swapRectToMinFirst(entry.data.i32 + i);
+ }
+ }
+ }
+
+ return OK;
+}
+
+void RotateAndCropMapper::transformPoints(int32_t* pts, size_t count, float transformMat[4],
+ float xShift, float yShift, float ox, float oy) {
+ for (size_t i = 0; i < count * 2; i += 2) {
+ float x0 = pts[i] - ox;
+ float y0 = pts[i + 1] - oy;
+ int32_t nx = std::round(transformMat[0] * x0 + transformMat[1] * y0 + xShift + ox);
+ int32_t ny = std::round(transformMat[2] * x0 + transformMat[3] * y0 + yShift + oy);
+
+ pts[i] = std::min(std::max(nx, 0), mArrayWidth);
+ pts[i + 1] = std::min(std::max(ny, 0), mArrayHeight);
+ }
+}
+
+void RotateAndCropMapper::swapRectToMinFirst(int32_t* rect) {
+ if (rect[0] > rect[2]) {
+ auto tmp = rect[0];
+ rect[0] = rect[2];
+ rect[2] = tmp;
+ }
+ if (rect[1] > rect[3]) {
+ auto tmp = rect[1];
+ rect[1] = rect[3];
+ rect[3] = tmp;
+ }
+}
+
+} // namespace camera3
+
+} // namespace android
diff --git a/services/camera/libcameraservice/device3/RotateAndCropMapper.h b/services/camera/libcameraservice/device3/RotateAndCropMapper.h
new file mode 100644
index 0000000..459e27f
--- /dev/null
+++ b/services/camera/libcameraservice/device3/RotateAndCropMapper.h
@@ -0,0 +1,68 @@
+/*
+ * Copyright (C) 2020 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_SERVERS_ROTATEANDCROPMAPPER_H
+#define ANDROID_SERVERS_ROTATEANDCROPMAPPER_H
+
+#include <utils/Errors.h>
+#include <array>
+#include <mutex>
+
+#include "camera/CameraMetadata.h"
+#include "device3/CoordinateMapper.h"
+
+namespace android {
+
+namespace camera3 {
+
+/**
+ * Utilities to transform between unrotated and rotated-and-cropped coordinate systems
+ * for cameras that support SCALER_ROTATE_AND_CROP controls in AUTO mode.
+ */
+class RotateAndCropMapper : private CoordinateMapper {
+ public:
+ static bool isNeeded(const CameraMetadata* deviceInfo);
+
+ RotateAndCropMapper(const CameraMetadata* deviceInfo);
+
+ /**
+ * Adjust capture request assuming rotate and crop AUTO is enabled
+ */
+ status_t updateCaptureRequest(CameraMetadata *request);
+
+ /**
+ * Adjust capture result assuming rotate and crop AUTO is enabled
+ */
+ status_t updateCaptureResult(CameraMetadata *result);
+
+ private:
+ // Transform count's worth of x,y points passed in with 2x2 matrix + translate with transform
+ // origin (cx,cy)
+ void transformPoints(int32_t* pts, size_t count, float transformMat[4],
+ float xShift, float yShift, float cx, float cy);
+ // Take two corners of a rect as (x1,y1,x2,y2) and swap x and y components
+ // if needed so that x1 < x2, y1 < y2.
+ void swapRectToMinFirst(int32_t* rect);
+
+ int32_t mArrayWidth, mArrayHeight;
+ float mArrayAspect, mRotateAspect;
+}; // class RotateAndCroMapper
+
+} // namespace camera3
+
+} // namespace android
+
+#endif