Camera: fill in Camera3Device offline processing impl
Move shared method implementations to Camera3OutputUtils.cpp/h
Defined interfaces to handle Camera3Device/Camera3OfflineSession
behavior differences in Camera3OutputInterface.h.
Bug: 135142453
Test: N/A (not enough implementation yet)
Change-Id: I57476ca5a1edf69c02a22241ad776d6f02636033
diff --git a/services/camera/libcameraservice/device3/Camera3Device.h b/services/camera/libcameraservice/device3/Camera3Device.h
index 5faabd1..6810fea 100644
--- a/services/camera/libcameraservice/device3/Camera3Device.h
+++ b/services/camera/libcameraservice/device3/Camera3Device.h
@@ -43,10 +43,14 @@
#include <camera/CaptureResult.h>
#include "common/CameraDeviceBase.h"
+#include "device3/BufferUtils.h"
#include "device3/StatusTracker.h"
#include "device3/Camera3BufferManager.h"
#include "device3/DistortionMapper.h"
#include "device3/ZoomRatioMapper.h"
+#include "device3/InFlightRequest.h"
+#include "device3/Camera3OutputInterface.h"
+#include "device3/Camera3OfflineSession.h"
#include "utils/TagMonitor.h"
#include "utils/LatencyHistogram.h"
#include <camera_metadata_hidden.h>
@@ -69,7 +73,11 @@
*/
class Camera3Device :
public CameraDeviceBase,
- virtual public hardware::camera::device::V3_5::ICameraDeviceCallback {
+ virtual public hardware::camera::device::V3_5::ICameraDeviceCallback,
+ public camera3::SetErrorInterface,
+ public camera3::InflightRequestUpdateInterface,
+ public camera3::RequestBufferInterface,
+ public camera3::FlushBufferInterface {
public:
explicit Camera3Device(const String8& id);
@@ -201,6 +209,16 @@
status_t switchToOffline(const std::vector<int32_t>& streamsToKeep,
/*out*/ sp<CameraOfflineSessionBase>* session) override;
+ // RequestBufferInterface
+ bool startRequestBuffer() override;
+ void endRequestBuffer() override;
+ nsecs_t getWaitDuration() override;
+
+ // FlushBufferInterface
+ void getInflightBufferKeys(std::vector<std::pair<int32_t, int32_t>>* out) override;
+ void getInflightRequestBufferKeys(std::vector<uint64_t>* out) override;
+ std::vector<sp<camera3::Camera3StreamInterface>> getAllStreams() override;
+
/**
* Helper functions to map between framework and HIDL values
*/
@@ -213,8 +231,6 @@
// Returns a negative error code if the passed-in operation mode is not valid.
static status_t mapToStreamConfigurationMode(camera3_stream_configuration_mode_t operationMode,
/*out*/ hardware::camera::device::V3_2::StreamConfigurationMode *mode);
- static camera3_buffer_status_t mapHidlBufferStatus(
- hardware::camera::device::V3_2::BufferStatus status);
static int mapToFrameworkFormat(hardware::graphics::common::V1_0::PixelFormat pixelFormat);
static android_dataspace mapToFrameworkDataspace(
hardware::camera::device::V3_2::DataspaceFlags);
@@ -229,7 +245,6 @@
// internal typedefs
using RequestMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>;
- using ResultMetadataQueue = hardware::MessageQueue<uint8_t, hardware::kSynchronizedReadWrite>;
static const size_t kDumpLockAttempts = 10;
static const size_t kDumpSleepDuration = 100000; // 0.10 sec
@@ -283,7 +298,8 @@
* Adapter for legacy HAL / HIDL HAL interface calls; calls either into legacy HALv3 or the
* HIDL HALv3 interfaces.
*/
- class HalInterface : public camera3::Camera3StreamBufferFreedListener {
+ class HalInterface : public camera3::Camera3StreamBufferFreedListener,
+ public camera3::BufferRecordsInterface {
public:
HalInterface(sp<hardware::camera::device::V3_2::ICameraDeviceSession> &session,
std::shared_ptr<RequestMetadataQueue> queue,
@@ -321,27 +337,31 @@
bool isReconfigurationRequired(CameraMetadata& oldSessionParams,
CameraMetadata& newSessionParams);
+ // Upon successful return, HalInterface will return buffer maps needed for offline
+ // processing, and clear all its internal buffer maps.
status_t switchToOffline(
const std::vector<int32_t>& streamsToKeep,
/*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo,
- /*out*/sp<hardware::camera::device::V3_6::ICameraOfflineSession>* offlineSession);
+ /*out*/sp<hardware::camera::device::V3_6::ICameraOfflineSession>* offlineSession,
+ /*out*/camera3::BufferRecords* bufferRecords);
- // method to extract buffer's unique ID
- // return pair of (newlySeenBuffer?, bufferId)
- std::pair<bool, uint64_t> getBufferId(const buffer_handle_t& buf, int streamId);
+ /////////////////////////////////////////////////////////////////////
+ // Implements BufferRecordsInterface
- // Find a buffer_handle_t based on frame number and stream ID
+ std::pair<bool, uint64_t> getBufferId(
+ const buffer_handle_t& buf, int streamId) override;
+
status_t popInflightBuffer(int32_t frameNumber, int32_t streamId,
- /*out*/ buffer_handle_t **buffer);
+ /*out*/ buffer_handle_t **buffer) override;
- // Register a bufId (streamId, buffer_handle_t) to inflight request buffer
status_t pushInflightRequestBuffer(
- uint64_t bufferId, buffer_handle_t* buf, int32_t streamId);
+ uint64_t bufferId, buffer_handle_t* buf, int32_t streamId) override;
- // Find a buffer_handle_t based on bufferId
status_t popInflightRequestBuffer(uint64_t bufferId,
/*out*/ buffer_handle_t** buffer,
- /*optional out*/ int32_t* streamId = nullptr);
+ /*optional out*/ int32_t* streamId = nullptr) override;
+
+ /////////////////////////////////////////////////////////////////////
// Get a vector of (frameNumber, streamId) pair of currently inflight
// buffers
@@ -352,7 +372,6 @@
void onStreamReConfigured(int streamId);
- static const uint64_t BUFFER_ID_NO_BUFFER = 0;
private:
// Always valid
sp<hardware::camera::device::V3_2::ICameraDeviceSession> mHidlSession;
@@ -367,8 +386,6 @@
std::shared_ptr<RequestMetadataQueue> mRequestMetadataQueue;
- std::mutex mInflightLock;
-
// The output HIDL request still depends on input camera3_capture_request_t
// Do not free input camera3_capture_request_t before output HIDL request
status_t wrapAsHidlRequest(camera3_capture_request_t* in,
@@ -382,55 +399,20 @@
// Pop inflight buffers based on pairs of (frameNumber,streamId)
void popInflightBuffers(const std::vector<std::pair<int32_t, int32_t>>& buffers);
- // Cache of buffer handles keyed off (frameNumber << 32 | streamId)
- std::unordered_map<uint64_t, buffer_handle_t*> mInflightBufferMap;
+ // Return true if the input caches match what we have; otherwise false
+ bool verifyBufferIds(int32_t streamId, std::vector<uint64_t>& inBufIds);
// Delete and optionally close native handles and clear the input vector afterward
static void cleanupNativeHandles(
std::vector<native_handle_t*> *handles, bool closeFd = false);
- struct BufferHasher {
- size_t operator()(const buffer_handle_t& buf) const {
- if (buf == nullptr)
- return 0;
-
- size_t result = 1;
- result = 31 * result + buf->numFds;
- for (int i = 0; i < buf->numFds; i++) {
- result = 31 * result + buf->data[i];
- }
- return result;
- }
- };
-
- struct BufferComparator {
- bool operator()(const buffer_handle_t& buf1, const buffer_handle_t& buf2) const {
- if (buf1->numFds == buf2->numFds) {
- for (int i = 0; i < buf1->numFds; i++) {
- if (buf1->data[i] != buf2->data[i]) {
- return false;
- }
- }
- return true;
- }
- return false;
- }
- };
-
- std::mutex mBufferIdMapLock; // protecting mBufferIdMaps and mNextBufferId
- typedef std::unordered_map<const buffer_handle_t, uint64_t,
- BufferHasher, BufferComparator> BufferIdMap;
- // stream ID -> per stream buffer ID map
- std::unordered_map<int, BufferIdMap> mBufferIdMaps;
- uint64_t mNextBufferId = 1; // 0 means no buffer
-
virtual void onBufferFreed(int streamId, const native_handle_t* handle) override;
+ std::mutex mFreedBuffersLock;
std::vector<std::pair<int, uint64_t>> mFreedBuffers;
- // Buffers given to HAL through requestStreamBuffer API
- std::mutex mRequestedBuffersLock;
- std::unordered_map<uint64_t, std::pair<int32_t, buffer_handle_t*>> mRequestedBuffers;
+ // Keep track of buffer cache and inflight buffer records
+ camera3::BufferRecords mBufferRecords;
uint32_t mNextStreamConfigCounter = 1;
@@ -473,24 +455,7 @@
// Tracking cause of fatal errors when in STATUS_ERROR
String8 mErrorCause;
- // Synchronized mapping of stream IDs to stream instances
- class StreamSet {
- public:
- status_t add(int streamId, sp<camera3::Camera3OutputStreamInterface>);
- ssize_t remove(int streamId);
- sp<camera3::Camera3OutputStreamInterface> get(int streamId);
- // get by (underlying) vector index
- sp<camera3::Camera3OutputStreamInterface> operator[] (size_t index);
- size_t size() const;
- std::vector<int> getStreamIds();
- void clear();
-
- private:
- mutable std::mutex mLock;
- KeyedVector<int, sp<camera3::Camera3OutputStreamInterface>> mData;
- };
-
- StreamSet mOutputStreams;
+ camera3::StreamSet mOutputStreams;
sp<camera3::Camera3Stream> mInputStream;
int mNextStreamId;
bool mNeedConfig;
@@ -579,15 +544,6 @@
const hardware::hidl_vec<
hardware::camera::device::V3_2::StreamBuffer>& buffers) override;
- // Handle one capture result. Assume that mProcessCaptureResultLock is held.
- void processOneCaptureResultLocked(
- const hardware::camera::device::V3_2::CaptureResult& result,
- const hardware::hidl_vec<
- hardware::camera::device::V3_4::PhysicalCameraMetadata> physicalCameraMetadata);
- status_t readOneCameraMetadataLocked(uint64_t fmqResultSize,
- hardware::camera::device::V3_2::CameraMetadata& resultMetadata,
- const hardware::camera::device::V3_2::CameraMetadata& result);
-
// Handle one notify message
void notify(const hardware::camera::device::V3_2::NotifyMsg& msg);
@@ -710,11 +666,20 @@
* error message to indicate why. Only the first call's message will be
* used. The message is also sent to the log.
*/
- void setErrorState(const char *fmt, ...);
+ void setErrorState(const char *fmt, ...) override;
+ void setErrorStateLocked(const char *fmt, ...) override;
void setErrorStateV(const char *fmt, va_list args);
- void setErrorStateLocked(const char *fmt, ...);
void setErrorStateLockedV(const char *fmt, va_list args);
+ /////////////////////////////////////////////////////////////////////
+ // Implements InflightRequestUpdateInterface
+
+ void onInflightEntryRemovedLocked(nsecs_t duration) override;
+ void checkInflightMapLengthLocked() override;
+ void onInflightMapFlushedLocked() override;
+
+ /////////////////////////////////////////////////////////////////////
+
/**
* Debugging trylock/spin method
* Try to acquire a lock a few times with sleeps between before giving up.
@@ -722,12 +687,6 @@
bool tryLockSpinRightRound(Mutex& lock);
/**
- * Helper function to determine if an input size for implementation defined
- * format is supported.
- */
- bool isOpaqueInputSizeSupported(uint32_t width, uint32_t height);
-
- /**
* Helper function to get the largest Jpeg resolution (in area)
* Return Size(0, 0) if static metatdata is invalid
*/
@@ -860,7 +819,8 @@
status_t switchToOffline(
const std::vector<int32_t>& streamsToKeep,
/*out*/hardware::camera::device::V3_6::CameraOfflineSessionInfo* offlineSessionInfo,
- /*out*/sp<hardware::camera::device::V3_6::ICameraOfflineSession>* offlineSession);
+ /*out*/sp<hardware::camera::device::V3_6::ICameraOfflineSession>* offlineSession,
+ /*out*/camera3::BufferRecords* bufferRecords);
protected:
@@ -1021,119 +981,12 @@
/**
* In-flight queue for tracking completion of capture requests.
*/
+ std::mutex mInFlightLock;
+ camera3::InFlightRequestMap mInFlightMap;
+ nsecs_t mExpectedInflightDuration = 0;
+ // End of mInFlightLock protection scope
- struct InFlightRequest {
- // Set by notify() SHUTTER call.
- nsecs_t shutterTimestamp;
- // Set by process_capture_result().
- nsecs_t sensorTimestamp;
- int requestStatus;
- // Set by process_capture_result call with valid metadata
- bool haveResultMetadata;
- // Decremented by calls to process_capture_result with valid output
- // and input buffers
- int numBuffersLeft;
- CaptureResultExtras resultExtras;
- // If this request has any input buffer
- bool hasInputBuffer;
-
- // The last metadata that framework receives from HAL and
- // not yet send out because the shutter event hasn't arrived.
- // It's added by process_capture_result and sent when framework
- // receives the shutter event.
- CameraMetadata pendingMetadata;
-
- // The metadata of the partial results that framework receives from HAL so far
- // and has sent out.
- CameraMetadata collectedPartialResult;
-
- // Buffers are added by process_capture_result when output buffers
- // return from HAL but framework has not yet received the shutter
- // event. They will be returned to the streams when framework receives
- // the shutter event.
- Vector<camera3_stream_buffer_t> pendingOutputBuffers;
-
- // Whether this inflight request's shutter and result callback are to be
- // called. The policy is that if the request is the last one in the constrained
- // high speed recording request list, this flag will be true. If the request list
- // is not for constrained high speed recording, this flag will also be true.
- bool hasCallback;
-
- // Maximum expected frame duration for this request.
- // For manual captures, equal to the max of requested exposure time and frame duration
- // For auto-exposure modes, equal to 1/(lower end of target FPS range)
- nsecs_t maxExpectedDuration;
-
- // Whether the result metadata for this request is to be skipped. The
- // result metadata should be skipped in the case of
- // REQUEST/RESULT error.
- bool skipResultMetadata;
-
- // The physical camera ids being requested.
- std::set<String8> physicalCameraIds;
-
- // Map of physicalCameraId <-> Metadata
- std::vector<PhysicalCaptureResultInfo> physicalMetadatas;
-
- // Indicates a still capture request.
- bool stillCapture;
-
- // Indicates a ZSL capture request
- bool zslCapture;
-
- // Requested camera ids (both logical and physical) with zoomRatio != 1.0f
- std::set<std::string> cameraIdsWithZoom;
-
- // What shared surfaces an output should go to
- SurfaceMap outputSurfaces;
-
- // Default constructor needed by KeyedVector
- InFlightRequest() :
- shutterTimestamp(0),
- sensorTimestamp(0),
- requestStatus(OK),
- haveResultMetadata(false),
- numBuffersLeft(0),
- hasInputBuffer(false),
- hasCallback(true),
- maxExpectedDuration(kDefaultExpectedDuration),
- skipResultMetadata(false),
- stillCapture(false),
- zslCapture(false) {
- }
-
- InFlightRequest(int numBuffers, CaptureResultExtras extras, bool hasInput,
- bool hasAppCallback, nsecs_t maxDuration,
- const std::set<String8>& physicalCameraIdSet, bool isStillCapture,
- bool isZslCapture, const std::set<std::string>& idsWithZoom,
- const SurfaceMap& outSurfaces = SurfaceMap{}) :
- shutterTimestamp(0),
- sensorTimestamp(0),
- requestStatus(OK),
- haveResultMetadata(false),
- numBuffersLeft(numBuffers),
- resultExtras(extras),
- hasInputBuffer(hasInput),
- hasCallback(hasAppCallback),
- maxExpectedDuration(maxDuration),
- skipResultMetadata(false),
- physicalCameraIds(physicalCameraIdSet),
- stillCapture(isStillCapture),
- zslCapture(isZslCapture),
- cameraIdsWithZoom(idsWithZoom),
- outputSurfaces(outSurfaces) {
- }
- };
-
- // Map from frame number to the in-flight request state
- typedef KeyedVector<uint32_t, InFlightRequest> InFlightMap;
-
-
- Mutex mInFlightLock; // Protects mInFlightMap and
- // mExpectedInflightDuration
- InFlightMap mInFlightMap;
- nsecs_t mExpectedInflightDuration = 0;
- int mInFlightStatusId;
+ int mInFlightStatusId; // const after initialize
status_t registerInFlight(uint32_t frameNumber,
int32_t numBuffers, CaptureResultExtras resultExtras, bool hasInput,
@@ -1211,7 +1064,7 @@
*/
// Lock for output side of device
- Mutex mOutputLock;
+ std::mutex mOutputLock;
/**** Scope for mOutputLock ****/
// the minimal frame number of the next non-reprocess result
@@ -1226,61 +1079,18 @@
uint32_t mNextReprocessShutterFrameNumber;
// the minimal frame number of the next ZSL still capture shutter
uint32_t mNextZslStillShutterFrameNumber;
- List<CaptureResult> mResultQueue;
- Condition mResultSignal;
- wp<NotificationListener> mListener;
+ List<CaptureResult> mResultQueue;
+ std::condition_variable mResultSignal;
+ wp<NotificationListener> mListener;
/**** End scope for mOutputLock ****/
- /**
- * Callback functions from HAL device
- */
- void processCaptureResult(const camera3_capture_result *result);
-
- void notify(const camera3_notify_msg *msg);
-
- // Specific notify handlers
- void notifyError(const camera3_error_msg_t &msg,
- sp<NotificationListener> listener);
- void notifyShutter(const camera3_shutter_msg_t &msg,
- sp<NotificationListener> listener);
-
- // helper function to return the output buffers to the streams.
- void returnOutputBuffers(const camera3_stream_buffer_t *outputBuffers,
- size_t numBuffers, nsecs_t timestamp, bool timestampIncreasing = true,
- // The following arguments are only meant for surface sharing use case
- const SurfaceMap& outputSurfaces = SurfaceMap{},
- // Used to send buffer error callback when failing to return buffer
- const CaptureResultExtras &resultExtras = CaptureResultExtras{});
-
- // Send a partial capture result.
- void sendPartialCaptureResult(const camera_metadata_t * partialResult,
- const CaptureResultExtras &resultExtras, uint32_t frameNumber);
-
- // Send a total capture result given the pending metadata and result extras,
- // partial results, and the frame number to the result queue.
- void sendCaptureResult(CameraMetadata &pendingMetadata,
- CaptureResultExtras &resultExtras,
- CameraMetadata &collectedPartialResult, uint32_t frameNumber,
- bool reprocess, bool zslStillCapture,
- const std::set<std::string>& cameraIdsWithZoom,
- const std::vector<PhysicalCaptureResultInfo>& physicalMetadatas);
-
- bool isLastFullResult(const InFlightRequest& inFlightRequest);
-
- // Insert the result to the result queue after updating frame number and overriding AE
- // trigger cancel.
- // mOutputLock must be held when calling this function.
- void insertResultLocked(CaptureResult *result, uint32_t frameNumber);
-
/**** Scope for mInFlightLock ****/
// Remove the in-flight map entry of the given index from mInFlightMap.
// It must only be called with mInFlightLock held.
void removeInFlightMapEntryLocked(int idx);
- // Remove the in-flight request of the given index from mInFlightMap
- // if it's no longer needed. It must only be called with mInFlightLock held.
- void removeInFlightRequestIfReadyLocked(int idx);
+
// Remove all in-flight requests and return all buffers.
// This is used after HAL interface is closed to cleanup any request/buffers
// not returned by HAL.
@@ -1378,6 +1188,10 @@
void onSubmittingRequest();
void onRequestThreadPaused();
+ // Events triggered by successful switchToOffline call
+ // Return true is there is no ongoing requestBuffer call.
+ bool onSwitchToOfflineSuccess();
+
private:
void notifyTrackerLocked(bool active);
@@ -1391,6 +1205,7 @@
bool mRequestThreadPaused = true;
bool mInflightMapEmpty = true;
bool mRequestBufferOngoing = false;
+ bool mSwitchedToOffline = false;
wp<camera3::StatusTracker> mStatusTracker;
int mRequestBufferStatusId;
@@ -1398,7 +1213,6 @@
// Fix up result metadata for monochrome camera.
bool mNeedFixupMonochromeTags;
- status_t fixupMonochromeTags(const CameraMetadata& deviceInfo, CameraMetadata& resultMetadata);
// Whether HAL supports offline processing capability.
bool mSupportOfflineProcessing = false;