Camera2: Revamp recording support to allow opaque formats.
- Avoid memcpy of buffer data by using metadata mode
- Temporarily add MediaConsumer BufferQueue class
Bug: 6243944
Change-Id: I7a8c4222b7dbd14f1b1d86fda06d38eb640e87c1
diff --git a/services/camera/libcameraservice/Android.mk b/services/camera/libcameraservice/Android.mk
index 8cccf49..c14ae22 100644
--- a/services/camera/libcameraservice/Android.mk
+++ b/services/camera/libcameraservice/Android.mk
@@ -10,7 +10,8 @@
CameraService.cpp \
CameraClient.cpp \
Camera2Client.cpp \
- Camera2Device.cpp
+ Camera2Device.cpp \
+ MediaConsumer.cpp
LOCAL_SHARED_LIBRARIES:= \
libui \
diff --git a/services/camera/libcameraservice/Camera2Client.cpp b/services/camera/libcameraservice/Camera2Client.cpp
index f21a518..2e1940a 100644
--- a/services/camera/libcameraservice/Camera2Client.cpp
+++ b/services/camera/libcameraservice/Camera2Client.cpp
@@ -24,6 +24,7 @@
#include <cutils/properties.h>
#include <gui/SurfaceTextureClient.h>
#include <gui/Surface.h>
+#include <media/hardware/MetadataBufferType.h>
#include <math.h>
@@ -611,7 +612,21 @@
status_t Camera2Client::storeMetaDataInBuffers(bool enabled) {
ATRACE_CALL();
Mutex::Autolock icl(mICameraLock);
- return BAD_VALUE;
+ switch (mState) {
+ case RECORD:
+ case VIDEO_SNAPSHOT:
+ ALOGE("%s: Camera %d: Can't be called in state %s",
+ __FUNCTION__, mCameraId, getStateName(mState));
+ return INVALID_OPERATION;
+ default:
+ // OK
+ break;
+ }
+ Mutex::Autolock pl(mParamsLock);
+
+ mParameters.storeMetadataInBuffers = enabled;
+
+ return OK;
}
status_t Camera2Client::startRecording() {
@@ -640,6 +655,13 @@
Mutex::Autolock pl(mParamsLock);
+ if (!mParameters.storeMetadataInBuffers) {
+ ALOGE("%s: Camera %d: Recording only supported in metadata mode, but "
+ "non-metadata recording mode requested!", __FUNCTION__,
+ mCameraId);
+ return INVALID_OPERATION;
+ }
+
res = updateRecordingStream();
if (res != OK) {
ALOGE("%s: Camera %d: Unable to update recording stream: %s (%d)",
@@ -730,6 +752,7 @@
// Make sure this is for the current heap
ssize_t offset;
size_t size;
+ status_t res;
sp<IMemoryHeap> heap = mem->getMemory(&offset, &size);
if (heap->getHeapID() != mRecordingHeap->mHeap->getHeapID()) {
ALOGW("%s: Camera %d: Mismatched heap ID, ignoring release "
@@ -737,6 +760,24 @@
heap->getHeapID(), mRecordingHeap->mHeap->getHeapID());
return;
}
+ uint8_t *data = (uint8_t*)heap->getBase() + offset;
+ uint32_t type = *(uint32_t*)data;
+ if (type != kMetadataBufferTypeGrallocSource) {
+ ALOGE("%s: Camera %d: Recording frame type invalid (got %x, expected %x)",
+ __FUNCTION__, mCameraId, type, kMetadataBufferTypeGrallocSource);
+ return;
+ }
+ buffer_handle_t imgBuffer = *(buffer_handle_t*)(data + 4);
+ ALOGV("%s: Camera %d: Freeing buffer_handle_t %p", __FUNCTION__, mCameraId,
+ imgBuffer, *(uint32_t*)(data + 4));
+ res = mRecordingConsumer->freeBuffer(imgBuffer);
+ if (res != OK) {
+ ALOGE("%s: Camera %d: Unable to free recording frame (buffer_handle_t: %p):"
+ "%s (%d)",
+ __FUNCTION__, mCameraId, imgBuffer, strerror(-res), res);
+ return;
+ }
+
mRecordingHeapFree++;
}
@@ -1519,30 +1560,21 @@
discardData = true;
}
- CpuConsumer::LockedBuffer imgBuffer;
- res = mRecordingConsumer->lockNextBuffer(&imgBuffer);
+ buffer_handle_t imgBuffer;
+ res = mRecordingConsumer->getNextBuffer(&imgBuffer, ×tamp);
if (res != OK) {
ALOGE("%s: Camera %d: Error receiving recording buffer: %s (%d)",
__FUNCTION__, mCameraId, strerror(-res), res);
return;
}
- if (imgBuffer.format != (int)kRecordingFormat) {
- ALOGE("%s: Camera %d: Unexpected recording format: %x",
- __FUNCTION__, mCameraId, imgBuffer.format);
- discardData = true;
- }
-
if (discardData) {
- mRecordingConsumer->unlockBuffer(imgBuffer);
+ mRecordingConsumer->freeBuffer(imgBuffer);
return;
}
- size_t bufferSize = imgBuffer.width * imgBuffer.height * 3 / 2;
-
- if (mRecordingHeap == 0 ||
- bufferSize >
- mRecordingHeap->mHeap->getSize() / kRecordingHeapCount) {
+ if (mRecordingHeap == 0) {
+ const size_t bufferSize = 4 + sizeof(buffer_handle_t);
ALOGV("%s: Camera %d: Creating recording heap with %d buffers of "
"size %d bytes", __FUNCTION__, mCameraId,
kRecordingHeapCount, bufferSize);
@@ -1560,22 +1592,20 @@
if (mRecordingHeap->mHeap->getSize() == 0) {
ALOGE("%s: Camera %d: Unable to allocate memory for recording",
__FUNCTION__, mCameraId);
- mRecordingConsumer->unlockBuffer(imgBuffer);
+ mRecordingConsumer->freeBuffer(imgBuffer);
return;
}
mRecordingHeapHead = 0;
mRecordingHeapFree = kRecordingHeapCount;
}
- // TODO: Optimize this to avoid memcopy
if ( mRecordingHeapFree == 0) {
ALOGE("%s: Camera %d: No free recording buffers, dropping frame",
__FUNCTION__, mCameraId);
- mRecordingConsumer->unlockBuffer(imgBuffer);
+ mRecordingConsumer->freeBuffer(imgBuffer);
return;
}
heapIdx = mRecordingHeapHead;
- timestamp = imgBuffer.timestamp;
mRecordingHeapHead = (mRecordingHeapHead + 1) % kRecordingHeapCount;
mRecordingHeapFree--;
@@ -1588,10 +1618,12 @@
mRecordingHeap->mBuffers[heapIdx]->getMemory(&offset,
&size);
- memcpy((uint8_t*)heap->getBase() + offset, imgBuffer.data, size);
-
- mRecordingConsumer->unlockBuffer(imgBuffer);
-
+ uint8_t *data = (uint8_t*)heap->getBase() + offset;
+ uint32_t type = kMetadataBufferTypeGrallocSource;
+ memcpy(data, &type, 4);
+ memcpy(data + 4, &imgBuffer, sizeof(buffer_handle_t));
+ ALOGV("%s: Camera %d: Sending out buffer_handle_t %p",
+ __FUNCTION__, mCameraId, imgBuffer, *(uint32_t*)(data + 4));
currentClient = mCameraClient;
}
// Call outside mICameraLock to allow re-entrancy from notification
@@ -2306,7 +2338,7 @@
0);
params.set(CameraParameters::KEY_VIDEO_FRAME_FORMAT,
- formatEnumToString(kRecordingFormat));
+ CameraParameters::PIXEL_FORMAT_ANDROID_OPAQUE);
params.set(CameraParameters::KEY_RECORDING_HINT,
CameraParameters::FALSE);
@@ -2329,6 +2361,9 @@
CameraParameters::FALSE);
}
+ // Always use metadata mode for recording
+ mParameters.storeMetadataInBuffers = true;
+
mParamsFlattened = params.flatten();
return OK;
@@ -2580,7 +2615,7 @@
if (mRecordingConsumer == 0) {
// Create CPU buffer queue endpoint
- mRecordingConsumer = new CpuConsumer(1);
+ mRecordingConsumer = new MediaConsumer(4);
mRecordingConsumer->setFrameAvailableListener(new RecordingWaiter(this));
mRecordingConsumer->setName(String8("Camera2Client::RecordingConsumer"));
mRecordingWindow = new SurfaceTextureClient(
@@ -2615,7 +2650,7 @@
if (mRecordingStreamId == NO_STREAM) {
res = mDevice->createStream(mRecordingWindow,
mParameters.videoWidth, mParameters.videoHeight,
- kRecordingFormat, 0, &mRecordingStreamId);
+ CAMERA2_HAL_PIXEL_FORMAT_OPAQUE, 0, &mRecordingStreamId);
if (res != OK) {
ALOGE("%s: Camera %d: Can't create output stream for recording: "
"%s (%d)", __FUNCTION__, mCameraId, strerror(-res), res);
diff --git a/services/camera/libcameraservice/Camera2Client.h b/services/camera/libcameraservice/Camera2Client.h
index 8d410f1..d7836ca 100644
--- a/services/camera/libcameraservice/Camera2Client.h
+++ b/services/camera/libcameraservice/Camera2Client.h
@@ -23,6 +23,7 @@
#include <binder/MemoryBase.h>
#include <binder/MemoryHeapBase.h>
#include <gui/CpuConsumer.h>
+#include "MediaConsumer.h"
namespace android {
@@ -170,6 +171,8 @@
bool recordingHint;
bool videoStabilization;
+
+ bool storeMetadataInBuffers;
} mParameters;
/** Camera device-related private members */
@@ -219,11 +222,11 @@
/* Recording related members */
int mRecordingStreamId;
- sp<CpuConsumer> mRecordingConsumer;
+ sp<MediaConsumer> mRecordingConsumer;
sp<ANativeWindow> mRecordingWindow;
// Simple listener that forwards frame available notifications from
// a CPU consumer to the recording notification
- class RecordingWaiter: public CpuConsumer::FrameAvailableListener {
+ class RecordingWaiter: public MediaConsumer::FrameAvailableListener {
public:
RecordingWaiter(Camera2Client *parent) : mParent(parent) {}
void onFrameAvailable() { mParent->onRecordingFrameAvailable(); }
@@ -237,7 +240,6 @@
// TODO: This needs to be queried from somewhere, or the BufferQueue needs
// to be passed all the way to stagefright
static const size_t kRecordingHeapCount = 4;
- static const uint32_t kRecordingFormat = HAL_PIXEL_FORMAT_YCrCb_420_SP;
size_t mRecordingHeapHead, mRecordingHeapFree;
// Handle new recording image buffers
void onRecordingFrameAvailable();
diff --git a/services/camera/libcameraservice/MediaConsumer.cpp b/services/camera/libcameraservice/MediaConsumer.cpp
new file mode 100644
index 0000000..0d857cf
--- /dev/null
+++ b/services/camera/libcameraservice/MediaConsumer.cpp
@@ -0,0 +1,196 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+//#define LOG_NDEBUG 0
+#define LOG_TAG "MediaConsumer"
+#define ATRACE_TAG ATRACE_TAG_GRAPHICS
+#include <utils/Log.h>
+
+#include "MediaConsumer.h"
+
+#define MC_LOGV(x, ...) ALOGV("[%s] "x, mName.string(), ##__VA_ARGS__)
+#define MC_LOGD(x, ...) ALOGD("[%s] "x, mName.string(), ##__VA_ARGS__)
+#define MC_LOGI(x, ...) ALOGI("[%s] "x, mName.string(), ##__VA_ARGS__)
+#define MC_LOGW(x, ...) ALOGW("[%s] "x, mName.string(), ##__VA_ARGS__)
+#define MC_LOGE(x, ...) ALOGE("[%s] "x, mName.string(), ##__VA_ARGS__)
+
+namespace android {
+
+// Get an ID that's unique within this process.
+static int32_t createProcessUniqueId() {
+ static volatile int32_t globalCounter = 0;
+ return android_atomic_inc(&globalCounter);
+}
+
+MediaConsumer::MediaConsumer(uint32_t maxLockedBuffers) :
+ mMaxLockedBuffers(maxLockedBuffers),
+ mCurrentLockedBuffers(0)
+{
+ mName = String8::format("mc-unnamed-%d-%d", getpid(),
+ createProcessUniqueId());
+
+ mBufferQueue = new BufferQueue(true);
+
+ wp<BufferQueue::ConsumerListener> listener;
+ sp<BufferQueue::ConsumerListener> proxy;
+ listener = static_cast<BufferQueue::ConsumerListener*>(this);
+ proxy = new BufferQueue::ProxyConsumerListener(listener);
+
+ status_t err = mBufferQueue->consumerConnect(proxy);
+ if (err != NO_ERROR) {
+ ALOGE("MediaConsumer: error connecting to BufferQueue: %s (%d)",
+ strerror(-err), err);
+ } else {
+ mBufferQueue->setSynchronousMode(true);
+ mBufferQueue->setConsumerUsageBits(GRALLOC_USAGE_HW_VIDEO_ENCODER);
+ mBufferQueue->setConsumerName(mName);
+ }
+}
+
+MediaConsumer::~MediaConsumer()
+{
+ Mutex::Autolock _l(mMutex);
+ for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
+ freeBufferLocked(i);
+ }
+ mBufferQueue->consumerDisconnect();
+ mBufferQueue.clear();
+}
+
+void MediaConsumer::setName(const String8& name) {
+ Mutex::Autolock _l(mMutex);
+ mName = name;
+ mBufferQueue->setConsumerName(name);
+}
+
+status_t MediaConsumer::getNextBuffer(buffer_handle_t *buffer, nsecs_t *timestamp) {
+ status_t err;
+
+ if (!buffer) return BAD_VALUE;
+ if (mCurrentLockedBuffers == mMaxLockedBuffers) {
+ return INVALID_OPERATION;
+ }
+
+ BufferQueue::BufferItem b;
+
+ Mutex::Autolock _l(mMutex);
+
+ err = mBufferQueue->acquireBuffer(&b);
+ if (err != OK) {
+ if (err == BufferQueue::NO_BUFFER_AVAILABLE) {
+ return BAD_VALUE;
+ } else {
+ MC_LOGE("Error acquiring buffer: %s (%d)", strerror(err), err);
+ return err;
+ }
+ }
+
+ int buf = b.mBuf;
+
+ if (b.mGraphicBuffer != NULL) {
+ mBufferSlot[buf] = b.mGraphicBuffer;
+ }
+
+ if (b.mFence.get()) {
+ err = b.mFence->wait(Fence::TIMEOUT_NEVER);
+ if (err != OK) {
+ MC_LOGE("Failed to wait for fence of acquired buffer: %s (%d)",
+ strerror(-err), err);
+ return err;
+ }
+ }
+
+ *buffer = mBufferSlot[buf]->handle;
+ *timestamp = b.mTimestamp;
+
+ mCurrentLockedBuffers++;
+
+ return OK;
+}
+
+status_t MediaConsumer::freeBuffer(buffer_handle_t buffer) {
+ Mutex::Autolock _l(mMutex);
+ int buf = 0;
+ status_t err;
+
+ for (; buf < BufferQueue::NUM_BUFFER_SLOTS; buf++) {
+ if (buffer == mBufferSlot[buf]->handle) break;
+ }
+ if (buf == BufferQueue::NUM_BUFFER_SLOTS) {
+ MC_LOGE("%s: Can't find buffer to free", __FUNCTION__);
+ return BAD_VALUE;
+ }
+
+ err = mBufferQueue->releaseBuffer(buf, EGL_NO_DISPLAY, EGL_NO_SYNC_KHR,
+ Fence::NO_FENCE);
+ if (err == BufferQueue::STALE_BUFFER_SLOT) {
+ freeBufferLocked(buf);
+ } else if (err != OK) {
+ MC_LOGE("%s: Unable to release graphic buffer %d to queue", __FUNCTION__,
+ buf);
+ return err;
+ }
+
+ mCurrentLockedBuffers--;
+
+ return OK;
+}
+
+void MediaConsumer::setFrameAvailableListener(
+ const sp<FrameAvailableListener>& listener) {
+ MC_LOGV("setFrameAvailableListener");
+ Mutex::Autolock lock(mMutex);
+ mFrameAvailableListener = listener;
+}
+
+
+void MediaConsumer::onFrameAvailable() {
+ MC_LOGV("onFrameAvailable");
+ sp<FrameAvailableListener> listener;
+ { // scope for the lock
+ Mutex::Autolock _l(mMutex);
+ listener = mFrameAvailableListener;
+ }
+
+ if (listener != NULL) {
+ MC_LOGV("actually calling onFrameAvailable");
+ listener->onFrameAvailable();
+ }
+}
+
+void MediaConsumer::onBuffersReleased() {
+ MC_LOGV("onBuffersReleased");
+
+ Mutex::Autolock lock(mMutex);
+
+ uint32_t mask = 0;
+ mBufferQueue->getReleasedBuffers(&mask);
+ for (int i = 0; i < BufferQueue::NUM_BUFFER_SLOTS; i++) {
+ if (mask & (1 << i)) {
+ freeBufferLocked(i);
+ }
+ }
+
+}
+
+status_t MediaConsumer::freeBufferLocked(int buf) {
+ status_t err = OK;
+
+ mBufferSlot[buf] = NULL;
+ return err;
+}
+
+} // namespace android
diff --git a/services/camera/libcameraservice/MediaConsumer.h b/services/camera/libcameraservice/MediaConsumer.h
new file mode 100644
index 0000000..3377d94
--- /dev/null
+++ b/services/camera/libcameraservice/MediaConsumer.h
@@ -0,0 +1,126 @@
+/*
+ * Copyright (C) 2012 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef ANDROID_SERVERS_CAMERA_MEDIACONSUMER_H
+#define ANDROID_SERVERS_CAMERA_MEDIACONSUMER_H
+
+#include <gui/BufferQueue.h>
+
+#include <ui/GraphicBuffer.h>
+
+#include <utils/String8.h>
+#include <utils/Vector.h>
+#include <utils/threads.h>
+
+#define ANDROID_GRAPHICS_MEDIACONSUMER_JNI_ID "mMediaConsumer"
+
+namespace android {
+
+/**
+ * MediaConsumer is a BufferQueue consumer endpoint that makes it
+ * straightforward to bridge Camera 2 to the existing media recording framework.
+ * This queue is synchronous by default.
+ *
+ * TODO: This is a temporary replacement for the full camera->media recording
+ * path using SurfaceMediaEncoder or equivalent.
+ */
+
+class MediaConsumer: public virtual RefBase,
+ protected BufferQueue::ConsumerListener
+{
+ public:
+ struct FrameAvailableListener : public virtual RefBase {
+ // onFrameAvailable() is called each time an additional frame becomes
+ // available for consumption. A new frame queued will always trigger the
+ // callback, whether the queue is empty or not.
+ //
+ // This is called without any lock held and can be called concurrently
+ // by multiple threads.
+ virtual void onFrameAvailable() = 0;
+ };
+
+ // Create a new media consumer. The maxBuffers parameter specifies
+ // how many buffers can be locked for user access at the same time.
+ MediaConsumer(uint32_t maxBuffers);
+
+ virtual ~MediaConsumer();
+
+ // set the name of the MediaConsumer that will be used to identify it in
+ // log messages.
+ void setName(const String8& name);
+
+ // Gets the next graphics buffer from the producer. Returns BAD_VALUE if no
+ // new buffer is available, and INVALID_OPERATION if the maximum number of
+ // buffers is already in use.
+ //
+ // Only a fixed number of buffers can be available at a time, determined by
+ // the construction-time maxBuffers parameter. If INVALID_OPERATION is
+ // returned by getNextBuffer, then old buffers must be returned to the
+ // queue by calling freeBuffer before more buffers can be acquired.
+ status_t getNextBuffer(buffer_handle_t *buffer, nsecs_t *timestamp);
+
+ // Returns a buffer to the queue, allowing it to be reused. Since
+ // only a fixed number of buffers may be locked at a time, old buffers must
+ // be released by calling unlockBuffer to ensure new buffers can be acquired by
+ // lockNextBuffer.
+ status_t freeBuffer(buffer_handle_t buffer);
+
+ // setFrameAvailableListener sets the listener object that will be notified
+ // when a new frame becomes available.
+ void setFrameAvailableListener(const sp<FrameAvailableListener>& listener);
+
+ sp<ISurfaceTexture> getProducerInterface() const { return mBufferQueue; }
+ protected:
+
+ // Implementation of the BufferQueue::ConsumerListener interface. These
+ // calls are used to notify the MediaConsumer of asynchronous events in the
+ // BufferQueue.
+ virtual void onFrameAvailable();
+ virtual void onBuffersReleased();
+
+ private:
+ // Free local buffer state
+ status_t freeBufferLocked(int buf);
+
+ // Maximum number of buffers that can be locked at a time
+ uint32_t mMaxLockedBuffers;
+
+ // mName is a string used to identify the SurfaceTexture in log messages.
+ // It can be set by the setName method.
+ String8 mName;
+
+ // mFrameAvailableListener is the listener object that will be called when a
+ // new frame becomes available. If it is not NULL it will be called from
+ // queueBuffer.
+ sp<FrameAvailableListener> mFrameAvailableListener;
+
+ // Underlying buffer queue
+ sp<BufferQueue> mBufferQueue;
+
+ // Array for caching buffers from the buffer queue
+ sp<GraphicBuffer> mBufferSlot[BufferQueue::NUM_BUFFER_SLOTS];
+ // Count of currently outstanding buffers
+ uint32_t mCurrentLockedBuffers;
+
+ // mMutex is the mutex used to prevent concurrent access to the member
+ // variables of MediaConsumer objects. It must be locked whenever the
+ // member variables are accessed.
+ mutable Mutex mMutex;
+};
+
+} // namespace android
+
+#endif // ANDROID_SERVERS_CAMERA_MEDIACONSUMER_H