spatializer: integrate pose controller
Implement head tracking functions in Spatializer
class based on the SpatializerPoseController class.
Add new APIs to ISpatializer interface to:
- set head pose sensor
- set screen pose sensor
- set screen orientation
Add new APIs to INativeSpatializerCallback to:
- notify new head to stage pose
- notify new actual headtracking mode
Bug: 188502620
Test: manual test with mock spatializer
Change-Id: I4584df055cd91bc124d081861baa2fe83df69fa5
diff --git a/media/libaudioclient/Android.bp b/media/libaudioclient/Android.bp
index a1c1234..fbbd7ff 100644
--- a/media/libaudioclient/Android.bp
+++ b/media/libaudioclient/Android.bp
@@ -394,6 +394,7 @@
"aidl/android/media/DeviceRole.aidl",
"aidl/android/media/SoundTriggerSession.aidl",
"aidl/android/media/SpatializationLevel.aidl",
+ "aidl/android/media/SpatializationMode.aidl",
"aidl/android/media/SpatializerHeadTrackingMode.aidl",
],
imports: [
diff --git a/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl b/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl
index f34df05..73e897d 100644
--- a/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl
+++ b/media/libaudioclient/aidl/android/media/INativeSpatializerCallback.aidl
@@ -17,17 +17,29 @@
package android.media;
import android.media.SpatializationLevel;
+import android.media.SpatializerHeadTrackingMode;
/**
* The INativeSpatializerCallback interface is a callback associated to the
- * ISpatializer interface. The callback is used by the spatializer stage
- * implementation in native audio server to communicate stage changes to the
+ * ISpatializer interface. The callback is used by the spatializer
+ * implementation in native audio server to communicate state changes to the
* client controlling the spatializer with the ISpatializer interface.
* {@hide}
*/
-interface INativeSpatializerCallback {
- /** Called when the spatialization level applied by the vitualizer stage changes
+oneway interface INativeSpatializerCallback {
+ /** Called when the spatialization level applied by the spatializer changes
* (e.g. when the spatializer is enabled or disabled)
*/
- oneway void onLevelChanged(SpatializationLevel level);
+ void onLevelChanged(SpatializationLevel level);
+
+ /** Called when the head tracking mode has changed
+ */
+ void onHeadTrackingModeChanged(SpatializerHeadTrackingMode mode);
+
+ /** Called when the head to stage pose hase been updated
+ * The head to stage pose is conveyed as a vector of 6 elements,
+ * where the first three are a translation vector and
+ * the last three are a rotation vector.
+ */
+ void onHeadToSoundStagePoseUpdated(in float[] headToStage);
}
diff --git a/media/libaudioclient/aidl/android/media/ISpatializer.aidl b/media/libaudioclient/aidl/android/media/ISpatializer.aidl
index 3c2eda6..e5bf50c 100644
--- a/media/libaudioclient/aidl/android/media/ISpatializer.aidl
+++ b/media/libaudioclient/aidl/android/media/ISpatializer.aidl
@@ -17,8 +17,10 @@
package android.media;
import android.media.SpatializationLevel;
+import android.media.SpatializationMode;
import android.media.SpatializerHeadTrackingMode;
+
/**
* The ISpatializer interface is used to control the native audio service implementation
* of the spatializer stage with headtracking when present on a platform.
@@ -64,6 +66,41 @@
/** Reset the head tracking algorithm to consider current head pose as neutral */
void recenterHeadTracker();
- /** Set the screen to stage transform to use by the head tracking algorithm */
+ /** Set the screen to stage transform to use by the head tracking algorithm
+ * The screen to stage transform is conveyed as a vector of 6 elements,
+ * where the first three are a translation vector and
+ * the last three are a rotation vector.
+ */
void setGlobalTransform(in float[] screenToStage);
+
+ /**
+ * Set the sensor that is to be used for head-tracking.
+ * -1 can be used to disable head-tracking.
+ */
+ void setHeadSensor(int sensorHandle);
+
+ /**
+ * Set the sensor that is to be used for screen-tracking.
+ * -1 can be used to disable screen-tracking.
+ */
+ void setScreenSensor(int sensorHandle);
+
+ /**
+ * Sets the display orientation.
+ * Orientation is expressed in the angle of rotation from the physical "up" side of the screen
+ * to the logical "up" side of the content displayed the screen. Counterclockwise angles, as
+ * viewed while facing the screen are positive.
+ */
+ void setDisplayOrientation(float physicalToLogicalAngle);
+
+ /**
+ * Sets the hinge angle for foldable devices.
+ */
+ void setHingeAngle(float hingeAngle);
+
+ /** Reports the list of supported spatialization modess (see SpatializationMode.aidl).
+ * The list should never be empty if an ISpatializer interface was successfully
+ * retrieved with IAudioPolicyService.getSpatializer().
+ */
+ SpatializationMode[] getSupportedModes();
}
diff --git a/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl b/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl
index cef42bb..961c5a1 100644
--- a/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl
+++ b/media/libaudioclient/aidl/android/media/SpatializationLevel.aidl
@@ -17,7 +17,7 @@
package android.media;
/**
- * The spatialization level or mode supported by the spatializer stage effect implementation.
+ * The spatialization level supported by the spatializer stage effect implementation.
* Used by methods of the ISpatializer interface.
* {@hide}
*/
diff --git a/media/libaudioclient/aidl/android/media/SpatializationMode.aidl b/media/libaudioclient/aidl/android/media/SpatializationMode.aidl
new file mode 100644
index 0000000..5d8fd93
--- /dev/null
+++ b/media/libaudioclient/aidl/android/media/SpatializationMode.aidl
@@ -0,0 +1,30 @@
+/*
+ * Copyright 2021 The Android Open Source Project
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package android.media;
+
+/**
+ * The spatialization mode supported by the spatializer stage effect implementation.
+ * Used by methods of the ISpatializer interface.
+ * {@hide}
+ */
+@Backing(type="byte")
+enum SpatializationMode {
+ /** The spatializer supports binaural mode (over headphones type devices). */
+ SPATIALIZATER_BINAURAL = 0,
+ /** The spatializer supports transaural mode (over speaker type devices). */
+ SPATIALIZATER_TRANSAURAL = 1,
+}
diff --git a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
index dd91792..4bd1260 100644
--- a/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
+++ b/services/audiopolicy/service/AudioPolicyInterfaceImpl.cpp
@@ -2223,7 +2223,9 @@
const sp<media::INativeSpatializerCallback>& callback,
media::GetSpatializerResponse* _aidl_return) {
_aidl_return->spatializer = nullptr;
- LOG_ALWAYS_FATAL_IF(callback == nullptr);
+ if (callback == nullptr) {
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
if (mSpatializer != nullptr) {
RETURN_IF_BINDER_ERROR(
binderStatusFromStatusT(mSpatializer->registerCallback(callback)));
diff --git a/services/audiopolicy/service/Spatializer.cpp b/services/audiopolicy/service/Spatializer.cpp
index aa104a0..c5506a3 100644
--- a/services/audiopolicy/service/Spatializer.cpp
+++ b/services/audiopolicy/service/Spatializer.cpp
@@ -25,8 +25,10 @@
#include <sys/types.h>
#include <android/content/AttributionSourceState.h>
+#include <android/sensor.h>
#include <audio_utils/fixedfft.h>
#include <cutils/bitops.h>
+#include <hardware/sensors.h>
#include <media/ShmemCompat.h>
#include <media/audiohal/EffectsFactoryHalInterface.h>
#include <mediautils/ServiceUtilities.h>
@@ -40,8 +42,15 @@
using aidl_utils::binderStatusFromStatusT;
using android::content::AttributionSourceState;
using binder::Status;
+using media::HeadTrackingMode;
+using media::Pose3f;
using media::SpatializationLevel;
+using media::SpatializationMode;
using media::SpatializerHeadTrackingMode;
+using media::SensorPoseProvider;
+
+
+using namespace std::chrono_literals;
#define VALUE_OR_RETURN_BINDER_STATUS(x) \
({ auto _tmp = (x); \
@@ -84,18 +93,17 @@
if (status == NO_ERROR && effect != nullptr) {
spatializer = new Spatializer(descriptors[0], callback);
- // TODO: Read supported config from engine
- audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
- config.channel_mask = AUDIO_CHANNEL_OUT_5POINT1;
- spatializer->setAudioInConfig(config);
+ if (spatializer->loadEngineConfiguration(effect) != NO_ERROR) {
+ spatializer.clear();
+ }
}
return spatializer;
}
-Spatializer::Spatializer(effect_descriptor_t engineDescriptor,
- SpatializerPolicyCallback *callback)
- : mEngineDescriptor(engineDescriptor), mPolicyCallback(callback) {
+Spatializer::Spatializer(effect_descriptor_t engineDescriptor, SpatializerPolicyCallback* callback)
+ : mEngineDescriptor(engineDescriptor),
+ mPolicyCallback(callback) {
ALOGV("%s", __func__);
}
@@ -103,9 +111,51 @@
ALOGV("%s", __func__);
}
+status_t Spatializer::loadEngineConfiguration(sp<EffectHalInterface> effect) {
+ ALOGV("%s", __func__);
+
+ std::vector<bool> supportsHeadTracking;
+ status_t status = getHalParameter<false>(effect, SPATIALIZER_PARAM_HEADTRACKING_SUPPORTED,
+ &supportsHeadTracking);
+ if (status != NO_ERROR) {
+ return status;
+ }
+ mSupportsHeadTracking = supportsHeadTracking[0];
+
+ status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_LEVELS, &mLevels);
+ if (status != NO_ERROR) {
+ return status;
+ }
+ status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_SPATIALIZATION_MODES,
+ &mSpatializationModes);
+ if (status != NO_ERROR) {
+ return status;
+ }
+ status = getHalParameter<true>(effect, SPATIALIZER_PARAM_SUPPORTED_CHANNEL_MASKS,
+ &mChannelMasks);
+ if (status != NO_ERROR) {
+ return status;
+ }
+ return NO_ERROR;
+}
+
+/** Gets the channel mask, sampling rate and format set for the spatializer input. */
+audio_config_base_t Spatializer::getAudioInConfig() const {
+ std::lock_guard lock(mLock);
+ audio_config_base_t config = AUDIO_CONFIG_BASE_INITIALIZER;
+ // For now use highest supported channel count
+ uint32_t maxCount = 0;
+ for ( auto mask : mChannelMasks) {
+ if (audio_channel_count_from_out_mask(mask) > maxCount) {
+ config.channel_mask = mask;
+ }
+ }
+ return config;
+}
+
status_t Spatializer::registerCallback(
const sp<media::INativeSpatializerCallback>& callback) {
- Mutex::Autolock _l(mLock);
+ std::lock_guard lock(mLock);
if (callback == nullptr) {
return BAD_VALUE;
}
@@ -122,7 +172,7 @@
// IBinder::DeathRecipient
void Spatializer::binderDied(__unused const wp<IBinder> &who) {
{
- Mutex::Autolock _l(mLock);
+ std::lock_guard lock(mLock);
mLevel = SpatializationLevel::NONE;
mSpatializerCallback.clear();
}
@@ -136,25 +186,28 @@
if (levels == nullptr) {
return binderStatusFromStatusT(BAD_VALUE);
}
- //TODO: get this from engine
levels->push_back(SpatializationLevel::NONE);
- levels->push_back(SpatializationLevel::SPATIALIZER_MULTICHANNEL);
+ levels->insert(levels->end(), mLevels.begin(), mLevels.end());
return Status::ok();
}
-Status Spatializer::setLevel(media::SpatializationLevel level) {
+Status Spatializer::setLevel(SpatializationLevel level) {
ALOGV("%s level %d", __func__, (int)level);
if (level != SpatializationLevel::NONE
- && level != SpatializationLevel::SPATIALIZER_MULTICHANNEL) {
+ && std::find(mLevels.begin(), mLevels.end(), level) == mLevels.end()) {
return binderStatusFromStatusT(BAD_VALUE);
}
sp<media::INativeSpatializerCallback> callback;
bool levelChanged = false;
{
- Mutex::Autolock _l(mLock);
+ std::lock_guard lock(mLock);
levelChanged = mLevel != level;
mLevel = level;
callback = mSpatializerCallback;
+
+ if (levelChanged && mEngine != nullptr) {
+ setEffectParameter_l(SPATIALIZER_PARAM_LEVEL, std::vector<SpatializationLevel>{level});
+ }
}
if (levelChanged) {
@@ -166,61 +219,93 @@
return Status::ok();
}
-Status Spatializer::getLevel(media::SpatializationLevel *level) {
+Status Spatializer::getLevel(SpatializationLevel *level) {
if (level == nullptr) {
return binderStatusFromStatusT(BAD_VALUE);
}
- Mutex::Autolock _l(mLock);
+ std::lock_guard lock(mLock);
*level = mLevel;
ALOGV("%s level %d", __func__, (int)*level);
return Status::ok();
}
Status Spatializer::getSupportedHeadTrackingModes(
- std::vector<media::SpatializerHeadTrackingMode>* modes) {
+ std::vector<SpatializerHeadTrackingMode>* modes) {
+ std::lock_guard lock(mLock);
ALOGV("%s", __func__);
if (modes == nullptr) {
return binderStatusFromStatusT(BAD_VALUE);
}
- //TODO: get this from:
- // - The engine capabilities
- // - If a head tracking sensor is registered and linked to a connected audio device
- // - if we have indications on the screen orientation
- modes->push_back(SpatializerHeadTrackingMode::RELATIVE_WORLD);
- return Status::ok();
-}
-Status Spatializer::setDesiredHeadTrackingMode(media::SpatializerHeadTrackingMode mode) {
- ALOGV("%s level %d", __func__, (int)mode);
- if (mode != SpatializerHeadTrackingMode::DISABLED
- && mode != SpatializerHeadTrackingMode::RELATIVE_WORLD) {
- return binderStatusFromStatusT(BAD_VALUE);
- }
- {
- Mutex::Autolock _l(mLock);
- mHeadTrackingMode = mode;
+ modes->push_back(SpatializerHeadTrackingMode::DISABLED);
+ if (mSupportsHeadTracking) {
+ if (mHeadSensor != nullptr) {
+ modes->push_back(SpatializerHeadTrackingMode::RELATIVE_WORLD);
+ if (mScreenSensor != nullptr) {
+ modes->push_back(SpatializerHeadTrackingMode::RELATIVE_SCREEN);
+ }
+ }
}
return Status::ok();
}
-Status Spatializer::getActualHeadTrackingMode(media::SpatializerHeadTrackingMode *mode) {
+Status Spatializer::setDesiredHeadTrackingMode(SpatializerHeadTrackingMode mode) {
+ ALOGV("%s mode %d", __func__, (int)mode);
+
+ if (!mSupportsHeadTracking) {
+ return binderStatusFromStatusT(INVALID_OPERATION);
+ }
+ std::lock_guard lock(mLock);
+ switch (mode) {
+ case SpatializerHeadTrackingMode::OTHER:
+ return binderStatusFromStatusT(BAD_VALUE);
+ case SpatializerHeadTrackingMode::DISABLED:
+ mDesiredHeadTrackingMode = HeadTrackingMode::STATIC;
+ break;
+ case SpatializerHeadTrackingMode::RELATIVE_WORLD:
+ mDesiredHeadTrackingMode = HeadTrackingMode::WORLD_RELATIVE;
+ break;
+ case SpatializerHeadTrackingMode::RELATIVE_SCREEN:
+ mDesiredHeadTrackingMode = HeadTrackingMode::SCREEN_RELATIVE;
+ break;
+ }
+
+ if (mPoseController != nullptr) {
+ mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
+ }
+
+ return Status::ok();
+}
+
+Status Spatializer::getActualHeadTrackingMode(SpatializerHeadTrackingMode *mode) {
if (mode == nullptr) {
return binderStatusFromStatusT(BAD_VALUE);
}
- Mutex::Autolock _l(mLock);
- *mode = mHeadTrackingMode;
+ std::lock_guard lock(mLock);
+ *mode = mActualHeadTrackingMode;
ALOGV("%s mode %d", __func__, (int)*mode);
return Status::ok();
}
Status Spatializer::recenterHeadTracker() {
+ std::lock_guard lock(mLock);
+ if (mPoseController != nullptr) {
+ mPoseController->recenter();
+ }
return Status::ok();
}
Status Spatializer::setGlobalTransform(const std::vector<float>& screenToStage) {
- Mutex::Autolock _l(mLock);
- mScreenToStageTransform = screenToStage;
ALOGV("%s", __func__);
+ std::optional<Pose3f> maybePose = Pose3f::fromVector(screenToStage);
+ if (!maybePose.has_value()) {
+ ALOGW("Invalid screenToStage vector.");
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
+ std::lock_guard lock(mLock);
+ if (mPoseController != nullptr) {
+ mPoseController->setScreenToStagePose(maybePose.value());
+ }
return Status::ok();
}
@@ -228,7 +313,7 @@
ALOGV("%s", __func__);
bool levelChanged = false;
{
- Mutex::Autolock _l(mLock);
+ std::lock_guard lock(mLock);
if (mSpatializerCallback == nullptr) {
return binderStatusFromStatusT(INVALID_OPERATION);
}
@@ -247,56 +332,212 @@
return Status::ok();
}
+Status Spatializer::setHeadSensor(int sensorHandle) {
+ ALOGV("%s sensorHandle %d", __func__, sensorHandle);
+ std::lock_guard lock(mLock);
+ if (sensorHandle == ASENSOR_INVALID) {
+ mHeadSensor = nullptr;
+ } else {
+ mHeadSensor = VALUE_OR_RETURN_BINDER_STATUS(getSensorFromHandle(sensorHandle));
+ }
+ if (mPoseController != nullptr) {
+ mPoseController->setHeadSensor(mHeadSensor);
+ }
+ return Status::ok();
+}
+
+Status Spatializer::setScreenSensor(int sensorHandle) {
+ ALOGV("%s sensorHandle %d", __func__, sensorHandle);
+ std::lock_guard lock(mLock);
+ if (sensorHandle == ASENSOR_INVALID) {
+ mScreenSensor = nullptr;
+ } else {
+ mScreenSensor = VALUE_OR_RETURN_BINDER_STATUS(getSensorFromHandle(sensorHandle));
+ }
+ if (mPoseController != nullptr) {
+ mPoseController->setScreenSensor(mScreenSensor);
+ }
+ return Status::ok();
+}
+
+Status Spatializer::setDisplayOrientation(float physicalToLogicalAngle) {
+ ALOGV("%s physicalToLogicalAngle %f", __func__, physicalToLogicalAngle);
+ std::lock_guard lock(mLock);
+ mDisplayOrientation = physicalToLogicalAngle;
+ if (mPoseController != nullptr) {
+ mPoseController->setDisplayOrientation(mDisplayOrientation);
+ }
+ return Status::ok();
+}
+
+Status Spatializer::setHingeAngle(float hingeAngle) {
+ std::lock_guard lock(mLock);
+ ALOGV("%s hingeAngle %f", __func__, hingeAngle);
+ if (mEngine != nullptr) {
+ setEffectParameter_l(SPATIALIZER_PARAM_HINGE_ANGLE, std::vector<float>{hingeAngle});
+ }
+ return Status::ok();
+}
+
+Status Spatializer::getSupportedModes(std::vector<SpatializationMode> *modes) {
+ ALOGV("%s", __func__);
+ if (modes == nullptr) {
+ return binderStatusFromStatusT(BAD_VALUE);
+ }
+ *modes = mSpatializationModes;
+ return Status::ok();
+}
+
+// SpatializerPoseController::Listener
+void Spatializer::onHeadToStagePose(const Pose3f& headToStage) {
+ ALOGV("%s", __func__);
+ sp<media::INativeSpatializerCallback> callback;
+ auto vec = headToStage.toVector();
+ {
+ std::lock_guard lock(mLock);
+ callback = mSpatializerCallback;
+ if (mEngine != nullptr) {
+ setEffectParameter_l(SPATIALIZER_PARAM_HEAD_TO_STAGE, vec);
+ }
+ }
+
+ if (callback != nullptr) {
+ callback->onHeadToSoundStagePoseUpdated(vec);
+ }
+}
+
+void Spatializer::onActualModeChange(HeadTrackingMode mode) {
+ ALOGV("onActualModeChange(%d)", (int) mode);
+ sp<media::INativeSpatializerCallback> callback;
+ SpatializerHeadTrackingMode spatializerMode;
+ {
+ std::lock_guard lock(mLock);
+ if (!mSupportsHeadTracking) {
+ spatializerMode = SpatializerHeadTrackingMode::DISABLED;
+ } else {
+ switch (mode) {
+ case HeadTrackingMode::STATIC:
+ spatializerMode = SpatializerHeadTrackingMode::DISABLED;
+ break;
+ case HeadTrackingMode::WORLD_RELATIVE:
+ spatializerMode = SpatializerHeadTrackingMode::RELATIVE_WORLD;
+ break;
+ case HeadTrackingMode::SCREEN_RELATIVE:
+ spatializerMode = SpatializerHeadTrackingMode::RELATIVE_SCREEN;
+ break;
+ default:
+ LOG_ALWAYS_FATAL("Unknown mode: %d", mode);
+ }
+ }
+ mActualHeadTrackingMode = spatializerMode;
+ callback = mSpatializerCallback;
+ }
+ if (callback != nullptr) {
+ callback->onHeadTrackingModeChanged(spatializerMode);
+ }
+}
+
+/* static */
+ConversionResult<ASensorRef> Spatializer::getSensorFromHandle(int handle) {
+ ASensorManager* sensorManager =
+ ASensorManager_getInstanceForPackage("headtracker");
+ if (!sensorManager) {
+ ALOGE("Failed to get a sensor manager");
+ return base::unexpected(NO_INIT);
+ }
+ ASensorList sensorList;
+ int numSensors = ASensorManager_getSensorList(sensorManager, &sensorList);
+ for (int i = 0; i < numSensors; ++i) {
+ if (ASensor_getHandle(sensorList[i]) == handle) {
+ return sensorList[i];
+ }
+ }
+ return base::unexpected(BAD_VALUE);
+}
+
status_t Spatializer::attachOutput(audio_io_handle_t output) {
- Mutex::Autolock _l(mLock);
- ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
- if (mOutput != AUDIO_IO_HANDLE_NONE) {
- LOG_ALWAYS_FATAL_IF(mEngine != nullptr, "%s output set without FX engine", __func__);
- // remove FX instance
- mEngine->setEnabled(false);
- mEngine.clear();
+ std::shared_ptr<SpatializerPoseController> poseController;
+ {
+ std::lock_guard lock(mLock);
+ ALOGV("%s output %d mOutput %d", __func__, (int)output, (int)mOutput);
+ if (mOutput != AUDIO_IO_HANDLE_NONE) {
+ LOG_ALWAYS_FATAL_IF(mEngine == nullptr, "%s output set without FX engine", __func__);
+ // remove FX instance
+ mEngine->setEnabled(false);
+ mEngine.clear();
+ }
+ // create FX instance on output
+ AttributionSourceState attributionSource = AttributionSourceState();
+ mEngine = new AudioEffect(attributionSource);
+ mEngine->set(nullptr, &mEngineDescriptor.uuid, 0, Spatializer::engineCallback /* cbf */,
+ this /* user */, AUDIO_SESSION_OUTPUT_STAGE, output, {} /* device */,
+ false /* probe */, true /* notifyFramesProcessed */);
+ status_t status = mEngine->initCheck();
+ ALOGV("%s mEngine create status %d", __func__, (int)status);
+ if (status != NO_ERROR) {
+ return status;
+ }
+
+ setEffectParameter_l(SPATIALIZER_PARAM_LEVEL,
+ std::vector<SpatializationLevel>{mLevel});
+ setEffectParameter_l(SPATIALIZER_PARAM_HEADTRACKING_MODE,
+ std::vector<SpatializerHeadTrackingMode>{mActualHeadTrackingMode});
+
+ mEngine->setEnabled(true);
+ mOutput = output;
+
+ mPoseController = std::make_shared<SpatializerPoseController>(
+ static_cast<SpatializerPoseController::Listener*>(this), 10ms, 50ms);
+ LOG_ALWAYS_FATAL_IF(mPoseController == nullptr,
+ "%s could not allocate pose controller", __func__);
+
+ mPoseController->setDesiredMode(mDesiredHeadTrackingMode);
+ mPoseController->setHeadSensor(mHeadSensor);
+ mPoseController->setScreenSensor(mScreenSensor);
+ mPoseController->setDisplayOrientation(mDisplayOrientation);
+ poseController = mPoseController;
}
- // create FX instance on output
- AttributionSourceState attributionSource = AttributionSourceState();
- mEngine = new AudioEffect(attributionSource);
- mEngine->set(nullptr, &mEngineDescriptor.uuid, 0, Spatializer::engineCallback /* cbf */,
- this /* user */, AUDIO_SESSION_OUTPUT_STAGE, output, {} /* device */,
- false /* probe */, true /* notifyFramesProcessed */);
- status_t status = mEngine->initCheck();
- ALOGV("%s mEngine create status %d", __func__, (int)status);
- if (status != NO_ERROR) {
- return status;
- }
- mEngine->setEnabled(true);
- mOutput = output;
+ poseController->waitUntilCalculated();
return NO_ERROR;
}
audio_io_handle_t Spatializer::detachOutput() {
- Mutex::Autolock _l(mLock);
+ std::lock_guard lock(mLock);
ALOGV("%s mOutput %d", __func__, (int)mOutput);
+ audio_io_handle_t output = AUDIO_IO_HANDLE_NONE;
if (mOutput == AUDIO_IO_HANDLE_NONE) {
- return AUDIO_IO_HANDLE_NONE;
+ return output;
}
// remove FX instance
mEngine->setEnabled(false);
mEngine.clear();
- audio_io_handle_t output = mOutput;
+ output = mOutput;
mOutput = AUDIO_IO_HANDLE_NONE;
+ mPoseController.reset();
return output;
}
-void Spatializer::engineCallback(int32_t event, void *user, void *info) {
+void Spatializer::calculateHeadPose() {
+ ALOGV("%s", __func__);
+ std::lock_guard lock(mLock);
+ if (mPoseController != nullptr) {
+ mPoseController->calculateAsync();
+ }
+}
+void Spatializer::engineCallback(int32_t event, void *user, void *info) {
if (user == nullptr) {
return;
}
- const Spatializer * const me = reinterpret_cast<Spatializer *>(user);
+ Spatializer* const me = reinterpret_cast<Spatializer *>(user);
switch (event) {
case AudioEffect::EVENT_FRAMES_PROCESSED: {
- int frames = info == nullptr ? 0 : *(int *)info;
+ int frames = info == nullptr ? 0 : *(int*)info;
ALOGD("%s frames processed %d for me %p", __func__, frames, me);
- } break;
+ if (frames > 0) {
+ me->calculateHeadPose();
+ }
+ } break;
default:
ALOGD("%s event %d", __func__, event);
break;
diff --git a/services/audiopolicy/service/Spatializer.h b/services/audiopolicy/service/Spatializer.h
index 768170a..a45290b 100644
--- a/services/audiopolicy/service/Spatializer.h
+++ b/services/audiopolicy/service/Spatializer.h
@@ -19,12 +19,15 @@
#include <android/media/BnEffect.h>
#include <android/media/BnSpatializer.h>
-#include <android/media/SpatializerHeadTrackingMode.h>
#include <android/media/SpatializationLevel.h>
-
+#include <android/media/SpatializationMode.h>
+#include <android/media/SpatializerHeadTrackingMode.h>
+#include <android/sensor.h>
+#include <media/audiohal/EffectHalInterface.h>
#include <media/AudioEffect.h>
#include <system/audio_effects/effect_spatializer.h>
+#include "SpatializerPoseController.h"
namespace android {
@@ -80,9 +83,10 @@
* made to audio policy manager to release and close the spatializer output stream and the
* spatializer mixer thread is destroyed.
*/
-class Spatializer : public media::BnSpatializer, public IBinder::DeathRecipient {
-public:
-
+class Spatializer : public media::BnSpatializer,
+ public IBinder::DeathRecipient,
+ private SpatializerPoseController::Listener {
+ public:
static sp<Spatializer> create(SpatializerPolicyCallback *callback);
~Spatializer() override;
@@ -100,6 +104,12 @@
media::SpatializerHeadTrackingMode* mode) override;
binder::Status recenterHeadTracker() override;
binder::Status setGlobalTransform(const std::vector<float>& screenToStage) override;
+ binder::Status setHeadSensor(int sensorHandle) override;
+ binder::Status setScreenSensor(int sensorHandle) override;
+ binder::Status setDisplayOrientation(float physicalToLogicalAngle) override;
+ binder::Status setHingeAngle(float hingeAngle) override;
+ binder::Status getSupportedModes(std::vector<media::SpatializationMode>* modes) override;
+
/** IBinder::DeathRecipient. Listen to the death of the INativeSpatializerCallback. */
virtual void binderDied(const wp<IBinder>& who);
@@ -109,8 +119,10 @@
*/
status_t registerCallback(const sp<media::INativeSpatializerCallback>& callback);
+ status_t loadEngineConfiguration(sp<EffectHalInterface> effect);
+
/** Level getter for use by local classes. */
- media::SpatializationLevel getLevel() const { Mutex::Autolock _l(mLock); return mLevel; }
+ media::SpatializationLevel getLevel() const { std::lock_guard lock(mLock); return mLevel; }
/** Called by audio policy service when the special output mixer dedicated to spatialization
* is opened and the spatializer engine must be created.
@@ -121,19 +133,10 @@
*/
audio_io_handle_t detachOutput();
/** Returns the output stream the spatializer is attached to. */
- audio_io_handle_t getOutput() const { Mutex::Autolock _l(mLock); return mOutput; }
-
- /** Sets the channel mask, sampling rate and format for the spatializer input. */
- void setAudioInConfig(const audio_config_base_t& config) {
- Mutex::Autolock _l(mLock);
- mAudioInConfig = config;
- }
+ audio_io_handle_t getOutput() const { std::lock_guard lock(mLock); return mOutput; }
/** Gets the channel mask, sampling rate and format set for the spatializer input. */
- audio_config_base_t getAudioInConfig() const {
- Mutex::Autolock _l(mLock);
- return mAudioInConfig;
- }
+ audio_config_base_t getAudioInConfig() const;
/** An implementation of an IEffect interface that can be used to pass advanced parameters to
* the spatializer engine. All APis are noop (i.e. the interface cannot be used to control
@@ -164,41 +167,137 @@
};
private:
-
Spatializer(effect_descriptor_t engineDescriptor,
SpatializerPolicyCallback *callback);
-
static void engineCallback(int32_t event, void* user, void *info);
+ // From VirtualizerStageController::Listener
+ void onHeadToStagePose(const media::Pose3f& headToStage) override;
+ void onActualModeChange(media::HeadTrackingMode mode) override;
+
+ void calculateHeadPose();
+
+ static ConversionResult<ASensorRef> getSensorFromHandle(int handle);
+
+ static constexpr int kMaxEffectParamValues = 10;
+ /**
+ * Get a parameter from spatializer engine by calling the effect HAL command method directly.
+ * To be used when the engine instance mEngine is not yet created in the effect framework.
+ * When MULTI_VALUES is false, the expected reply is only one value of type T.
+ * When MULTI_VALUES is true, the expected reply is made of a number (of type T) indicating
+ * how many values are returned, followed by this number for values of type T.
+ */
+ template<bool MULTI_VALUES, typename T>
+ status_t getHalParameter(sp<EffectHalInterface> effect, uint32_t type,
+ std::vector<T> *values) {
+ static_assert(sizeof(T) <= sizeof(uint32_t), "The size of T must less than 32 bits");
+
+ uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 1];
+ uint32_t reply[sizeof(effect_param_t) / sizeof(uint32_t) + 2 + kMaxEffectParamValues];
+
+ effect_param_t *p = (effect_param_t *)cmd;
+ p->psize = sizeof(uint32_t);
+ if (MULTI_VALUES) {
+ p->vsize = (kMaxEffectParamValues + 1) * sizeof(T);
+ } else {
+ p->vsize = sizeof(T);
+ }
+ *(uint32_t *)p->data = type;
+ uint32_t replySize = sizeof(effect_param_t) + p->psize + p->vsize;
+
+ status_t status = effect->command(EFFECT_CMD_GET_PARAM,
+ sizeof(effect_param_t) + sizeof(uint32_t), cmd,
+ &replySize, reply);
+ if (status != NO_ERROR) {
+ return status;
+ }
+ if (p->status != NO_ERROR) {
+ return p->status;
+ }
+ if (replySize <
+ sizeof(effect_param_t) + sizeof(uint32_t) + (MULTI_VALUES ? 2 : 1) * sizeof(T)) {
+ return BAD_VALUE;
+ }
+
+ T *params = (T *)((uint8_t *)reply + sizeof(effect_param_t) + sizeof(uint32_t));
+ int numParams = 1;
+ if (MULTI_VALUES) {
+ numParams = (int)*params++;
+ }
+ if (numParams > kMaxEffectParamValues) {
+ return BAD_VALUE;
+ }
+ std::copy(¶ms[0], ¶ms[numParams], back_inserter(*values));
+ return NO_ERROR;
+ }
+
+ /**
+ * Set a parameter to spatializer engine by calling setParameter on mEngine AudioEffect object.
+ * It is possible to pass more than one value of type T according to the parameter type
+ * according to values vector size.
+ */
+ template<typename T>
+ status_t setEffectParameter_l(uint32_t type, const std::vector<T>& values) REQUIRES(mLock) {
+ static_assert(sizeof(T) <= sizeof(uint32_t), "The size of T must less than 32 bits");
+
+ uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 1 + values.size()];
+ effect_param_t *p = (effect_param_t *)cmd;
+ p->psize = sizeof(uint32_t);
+ p->vsize = sizeof(T) * values.size();
+ *(uint32_t *)p->data = type;
+ memcpy((uint32_t *)p->data + 1, values.data(), sizeof(T) * values.size());
+
+ return mEngine->setParameter(p);
+ }
+
/** Effect engine descriptor */
const effect_descriptor_t mEngineDescriptor;
/** Callback interface to parent audio policy service */
SpatializerPolicyCallback* mPolicyCallback;
/** Mutex protecting internal state */
- mutable Mutex mLock;
+ mutable std::mutex mLock;
/** Client AudioEffect for the engine */
sp<AudioEffect> mEngine GUARDED_BY(mLock);
/** Output stream the spatializer mixer thread is attached to */
audio_io_handle_t mOutput GUARDED_BY(mLock) = AUDIO_IO_HANDLE_NONE;
- /** Virtualizer engine input configuration */
- audio_config_base_t mAudioInConfig GUARDED_BY(mLock) = AUDIO_CONFIG_BASE_INITIALIZER;
/** Callback interface to the client (AudioService) controlling this`Spatializer */
sp<media::INativeSpatializerCallback> mSpatializerCallback GUARDED_BY(mLock);
/** Requested spatialization level */
media::SpatializationLevel mLevel GUARDED_BY(mLock) = media::SpatializationLevel::NONE;
- /** Requested head tracking mode */
- media::SpatializerHeadTrackingMode mHeadTrackingMode GUARDED_BY(mLock)
- = media::SpatializerHeadTrackingMode::DISABLED;
- /** Configured screen to stage transform */
- std::vector<float> mScreenToStageTransform GUARDED_BY(mLock);
/** Extended IEffect interface is one has been created */
sp<EffectClient> mEffectClient GUARDED_BY(mLock);
+
+ /** Control logic for head-tracking, etc. */
+ std::shared_ptr<SpatializerPoseController> mPoseController GUARDED_BY(mLock);
+
+ /** Last requested head tracking mode */
+ media::HeadTrackingMode mDesiredHeadTrackingMode GUARDED_BY(mLock)
+ = media::HeadTrackingMode::STATIC;
+
+ /** Last-reported actual head-tracking mode. */
+ media::SpatializerHeadTrackingMode mActualHeadTrackingMode GUARDED_BY(mLock)
+ = media::SpatializerHeadTrackingMode::DISABLED;
+
+ /** Selected Head pose sensor */
+ ASensorRef mHeadSensor GUARDED_BY(mLock) = nullptr;
+
+ /** Selected Screen pose sensor */
+ ASensorRef mScreenSensor GUARDED_BY(mLock) = nullptr;
+
+ /** Last display orientation received */
+ static constexpr float kDisplayOrientationInvalid = 1000;
+ float mDisplayOrientation GUARDED_BY(mLock) = kDisplayOrientationInvalid;
+
+ std::vector<media::SpatializationLevel> mLevels;
+ std::vector<media::SpatializationMode> mSpatializationModes;
+ std::vector<audio_channel_mask_t> mChannelMasks;
+ bool mSupportsHeadTracking;
};